diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9c105f4c4c..dbf3f28d91b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -590,39 +590,6 @@ jobs: run: | cargo ci cli-docs - llm_ci_check: - name: Verify LLM benchmark is up to date - permissions: - contents: read - runs-on: ubuntu-latest - # Disable the tests because they are causing us headaches with merge conflicts and re-runs etc. - if: false - steps: - # Build the tool from master to ensure consistent hash computation - # with the llm-benchmark-update workflow (which also uses master's tool). - - name: Checkout master (build tool from trusted code) - uses: actions/checkout@v4 - with: - ref: master - fetch-depth: 1 - - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - - name: Install llm-benchmark tool from master - run: | - cargo install --path tools/xtask-llm-benchmark --locked - command -v llm_benchmark - - # Now checkout the PR branch to verify its benchmark files - - name: Checkout PR branch - uses: actions/checkout@v4 - with: - clean: false - - - name: Run hash check (both langs) - run: llm_benchmark ci-check - unity-testsuite: needs: [lints] # Skip if this is an external contribution. diff --git a/.github/workflows/docs-update-llms.yaml b/.github/workflows/docs-update-llms.yaml new file mode 100644 index 00000000000..d7d2751cd68 --- /dev/null +++ b/.github/workflows/docs-update-llms.yaml @@ -0,0 +1,67 @@ +name: Docs / Update llms files + +permissions: + contents: write + +on: + push: + branches: + - docs/release + paths: + - 'docs/docs/**' + - 'skills/**' + workflow_dispatch: # Allow manual trigger + +jobs: + update-llms: + runs-on: spacetimedb-new-runner-2 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + ref: docs/release + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '22' + + - uses: pnpm/action-setup@v4 + with: + run_install: true + + - name: Get pnpm store directory + working-directory: sdks/typescript + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - uses: actions/cache@v4 + name: Setup pnpm cache + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + working-directory: docs + run: pnpm install + + - name: Docusaurus build + working-directory: docs + run: pnpm build + + - name: Generate llms files + working-directory: docs + run: node scripts/generate-llms.mjs + + - name: Commit updated llms files + working-directory: docs + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add static/llms.md + git diff --staged --quiet && echo "No changes" && exit 0 + git commit -m "Update llms files from docs build" + git push diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml new file mode 100644 index 00000000000..c043611a49a --- /dev/null +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -0,0 +1,118 @@ +name: Periodic LLM benchmarks + +on: + schedule: + # Daily at midnight UTC. Change to '0 */6 * * *' for every 6h, + # or '0 */4 * * *' for every 4h. + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + models: + description: 'Models to run (provider:model format, comma-separated, or "all")' + required: false + default: 'all' + languages: + description: 'Languages to benchmark (comma-separated: rust,csharp,typescript)' + required: false + default: 'rust,csharp,typescript' + modes: + description: 'Modes to run (comma-separated: guidelines,no_context,docs,...)' + required: false + default: 'guidelines,no_context' + +permissions: + contents: read + +concurrency: + group: llm-benchmark-periodic + cancel-in-progress: true + +jobs: + run-benchmarks: + runs-on: spacetimedb-new-runner + container: + image: localhost:5000/spacetimedb-ci:latest + options: >- + --privileged + timeout-minutes: 180 + + steps: + - name: Install spacetime CLI + run: | + curl -sSf https://install.spacetimedb.com | sh -s -- -y + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Checkout master + uses: actions/checkout@v4 + with: + ref: master + fetch-depth: 1 + + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - name: Install WASI workload + env: + DOTNET_MULTILEVEL_LOOKUP: "0" + DOTNET_CLI_HOME: ${{ runner.temp }}/dotnet-home + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" + run: | + dotnet workload install wasi-experimental --skip-manifest-update --disable-parallel + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Build llm-benchmark tool + run: cargo install --path tools/xtask-llm-benchmark --locked + + - name: Run benchmarks + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + LLM_BENCHMARK_API_KEY: ${{ secrets.LLM_BENCHMARK_API_KEY }} + LLM_BENCHMARK_UPLOAD_URL: ${{ secrets.LLM_BENCHMARK_UPLOAD_URL }} + MSBUILDDISABLENODEREUSE: "1" + DOTNET_CLI_USE_MSBUILD_SERVER: "0" + INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }} + INPUT_MODELS: ${{ inputs.models || 'all' }} + INPUT_MODES: ${{ inputs.modes || 'guidelines,no_context' }} + run: | + LANGS="$INPUT_LANGUAGES" + MODELS="$INPUT_MODELS" + MODES="$INPUT_MODES" + + SUCCEEDED=0 + FAILED=0 + for LANG in $(echo "$LANGS" | tr ',' ' '); do + if [ "$MODELS" = "all" ]; then + if llm_benchmark run --lang "$LANG" --modes "$MODES"; then + SUCCEEDED=$((SUCCEEDED + 1)) + else + echo "::warning::Benchmark run failed for lang=$LANG" + FAILED=$((FAILED + 1)) + fi + else + if llm_benchmark run --lang "$LANG" --modes "$MODES" --models "$MODELS"; then + SUCCEEDED=$((SUCCEEDED + 1)) + else + echo "::warning::Benchmark run failed for lang=$LANG models=$MODELS" + FAILED=$((FAILED + 1)) + fi + fi + done + echo "Benchmark runs: $SUCCEEDED succeeded, $FAILED failed" + if [ "$SUCCEEDED" -eq 0 ] && [ "$FAILED" -gt 0 ]; then + echo "::error::All benchmark runs failed" + exit 1 + fi diff --git a/.github/workflows/llm-benchmark-update.yml b/.github/workflows/llm-benchmark-update.yml deleted file mode 100644 index 8d16cd662ed..00000000000 --- a/.github/workflows/llm-benchmark-update.yml +++ /dev/null @@ -1,312 +0,0 @@ -name: Update LLM benchmarks - -on: - workflow_dispatch: - inputs: - pr_number: - description: "Pull Request Number" - required: true - issue_comment: - types: [created] # only run when the comment is first created - -permissions: - contents: read - pull-requests: write - issues: write - -concurrency: - group: >- - llm-benchmark - -${{ github.event_name == 'issue_comment' && github.event.issue.number || inputs.pr_number }} - ${{ github.event_name == 'issue_comment' && !startsWith(github.event.comment.body, '/update-llm-benchmark') && '-unrelated-comment' }} - cancel-in-progress: true - -jobs: - update-llm-benchmark: - # Runnable either with a comment that starts with /update-llm-benchmark - # or by manually dispatching - if: | - (github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, '/update-llm-benchmark')) || - (github.event_name == 'workflow_dispatch') - runs-on: spacetimedb-new-runner - container: - image: localhost:5000/spacetimedb-ci:latest - options: >- - --privileged - steps: - # Here we install the spacetime CLI for faster execution of the tests - # SpacetimeDB itself is not under test here, rather it's the docs. - # If we want to change that it is possible to have the benchmark compile - # SpacetimeDB from source. - - name: Install spacetime CLI - run: | - curl -sSf https://install.spacetimedb.com | sh -s -- -y - echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Load PR info - id: pr - uses: actions/github-script@v7 - with: - script: | - let prNumber; - if (context.eventName === 'issue_comment') { - prNumber = context.payload.issue.number; - } else if (context.eventName === 'workflow_dispatch') { - const raw = context.payload.inputs?.pr_number; - if (!raw || !/^\d+$/.test(raw)) { - core.setFailed(`Invalid pr_number input: '${raw}'.`); - return; - } - prNumber = Number(raw); - } else { - core.setFailed(`Unsupported event: ${context.eventName}`); - return; - } - - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - - core.setOutput('number', String(prNumber)); - core.setOutput('head_ref', pr.head.ref); - core.setOutput('head_sha', pr.head.sha); - core.setOutput('head_repo_full_name', pr.head.repo.full_name); - core.setOutput('head_owner_type', pr.head.repo.owner.type); // "User"|"Organization" - core.setOutput('maintainer_can_modify', String(pr.maintainer_can_modify)); - - # If this was kicked off by a comment, ensure that the commenter is - # a collaborator on the repo. We don't want unprivileged users to run benchmarks. - # Note that the workflow that will be run will be the one that is on the `master` - # branch, NOT the one from the PR. This is important so that the PR author can't - # sneak in an exfiltration exploit. - - name: Check commenter permission - if: github.event_name == 'issue_comment' - uses: actions/github-script@v7 - with: - script: | - const user = context.payload.comment.user.login; - const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner: context.repo.owner, - repo: context.repo.repo, - username: user, - }); - - const allowed = new Set(['admin', 'maintain', 'write', 'triage']); - if (!allowed.has(data.permission)) { - core.setFailed(`User ${user} has permission '${data.permission}', not allowed to run benchmarks.`); - } - - # If the PR is from a fork, we need to be able to have GitHub actions commit back - # to the forked repo, so that we can update the benchmark results. - # In order to do this we need to ensure that the PR is configured to allow the maintainers - # of the SpacetimeDB repo to commit back ot the fork. - - name: Check fork pushability (and comment if not) - if: steps.pr.outputs.head_repo_full_name != github.repository - uses: actions/github-script@v7 - env: - PR_NUMBER: ${{ steps.pr.outputs.number }} - HEAD_OWNER_TYPE: ${{ steps.pr.outputs.head_owner_type }} - MAINTAINER_CAN_MODIFY: ${{ steps.pr.outputs.maintainer_can_modify }} - with: - script: | - const issue_number = Number(process.env.PR_NUMBER); - const headOwnerType = process.env.HEAD_OWNER_TYPE; - const canModify = process.env.MAINTAINER_CAN_MODIFY === 'true'; - - if (headOwnerType === 'Organization') { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number, - body: [ - "I can’t push benchmark updates to this PR because it comes from an **organization-owned fork**.", - "GitHub doesn’t allow granting upstream maintainers push permissions to org-owned forks.", - "", - "Options:", - "- Reopen the PR from a **personal fork** with **Allow edits from maintainers** enabled, or", - "- A maintainer can apply the benchmark update on an internal branch." - ].join("\n"), - }); - core.setFailed("Org-owned fork PR is not pushable by maintainers."); - return; - } - - if (!canModify) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number, - body: [ - "I can’t push benchmark updates to this PR branch until you enable **Allow edits from maintainers**.", - "Please check the box on the PR page, then re-comment `/update-llm-benchmark`.", - "See https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork" - ].join("\n"), - }); - core.setFailed("maintainer_can_modify is false; author must enable 'Allow edits from maintainers'."); - } - - # Run the benchmark that is already checked into master to prevent - # an exfiltration attack whereby the PR author tries to sneak in an exploit - # and get a maintainer to run the modified benchmark without looking at the - # PR first. This ensure that we only ever execute code that is checked into - # master. - - name: Checkout master (build/install tool from trusted code) - uses: actions/checkout@v4 - with: - ref: master - fetch-depth: 0 - persist-credentials: false - - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - # Ensure we use a user-writable .NET install (not /usr/share/dotnet), - # so workload installs don't require sudo. - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4 - with: - dotnet-version: "8.0.x" - - - name: Install WASI workload (wasi-experimental) - env: - DOTNET_MULTILEVEL_LOOKUP: "0" - DOTNET_CLI_HOME: ${{ runner.temp }}/dotnet-home - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" - run: | - dotnet --info - dotnet workload install wasi-experimental --skip-manifest-update --disable-parallel - - - name: Install llm-benchmark tool from master - run: | - cargo install --path tools/xtask-llm-benchmark --locked - command -v llm_benchmark - - # Check out the repo on the branch, but ONLY use this code as data! - # Never execute code that is on the PR branch. - - name: Checkout PR head (branch) - uses: actions/checkout@v4 - with: - repository: ${{ steps.pr.outputs.head_repo_full_name }} - ref: ${{ steps.pr.outputs.head_sha }} - fetch-depth: 0 - persist-credentials: false - - # Run the benchmark against the PR using the installed tool from the - # master branch. - - name: Run benchmark (with provider keys) - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - # Prevent MSBuild node reuse issues that cause "Pipe is broken" errors - # when running multiple dotnet publish commands in parallel. - # See: https://github.com/dotnet/msbuild/issues/6657 - MSBUILDDISABLENODEREUSE: "1" - DOTNET_CLI_USE_MSBUILD_SERVER: "0" - run: | - llm_benchmark ci-quickfix - llm_benchmark ci-check - - # Generate failure analysis if there are any failures - - name: Generate failure analysis - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - run: | - llm_benchmark analyze -o docs/llms/docs-benchmark-analysis.md || true - - # Generate PR comment markdown (compares against master baseline) - - name: Generate PR comment markdown - run: | - llm_benchmark ci-comment - - - name: Ensure only docs/llms changed - run: | - set -euo pipefail - CHANGED="$(git diff --name-only)" - - if [ -z "$CHANGED" ]; then - echo "No changes." - exit 0 - fi - - if echo "$CHANGED" | grep -qvE '^docs/llms/'; then - echo "Benchmark produced changes outside docs/llms:" - echo "$CHANGED" | grep -vE '^docs/llms/' - exit 1 - fi - - # Comment the benchmark results on the PR - - name: Comment benchmark results on PR - uses: actions/github-script@v7 - env: - PR_NUMBER: ${{ steps.pr.outputs.number }} - with: - github-token: ${{ secrets.CLOCKWORK_LABS_BOT_PAT }} - script: | - const fs = require('fs'); - - // Read the pre-generated comment markdown - const commentPath = 'docs/llms/docs-benchmark-comment.md'; - if (!fs.existsSync(commentPath)) { - core.setFailed(`Comment file not found: ${commentPath}`); - return; - } - let body = fs.readFileSync(commentPath, 'utf8'); - - // Check if failure analysis exists and append it - const analysisPath = 'docs/llms/docs-benchmark-analysis.md'; - if (fs.existsSync(analysisPath)) { - const analysis = fs.readFileSync(analysisPath, 'utf8'); - // Only include if there's meaningful content (not just "no failures") - if (!analysis.includes('No failures found')) { - body += `\n
\nFailure Analysis (click to expand)\n\n${analysis}\n
`; - } - } - - const issue_number = Number(process.env.PR_NUMBER); - - // Always post a new comment - console.log(`Posting new comment on PR #${issue_number}...`); - try { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number, - body, - }); - console.log('Comment created successfully'); - } catch (err) { - console.error('Failed to post comment:', err.message); - console.error('Full error:', JSON.stringify(err, null, 2)); - throw err; - } - - # The benchmarks only modify the docs/llms directory. - # Commit the changes. - - name: Commit changes - run: | - git config user.name "clockwork-labs-bot" - git config user.email "clockwork-labs-bot@users.noreply.github.com" - - # Prefer staging only the benchmark output area (adjust as needed) - git add docs/llms - - git diff --cached --quiet && exit 0 - git commit -m "Update LLM benchmark results" - - # Here we use the https://github.com/clockwork-labs-bot user's - # personal access token to commit back to the PR branch. This is necessary - # if we want to be able to push back to external contributor forks. - - name: Push back to PR branch (same repo or fork) - env: - GH_TOKEN: ${{ secrets.CLOCKWORK_LABS_BOT_PAT }} - run: | - git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ steps.pr.outputs.head_repo_full_name }}.git" - # Fetch and rebase in case branch moved since workflow started (e.g., previous benchmark run) - git fetch origin "${{ steps.pr.outputs.head_ref }}" - if ! git rebase "origin/${{ steps.pr.outputs.head_ref }}"; then - git rebase --abort - echo "::error::Rebase failed due to conflicts. The PR branch may have been updated during the benchmark run. Please re-run /update-llm-benchmark." - exit 1 - fi - git push origin "HEAD:${{ steps.pr.outputs.head_ref }}" diff --git a/.github/workflows/llm-benchmark-validate-goldens.yml b/.github/workflows/llm-benchmark-validate-goldens.yml new file mode 100644 index 00000000000..22866428f16 --- /dev/null +++ b/.github/workflows/llm-benchmark-validate-goldens.yml @@ -0,0 +1,78 @@ +name: Validate LLM benchmark golden answers + +on: + schedule: + # Nightly at 2 AM UTC + - cron: '0 2 * * *' + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: llm-benchmark-validate-goldens + cancel-in-progress: true + +jobs: + validate-goldens: + runs-on: spacetimedb-new-runner + container: + image: localhost:5000/spacetimedb-ci:latest + options: >- + --privileged + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + lang: [rust, csharp, typescript] + + steps: + - name: Install spacetime CLI + run: | + curl -sSf https://install.spacetimedb.com | sh -s -- -y + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Checkout master + uses: actions/checkout@v4 + with: + ref: master + fetch-depth: 1 + + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + + - name: Setup .NET SDK + if: matrix.lang == 'csharp' + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "8.0.x" + + - name: Install WASI workload + if: matrix.lang == 'csharp' + env: + DOTNET_MULTILEVEL_LOOKUP: "0" + DOTNET_CLI_HOME: ${{ runner.temp }}/dotnet-home + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" + run: | + dotnet workload install wasi-experimental --skip-manifest-update --disable-parallel + + - name: Set up Node.js + if: matrix.lang == 'typescript' + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install pnpm + if: matrix.lang == 'typescript' + uses: pnpm/action-setup@v4 + + - name: Build llm-benchmark tool + run: cargo install --path tools/xtask-llm-benchmark --locked + + - name: Validate golden answers (${{ matrix.lang }}) + env: + MSBUILDDISABLENODEREUSE: "1" + DOTNET_CLI_USE_MSBUILD_SERVER: "0" + run: | + llm_benchmark run --goldens-only --lang ${{ matrix.lang }} diff --git a/Cargo.lock b/Cargo.lock index 4a42447a83f..6d474123a49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11427,7 +11427,6 @@ dependencies = [ "chrono", "clap 4.5.50", "dotenvy", - "fs2", "futures", "heck 0.5.0", "regex", @@ -11437,7 +11436,6 @@ dependencies = [ "spacetimedb 2.1.0", "spacetimedb-data-structures", "spacetimedb-guard", - "tempfile", "thiserror 2.0.17", "tokio", "urlencoding", diff --git a/crates/cli/build.rs b/crates/cli/build.rs index 06b962c6066..684cfa6fb50 100644 --- a/crates/cli/build.rs +++ b/crates/cli/build.rs @@ -56,10 +56,7 @@ fn get_manifest_dir() -> PathBuf { // * `get_templates_json` - returns contents of the JSON file with the list of templates // * `get_template_files` - returns a HashMap with templates contents based on the // templates list at templates/templates-list.json -// * `get_ai_rules_base` - returns base AI rules for all languages -// * `get_ai_rules_typescript` - returns TypeScript-specific AI rules -// * `get_ai_rules_rust` - returns Rust-specific AI rules -// * `get_ai_rules_csharp` - returns C#-specific AI rules +// * `get_skill` - returns the content of a skill file by name fn generate_template_files() { let manifest_dir = get_manifest_dir(); let repo_root = get_repo_root(); @@ -111,64 +108,24 @@ fn generate_template_files() { let ts_bindings_version = extract_ts_bindings_version(&ts_bindings_package).expect("Failed to read TypeScript bindings version"); - // Embed AI rules files from docs/static/ai-rules/ - let ai_rules_dir = repo_root.join("docs/static/ai-rules"); + // Embed skill files from skills/*/SKILL.md + let skills_dir = repo_root.join("skills"); + let skill_names = discover_skill_names(&skills_dir); - // Base rules (all languages) - let base_rules_path = ai_rules_dir.join("spacetimedb.mdc"); - if base_rules_path.exists() { - generated_code.push_str("pub fn get_ai_rules_base() -> &'static str {\n"); - generated_code.push_str(&format!( - " include_str!(\"{}\")\n", - base_rules_path.to_str().unwrap().replace('\\', "\\\\") - )); - generated_code.push_str("}\n\n"); - println!("cargo:rerun-if-changed={}", base_rules_path.display()); - } else { - panic!("Could not find \"docs/static/ai-rules/spacetimedb.mdc\" file."); - } - - // TypeScript-specific rules - let ts_rules_path = ai_rules_dir.join("spacetimedb-typescript.mdc"); - if ts_rules_path.exists() { - generated_code.push_str("pub fn get_ai_rules_typescript() -> &'static str {\n"); - generated_code.push_str(&format!( - " include_str!(\"{}\")\n", - ts_rules_path.to_str().unwrap().replace('\\', "\\\\") - )); - generated_code.push_str("}\n\n"); - println!("cargo:rerun-if-changed={}", ts_rules_path.display()); - } else { - panic!("Could not find \"docs/static/ai-rules/spacetimedb-typescript.mdc\" file."); - } - - // Rust-specific rules - let rust_rules_path = ai_rules_dir.join("spacetimedb-rust.mdc"); - if rust_rules_path.exists() { - generated_code.push_str("pub fn get_ai_rules_rust() -> &'static str {\n"); - generated_code.push_str(&format!( - " include_str!(\"{}\")\n", - rust_rules_path.to_str().unwrap().replace('\\', "\\\\") - )); - generated_code.push_str("}\n\n"); - println!("cargo:rerun-if-changed={}", rust_rules_path.display()); - } else { - panic!("Could not find \"docs/static/ai-rules/spacetimedb-rust.mdc\" file."); - } - - // C#-specific rules - let csharp_rules_path = ai_rules_dir.join("spacetimedb-csharp.mdc"); - if csharp_rules_path.exists() { - generated_code.push_str("pub fn get_ai_rules_csharp() -> &'static str {\n"); + generated_code.push_str("pub fn get_skill(name: &str) -> Option<&'static str> {\n"); + generated_code.push_str(" match name {\n"); + for skill_name in &skill_names { + let skill_path = skills_dir.join(skill_name).join("SKILL.md"); generated_code.push_str(&format!( - " include_str!(\"{}\")\n", - csharp_rules_path.to_str().unwrap().replace('\\', "\\\\") + " \"{}\" => Some(include_str!(\"{}\")),\n", + skill_name, + skill_path.to_str().unwrap().replace('\\', "\\\\") )); - generated_code.push_str("}\n\n"); - println!("cargo:rerun-if-changed={}", csharp_rules_path.display()); - } else { - panic!("Could not find \"docs/static/ai-rules/spacetimedb-csharp.mdc\" file."); + println!("cargo:rerun-if-changed={}", skill_path.display()); } + generated_code.push_str(" _ => None,\n"); + generated_code.push_str(" }\n"); + generated_code.push_str("}\n\n"); // Expose workspace metadata so `spacetime init` can rewrite template manifests without hardcoding versions. generated_code.push_str("pub fn get_workspace_edition() -> &'static str {\n"); @@ -619,3 +576,31 @@ fn copy_if_changed(src: &Path, dst: &Path) -> io::Result<()> { let mut file = fs::File::create(dst)?; file.write_all(&src_bytes) } + +/// Discover skill directories under skills/. Each directory containing a SKILL.md +/// file is considered a skill. Returns sorted skill names. +fn discover_skill_names(skills_dir: &Path) -> Vec { + let mut names = Vec::new(); + + let entries = match fs::read_dir(skills_dir) { + Ok(entries) => entries, + Err(_) => { + panic!( + "Could not read skills directory at {}. Ensure skills/ exists at the repo root.", + skills_dir.display() + ); + } + }; + + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() && path.join("SKILL.md").exists() { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + names.push(name.to_string()); + } + } + } + + names.sort(); + names +} diff --git a/crates/cli/src/subcommands/init.rs b/crates/cli/src/subcommands/init.rs index b4701b41164..7d706a1843d 100644 --- a/crates/cli/src/subcommands/init.rs +++ b/crates/cli/src/subcommands/init.rs @@ -1872,61 +1872,73 @@ fn set_dependency_version(item: &mut Item, version: &str, remove_path: bool) { } /// Install AI assistant rules for multiple editors/tools. -/// Writes rules to: -/// - .cursor/rules/ (Cursor) +/// Reads from embedded skill files (skills/*/SKILL.md) and writes to: +/// - .cursor/rules/ (Cursor) — with .mdc frontmatter /// - CLAUDE.md (Claude Code) /// - AGENTS.md (Opencode) /// - .windsurfrules (Windsurf) /// - .github/copilot-instructions.md (VS Code Copilot) fn install_ai_rules(config: &TemplateConfig, project_path: &Path) -> anyhow::Result<()> { - let base_rules = embedded::get_ai_rules_base(); - let ts_rules = embedded::get_ai_rules_typescript(); - let rust_rules = embedded::get_ai_rules_rust(); - let csharp_rules = embedded::get_ai_rules_csharp(); - - // Check which languages are used in server or client - let uses_typescript = config.server_lang == Some(ServerLanguage::TypeScript) - || config.client_lang == Some(ClientLanguage::TypeScript); - let uses_rust = - config.server_lang == Some(ServerLanguage::Rust) || config.client_lang == Some(ClientLanguage::Rust); - let uses_csharp = - config.server_lang == Some(ServerLanguage::Csharp) || config.client_lang == Some(ClientLanguage::Csharp); - - // 1. Cursor: .cursor/rules/ directory with separate files - let cursor_dir = project_path.join(".cursor/rules"); - fs::create_dir_all(&cursor_dir)?; - fs::write(cursor_dir.join("spacetimedb.mdc"), base_rules)?; - if uses_typescript { - fs::write(cursor_dir.join("spacetimedb-typescript.mdc"), ts_rules)?; - } - if uses_rust { - fs::write(cursor_dir.join("spacetimedb-rust.mdc"), rust_rules)?; + // Collect relevant skills based on server and client languages + let mut skills: Vec<(&str, &str)> = Vec::new(); + + // Always include shared skills + if let Some(content) = embedded::get_skill("concepts") { + skills.push(("concepts", content)); } - if uses_csharp { - fs::write(cursor_dir.join("spacetimedb-csharp.mdc"), csharp_rules)?; + if let Some(content) = embedded::get_skill("cli") { + skills.push(("cli", content)); } - // Build combined content for single-file AI assistants - // Strip the YAML frontmatter from the .mdc files for non-Cursor tools - let base_content = strip_mdc_frontmatter(base_rules); - let mut combined_content = base_content.to_string(); - - if uses_typescript { - let ts_content = strip_mdc_frontmatter(ts_rules); - combined_content.push_str("\n\n"); - combined_content.push_str(ts_content); + // Server language skill + if let Some(server_lang) = config.server_lang { + let name = match server_lang { + ServerLanguage::Rust => "rust-server", + ServerLanguage::TypeScript => "typescript-server", + ServerLanguage::Csharp => "csharp-server", + ServerLanguage::Cpp => "cpp-server", + }; + if let Some(content) = embedded::get_skill(name) { + skills.push((name, content)); + } + // C++ server projects use Unreal as their client SDK + if server_lang == ServerLanguage::Cpp { + if let Some(content) = embedded::get_skill("unreal") { + skills.push(("unreal", content)); + } + } } - if uses_rust { - let rust_content = strip_mdc_frontmatter(rust_rules); - combined_content.push_str("\n\n"); - combined_content.push_str(rust_content); + + // Client language skill(s) + if let Some(client_lang) = config.client_lang { + let names: &[&str] = match client_lang { + ClientLanguage::Rust => &[], // no Rust client skill yet + ClientLanguage::TypeScript => &["typescript-client"], + ClientLanguage::Csharp => &["csharp-client", "unity"], + }; + for name in names { + if let Some(content) = embedded::get_skill(name) { + skills.push((name, content)); + } + } } - if uses_csharp { - let csharp_content = strip_mdc_frontmatter(csharp_rules); - combined_content.push_str("\n\n"); - combined_content.push_str(csharp_content); + + // 1. Cursor: .cursor/rules/ directory with separate .mdc files + let cursor_dir = project_path.join(".cursor/rules"); + fs::create_dir_all(&cursor_dir)?; + for (name, content) in &skills { + let mdc_content = skill_to_mdc(content); + fs::write(cursor_dir.join(format!("{}.mdc", name)), &mdc_content)?; } + // Build combined content for single-file AI assistants + // Strip the YAML frontmatter from skills for non-Cursor tools + let combined_content: String = skills + .iter() + .map(|(_, content)| strip_frontmatter(content)) + .collect::>() + .join("\n\n"); + // 2. Claude Code: CLAUDE.md fs::write(project_path.join("CLAUDE.md"), &combined_content)?; @@ -1944,18 +1956,58 @@ fn install_ai_rules(config: &TemplateConfig, project_path: &Path) -> anyhow::Res Ok(()) } -/// Strip YAML frontmatter from .mdc files (the --- delimited section at the start) -fn strip_mdc_frontmatter(content: &str) -> &str { - // Look for frontmatter: starts with --- and ends with --- +/// Convert a SKILL.md file to Cursor .mdc format. +/// Parses the SKILL.md frontmatter for cursor_globs and cursor_always_apply, +/// then generates Cursor-compatible frontmatter. +fn skill_to_mdc(content: &str) -> String { + let (frontmatter, body) = split_frontmatter(content); + + // Parse cursor-specific fields from frontmatter + let mut description = String::new(); + let mut cursor_globs = String::new(); + let mut cursor_always_apply = false; + + for line in frontmatter.lines() { + let line = line.trim(); + if let Some(val) = line.strip_prefix("description:") { + description = val.trim().trim_matches('"').to_string(); + } else if let Some(val) = line.strip_prefix("cursor_globs:") { + cursor_globs = val.trim().trim_matches('"').to_string(); + } else if let Some(val) = line.strip_prefix("cursor_always_apply:") { + cursor_always_apply = val.trim() == "true"; + } + } + + let mut mdc = String::new(); + mdc.push_str("---\n"); + if !description.is_empty() { + mdc.push_str(&format!("description: \"{}\"\n", description)); + } + if !cursor_globs.is_empty() { + mdc.push_str(&format!("globs: {}\n", cursor_globs)); + } + mdc.push_str(&format!("alwaysApply: {}\n", cursor_always_apply)); + mdc.push_str("---\n\n"); + mdc.push_str(body); + mdc +} + +/// Split content into (frontmatter, body). Frontmatter is the text between --- delimiters. +fn split_frontmatter(content: &str) -> (&str, &str) { if let Some(after_opening) = content.strip_prefix("---") && let Some(end_idx) = after_opening.find("\n---") { - // Skip past the closing --- and the newline after it - let remaining = &after_opening[end_idx + 4..]; // 4 for \n--- - // Skip any leading newlines after frontmatter - return remaining.trim_start_matches('\n'); + let frontmatter = &after_opening[..end_idx]; + let body = &after_opening[end_idx + 4..]; // 4 for \n--- + let body = body.trim_start_matches('\n'); + return (frontmatter, body); } - content + ("", content) +} + +/// Strip YAML frontmatter (the --- delimited section at the start) +fn strip_frontmatter(content: &str) -> &str { + split_frontmatter(content).1 } /// Check if Emscripten and CMake tooling are available in PATH. diff --git a/docs/DEVELOP.md b/docs/DEVELOP.md index fd97525f270..3ee7a261572 100644 --- a/docs/DEVELOP.md +++ b/docs/DEVELOP.md @@ -126,21 +126,9 @@ $env:LLM_BENCH_ROUTE_CONCURRENCY="4" Results directory: `docs/llms` -### Result Files +### Results Storage -There are two sets of result files, each serving a different purpose: - -| Files | Purpose | Updated By | -|-------|---------|------------| -| `docs-benchmark-details.json`
`docs-benchmark-summary.json` | Test documentation quality with a single reference model (GPT-5) | `cargo llm ci-quickfix` | -| `llm-comparison-details.json`
`llm-comparison-summary.json` | Compare all LLMs against the same documentation | `cargo llm run` | - -- **docs-benchmark**: Used by CI to ensure documentation quality. Contains only GPT-5 results. -- **llm-comparison**: Used for manual benchmark runs to compare LLM performance. Contains results from all configured models. - -> Results writes are lock-safe and atomic. The tool takes an exclusive lock and writes via a temp file, then renames it, so concurrent runs won't corrupt results. - -Open `llm_benchmark_stats_viewer.html` in a browser to inspect merged results locally. +Benchmark results are stored in a remote PostgreSQL database via the spacetime-web API. Results are uploaded automatically after each benchmark batch when `LLM_BENCHMARK_UPLOAD_URL` and `LLM_BENCHMARK_API_KEY` environment variables are set. Use `--dry-run` to skip uploading. ### Current Benchmarks **basics** diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index cb921b5dcf7..141c0e803cf 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -217,6 +217,20 @@ const config: Config = { }, }, ], + [ + '@signalwire/docusaurus-plugin-llms-txt', + { + siteTitle: 'SpacetimeDB', + siteDescription: + 'SpacetimeDB is a database that lets you write your entire application as a database module. Server logic runs inside the database as WebAssembly. Clients subscribe to queries and get real-time updates over WebSocket. No separate server needed.', + depth: 2, + content: { + enableLlmsFullTxt: true, + enableMarkdownFiles: false, + includeVersionedDocs: false, + }, + }, + ], ], }; diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md deleted file mode 100644 index 459875a4ebb..00000000000 --- a/docs/llms/docs-benchmark-analysis.md +++ /dev/null @@ -1,250 +0,0 @@ -# Benchmark Failure Analysis - -Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../docs/llms/docs-benchmark-details.json` - -## Summary - -- **Total failures analyzed**: 33 - ---- - -## Analysis of SpacetimeDB Benchmark Test Failures - -### Rust / rustdoc_json Failures - -#### Compile/Publish Errors (2 Failures) - -##### Failure Group 1: `t_002_scheduled_table` and `t_017_scheduled_columns` -1. **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - - #[table(accessor = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::RepeatMicros(50_000), - }); - } - } - - #[reducer] - pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { - } - ``` - -2. **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; - - #[table(accessor = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: every_50ms, - }); - } - ``` - -3. **The error**: - - `publish_error: spacetime publish failed (exit=1)` - -4. **Explain the difference**: - - The generated code used `ScheduleAt::RepeatMicros(50_000)` instead of the correct `ScheduleAt::Interval(Duration::from_millis(50).into())`. The way the scheduling was set up was incorrect. - -5. **Root cause**: - - The documentation does not clearly specify the constructor syntax for `ScheduleAt` nor how to correctly set up the scheduled tasks in this context. - -6. **Recommendation**: - - Update documentation to provide examples of different constructors for `ScheduleAt`, specifically emphasizing how to define intervals correctly. - ---- - -#### Other Failures (5 failures) - -##### Failure Group 2: `t_013_spacetime_sum_type`, `t_015_product_type_columns`, `t_016_sum_type_columns`, `t_018_constraints`, `t_020_ecs` -1. **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; - - #[derive(SpacetimeType)] - pub struct Rect { - width: i32, - height: i32, - } - - #[table(accessor = result)] - pub struct ResultRow { - #[primary_key] - id: i32, - value: Shape, - } - - #[reducer] - pub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { - ctx.db.result().insert(ResultRow { - id, - value: Shape::Circle(radius), - }); - } - ``` - -2. **The golden example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; - - #[derive(SpacetimeType, Clone, Debug)] - pub struct Rect { - pub width: i32, - pub height: i32, - } - - #[table(accessor = result)] - pub struct ResultRow { - #[primary_key] - pub id: i32, - pub value: Shape, - } - - #[reducer] - pub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) { - ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) }); - } - ``` - -3. **The error**: - - `spacetime sql failed: no such table: result` - - `spacetime sql failed: no such table: profile` - - `spacetime sql failed: no such table: drawings` - -4. **Explain the difference**: - - The generated code omits the `pub` visibility keyword for fields and structs, which prevents proper access by the macros that generate the expected database schema. Additionally, the enum `Shape` wasn't declared correctly in the generated code. - -5. **Root cause**: - - Lack of proper visibility (missing `pub`) for structs and enum fields was not clearly emphasized in the documentation, leading to access issues. - -6. **Recommendation**: - - Provide clear guidelines in the documentation regarding the necessity of using `pub` for struct and enum fields when working with SpacetimeDB components. Include example schemas with visibility marked. - ---- - -### Rust / docs Failures (22 total) - -#### Timeout Issues (1 failure) - -1. **Failure Group**: `t_015_product_type_columns` - - **Expected**: Modify the query logic to ensure no unnecessary long-running operations exist. - - **Recommendation**: Provide timeout considerations in the documentation to ensure optimization options are explored to prevent long-running tasks. - ---- - -### C# / docs Failures (4 total) - -#### C# Failure Group: `t_014_elementary_columns`, `t_016_sum_type_columns`, `t_017_scheduled_columns`, `t_020_ecs` -1. **The generated code**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [SpacetimeDB.Table(Accessor = "Primitive", Public = true)] - public partial struct Primitive - { - [SpacetimeDB.PrimaryKey] - public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [SpacetimeDB.Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive - { - Id = 1, - Count = 2, - Total = 3000000000L, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } - ``` - -2. **The golden example**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [Table(Accessor = "Primitive")] - public partial struct Primitive - { - [PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; - public string Name; - } - - [Reducer] - public static void Seed(ReducerContext ctx) - { - ctx.Db.Primitive.Insert(new Primitive { - Id = 1, - Count = 2, - Total = 3000000000, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); - } - } - ``` - -3. **The error**: `no such table: primitive` - -4. **Explain the difference**: - - Missing the `public` access modifier in the declaration of the `Table` attribute. The expected syntax properly utilizes attributes defined in the library. - -5. **Root cause**: - - Documentation may lack clarity about access modifiers, especially when it comes to how they affect visibility in entities. - -6. **Recommendation**: - - Ensure C# documentation includes explicit examples where `public` is required in class and struct declarations to prevent access issues with tables. - ---- - -By addressing the above gaps in documentation and ensuring that generated samples adhere to the expected outcomes, we can significantly reduce the number of failures in future benchmarks. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md deleted file mode 100644 index b26d4820585..00000000000 --- a/docs/llms/docs-benchmark-comment.md +++ /dev/null @@ -1,17 +0,0 @@ -## LLM Benchmark Results (ci-quickfix) - -| Language | Mode | Category | Tests Passed | Task Pass % | -|----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 26/27 | 91.7% ⬆️ +8.3% | -| Rust | rustdoc_json | schema | 23/34 | 65.3% ⬆️ +10.0% | -| Rust | rustdoc_json | **total** | 49/61 | **79.7%** ⬆️ +9.1% | -| Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/32 | 20.5% ⬆️ +8.0% | -| Rust | docs | **total** | 13/59 | **15.4%** ⬆️ +3.6% | -| C# | docs | basics | 27/27 | 100.0% | -| C# | docs | schema | 25/34 | 73.7% | -| C# | docs | **total** | 52/61 | **88.0%** | - -_Compared against master branch baseline_ - -Generated at: 2026-01-27T20:01:19.767Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json deleted file mode 100644 index b91afc0d43a..00000000000 --- a/docs/llms/docs-benchmark-details.json +++ /dev/null @@ -1,3634 +0,0 @@ -{ - "languages": [ - { - "lang": "rust", - "modes": [ - { - "mode": "rustdoc_json", - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "models": [ - { - "name": "GPT-5", - "route_api_model": "gpt-5", - "tasks": { - "t_000_empty_reducers": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n let _ = count;\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n let _ = name;\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n let _ = (count, name);\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n let _ = (active, ratio, label);\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-000-empty-reducers-golden", - "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_000_empty_reducers/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_000_empty_reducers/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-000-empty-reducers-golden", - "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:33.923590167Z", - "finished_at": "2026-01-27T19:51:02.622615452Z" - }, - "t_001_basic_tables": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-001-basic-tables-golden", - "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_001_basic_tables/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-001-basic-tables-golden", - "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:58.838771838Z", - "finished_at": "2026-01-27T19:51:33.707208916Z" - }, - "t_002_scheduled_table": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": false, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db\n .tick_timer()\n .insert(TickTimer { scheduled_id: 0, scheduled_at: ScheduleAt::repeat_micros(50_000) });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling zerocopy v0.8.34\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.17\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling getrandom v0.2.17\n Compiling itoa v1.0.17\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.11.1\n Compiling constant_time_eq v0.4.2\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling rand_core v0.6.4\n Compiling cc v1.2.54\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:17:21\n |\n17 | .insert(TickTimer { scheduled_id: 0, scheduled_at: ScheduleAt::repeat_micros(50_000) });\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:16:14\n |\n15 | / ctx.db\n16 | | .tick_timer()\n | | -^^^^^^^^^^ method not found in `Local`\n | |_____________|\n |\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:76\n |\n17 | .insert(TickTimer { scheduled_id: 0, scheduled_at: ScheduleAt::repeat_micros(50_000) });\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:00.465423417Z", - "finished_at": "2026-01-27T19:51:47.339587032Z" - }, - "t_003_struct_in_table": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-003-struct-in-table-golden", - "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-003-struct-in-table-golden", - "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:43.120116357Z", - "finished_at": "2026-01-27T19:51:34.789700080Z" - }, - "t_004_insert": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 2, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_db": "basics-t-004-insert-golden", - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user", - "server": "http://127.0.0.1:33615" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:00.012104606Z", - "finished_at": "2026-01-27T19:51:27.169795394Z" - }, - "t_005_update": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let row = User { id, name, age, active };\n ctx.db.user().id().update(row);\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user", - "server": "http://127.0.0.1:33615" - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:22.268945837Z", - "finished_at": "2026-01-27T19:51:01.660325703Z" - }, - "t_006_delete": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", - "scorer_details": { - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:18.814239252Z", - "finished_at": "2026-01-27T19:51:00.465380864Z" - }, - "t_007_crud": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n // insert (id=1, name=\"Alice\", age=30, active=true)\n users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n // insert (id=2, name=\"Bob\", age=22, active=false)\n users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n // update (id=1, name=\"Alice2\", age=31, active=false)\n users.id().update(User {\n id: 1,\n name: \"Alice2\".to_string(),\n age: 31,\n active: false,\n });\n\n // delete id=2\n users.id().delete(&2);\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "basics-t-007-crud-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud", - "server": "http://127.0.0.1:33615" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:19.572189963Z", - "finished_at": "2026-01-27T19:51:00.012056746Z" - }, - "t_008_index_lookup": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n let _ = ctx.db.result().try_insert(ResultRow {\n id: user.id,\n name: user.name.clone(),\n });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", - "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_db": "basics-t-008-index-lookup-golden", - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name", - "server": "http://127.0.0.1:33615" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:59.032238062Z", - "finished_at": "2026-01-27T19:51:34.520084103Z" - }, - "t_009_init": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", - "scorer_details": { - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - }, - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:21.549783Z", - "finished_at": "2026-01-27T19:50:59.032183836Z" - }, - "t_010_connect": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n let _ = ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n let _ = ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-010-connect-golden", - "llm_db": "basics-t-010-connect-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_010_connect/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_010_connect/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-010-connect-golden", - "llm_db": "basics-t-010-connect-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:19.565630879Z", - "finished_at": "2026-01-27T19:50:43.120012892Z" - }, - "t_011_helper_function": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "basics-t-011-helper-function-golden", - "golden_out": "id | sum ----+----- 1 | 5", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum", - "server": "http://127.0.0.1:33615" - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:00.227420335Z", - "finished_at": "2026-01-27T19:51:29.538171947Z" - }, - "t_012_spacetime_product_type": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score", - "server": "http://127.0.0.1:33615" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:17.029809591Z", - "finished_at": "2026-01-27T19:50:58.838714798Z" - }, - "t_013_spacetime_sum_type": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", - "scorer_details": { - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c200ef5026b116b67d90edbd0acaca12a273f2a7e6e888cc7265a5d4a4fe7173/sql)\n", - "phase": "sql_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.665478058Z", - "finished_at": "2026-01-27T19:50:21.549671530Z" - }, - "t_014_elementary_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", - "scorer_details": { - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-014-elementary-columns-golden", - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:33615" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:50:12.818589409Z", - "finished_at": "2026-01-27T19:51:00.227383147Z" - }, - "t_015_product_type_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\nstruct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\nstruct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\nstruct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.profile();\n if tbl.count() == 0 {\n let home = Address { street: \"1 Main\".to_string(), zip: 11111 };\n let work = Address { street: \"2 Broad\".to_string(), zip: 22222 };\n let pos = Position { x: 7, y: 9 };\n tbl.insert(Profile { id: 1, home, work, pos });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", - "scorer_details": { - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - }, - "product_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `profile`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c200171a9841d8ed2a9c762ac4e081fe93ed52fe5c9e32ae9bbd874bb7e1a8b5/sql)\n", - "phase": "sql_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.666090278Z", - "finished_at": "2026-01-27T19:50:19.565536981Z" - }, - "t_016_sum_type_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().id().find(1i32).is_none() {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c200881b5220df4b060ef4c2656171f2f8326cecff55db6c3c11d3ebbd2f5cec/sql)\n", - "phase": "sql_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c20074c3ee57884fdd34989755bdcfa11644a8f79ecbbdbf8f5182f2be46b5ec/sql)\n", - "phase": "sql" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.665270093Z", - "finished_at": "2026-01-27T19:50:17.767945290Z" - }, - "t_017_scheduled_columns": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": false, - "model_name": "GPT-5", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-017-scheduled-columns-golden", - "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.34\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.17\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling heck v0.4.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling constant_time_eq v0.4.2\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.54\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling itertools v0.12.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.664841535Z", - "finished_at": "2026-01-27T19:50:33.923572352Z" - }, - "t_018_constraints": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let accounts = ctx.db.account();\n let _ = accounts.try_insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n let _ = accounts.try_insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - }, - "constraints_row_parity_after_seed": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `account`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c2002f3e10afac17bd0350f8d8feb90d1ae94d73c7c44c6824e7324679a8e5b9/sql)\n", - "phase": "sql_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.664543506Z", - "finished_at": "2026-01-27T19:50:17.029704757Z" - }, - "t_019_many_to_many": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 5, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure the database contains exactly the specified rows.\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", - "scorer_details": { - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.665682949Z", - "finished_at": "2026-01-27T19:50:22.268882588Z" - }, - "t_020_ecs": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n let _ = ctx.db.next_position().entity_id().update(existing);\n } else {\n let _ = ctx.db.next_position().try_insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", - "scorer_details": { - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c200ef190fe9bac0d026751d082f663c75cb0435d300fdc35e1bc1f038529566/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c200ef190fe9bac0d026751d082f663c75cb0435d300fdc35e1bc1f038529566/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c200ef190fe9bac0d026751d082f663c75cb0435d300fdc35e1bc1f038529566/sql)\n", - "phase": "sql" - } - }, - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:33615/v1/database/c200ef190fe9bac0d026751d082f663c75cb0435d300fdc35e1bc1f038529566/sql)\n", - "phase": "sql" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.665885964Z", - "finished_at": "2026-01-27T19:50:18.814108696Z" - }, - "t_021_multi_column_index": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let table = ctx.db.log();\n table.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n table.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n table.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:33615", - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:49:28.665048738Z", - "finished_at": "2026-01-27T19:50:12.818516478Z" - } - } - } - ] - }, - { - "mode": "docs", - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "models": [ - { - "name": "GPT-5", - "route_api_model": "gpt-5", - "tasks": { - "t_000_empty_reducers": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-000-empty-reducers-golden", - "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_000_empty_reducers/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_000_empty_reducers/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-000-empty-reducers-golden`.\n", - "phase": "describe_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:02.635478741Z", - "finished_at": "2026-01-27T19:53:40.787269737Z" - }, - "t_001_basic_tables": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-001-basic-tables-golden", - "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_001_basic_tables/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-001-basic-tables-golden`.\n", - "phase": "describe_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:04.338845678Z", - "finished_at": "2026-01-27T19:53:39.243226783Z" - }, - "t_002_scheduled_table": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n if ctx.db.tick_timer().count() == 0 {\n use std::time::Duration;\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n }\n Ok(())\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n", - "phase": "describe_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:38.403872273Z", - "finished_at": "2026-01-27T19:54:31.831288022Z" - }, - "t_003_struct_in_table": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-003-struct-in-table-golden", - "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-003-struct-in-table-golden`.\n", - "phase": "describe_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:02.638916802Z", - "finished_at": "2026-01-27T19:53:40.055864699Z" - }, - "t_004_insert": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "describe_golden" - } - }, - "data_parity_insert_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "call_reducer_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:33.748279444Z", - "finished_at": "2026-01-27T19:53:56.014195443Z" - }, - "t_005_update": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "describe_golden" - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:00.578961998Z", - "finished_at": "2026-01-27T19:53:38.403844461Z" - }, - "t_006_delete": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "describe_golden" - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:52:31.910165978Z", - "finished_at": "2026-01-27T19:52:58.192517311Z" - }, - "t_007_crud": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.user().id().delete(2);\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", - "scorer_details": { - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:52:42.328837521Z", - "finished_at": "2026-01-27T19:53:33.740284067Z" - }, - "t_008_index_lookup": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n let row = ResultRow { id: user.id, name: user.name.clone() };\n if ctx.db.result().id().find(user.id).is_some() {\n ctx.db.result().id().update(row);\n } else {\n ctx.db.result().insert(row);\n }\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "describe_golden" - } - }, - "index_lookup_projection_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "seed_user_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:17.657026192Z", - "finished_at": "2026-01-27T19:53:54.093778910Z" - }, - "t_009_init": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", - "scorer_details": { - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", - "phase": "describe_golden" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:52:58.192552680Z", - "finished_at": "2026-01-27T19:53:38.342926441Z" - }, - "t_010_connect": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-010-connect-golden", - "llm_db": "basics-t-010-connect-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_010_connect/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_010_connect/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-010-connect-golden`.\n", - "phase": "describe_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:52:35.522740907Z", - "finished_at": "2026-01-27T19:53:02.635440935Z" - }, - "t_011_helper_function": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", - "scorer_details": { - "helper_func_sum_abs": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "describe_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:53:37.773674958Z", - "finished_at": "2026-01-27T19:53:57.689433966Z" - }, - "t_012_spacetime_product_type": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", - "scorer_details": { - "product_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" - } - }, - "product_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:52:29.436388253Z", - "finished_at": "2026-01-27T19:53:04.338818518Z" - }, - "t_013_spacetime_sum_type": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::Table;\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &spacetimedb::ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "describe_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:47.638500741Z", - "finished_at": "2026-01-27T19:53:00.578888253Z" - }, - "t_014_elementary_columns": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", - "scorer_details": { - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "describe_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:52:27.568323309Z", - "finished_at": "2026-01-27T19:53:02.605112864Z" - }, - "t_015_product_type_columns": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": false, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:54:31.831326960Z", - "finished_at": "2026-01-27T19:54:31.831326960Z" - }, - "t_016_sum_type_columns": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().count() == 0 {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" - } - }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38927/v1/database/c200596b7e7f2350abeb00119177ef3448e7673481a61aeaa7c22e4dd952cb52/sql)\n", - "phase": "sql" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:47.634170474Z", - "finished_at": "2026-01-27T19:52:27.568235316Z" - }, - "t_017_scheduled_columns": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 2, - "passed_tests": 1, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _tick_timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n }\n Ok(())\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-017-scheduled-columns-golden", - "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:47.625840249Z", - "finished_at": "2026-01-27T19:52:35.522695487Z" - }, - "t_018_constraints": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.account().iter().map(|a| a.id).collect();\n for id in ids {\n ctx.db.account().id().delete(id);\n }\n\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", - "scorer_details": { - "constraints_row_parity_after_seed": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "describe_golden" - } - }, - "constraints_seed_two_rows": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:47.622424031Z", - "finished_at": "2026-01-27T19:53:37.773579225Z" - }, - "t_019_many_to_many": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group, public)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n public,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &spacetimedb::ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", - "scorer_details": { - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", - "phase": "describe_golden" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:47.642788029Z", - "finished_at": "2026-01-27T19:52:31.910080797Z" - }, - "t_020_ecs": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // First entity: id=1, pos=(0,0), vel=(1,0)\n ctx.db.entity().id().delete(1);\n ctx.db.position().entity_id().delete(1);\n ctx.db.velocity().entity_id().delete(1);\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n // Second entity: id=2, pos=(10,0), vel=(-2,3)\n ctx.db.entity().id().delete(2);\n ctx.db.position().entity_id().delete(2);\n ctx.db.velocity().entity_id().delete(2);\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(_existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n ctx.db\n .next_position()\n .entity_id()\n .update(NextPosition { entity_id: pos.entity_id, x: nx, y: ny });\n } else {\n ctx.db\n .next_position()\n .insert(NextPosition { entity_id: pos.entity_id, x: nx, y: ny });\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", - "phase": "describe_golden" - } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38927/v1/database/c2000dfb254023618a83b1182de85ff2f9590dae5ce1c9301d7d8cc01c8bff6e/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38927/v1/database/c2000dfb254023618a83b1182de85ff2f9590dae5ce1c9301d7d8cc01c8bff6e/sql)\n", - "phase": "sql" - } - }, - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38927/v1/database/c2000dfb254023618a83b1182de85ff2f9590dae5ce1c9301d7d8cc01c8bff6e/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38927/v1/database/c2000dfb254023618a83b1182de85ff2f9590dae5ce1c9301d7d8cc01c8bff6e/sql)\n", - "phase": "sql" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:47.647113172Z", - "finished_at": "2026-01-27T19:52:42.328754582Z" - }, - "t_021_multi_column_index": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" - } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:51:47.629929093Z", - "finished_at": "2026-01-27T19:52:29.436282461Z" - } - } - } - ] - } - ], - "golden_answers": { - "basics/t_000_empty_reducers": { - "answer": "use spacetimedb::{reducer, ReducerContext};\n\n#[reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_three_args(\n ctx: &ReducerContext,\n active: bool,\n ratio: f32,\n label: String,\n) -> Result<(), String> {\n Ok(())\n}\n", - "syntax": "rust" - }, - "basics/t_001_basic_tables": { - "answer": "use spacetimedb::table;\n\n#[table(name = users)]\npub struct Users {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = products)]\npub struct Products {\n #[primary_key]\n pub id: i32,\n pub title: String,\n pub price: f32,\n pub in_stock: bool,\n}\n\n#[table(name = notes)]\npub struct Notes {\n #[primary_key]\n pub id: i32,\n pub body: String,\n pub rating: i64,\n pub pinned: bool,\n}\n", - "syntax": "rust" - }, - "basics/t_002_scheduled_table": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()),\n });\n Ok(())\n}\n", - "syntax": "rust" - }, - "basics/t_003_struct_in_table": { - "answer": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n", - "syntax": "rust" - }, - "basics/t_004_insert": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct Users {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n ctx.db.users().insert(Users { id, name, age, active });\n Ok(())\n}\n", - "syntax": "rust" - }, - "basics/t_005_update": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.users().id().update(User { id, name, age, active });\n}", - "syntax": "rust" - }, - "basics/t_006_delete": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.users().id().delete(id);\n}\n", - "syntax": "rust" - }, - "basics/t_007_crud": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.users().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.users().id().delete(2);\n}\n", - "syntax": "rust" - }, - "basics/t_008_index_lookup": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.users().id().find(id) {\n ctx.db.results().insert(ResultRow { id: u.id, name: u.name });\n }\n}\n", - "syntax": "rust" - }, - "basics/t_009_init": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}\n", - "syntax": "rust" - }, - "basics/t_010_connect": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n pub id: u64,\n pub kind: String,\n}\n\n#[reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".into() });\n}\n", - "syntax": "rust" - }, - "basics/t_011_helper_function": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 { a + b }\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.results().insert(ResultRow { id, sum: add(a, b) });\n}\n", - "syntax": "rust" - }, - "schema/t_012_spacetime_product_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.results().insert(ResultRow { id, value: Score { left, right } });\n}\n", - "syntax": "rust" - }, - "schema/t_013_spacetime_sum_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = results)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.results().insert(ResultRow { id, value: Shape::Circle(radius) });\n}\n", - "syntax": "rust" - }, - "schema/t_014_elementary_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n pub id: i32,\n pub count: i32,\n pub total: i64,\n pub price: f32,\n pub ratio: f64,\n pub active: bool,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}\n", - "syntax": "rust" - }, - "schema/t_015_product_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n pub street: String,\n pub zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = profiles)]\npub struct Profile {\n #[primary_key]\n pub id: i32,\n pub home: Address,\n pub work: Address,\n pub pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profiles().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}\n", - "syntax": "rust" - }, - "schema/t_016_sum_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawings)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawings().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}\n", - "syntax": "rust" - }, - "schema/t_017_scheduled_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n pub scheduled_id: u64,\n pub scheduled_at: ScheduleAt,\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let every_50ms: ScheduleAt = Duration::from_millis(50).into();\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: every_50ms,\n });\n}\n", - "syntax": "rust" - }, - "schema/t_018_constraints": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = accounts,\n index(name = by_name, btree(columns = [name]))\n)]\npub struct Account {\n #[primary_key]\n pub id: i32,\n #[unique]\n pub email: String,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.accounts().insert(Account { id: 1, email: \"a@example.com\".into(), name: \"Alice\".into() });\n ctx.db.accounts().insert(Account { id: 2, email: \"b@example.com\".into(), name: \"Bob\".into() });\n}\n", - "syntax": "rust" - }, - "schema/t_019_many_to_many": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n pub user_id: i32,\n pub name: String,\n}\n\n#[table(name = groups)]\npub struct Group {\n #[primary_key]\n pub group_id: i32,\n pub title: String,\n}\n\n#[table(\n name = memberships,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.users().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.users().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.groups().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.groups().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.memberships().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.memberships().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.memberships().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}\n", - "syntax": "rust" - }, - "schema/t_020_ecs": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = entities)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n}\n\n#[table(name = positions)]\npub struct Position {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = velocities)]\npub struct Velocity {\n #[primary_key]\n pub entity_id: i32,\n pub vx: i32,\n pub vy: i32,\n}\n\n#[table(name = next_positions)]\npub struct NextPosition {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entities().insert(Entity { id: 1 });\n ctx.db.entities().insert(Entity { id: 2 });\n\n ctx.db.positions().insert(Position {\n entity_id: 1,\n x: 1,\n y: 0,\n });\n ctx.db.positions().insert(Position {\n entity_id: 2,\n x: 10,\n y: 0,\n });\n\n ctx.db.velocities().insert(Velocity {\n entity_id: 1,\n vx: 1,\n vy: 0,\n });\n ctx.db.velocities().insert(Velocity {\n entity_id: 2,\n vx: -2,\n vy: 3,\n });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.positions().iter() {\n if let Some(v) = ctx.db.velocities().entity_id().find(p.entity_id) {\n let np = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n\n if ctx.db.next_positions().entity_id().find(p.entity_id).is_some() {\n ctx.db.next_positions().entity_id().update(np);\n } else {\n ctx.db.next_positions().insert(np);\n }\n }\n }\n}\n", - "syntax": "rust" - }, - "schema/t_021_multi_column_index": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = logs,\n index(name = by_user_day, btree(columns = [user_id, day]))\n)]\npub struct Log {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub day: i32,\n pub message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.logs().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.logs().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.logs().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}\n", - "syntax": "rust" - }, - "t_000_empty_reducers": { - "answer": "use spacetimedb::{reducer, ReducerContext};\n\n#[reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer]\npub fn empty_reducer_with_three_args(\n ctx: &ReducerContext,\n active: bool,\n ratio: f32,\n label: String,\n) -> Result<(), String> {\n Ok(())\n}\n", - "syntax": "rust" - }, - "t_001_basic_tables": { - "answer": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n pub id: i32,\n pub title: String,\n pub price: f32,\n pub in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n pub id: i32,\n pub body: String,\n pub rating: i64,\n pub pinned: bool,\n}\n", - "syntax": "rust" - }, - "t_002_scheduled_table": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()),\n });\n Ok(())\n}\n", - "syntax": "rust" - }, - "t_003_struct_in_table": { - "answer": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n", - "syntax": "rust" - }, - "t_004_insert": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n ctx.db.user().insert(User { id, name, age, active });\n Ok(())\n}\n", - "syntax": "rust" - }, - "t_005_update": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", - "syntax": "rust" - }, - "t_006_delete": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(id);\n}\n", - "syntax": "rust" - }, - "t_007_crud": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.user().id().delete(2);\n}\n", - "syntax": "rust" - }, - "t_008_index_lookup": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}\n", - "syntax": "rust" - }, - "t_009_init": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}\n", - "syntax": "rust" - }, - "t_010_connect": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n pub id: u64,\n pub kind: String,\n}\n\n#[reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n}\n", - "syntax": "rust" - }, - "t_011_helper_function": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 { a + b }\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}\n", - "syntax": "rust" - }, - "t_012_spacetime_product_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}\n", - "syntax": "rust" - }, - "t_013_spacetime_sum_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) });\n}\n", - "syntax": "rust" - }, - "t_014_elementary_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n pub id: i32,\n pub count: i32,\n pub total: i64,\n pub price: f32,\n pub ratio: f64,\n pub active: bool,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}\n", - "syntax": "rust" - }, - "t_015_product_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n pub street: String,\n pub zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n pub id: i32,\n pub home: Address,\n pub work: Address,\n pub pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}\n", - "syntax": "rust" - }, - "t_016_sum_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}\n", - "syntax": "rust" - }, - "t_017_scheduled_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n pub scheduled_id: u64,\n pub scheduled_at: ScheduleAt,\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let every_50ms: ScheduleAt = Duration::from_millis(50).into();\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: every_50ms,\n });\n}\n", - "syntax": "rust" - }, - "t_018_constraints": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = account,\n index(name = by_name, btree(columns = [name]))\n)]\npub struct Account {\n #[primary_key]\n pub id: i32,\n #[unique]\n pub email: String,\n pub name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account { id: 1, email: \"a@example.com\".into(), name: \"Alice\".into() });\n ctx.db.account().insert(Account { id: 2, email: \"b@example.com\".into(), name: \"Bob\".into() });\n}\n", - "syntax": "rust" - }, - "t_019_many_to_many": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n pub user_id: i32,\n pub name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n pub group_id: i32,\n pub title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}\n", - "syntax": "rust" - }, - "t_020_ecs": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n pub entity_id: i32,\n pub vx: i32,\n pub vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n pub entity_id: i32,\n pub x: i32,\n pub y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.entity().insert(Entity { id: 2 });\n\n ctx.db.position().insert(Position {\n entity_id: 1,\n x: 1,\n y: 0,\n });\n ctx.db.position().insert(Position {\n entity_id: 2,\n x: 10,\n y: 0,\n });\n\n ctx.db.velocity().insert(Velocity {\n entity_id: 1,\n vx: 1,\n vy: 0,\n });\n ctx.db.velocity().insert(Velocity {\n entity_id: 2,\n vx: -2,\n vy: 3,\n });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let np = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n\n if ctx.db.next_position().entity_id().find(p.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(np);\n }\n }\n }\n}\n", - "syntax": "rust" - }, - "t_021_multi_column_index": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(\n name = log,\n index(name = by_user_day, btree(columns = [user_id, day]))\n)]\npub struct Log {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub day: i32,\n pub message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}\n", - "syntax": "rust" - } - } - }, - { - "lang": "csharp", - "modes": [ - { - "mode": "docs", - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "models": [ - { - "name": "GPT-5", - "route_api_model": "gpt-5", - "tasks": { - "t_000_empty_reducers": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-000-empty-reducers-golden", - "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_000_empty_reducers/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-000-empty-reducers-golden", - "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:59:45.341740044Z", - "finished_at": "2026-01-27T20:00:17.110354447Z" - }, - "t_001_basic_tables": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Accessor = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Accessor = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-001-basic-tables-golden", - "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_001_basic_tables/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-001-basic-tables-golden", - "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T20:00:10.594319009Z", - "finished_at": "2026-01-27T20:00:39.454473930Z" - }, - "t_002_scheduled_table": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.TickTimer.Iter())\n {\n ctx.Db.TickTimer.ScheduledId.Delete(row.ScheduledId);\n }\n\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n // Tick scheduled reducer. Intentionally left blank.\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T20:00:39.454500245Z", - "finished_at": "2026-01-27T20:01:19.695962602Z" - }, - "t_003_struct_in_table": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Accessor = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Position Pos;\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-003-struct-in-table-golden", - "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-003-struct-in-table-golden", - "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T20:00:06.267861248Z", - "finished_at": "2026-01-27T20:00:37.823969244Z" - }, - "t_004_insert": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 2, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_db": "basics-t-004-insert-golden", - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "InsertUser", - "server": "http://127.0.0.1:34379" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T20:00:17.336276015Z", - "finished_at": "2026-01-27T20:00:48.236577365Z" - }, - "t_005_update": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.User.Id.Find(id) ?? throw new System.Exception(\"User not found\");\n user.Name = name;\n user.Age = age;\n user.Active = active;\n user.Id = id;\n ctx.Db.User.Id.Update(user);\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_db": "basics-t-005-update-golden", - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "UpdateUser", - "server": "http://127.0.0.1:34379" - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:59:43.681559749Z", - "finished_at": "2026-01-27T20:00:17.332627669Z" - }, - "t_006_delete": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm", - "scorer_details": { - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:58:58.264033858Z", - "finished_at": "2026-01-27T19:59:39.007925686Z" - }, - "t_007_crud": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n var found = ctx.Db.User.Id.Find(1);\n if (found is User user)\n {\n user.Name = \"Alice2\";\n user.Age = 31;\n user.Active = false;\n ctx.Db.User.Id.Update(user);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "basics-t-007-crud-golden", - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "Crud", - "server": "http://127.0.0.1:34379" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:59:29.414134061Z", - "finished_at": "2026-01-27T20:00:06.267814536Z" - }, - "t_008_index_lookup": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial class User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Accessor = \"Result\")]\n public partial class Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_db": "basics-t-008-index-lookup-golden", - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "LookupUserName", - "server": "http://127.0.0.1:34379" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T20:00:17.332669093Z", - "finished_at": "2026-01-27T20:00:55.597420674Z" - }, - "t_009_init": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - }, - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:59:39.007990909Z", - "finished_at": "2026-01-27T20:00:10.594272918Z" - }, - "t_010_connect": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-010-connect-golden", - "llm_db": "basics-t-010-connect-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_010_connect/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_010_connect/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-010-connect-golden", - "llm_db": "basics-t-010-connect-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:59:08.068058110Z", - "finished_at": "2026-01-27T19:59:45.341706421Z" - }, - "t_011_helper_function": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b),\n });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5", - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "basics-t-011-helper-function-golden", - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "ComputeSum", - "server": "http://127.0.0.1:34379" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T20:00:37.824007777Z", - "finished_at": "2026-01-27T20:01:12.698364118Z" - }, - "t_012_spacetime_product_type": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Accessor = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", - "scorer_details": { - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetScore", - "server": "http://127.0.0.1:34379" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:58:50.225754898Z", - "finished_at": "2026-01-27T19:59:29.413988946Z" - }, - "t_013_spacetime_sum_type": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Accessor = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetCircle", - "server": "http://127.0.0.1:34379" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:57:55.737450825Z", - "finished_at": "2026-01-27T19:58:46.370198569Z" - }, - "t_014_elementary_columns": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c2006db3efe118f9e2be4d0996245799216709f62c09af27307321501d4ca96f/sql)\n", - "phase": "sql" - } - }, - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c2008bda321ec5b24f06db6690e57eaa2c1b3a9bada2318accf99feb7910837d/sql)\n", - "phase": "sql_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:58:46.370277696Z", - "finished_at": "2026-01-27T19:59:43.681502253Z" - }, - "t_015_product_type_columns": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Accessor = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n if (ctx.Db.Profile.Id.Find(1) == null)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 },\n });\n }\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:34379" - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:58:04.285909374Z", - "finished_at": "2026-01-27T19:58:50.225511855Z" - }, - "t_016_sum_type_columns": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Accessor = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", - "scorer_details": { - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c200a38bd5eaec739ca1bb5f49423cf455fb484e997f330a37dbe10a2f09faac/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c20028da21a53491f84d8895dbcf4906c6c844846426c39ae32b5fd1689a2641/sql)\n", - "phase": "sql_golden" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:57:09.422528630Z", - "finished_at": "2026-01-27T19:58:04.281919840Z" - }, - "t_017_scheduled_columns": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 2, - "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-017-scheduled-columns-golden", - "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", - "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c200bf2221f01a62dfa1acb5a397b4271a57cfc2f171562204bb3bcee2259214/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-017-scheduled-columns-golden", - "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:57:09.413847953Z", - "finished_at": "2026-01-27T19:57:55.737395054Z" - }, - "t_018_constraints": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Account\", Public = true)]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n\n [SpacetimeDB.Unique]\n public string Email;\n\n [SpacetimeDB.Index.BTree(Name = \"by_name\")]\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", - "scorer_details": { - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-018-constraints-golden", - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:34379" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:57:09.410769526Z", - "finished_at": "2026-01-27T19:58:03.584746288Z" - }, - "t_019_many_to_many": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Accessor = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Accessor = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", - "scorer_details": { - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:58:04.046511986Z", - "finished_at": "2026-01-27T19:58:58.263921552Z" - }, - "t_020_ecs": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Entity\")]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Accessor = \"Position\")]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Accessor = \"Velocity\")]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Accessor = \"NextPosition\")]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entities\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n // Positions\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n // Velocities\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n int vx = 0, vy = 0;\n if (vel != null)\n {\n vx = vel.Value.VX;\n vy = vel.Value.VY;\n }\n\n var next = new NextPosition\n {\n EntityId = pos.EntityId,\n X = pos.X + vx,\n Y = pos.Y + vy\n };\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n ctx.Db.NextPosition.EntityId.Update(next);\n }\n else\n {\n ctx.Db.NextPosition.Insert(next);\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", - "scorer_details": { - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c20015874a697116ed39e7f80d0fb9c484518ec19d2cf28ab6bbc9a525d697e0/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c20015874a697116ed39e7f80d0fb9c484518ec19d2cf28ab6bbc9a525d697e0/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c20015874a697116ed39e7f80d0fb9c484518ec19d2cf28ab6bbc9a525d697e0/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34379/v1/database/c20015874a697116ed39e7f80d0fb9c484518ec19d2cf28ab6bbc9a525d697e0/sql)\n", - "phase": "sql" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:58:04.281975972Z", - "finished_at": "2026-01-27T19:59:08.067949434Z" - }, - "t_021_multi_column_index": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5", - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", - "scorer_details": { - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34379", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai", - "started_at": "2026-01-27T19:57:09.418092225Z", - "finished_at": "2026-01-27T19:58:04.046455117Z" - } - } - } - ] - } - ], - "golden_answers": { - "basics/t_000_empty_reducers": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\n}", - "syntax": "csharp" - }, - "basics/t_001_basic_tables": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct Users\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Accessor = \"products\")]\n public partial struct Products\n {\n [PrimaryKey] public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [Table(Accessor = \"notes\")]\n public partial struct Notes\n {\n [PrimaryKey] public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_002_scheduled_table": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}", - "syntax": "csharp" - }, - "basics/t_003_struct_in_table": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Accessor = \"entities\")]\n public partial struct Entity\n {\n [PrimaryKey] public int Id;\n public Position Pos;\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_004_insert": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_005_update": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.users.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_006_delete": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.users.Id.Delete(id);\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_007_crud": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.users.Id.Delete(2);\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_008_index_lookup": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Accessor = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var u = ctx.Db.users.Id.Find(id);\n if (u.HasValue)\n {\n var row = u.Value;\n ctx.Db.results.Insert(new Result { Id = row.Id, Name = row.Name });\n }\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_009_init": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_010_connect": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"events\")]\n public partial struct Event\n {\n [PrimaryKey, AutoInc] public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\n }\n}\n", - "syntax": "csharp" - }, - "basics/t_011_helper_function": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public int Sum;\n }\n\n static int Add(int a, int b) => a + b;\n\n [Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_012_spacetime_product_type": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [Table(Accessor = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Score Value;\n }\n\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_013_spacetime_sum_type": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Accessor = \"results\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Shape Value;\n }\n\n [Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_014_elementary_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"primitives\")]\n public partial struct Primitive\n {\n [PrimaryKey] public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.primitives.Insert(new Primitive {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_015_product_type_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Accessor = \"profiles\")]\n public partial struct Profile\n {\n [PrimaryKey] public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.profiles.Insert(new Profile {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_016_sum_type_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Accessor = \"drawings\")]\n public partial struct Drawing\n {\n [PrimaryKey] public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.drawings.Insert(new Drawing {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_017_scheduled_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.tick_timer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_018_constraints": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"accounts\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey] public int Id;\n [SpacetimeDB.Unique] public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", - "syntax": "csharp" - }, - "schema/t_019_many_to_many": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"users\")]\n public partial struct User\n {\n [PrimaryKey] public int UserId;\n public string Name;\n }\n\n [Table(Accessor = \"groups\")]\n public partial struct Group\n {\n [PrimaryKey] public int GroupId;\n public string Title;\n }\n\n [Table(Accessor = \"memberships\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(UserId) })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(GroupId) })]\n public partial struct Membership\n {\n [PrimaryKey] public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_020_ecs": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"entities\")]\n public partial struct Entity { [PrimaryKey] public int Id; }\n\n [Table(Accessor = \"positions\")]\n public partial struct Position\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Accessor = \"velocities\")]\n public partial struct Velocity\n {\n [PrimaryKey] public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Accessor = \"next_positions\")]\n public partial struct NextPosition\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.entities.Insert(new Entity { Id = 1 });\n ctx.Db.entities.Insert(new Entity { Id = 2 });\n\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var p in ctx.Db.positions.Iter())\n {\n var velOpt = ctx.Db.velocities.EntityId.Find(p.EntityId);\n if (!velOpt.HasValue) continue;\n\n var np = new NextPosition {\n EntityId = p.EntityId,\n X = p.X + velOpt.Value.VX,\n Y = p.Y + velOpt.Value.VY\n };\n\n if (ctx.Db.next_positions.EntityId.Find(p.EntityId).HasValue)\n ctx.Db.next_positions.EntityId.Update(np);\n else\n ctx.Db.next_positions.Insert(np);\n }\n }\n}\n", - "syntax": "csharp" - }, - "schema/t_021_multi_column_index": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"logs\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [PrimaryKey] public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}\n", - "syntax": "csharp" - }, - "t_000_empty_reducers": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\n\n [Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\n\n [Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\n\n [Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\n}", - "syntax": "csharp" - }, - "t_001_basic_tables": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Accessor = \"Product\")]\n public partial struct Product\n {\n [PrimaryKey] public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [Table(Accessor = \"Note\")]\n public partial struct Note\n {\n [PrimaryKey] public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}\n", - "syntax": "csharp" - }, - "t_002_scheduled_table": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}\n", - "syntax": "csharp" - }, - "t_003_struct_in_table": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Accessor = \"Entity\")]\n public partial struct Entity\n {\n [PrimaryKey] public int Id;\n public Position Pos;\n }\n}\n", - "syntax": "csharp" - }, - "t_004_insert": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}\n", - "syntax": "csharp" - }, - "t_005_update": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\n }\n}\n", - "syntax": "csharp" - }, - "t_006_delete": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}\n", - "syntax": "csharp" - }, - "t_007_crud": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}\n", - "syntax": "csharp" - }, - "t_008_index_lookup": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Table(Accessor = \"Result\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public string Name;\n }\n\n [Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var u = ctx.Db.User.Id.Find(id);\n if (u.HasValue)\n {\n var row = u.Value;\n ctx.Db.Result.Insert(new Result { Id = row.Id, Name = row.Name });\n }\n }\n}\n", - "syntax": "csharp" - }, - "t_009_init": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}\n", - "syntax": "csharp" - }, - "t_010_connect": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"Event\")]\n public partial struct Event\n {\n [PrimaryKey, AutoInc] public int Id;\n public string Kind;\n }\n\n [Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\n }\n\n [Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\n }\n}\n", - "syntax": "csharp" - }, - "t_011_helper_function": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"Result\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public int Sum;\n }\n\n static int Add(int a, int b) => a + b;\n\n [Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}\n", - "syntax": "csharp" - }, - "t_012_spacetime_product_type": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n \n [Table(Accessor = \"Result\", Public = true)]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Score Value;\n }\n\n [Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\n }\n}\n", - "syntax": "csharp" - }, - "t_013_spacetime_sum_type": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Accessor = \"Result\")]\n public partial struct Result\n {\n [PrimaryKey] public int Id;\n public Shape Value;\n }\n\n [Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\n }\n}\n", - "syntax": "csharp" - }, - "t_014_elementary_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"Primitive\")]\n public partial struct Primitive\n {\n [PrimaryKey] public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive {\n Id = 1,\n Count = 2,\n Total = 3000000000,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}\n", - "syntax": "csharp" - }, - "t_015_product_type_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [Table(Accessor = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [PrimaryKey] public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}\n", - "syntax": "csharp" - }, - "t_016_sum_type_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Type]\n public partial struct Circle { public int Radius; }\n\n [Type]\n public partial struct Rectangle { public int Width; public int Height; }\n\n [Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\n\n [Table(Accessor = \"Drawing\")]\n public partial struct Drawing\n {\n [PrimaryKey] public int Id;\n public Shape A;\n public Shape B;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}\n", - "syntax": "csharp" - }, - "t_017_scheduled_columns": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc] public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n var interval = new TimeDuration { Microseconds = 50_000 };\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(interval)\n });\n }\n}\n", - "syntax": "csharp" - }, - "t_018_constraints": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey] public int Id;\n [SpacetimeDB.Unique] public string Email;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}\n", - "syntax": "csharp" - }, - "t_019_many_to_many": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\")]\n public partial struct User\n {\n [PrimaryKey] public int UserId;\n public string Name;\n }\n\n [Table(Accessor = \"Group\")]\n public partial struct Group\n {\n [PrimaryKey] public int GroupId;\n public string Title;\n }\n\n [Table(Accessor = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(UserId) })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(GroupId) })]\n public partial struct Membership\n {\n [PrimaryKey] public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}\n", - "syntax": "csharp" - }, - "t_020_ecs": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"Entity\")]\n public partial struct Entity { [PrimaryKey] public int Id; }\n\n [Table(Accessor = \"Position\")]\n public partial struct Position\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Table(Accessor = \"Velocity\")]\n public partial struct Velocity\n {\n [PrimaryKey] public int EntityId;\n public int VX;\n public int VY;\n }\n\n [Table(Accessor = \"NextPosition\")]\n public partial struct NextPosition\n {\n [PrimaryKey] public int EntityId;\n public int X;\n public int Y;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var p in ctx.Db.Position.Iter())\n {\n var velOpt = ctx.Db.Velocity.EntityId.Find(p.EntityId);\n if (!velOpt.HasValue) continue;\n\n var np = new NextPosition {\n EntityId = p.EntityId,\n X = p.X + velOpt.Value.VX,\n Y = p.Y + velOpt.Value.VY\n };\n\n if (ctx.Db.NextPosition.EntityId.Find(p.EntityId).HasValue)\n ctx.Db.NextPosition.EntityId.Update(np);\n else\n ctx.Db.NextPosition.Insert(np);\n }\n }\n}\n", - "syntax": "csharp" - }, - "t_021_multi_column_index": { - "answer": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [PrimaryKey] public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}\n", - "syntax": "csharp" - } - } - } - ] -} diff --git a/docs/llms/docs-benchmark-details.lock b/docs/llms/docs-benchmark-details.lock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json deleted file mode 100644 index bf684185512..00000000000 --- a/docs/llms/docs-benchmark-summary.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "version": 1, - "generated_at": "2026-01-27T20:01:19.767Z", - "by_language": { - "csharp": { - "modes": { - "docs": { - "hash": "e3621eaacfd5238fb514ff8c09c277ccd66170773959e5b391507110bf3ac17b", - "models": { - "GPT-5": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 27, - "pass_pct": 100.0, - "task_pass_equiv": 12.0, - "task_pass_pct": 100.0 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 25, - "pass_pct": 73.52941, - "task_pass_equiv": 7.3666663, - "task_pass_pct": 73.666664 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 52, - "pass_pct": 85.2459, - "task_pass_equiv": 19.366667, - "task_pass_pct": 88.030304 - } - } - } - } - } - }, - "rust": { - "modes": { - "docs": { - "hash": "d1bf0b033db905c668d016082c762eb4b91edd1299f7b8d4bcc6def5398610f9", - "models": { - "GPT-5": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 5, - "pass_pct": 18.518518, - "task_pass_equiv": 1.3333334, - "task_pass_pct": 11.111112 - }, - "schema": { - "tasks": 10, - "total_tests": 32, - "passed_tests": 8, - "pass_pct": 25.0, - "task_pass_equiv": 2.05, - "task_pass_pct": 20.5 - } - }, - "totals": { - "tasks": 22, - "total_tests": 59, - "passed_tests": 13, - "pass_pct": 22.033897, - "task_pass_equiv": 3.3833334, - "task_pass_pct": 15.378788 - } - } - } - }, - "rustdoc_json": { - "hash": "2a0f4ec42cac03fb3da542793b311e4b399845c688bb0660a6c231c82e96c56a", - "models": { - "GPT-5": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 26, - "pass_pct": 96.296295, - "task_pass_equiv": 11.0, - "task_pass_pct": 91.66667 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 23, - "pass_pct": 67.64706, - "task_pass_equiv": 6.5333333, - "task_pass_pct": 65.33333 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 49, - "pass_pct": 80.327866, - "task_pass_equiv": 17.533335, - "task_pass_pct": 79.696976 - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/docs/llms/llm-comparison-details.json b/docs/llms/llm-comparison-details.json deleted file mode 100644 index 186a2936080..00000000000 --- a/docs/llms/llm-comparison-details.json +++ /dev/null @@ -1,28839 +0,0 @@ -{ - "languages": [ - { - "lang": "csharp", - "modes": [ - { - "mode": "docs", - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "models": [ - { - "name": "Grok 4", - "route_api_model": "grok-4", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "xai" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [SpacetimeDB.PrimaryKey]\r\n public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n var user = ctx.Db.User.Id.Find(id).Value;\r\n user.Name = name;\r\n user.Age = age;\r\n user.Active = active;\r\n ctx.Db.User.Id.Update(user);\r\n }\r\n}", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: InvalidOperation_NoValue\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200aa3b9311f7739950b10c863ca1ecbd3638a7be9b433e4255b989c306f4d3/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [SpacetimeDB.PrimaryKey]\r\n public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void Crud(ReducerContext ctx)\r\n {\r\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n var user = ctx.Db.User.Id.Find(1).Value;\r\n user.Name = \"Alice2\";\r\n user.Age = 31;\r\n user.Active = false;\r\n ctx.Db.User.Id.Update(user);\r\n ctx.Db.User.Id.Delete(2);\r\n }\r\n}", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "xai" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "xai" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "llm_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "query": "SELECT Id, A, B FROM Drawing WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Drawing WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "xai" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "xai" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "xai" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=2 AND X=8 AND Y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=1 AND X=1 AND Y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM NextPosition" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "xai" - } - } - }, - { - "name": "Claude Opus 4.6", - "route_api_model": "claude-opus-4-6", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "anthropic" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user is not null)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active,\n });\n }\n }\n}", - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "Id | Name | Age | Active ----+------+-----+--------", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "anthropic" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "anthropic" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - } - }, - "vendor": "anthropic" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "anthropic" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": false, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>;\n\n[SpacetimeDB.Table(Accessor = \"Result\", Public = true)]\npublic partial struct Result\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-opus-4-6\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-opus-4-6\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-opus-4-6\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\claude-opus-4-6\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "anthropic" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": false, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)>;\n\n[SpacetimeDB.Table(Accessor = \"Drawing\", Public = true)]\npublic partial struct Drawing\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-opus-4-6\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-opus-4-6\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-opus-4-6\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\claude-opus-4-6\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "anthropic" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "anthropic" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "anthropic" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "anthropic" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=2 AND X=8 AND Y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=1 AND X=1 AND Y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM NextPosition" - } - } - }, - "vendor": "anthropic" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "anthropic" - } - } - }, - { - "name": "Claude Sonnet 4.6", - "route_api_model": "claude-sonnet-4-6", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "anthropic" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n var user1 = ctx.Db.User.Id.Find(1) ?? throw new System.Exception(\"User 1 not found\");\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "anthropic" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "anthropic" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "anthropic" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "anthropic" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "llm_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "query": "SELECT Id, A, B FROM Drawing WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "drawing": {} - }, - "only_llm": { - "Drawing": {} - } - }, - "tables_equal": false - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Drawing WHERE Id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "anthropic" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "anthropic" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "anthropic" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=2 AND X=8 AND Y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=1 AND X=1 AND Y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM NextPosition" - } - } - }, - "vendor": "anthropic" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "anthropic" - } - } - }, - { - "name": "GPT-5.2-Codex", - "route_api_model": "gpt-5.2-codex", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"EmptyTable\")]\n public partial struct EmptyTable\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_NoArgs(ReducerContext ctx)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithString(ReducerContext ctx, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name)\n {\n }\n\n [SpacetimeDB.Reducer]\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label)\n {\n }\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "empty_table": {} - }, - "only_llm": { - "EmptyTable": {} - } - }, - "tables_equal": false - } - } - }, - "vendor": "openai" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "note": {}, - "product": {}, - "user": {} - }, - "only_llm": { - "Note": {}, - "Product": {}, - "User": {} - } - }, - "tables_equal": false - } - } - }, - "vendor": "openai" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "user": {} - }, - "only_llm": { - "User": {} - } - }, - "tables_equal": false - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "openai" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user == null)\n {\n throw new System.Exception(\"User not found\");\n }\n\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: User not found\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c2007cc5b4cc7151bdd0e1be650a69e8154c8656fdd4b50483331748cd393975/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "user": {} - }, - "only_llm": { - "User": {} - } - }, - "tables_equal": false - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "user": {} - }, - "only_llm": { - "User": {} - } - }, - "tables_equal": false - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "user": {} - }, - "only_llm": { - "User": {} - } - }, - "tables_equal": false - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "openai" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "openai" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "event": {} - }, - "only_llm": { - "Event": {} - } - }, - "tables_equal": false - } - } - }, - "vendor": "openai" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - } - }, - "vendor": "openai" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "openai" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "llm_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "primitive": {} - }, - "only_llm": { - "Primitive": {} - } - }, - "tables_equal": false - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "query": "SELECT Id, A, B FROM Drawing WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "drawing": {} - }, - "only_llm": { - "Drawing": {} - } - }, - "tables_equal": false - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Drawing WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "openai" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Account\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = new[] { nameof(Name) })]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n\n [SpacetimeDB.Unique]\n public string Email;\n\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "account": {} - }, - "only_llm": { - "Account": {} - } - }, - "tables_equal": false - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "openai" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "openai" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is Velocity vel)\n {\n var newX = pos.X + vel.VX;\n var newY = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition existing)\n {\n existing.X = newX;\n existing.Y = newY;\n ctx.Db.NextPosition.EntityId.Update(existing);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = newX,\n Y = newY\n });\n }\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=2 AND X=8 AND Y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=1 AND X=1 AND Y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "entity": {}, - "next_position": {}, - "position": {}, - "velocity": {} - }, - "only_llm": { - "Entity": {}, - "NextPosition": {}, - "Position": {}, - "Velocity": {} - } - }, - "tables_equal": false - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM NextPosition" - } - } - }, - "vendor": "openai" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "log": {} - }, - "only_llm": { - "Log": {} - } - }, - "tables_equal": false - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "openai" - } - } - }, - { - "name": "GPT-5-mini", - "route_api_model": "gpt-5-mini", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "openai" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n else\n {\n throw new System.Exception(\"User not found\");\n }\n }\n}", - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: User not found\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c20048d2fdae4756e4aaa130f4b085ba3c9bb64f9f0610cfcd04c4f66a98f06e/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user)\n {\n user.Name = \"Alice2\";\n user.Age = 31;\n user.Active = false;\n ctx.Db.User.Id.Update(user);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "openai" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "openai" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "openai" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "openai" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Accessor = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-5-mini\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\Module.Result.cs(25,47): error CS0426: The type name 'BSATN' does not exist in the type 'Module.Shape' [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gpt-5-mini\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "openai" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(\n Circle Circle,\n Rectangle Rectangle\n )>\n {\n }\n\n [SpacetimeDB.Table(Accessor = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Insert exactly one row into Drawing:\n // (Id=1, A=Circle{Radius=10}, B=Rectangle{Width=4, Height=6})\n // Construct sum variants using the generated TaggedEnum variants.\n // Depending on codegen, the factory method names may vary;\n // common pattern is Variant constructors accessible via the record.\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = Shape.Circle(new Circle { Radius = 10 }),\n B = Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gpt-5-mini\\llm\\Lib.cs(47,23): error CS1955: Non-invocable member 'Module.Shape.Circle' cannot be used like a method. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gpt-5-mini\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gpt-5-mini\\llm\\Lib.cs(48,23): error CS1955: Non-invocable member 'Module.Shape.Rectangle' cannot be used like a method. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gpt-5-mini\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "openai" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Index.BTree(Accessor = \"by_name\", Columns = new[] { nameof(Name) })]\n [SpacetimeDB.Table(Accessor = \"Account\", Public = true)]\n public partial struct Account\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n [SpacetimeDB.Unique]\n public string Email = \"\";\n public string Name = \"\";\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\n }\n}", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gpt-5-mini\\llm\\Lib.cs(8,27): error CS8983: A 'struct' with field initializers must include an explicitly declared constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_018_constraints\\csharp\\server\\gpt-5-mini\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "openai" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=2 AND X=8 AND Y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=1 AND X=1 AND Y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM NextPosition" - } - } - }, - "vendor": "openai" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "openai" - } - } - }, - { - "name": "Grok Code", - "route_api_model": "grok-code-fast-1", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "xai" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": false, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n }\n}", - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\Lib.cs(22,18): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\Lib.cs(23,18): error CS1061: 'Module.User?' does not contain a definition for 'Age' and no accessible extension method 'Age' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\Lib.cs(24,18): error CS1061: 'Module.User?' does not contain a definition for 'Active' and no accessible extension method 'Active' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\Lib.cs(25,35): error CS1503: Argument 1: cannot convert from 'Module.User?' to 'Module.User' [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_005_update\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "xai" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "xai" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "llm_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "llm_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "query": "SELECT Id, A, B FROM Drawing WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Drawing WHERE Id=1" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "xai" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "xai" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Accessor = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Accessor = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Accessor = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Accessor = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c200bbac8a83a1bd5dc662ac727908d6998563a26278c22b4767396494e5791f/call/seed)\n", - "phase": "call_reducer" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "xai" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": false, - "model_name": "Grok Code", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Accessor = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Accessor = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Accessor = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel != null)\n {\n int newX = pos.X + vel.VX;\n int newY = pos.Y + vel.VY;\n var nextPos = new NextPosition { EntityId = pos.EntityId, X = newX, Y = newY };\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n existing.X = newX;\n existing.Y = newY;\n ctx.Db.NextPosition.EntityId.Update(existing);\n }\n else\n {\n ctx.Db.NextPosition.Insert(nextPos);\n }\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\Lib.cs(59,40): error CS1061: 'Module.Velocity?' does not contain a definition for 'VX' and no accessible extension method 'VX' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\Lib.cs(60,40): error CS1061: 'Module.Velocity?' does not contain a definition for 'VY' and no accessible extension method 'VY' accepting a first argument of type 'Module.Velocity?' could be found (are you missing a using directive or an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\Lib.cs(65,30): error CS1061: 'Module.NextPosition?' does not contain a definition for 'X' and no accessible extension method 'X' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\Lib.cs(66,30): error CS1061: 'Module.NextPosition?' does not contain a definition for 'Y' and no accessible extension method 'Y' accepting a first argument of type 'Module.NextPosition?' could be found (are you missing a using directive or an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\Lib.cs(67,57): error CS1503: Argument 1: cannot convert from 'Module.NextPosition?' to 'Module.NextPosition' [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_020_ecs\\csharp\\server\\grok-code\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "xai" - } - } - }, - { - "name": "Gemini 3.1 Pro", - "route_api_model": "google/gemini-3.1-pro-preview", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "meta" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The row was not found, e.g., in an update call\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200c507051814efa1a26fabe22102c4e36fc055d14106ab65ac9c03fec8d3da/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "meta" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User\n {\n Id = 1,\n Name = \"Alice\",\n Age = 30,\n Active = true\n });\n\n ctx.Db.User.Insert(new User\n {\n Id = 2,\n Name = \"Bob\",\n Age = 22,\n Active = false\n });\n\n ctx.Db.User.Id.Update(new User\n {\n Id = 1,\n Name = \"Alice2\",\n Age = 31,\n Active = false\n });\n\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "meta" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "meta" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "meta" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "meta" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": false, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n[SpacetimeDB.Table(Accessor = \"Result\", Public = true)]\npublic partial struct Result\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-1-pro\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-1-pro\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-1-pro\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-1-pro\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "meta" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "meta" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": false, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n[SpacetimeDB.Table(Accessor = \"Drawing\", Public = true)]\npublic partial struct Drawing\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = Shape.Circle(new Circle { Radius = 10 }),\n B = Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-1-pro\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-1-pro\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-1-pro\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-1-pro\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "meta" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "meta" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "meta" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=2 AND X=8 AND Y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=1 AND X=1 AND Y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM NextPosition" - } - } - }, - "vendor": "meta" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "meta" - } - } - }, - { - "name": "Gemini 3 Flash", - "route_api_model": "google/gemini-3-flash-preview", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "meta" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Table(Accessor = \"User\", Public = true)]\npublic partial struct User\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n }\n}", - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "Id | Name | Age | Active ----+------+-----+--------", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "meta" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Table(Accessor = \"User\", Public = true)]\npublic partial struct User\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User\n {\n Id = 1,\n Name = \"Alice\",\n Age = 30,\n Active = true\n });\n\n ctx.Db.User.Insert(new User\n {\n Id = 2,\n Name = \"Bob\",\n Age = 22,\n Active = false\n });\n\n var alice = ctx.Db.User.Id.Find(1);\n if (alice.HasValue)\n {\n var updatedAlice = alice.Value;\n updatedAlice.Name = \"Alice2\";\n updatedAlice.Age = 31;\n updatedAlice.Active = false;\n ctx.Db.User.Id.Update(updatedAlice);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "meta" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "meta" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "meta" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "meta" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": false, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n[SpacetimeDB.Table(Accessor = \"Result\", Public = true)]\npublic partial struct Result\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-flash\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\gemini-3-flash\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c200c52fb7b64f38ac169672ccaaf9aa528624106da48843b0f078bfbec22654/call/seed)\n", - "phase": "call_reducer_llm" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "meta" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "meta" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": false, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic partial struct Circle\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic partial struct Rectangle\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Type]\npublic partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n[SpacetimeDB.Table(Accessor = \"Drawing\", Public = true)]\npublic partial struct Drawing\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = Shape.Circle(new Circle { Radius = 10 }),\n B = Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(6,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-flash\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-flash\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Shape.cs(12,26): error CS8910: The primary constructor conflicts with the synthesized copy constructor. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\gemini-3-flash\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "meta" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "meta" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": false, - "model_name": "Gemini 3 Flash", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [Table(Accessor = \"Group\", Public = true)]\n public partial struct Group\n {\n [PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [Table(Accessor = \"Membership\", Public = true)]\n [Index.BTree(Accessor = \"by_user\", Columns = new[] { \"UserId\" })]\n [Index.BTree(Accessor = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-3-flash\\llm\\Lib.cs(23,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-3-flash\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-3-flash\\llm\\Lib.cs(24,6): error CS0104: 'Index' is an ambiguous reference between 'SpacetimeDB.Index' and 'System.Index' [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\gemini-3-flash\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=2 AND X=8 AND Y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM NextPosition WHERE EntityId=1 AND X=1 AND Y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM NextPosition" - } - } - }, - "vendor": "meta" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "meta" - } - } - }, - { - "name": "DeepSeek Chat", - "route_api_model": "deepseek-chat", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "deepseek" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user.HasValue)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n }\n}", - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "Id | Name | Age | Active ----+------+-----+--------", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "deepseek" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "deepseek" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "deepseek" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "deepseek" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial struct Shape\n {\n public Circle? Circle;\n public Rectangle? Rectangle;\n }\n\n [SpacetimeDB.Table(Accessor = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape { Circle = new Circle { Radius = radius } }\n });\n }\n}", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "Id | Value ----+-------------------------- 1 | (circle = (radius = 10))", - "llm_out": "Id | Value ----+------------------------------------------------------------ 1 | (circle = (some = (radius = 10)), rectangle = (none = ()))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial struct Shape\n {\n public Circle? Circle;\n public Rectangle? Rectangle;\n }\n\n [SpacetimeDB.Table(Accessor = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape { Circle = new Circle { Radius = 10 } },\n B = new Shape { Rectangle = new Rectangle { Width = 4, Height = 6 } }\n });\n }\n}", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [], - "golden_out": "Id | A | B ----+--------------------------+--------------------------------------- 1 | (circle = (radius = 10)) | (rectangle = (width = 4, height = 6))", - "llm_out": "Id | A | B ----+------------------------------------------------------------+---------------------------------------------------------------------- 1 | (circle = (some = (radius = 10)), rectangle = (none = ())) | (circle = (none = ()), rectangle = (some = (width = 4, height = 6)))", - "query": "SELECT Id, A, B FROM Drawing WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Drawing WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "deepseek" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "deepseek" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" - } - } - }, - "vendor": "deepseek" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 5, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Accessor = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Accessor = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Accessor = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel.HasValue)\n {\n var nextPos = new NextPosition\n {\n EntityId = pos.EntityId,\n X = pos.X + vel.Value.VX,\n Y = pos.Y + vel.Value.VY\n };\n ctx.Db.NextPosition.EntityId.Update(nextPos);\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The row was not found, e.g., in an update call\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c20037a8f99a52ad0d30979e9235082416df0a4c9ac89afa9c7c4d3bf295770b/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The row was not found, e.g., in an update call\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c20037a8f99a52ad0d30979e9235082416df0a4c9ac89afa9c7c4d3bf295770b/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The row was not found, e.g., in an update call\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c20037a8f99a52ad0d30979e9235082416df0a4c9ac89afa9c7c4d3bf295770b/call/step)\n", - "phase": "call_reducer" - } - } - }, - "vendor": "deepseek" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" - } - } - }, - "vendor": "deepseek" - } - } - }, - { - "name": "DeepSeek Reasoner", - "route_api_model": "deepseek-reasoner", - "tasks": { - "t_000_empty_reducers": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_000_empty_reducers", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_001_basic_tables": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_001_basic_tables", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_002_scheduled_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_002_scheduled_table", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_003_struct_in_table": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_003_struct_in_table", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_004_insert": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_004_insert", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "deepseek" - }, - "t_005_update": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_005_update", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.User.Id.Find(id);\n if (existing.HasValue)\n {\n ctx.Db.User.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\n }\n }\n}", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "Id | Name | Age | Active ----+------+-----+--------", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:56242/v1/database/c2003fa012003c49d80084f437bb713f6a093beeaea5587a135d5ff1731afde4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_006_delete": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_006_delete", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_007_crud": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_007_crud", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: Value with given unique identifier already exists\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c200bae1f2bdd27271e560dbfda8814cfd2302b7e12fa45c11457037728b852a/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - } - }, - "vendor": "deepseek" - }, - "t_008_index_lookup": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_008_index_lookup", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_009_init": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_009_init", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Table(Name = \"User\", Public = true)]\npublic partial struct User\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": { - "changed": {}, - "only_golden": { - "user": {} - }, - "only_llm": { - "User": {} - } - }, - "tables_equal": false - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } - } - }, - "vendor": "deepseek" - }, - "t_010_connect": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_010_connect", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_011_helper_function": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_011_helper_function", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - } - }, - "vendor": "deepseek" - }, - "t_012_spacetime_product_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_012_spacetime_product_type", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "Id | Value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "set_score" - } - } - }, - "vendor": "deepseek" - }, - "t_013_spacetime_sum_type": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_013_spacetime_sum_type", - "lang": "csharp", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Type]\npublic abstract class Shape { }\n\n[SpacetimeDB.Type]\npublic class Circle : Shape\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic class Rectangle : Shape\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Table(Accessor = \"Result\")]\npublic partial struct Result\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Circle { Radius = radius }\n });\n }\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(5,23): error CS0260: Missing partial modifier on declaration of type 'Shape'; another partial declaration of this type exists [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(8,14): error CS0260: Missing partial modifier on declaration of type 'Circle'; another partial declaration of this type exists [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(14,14): error CS0260: Missing partial modifier on declaration of type 'Rectangle'; another partial declaration of this type exists [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Circle.cs(6,17): warning CS0108: 'Circle.ReadFields(BinaryReader)' hides inherited member 'Shape.ReadFields(BinaryReader)'. Use the new keyword if hiding was intended. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Circle.cs(10,17): warning CS0108: 'Circle.WriteFields(BinaryWriter)' hides inherited member 'Shape.WriteFields(BinaryWriter)'. Use the new keyword if hiding was intended. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Circle.cs(19,36): warning CS0108: 'Circle.BSATN' hides inherited member 'Shape.BSATN'. Use the new keyword if hiding was intended. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Rectangle.cs(6,17): warning CS0108: 'Rectangle.ReadFields(BinaryReader)' hides inherited member 'Shape.ReadFields(BinaryReader)'. Use the new keyword if hiding was intended. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Rectangle.cs(11,17): warning CS0108: 'Rectangle.WriteFields(BinaryWriter)' hides inherited member 'Shape.WriteFields(BinaryWriter)'. Use the new keyword if hiding was intended. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.BSATN.Codegen\\SpacetimeDB.Codegen.Type\\Rectangle.cs(21,36): warning CS0108: 'Rectangle.BSATN' hides inherited member 'Shape.BSATN'. Use the new keyword if hiding was intended. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_014_elementary_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_014_elementary_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "Id | Count | Total | Price | Ratio | Active | Name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT Id, Count, Total, Price, Ratio, Active, Name FROM Primitive WHERE Id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Primitive WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_015_product_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_015_product_type_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_016_sum_type_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_016_sum_type_columns", - "lang": "csharp", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\n[SpacetimeDB.Sum]\npublic interface Shape { }\n\n[SpacetimeDB.Type]\npublic struct Circle : Shape\n{\n public int Radius;\n}\n\n[SpacetimeDB.Type]\npublic struct Rectangle : Shape\n{\n public int Width;\n public int Height;\n}\n\n[SpacetimeDB.Table(Accessor = \"Drawing\")]\npublic partial struct Drawing\n{\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n}\n\npublic static partial class Module\n{\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Circle { Radius = 10 },\n B = new Rectangle { Width = 4, Height = 6 }\n });\n }\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(8,15): error CS0260: Missing partial modifier on declaration of type 'Circle'; another partial declaration of this type exists [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(14,15): error CS0260: Missing partial modifier on declaration of type 'Rectangle'; another partial declaration of this type exists [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(4,14): error CS0234: The type or namespace name 'SumAttribute' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(4,14): error CS0234: The type or namespace name 'Sum' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\Drawing.cs(26,40): error CS0426: The type name 'BSATN' does not exist in the type 'Shape' [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\obj\\Release\\net8.0\\wasi-wasm\\SpacetimeDB.Codegen\\SpacetimeDB.Codegen.Module\\Drawing.cs(27,40): error CS0426: The type name 'BSATN' does not exist in the type 'Shape' [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_017_scheduled_columns": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_017_scheduled_columns", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM TickTimer WHERE ScheduledId>=0" - } - } - }, - "vendor": "deepseek" - }, - "t_018_constraints": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_018_constraints", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" - } - } - }, - "vendor": "deepseek" - }, - "t_019_many_to_many": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_019_many_to_many", - "lang": "csharp", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Accessor = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Accessor = \"Membership\", Public = true)]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Index.BTree(Accessor = \"by_user\", Columns = new[] { nameof(Membership.UserId) })]\n [SpacetimeDB.Index.BTree(Accessor = \"by_group\", Columns = new[] { nameof(Membership.GroupId) })]\n private static void MembershipIndices() { }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(31,6): error CS0592: Attribute 'SpacetimeDB.Index.BTree' is not valid on this declaration type. It is only valid on 'class, struct, field' declarations. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(32,6): error CS0592: Attribute 'SpacetimeDB.Index.BTree' is not valid on this declaration type. It is only valid on 'class, struct, field' declarations. [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_020_ecs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_020_ecs", - "lang": "csharp", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 5, - "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Accessor = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Accessor = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Accessor = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var position in ctx.Db.Position.Iter())\n {\n var velocity = ctx.Db.Velocity.EntityId.Find(position.EntityId);\n if (velocity.HasValue)\n {\n ctx.Db.NextPosition.EntityId.Update(new NextPosition\n {\n EntityId = position.EntityId,\n X = position.X + velocity.Value.VX,\n Y = position.Y + velocity.Value.VY\n });\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The row was not found, e.g., in an update call\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c20099b3d0c2458756ced30eddcd5b6a159367cc5ed1f56ae744696f8ce77b24/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The row was not found, e.g., in an update call\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c20099b3d0c2458756ced30eddcd5b6a159367cc5ed1f56ae744696f8ce77b24/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM Position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The row was not found, e.g., in an update call\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:56242/v1/database/c20099b3d0c2458756ced30eddcd5b6a159367cc5ed1f56ae744696f8ce77b24/call/step)\n", - "phase": "call_reducer" - } - } - }, - "vendor": "deepseek" - }, - "t_021_multi_column_index": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "task": "t_021_multi_column_index", - "lang": "csharp", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Accessor = \"Log\")]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n\n [SpacetimeDB.Index.BTree(Accessor = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 0, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 0, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 0, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\nD:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-reasoner\\llm\\Lib.cs(17,5): error CS1519: Invalid token '}' in class, record, struct, or interface member declaration [D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_021_multi_column_index\\csharp\\server\\deepseek-reasoner\\llm\\StdbModule.csproj]\r\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - } - } - } - ] - }, - { - "mode": "llms.md", - "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4", - "models": [] - } - ], - "golden_answers": { - "basics/t_000_empty_reducers": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Reducer]\r\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\r\n}", - "syntax": "csharp" - }, - "basics/t_001_basic_tables": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct Users\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Name = \"products\")]\r\n public partial struct Products\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Title;\r\n public float Price;\r\n public bool InStock;\r\n }\r\n\r\n [Table(Name = \"notes\")]\r\n public partial struct Notes\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Body;\r\n public long Rating;\r\n public bool Pinned;\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_002_scheduled_table": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.tick_timer.Insert(new TickTimer\r\n {\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}", - "syntax": "csharp" - }, - "basics/t_003_struct_in_table": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"entities\")]\r\n public partial struct Entity\r\n {\r\n [PrimaryKey] public int Id;\r\n public Position Pos;\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_004_insert": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.users.Insert(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_005_update": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.users.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_006_delete": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void DeleteUser(ReducerContext ctx, int id)\r\n {\r\n ctx.Db.users.Id.Delete(id);\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_007_crud": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void Crud(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n ctx.Db.users.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\r\n ctx.Db.users.Id.Delete(2);\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_008_index_lookup": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void LookupUserName(ReducerContext ctx, int id)\r\n {\r\n var u = ctx.Db.users.Id.Find(id);\r\n if (u.HasValue)\r\n {\r\n var row = u.Value;\r\n ctx.Db.results.Insert(new Result { Id = row.Id, Name = row.Name });\r\n }\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_009_init": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.users.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_010_connect": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"events\")]\r\n public partial struct Event\r\n {\r\n [PrimaryKey, AutoInc] public int Id;\r\n public string Kind;\r\n }\r\n\r\n [Reducer(ReducerKind.ClientConnected)]\r\n public static void ClientConnected(ReducerContext ctx)\r\n {\r\n ctx.Db.events.Insert(new Event { Kind = \"connected\" });\r\n }\r\n\r\n [Reducer(ReducerKind.ClientDisconnected)]\r\n public static void ClientDisconnected(ReducerContext ctx)\r\n {\r\n ctx.Db.events.Insert(new Event { Kind = \"disconnected\" });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "basics/t_011_helper_function": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Sum;\r\n }\r\n\r\n static int Add(int a, int b) => a + b;\r\n\r\n [Reducer]\r\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Sum = Add(a, b) });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_012_spacetime_product_type": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Score\r\n {\r\n public int Left;\r\n public int Right;\r\n }\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Score Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_013_spacetime_sum_type": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Name = \"results\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetCircle(ReducerContext ctx, int id, int radius)\r\n {\r\n ctx.Db.results.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_014_elementary_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"primitives\")]\r\n public partial struct Primitive\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Count;\r\n public long Total;\r\n public float Price;\r\n public double Ratio;\r\n public bool Active;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.primitives.Insert(new Primitive {\r\n Id = 1,\r\n Count = 2,\r\n Total = 3000000000,\r\n Price = 1.5f,\r\n Ratio = 2.25,\r\n Active = true,\r\n Name = \"Alice\"\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_015_product_type_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Address\r\n {\r\n public string Street;\r\n public int Zip;\r\n }\r\n\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"profiles\")]\r\n public partial struct Profile\r\n {\r\n [PrimaryKey] public int Id;\r\n public Address Home;\r\n public Address Work;\r\n public Position Pos;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.profiles.Insert(new Profile {\r\n Id = 1,\r\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\r\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\r\n Pos = new Position { X = 7, Y = 9 }\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_016_sum_type_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Name = \"drawings\")]\r\n public partial struct Drawing\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape A;\r\n public Shape B;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.drawings.Insert(new Drawing {\r\n Id = 1,\r\n A = new Shape.Circle(new Circle { Radius = 10 }),\r\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_017_scheduled_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"tick_timer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.tick_timer.Insert(new TickTimer\r\n {\r\n ScheduledId = 0,\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_018_constraints": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Table(Accessor = \"accounts\", Public = true)]\r\n [SpacetimeDB.Index.BTree(Name = \"by_name\", Columns = [nameof(Name)])]\r\n public partial struct Account\r\n {\r\n [SpacetimeDB.PrimaryKey] public int Id;\r\n [SpacetimeDB.Unique] public string Email;\r\n public string Name;\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.accounts.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\r\n ctx.Db.accounts.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\r\n }\r\n}", - "syntax": "csharp" - }, - "schema/t_019_many_to_many": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"users\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int UserId;\r\n public string Name;\r\n }\r\n\r\n [Table(Name = \"groups\")]\r\n public partial struct Group\r\n {\r\n [PrimaryKey] public int GroupId;\r\n public string Title;\r\n }\r\n\r\n [Table(Name = \"memberships\")]\r\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { nameof(UserId) })]\r\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { nameof(GroupId) })]\r\n public partial struct Membership\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int GroupId;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.users.Insert(new User { UserId = 1, Name = \"Alice\" });\r\n ctx.Db.users.Insert(new User { UserId = 2, Name = \"Bob\" });\r\n\r\n ctx.Db.groups.Insert(new Group { GroupId = 10, Title = \"Admin\" });\r\n ctx.Db.groups.Insert(new Group { GroupId = 20, Title = \"Dev\" });\r\n\r\n ctx.Db.memberships.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\r\n ctx.Db.memberships.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\r\n ctx.Db.memberships.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_020_ecs": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"entities\")]\r\n public partial struct Entity { [PrimaryKey] public int Id; }\r\n\r\n [Table(Name = \"positions\")]\r\n public partial struct Position\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Name = \"velocities\")]\r\n public partial struct Velocity\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int VX;\r\n public int VY;\r\n }\r\n\r\n [Table(Name = \"next_positions\")]\r\n public partial struct NextPosition\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.entities.Insert(new Entity { Id = 1 });\r\n ctx.Db.entities.Insert(new Entity { Id = 2 });\r\n\r\n ctx.Db.positions.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\r\n ctx.Db.positions.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\r\n\r\n ctx.Db.velocities.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\r\n ctx.Db.velocities.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\r\n }\r\n\r\n [Reducer]\r\n public static void Step(ReducerContext ctx)\r\n {\r\n foreach (var p in ctx.Db.positions.Iter())\r\n {\r\n var velOpt = ctx.Db.velocities.EntityId.Find(p.EntityId);\r\n if (!velOpt.HasValue) continue;\r\n\r\n var np = new NextPosition {\r\n EntityId = p.EntityId,\r\n X = p.X + velOpt.Value.VX,\r\n Y = p.Y + velOpt.Value.VY\r\n };\r\n\r\n if (ctx.Db.next_positions.EntityId.Find(p.EntityId).HasValue)\r\n ctx.Db.next_positions.EntityId.Update(np);\r\n else\r\n ctx.Db.next_positions.Insert(np);\r\n }\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "schema/t_021_multi_column_index": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Name = \"logs\")]\r\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\r\n public partial struct Log\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int Day;\r\n public string Message;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.logs.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\r\n ctx.Db.logs.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\r\n ctx.Db.logs.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_000_empty_reducers": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"EmptyTable\")]\r\n public partial struct EmptyTable\r\n {\r\n [PrimaryKey] public int Id;\r\n }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_NoArgs(ReducerContext ctx) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithInt(ReducerContext ctx, int count) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithString(ReducerContext ctx, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithTwoArgs(ReducerContext ctx, int count, string name) { }\r\n\r\n [Reducer]\r\n public static void EmptyReducer_WithThreeArgs(ReducerContext ctx, bool active, float ratio, string label) { }\r\n}", - "syntax": "csharp" - }, - "t_001_basic_tables": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Accessor = \"Product\")]\r\n public partial struct Product\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Title;\r\n public float Price;\r\n public bool InStock;\r\n }\r\n\r\n [Table(Accessor = \"Note\")]\r\n public partial struct Note\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Body;\r\n public long Rating;\r\n public bool Pinned;\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_002_scheduled_table": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer timer) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.TickTimer.Insert(new TickTimer\r\n {\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_003_struct_in_table": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Accessor = \"Entity\")]\r\n public partial struct Entity\r\n {\r\n [PrimaryKey] public int Id;\r\n public Position Pos;\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_004_insert": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.User.Insert(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_005_update": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\r\n {\r\n ctx.Db.User.Id.Update(new User { Id = id, Name = name, Age = age, Active = active });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_006_delete": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void DeleteUser(ReducerContext ctx, int id)\r\n {\r\n ctx.Db.User.Id.Delete(id);\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_007_crud": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer]\r\n public static void Crud(ReducerContext ctx)\r\n {\r\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\r\n ctx.Db.User.Id.Delete(2);\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_008_index_lookup": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Table(Accessor = \"Result\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void LookupUserName(ReducerContext ctx, int id)\r\n {\r\n var u = ctx.Db.User.Id.Find(id);\r\n if (u.HasValue)\r\n {\r\n var row = u.Value;\r\n ctx.Db.Result.Insert(new Result { Id = row.Id, Name = row.Name });\r\n }\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_009_init": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int Id;\r\n public string Name;\r\n public int Age;\r\n public bool Active;\r\n }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\r\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_010_connect": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"Event\")]\r\n public partial struct Event\r\n {\r\n [PrimaryKey, AutoInc] public int Id;\r\n public string Kind;\r\n }\r\n\r\n [Reducer(ReducerKind.ClientConnected)]\r\n public static void ClientConnected(ReducerContext ctx)\r\n {\r\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\r\n }\r\n\r\n [Reducer(ReducerKind.ClientDisconnected)]\r\n public static void ClientDisconnected(ReducerContext ctx)\r\n {\r\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_011_helper_function": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"Result\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Sum;\r\n }\r\n\r\n static int Add(int a, int b) => a + b;\r\n\r\n [Reducer]\r\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\r\n {\r\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_012_spacetime_product_type": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Score\r\n {\r\n public int Left;\r\n public int Right;\r\n }\r\n \r\n [Table(Accessor = \"Result\", Public = true)]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Score Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\r\n {\r\n ctx.Db.Result.Insert(new Result { Id = id, Value = new Score { Left = left, Right = right } });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_013_spacetime_sum_type": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Accessor = \"Result\")]\r\n public partial struct Result\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape Value;\r\n }\r\n\r\n [Reducer]\r\n public static void SetCircle(ReducerContext ctx, int id, int radius)\r\n {\r\n ctx.Db.Result.Insert(new Result { Id = id, Value = new Shape.Circle(new Circle { Radius = radius }) });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_014_elementary_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"Primitive\")]\r\n public partial struct Primitive\r\n {\r\n [PrimaryKey] public int Id;\r\n public int Count;\r\n public long Total;\r\n public float Price;\r\n public double Ratio;\r\n public bool Active;\r\n public string Name;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.Primitive.Insert(new Primitive {\r\n Id = 1,\r\n Count = 2,\r\n Total = 3000000000,\r\n Price = 1.5f,\r\n Ratio = 2.25,\r\n Active = true,\r\n Name = \"Alice\"\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_015_product_type_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Address\r\n {\r\n public string Street;\r\n public int Zip;\r\n }\r\n\r\n [Type]\r\n public partial struct Position\r\n {\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Accessor = \"Profile\", Public = true)]\r\n public partial struct Profile\r\n {\r\n [PrimaryKey] public int Id;\r\n public Address Home;\r\n public Address Work;\r\n public Position Pos;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.Profile.Insert(new Profile {\r\n Id = 1,\r\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\r\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\r\n Pos = new Position { X = 7, Y = 9 }\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_016_sum_type_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Type]\r\n public partial struct Circle { public int Radius; }\r\n\r\n [Type]\r\n public partial struct Rectangle { public int Width; public int Height; }\r\n\r\n [Type]\r\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {}\r\n\r\n [Table(Accessor = \"Drawing\")]\r\n public partial struct Drawing\r\n {\r\n [PrimaryKey] public int Id;\r\n public Shape A;\r\n public Shape B;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.Drawing.Insert(new Drawing {\r\n Id = 1,\r\n A = new Shape.Circle(new Circle { Radius = 10 }),\r\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_017_scheduled_columns": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\r\n public partial struct TickTimer\r\n {\r\n [PrimaryKey, AutoInc] public ulong ScheduledId;\r\n public ScheduleAt ScheduledAt;\r\n }\r\n\r\n [Reducer]\r\n public static void Tick(ReducerContext ctx, TickTimer schedule) { }\r\n\r\n [Reducer(ReducerKind.Init)]\r\n public static void Init(ReducerContext ctx)\r\n {\r\n var interval = new TimeDuration { Microseconds = 50_000 };\r\n ctx.Db.TickTimer.Insert(new TickTimer\r\n {\r\n ScheduledId = 0,\r\n ScheduledAt = new ScheduleAt.Interval(interval)\r\n });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_018_constraints": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [SpacetimeDB.Table(Accessor = \"Account\", Public = true)]\r\n [SpacetimeDB.Index.BTree(Accessor = \"by_name\", Columns = [nameof(Name)])]\r\n public partial struct Account\r\n {\r\n [SpacetimeDB.PrimaryKey] public int Id;\r\n [SpacetimeDB.Unique] public string Email;\r\n public string Name;\r\n }\r\n\r\n [SpacetimeDB.Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.Account.Insert(new Account { Id = 1, Email = \"a@example.com\", Name = \"Alice\" });\r\n ctx.Db.Account.Insert(new Account { Id = 2, Email = \"b@example.com\", Name = \"Bob\" });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_019_many_to_many": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"User\")]\r\n public partial struct User\r\n {\r\n [PrimaryKey] public int UserId;\r\n public string Name;\r\n }\r\n\r\n [Table(Accessor = \"Group\")]\r\n public partial struct Group\r\n {\r\n [PrimaryKey] public int GroupId;\r\n public string Title;\r\n }\r\n\r\n [Table(Accessor = \"Membership\")]\r\n [SpacetimeDB.Index.BTree(Accessor = \"by_user\", Columns = new[] { nameof(UserId) })]\r\n [SpacetimeDB.Index.BTree(Accessor = \"by_group\", Columns = new[] { nameof(GroupId) })]\r\n public partial struct Membership\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int GroupId;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\r\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\r\n\r\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\r\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\r\n\r\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\r\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\r\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_020_ecs": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"Entity\")]\r\n public partial struct Entity { [PrimaryKey] public int Id; }\r\n\r\n [Table(Accessor = \"Position\")]\r\n public partial struct Position\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Table(Accessor = \"Velocity\")]\r\n public partial struct Velocity\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int VX;\r\n public int VY;\r\n }\r\n\r\n [Table(Accessor = \"NextPosition\")]\r\n public partial struct NextPosition\r\n {\r\n [PrimaryKey] public int EntityId;\r\n public int X;\r\n public int Y;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\r\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\r\n\r\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\r\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\r\n\r\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\r\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\r\n }\r\n\r\n [Reducer]\r\n public static void Step(ReducerContext ctx)\r\n {\r\n foreach (var p in ctx.Db.Position.Iter())\r\n {\r\n var velOpt = ctx.Db.Velocity.EntityId.Find(p.EntityId);\r\n if (!velOpt.HasValue) continue;\r\n\r\n var np = new NextPosition {\r\n EntityId = p.EntityId,\r\n X = p.X + velOpt.Value.VX,\r\n Y = p.Y + velOpt.Value.VY\r\n };\r\n\r\n if (ctx.Db.NextPosition.EntityId.Find(p.EntityId).HasValue)\r\n ctx.Db.NextPosition.EntityId.Update(np);\r\n else\r\n ctx.Db.NextPosition.Insert(np);\r\n }\r\n }\r\n}\r\n", - "syntax": "csharp" - }, - "t_021_multi_column_index": { - "answer": "using SpacetimeDB;\r\n\r\npublic static partial class Module\r\n{\r\n [Table(Accessor = \"Log\")]\r\n [SpacetimeDB.Index.BTree(Accessor = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\r\n public partial struct Log\r\n {\r\n [PrimaryKey] public int Id;\r\n public int UserId;\r\n public int Day;\r\n public string Message;\r\n }\r\n\r\n [Reducer]\r\n public static void Seed(ReducerContext ctx)\r\n {\r\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\r\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\r\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\r\n }\r\n}\r\n", - "syntax": "csharp" - } - } - }, - { - "lang": "rust", - "modes": [ - { - "mode": "docs", - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "models": [ - { - "name": "Grok 4", - "route_api_model": "grok-4", - "tasks": { - "t_000_empty_reducers": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_001_basic_tables": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_002_scheduled_table": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_003_struct_in_table": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_004_insert": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "xai" - }, - "t_005_update": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_006_delete": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_007_crud": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "xai" - }, - "t_008_index_lookup": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_009_init": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "xai" - }, - "t_010_connect": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_011_helper_function": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[spacetimedb::table(accessor = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling serde v1.0.228\n Compiling itoa v1.0.17\n Compiling find-msvc-tools v0.1.9\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling heck v0.4.1\n Compiling bytes v1.11.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.25.0\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling itertools v0.12.1\n Compiling hex v0.4.3\n Compiling zmij v1.0.21\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling uuid v1.21.0\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling num-traits v0.2.19\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling generic-array v0.14.7\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling rand v0.8.5\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\grok-4\\llm)\nwarning: unused import: `table`\n --> src\\lib.rs:2:19\n |\n2 | use spacetimedb::{table, reducer, ReducerContext};\n | ^^^^^\n |\n = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default\n\nerror[E0599]: no method named `insert` found for reference `&result__TableHandle` in the current scope\n --> src\\lib.rs:17:21\n |\n17 | ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | ctx.db.result().try_insert(ResultRow { id, sum: add(a, b) });\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 1 previous error; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_013_spacetime_sum_type": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_014_elementary_columns": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(accessor = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling find-msvc-tools v0.1.9\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling itoa v1.0.17\n Compiling shlex v1.3.0\n Compiling bitflags v2.11.0\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling bytes v1.11.1\n Compiling convert_case v0.4.0\n Compiling bytemuck v1.25.0\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling zmij v1.0.21\n Compiling smallvec v1.15.1\n Compiling uuid v1.21.0\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling http v1.4.0\n Compiling rand_core v0.6.4\n Compiling cc v1.2.56\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\grok-4\\llm)\nerror[E0599]: no method named `insert` found for reference `&primitive__TableHandle` in the current scope\n --> src\\lib.rs:18:24\n |\n18 | ctx.db.primitive().insert(Primitive {\n | -------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n18 | ctx.db.primitive().try_insert(Primitive {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_015_product_type_columns": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_016_sum_type_columns": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "xai" - }, - "t_018_constraints": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "xai" - }, - "t_019_many_to_many": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "xai" - }, - "t_020_ecs": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "9d3acb3d6846ea1568d76e4b62ded3705f5a1dea72331bd2cbcfb5aae9d6761c", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "xai" - } - } - }, - { - "name": "Grok Code", - "route_api_model": "grok-code-fast-1", - "tasks": { - "t_000_empty_reducers": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_001_basic_tables": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_002_scheduled_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": false, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ScheduleAt, ReducerContext, Table};\nuse std::time::Duration;\n\n#[table(accessor = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n pub scheduled_id: u64,\n pub scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}\n\n#[reducer]\npub fn tick(ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n log::info!(\"Tick!\");\n Ok(())\n}", - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.9\n Compiling itoa v1.0.17\n Compiling anyhow v1.0.102\n Compiling nohash-hasher v0.2.0\n Compiling zerocopy v0.8.39\n Compiling thiserror v1.0.69\n Compiling bitflags v2.11.0\n Compiling bytes v1.11.1\n Compiling ryu v1.0.23\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling convert_case v0.4.0\n Compiling uuid v1.21.0\n Compiling bytemuck v1.25.0\n Compiling getrandom v0.2.17\n Compiling zmij v1.0.21\n Compiling itertools v0.12.1\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling arrayref v0.3.9\n Compiling hex v0.4.3\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling log v0.4.29\n Compiling cc v1.2.56\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling blake3 v1.8.3\n Compiling syn v2.0.117\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\grok-code\\llm)\nerror[E0277]: the `?` operator can only be applied to values that implement `Try`\n --> src\\lib.rs:15:5\n |\n15 | / ctx.db.tick_timer().insert(TickTimer {\n16 | | scheduled_id: 0,\n17 | | scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n18 | | })?;\n | |_______^ the `?` operator cannot be applied to type `TickTimer`\n |\nhelp: the trait `Try` is not implemented for `TickTimer`\n --> src\\lib.rs:6:1\n |\n 6 | pub struct TickTimer {\n | ^^^^^^^^^^^^^^^^^^^^\n\nwarning: unused variable: `ctx`\n --> src\\lib.rs:23:13\n |\n23 | pub fn tick(ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n | ^^^ help: if this is intentional, prefix it with an underscore: `_ctx`\n |\n = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default\n\nFor more information about this error, try `rustc --explain E0277`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 1 previous error; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_003_struct_in_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_004_insert": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "xai" - }, - "t_005_update": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(accessor = user, public)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n if let Some(mut user) = ctx.db.user().id().find(id) {\n user.name = name;\n user.age = age;\n user.active = active;\n ctx.db.user().id().update(user);\n Ok(())\n } else {\n Err(\"User not found\".to_string())\n }\n}", - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: User not found\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:52560/v1/database/c200b4b6b681336f3d490f150af6142028dda5d692b4617ae373d927c932a73c/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:52560/v1/database/c2001d6dfa10edf536fccc3c29d6f6cb7b463b8da0a3dd77d4fb71c99c7750b4/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_006_delete": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_007_crud": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[spacetimedb::table(accessor = user, public)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) -> Result<(), String> {\n // insert (id=1, name=\"Alice\", age=30, active=true)\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n // insert (id=2, name=\"Bob\", age=22, active=false)\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n // update (id=1, name=\"Alice2\", age=31, active=false)\n let mut user = ctx.db.user().id().find(1).unwrap();\n user.name = \"Alice2\".to_string();\n user.age = 31;\n user.active = false;\n ctx.db.user().id().update(user);\n\n // delete id=2\n ctx.db.user().id().delete(2);\n\n Ok(())\n}", - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:52560/v1/database/c20066687897c66c0ddc228392f529f52e38e116fa0210b3a1a7c200f6511192/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "xai" - }, - "t_008_index_lookup": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_009_init": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "xai" - }, - "t_010_connect": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_011_helper_function": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_013_spacetime_sum_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_014_elementary_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_015_product_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_016_sum_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "xai" - }, - "t_018_constraints": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "xai" - }, - "t_019_many_to_many": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[spacetimedb::table(accessor = user, public)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(accessor = group, public)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(accessor = membership, index(accessor = by_user, btree(columns = [user_id])), index(accessor = by_group, btree(columns = [group_id])), public)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer(init)]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n Ok(())\n}", - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: can't directly call special Init lifecycle reducer\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:52560/v1/database/c200a51c04585e87f6869330cab3dc6e8908e5185bc174fc786acd0c2f65c478/call/seed)\n", - "phase": "call_reducer" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "xai" - }, - "t_020_ecs": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "xai" - } - } - }, - { - "name": "Claude Sonnet 4.6", - "route_api_model": "claude-sonnet-4-6", - "tasks": { - "t_000_empty_reducers": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_001_basic_tables": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_002_scheduled_table": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_003_struct_in_table": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_004_insert": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "anthropic" - }, - "t_005_update": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_006_delete": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_007_crud": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "anthropic" - }, - "t_008_index_lookup": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_009_init": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "anthropic" - }, - "t_010_connect": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_011_helper_function": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "anthropic" - }, - "t_012_spacetime_product_type": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "anthropic" - }, - "t_013_spacetime_sum_type": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_014_elementary_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_015_product_type_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_016_sum_type_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_017_scheduled_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "anthropic" - }, - "t_018_constraints": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "anthropic" - }, - "t_019_many_to_many": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "anthropic" - }, - "t_020_ecs": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "anthropic" - }, - "t_021_multi_column_index": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "anthropic" - } - } - }, - { - "name": "Claude Opus 4.6", - "route_api_model": "claude-opus-4-6", - "tasks": { - "t_000_empty_reducers": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_001_basic_tables": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_002_scheduled_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_003_struct_in_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_004_insert": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "anthropic" - }, - "t_005_update": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_006_delete": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_007_crud": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "anthropic" - }, - "t_008_index_lookup": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_009_init": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "anthropic" - }, - "t_010_connect": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_011_helper_function": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "anthropic" - }, - "t_012_spacetime_product_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "anthropic" - }, - "t_013_spacetime_sum_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_014_elementary_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_015_product_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_016_sum_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_017_scheduled_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "anthropic" - }, - "t_018_constraints": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "anthropic" - }, - "t_019_many_to_many": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "anthropic" - }, - "t_020_ecs": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "anthropic" - }, - "t_021_multi_column_index": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "anthropic" - } - } - }, - { - "name": "Gemini 3.1 Pro", - "route_api_model": "gemini-3.1-pro-preview", - "tasks": { - "t_000_empty_reducers": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_001_basic_tables": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_002_scheduled_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_003_struct_in_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": false, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity, public)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling itoa v1.0.17\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.9\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling anyhow v1.0.102\n Compiling thiserror v1.0.69\n Compiling zerocopy v0.8.39\n Compiling ryu v1.0.23\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling keccak v0.1.6\n Compiling bytes v1.11.1\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling second-stack v0.3.5\n Compiling itertools v0.12.1\n Compiling bytemuck v1.25.0\n Compiling uuid v1.21.0\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling zmij v1.0.21\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling generic-array v0.14.7\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\gemini-3-1-pro\\llm)\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = entity` with `accessor = entity`.\n --> src\\lib.rs:10:22\n |\n10 | #[spacetimedb::table(name = entity, public)]\n | ^^^^^^\n\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "google" - }, - "t_004_insert": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "google" - }, - "t_005_update": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "google" - }, - "t_006_delete": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_007_crud": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "google" - }, - "t_008_index_lookup": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": false, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[spacetimedb::table(name = result, public)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user_row) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow {\n id: user_row.id,\n name: user_row.name,\n });\n }\n}", - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling find-msvc-tools v0.1.9\n Compiling itoa v1.0.17\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.39\n Compiling anyhow v1.0.102\n Compiling bitflags v2.11.0\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.1\n Compiling heck v0.4.1\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling zmij v1.0.21\n Compiling arrayref v0.3.9\n Compiling uuid v1.21.0\n Compiling itertools v0.12.1\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling generic-array v0.14.7\n Compiling bytemuck v1.25.0\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\gemini-3-1-pro\\llm)\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = user` with `accessor = user`.\n --> src\\lib.rs:4:22\n |\n4 | #[spacetimedb::table(name = user, public)]\n | ^^^^^^\n\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = result` with `accessor = result`.\n --> src\\lib.rs:13:22\n |\n13 | #[spacetimedb::table(name = result, public)]\n | ^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `ResultRow` in this scope\n --> src\\lib.rs:23:32\n |\n23 | ctx.db.result().insert(ResultRow {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `user` found for struct `Local` in the current scope\n --> src\\lib.rs:22:36\n |\n22 | if let Some(user_row) = ctx.db.user().id().find(id) {\n | ^^^^ method not found in `Local`\n\nerror[E0599]: no method named `result` found for struct `Local` in the current scope\n --> src\\lib.rs:23:16\n |\n23 | ctx.db.result().insert(ResultRow {\n | ^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "google" - }, - "t_009_init": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "google" - }, - "t_010_connect": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_010_connect", - "lang": "rust", - "golden_published": false, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event, public)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n pub id: i32,\n pub kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.9\n Compiling either v1.15.0\n Compiling itoa v1.0.17\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.1\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.17\n Compiling zmij v1.0.21\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling itertools v0.12.1\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.25.0\n Compiling hex v0.4.3\n Compiling uuid v1.21.0\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling num-traits v0.2.19\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_010_connect\\rust\\server\\gemini-3-1-pro\\llm)\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = event` with `accessor = event`.\n --> src\\lib.rs:4:22\n |\n4 | #[spacetimedb::table(name = event, public)]\n | ^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `Event` in this scope\n --> src\\lib.rs:14:27\n |\n14 | ctx.db.event().insert(Event {\n | ^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Event` in this scope\n --> src\\lib.rs:22:27\n |\n22 | ctx.db.event().insert(Event {\n | ^^^^^ not found in this scope\n\nerror[E0599]: no method named `event` found for struct `Local` in the current scope\n --> src\\lib.rs:14:12\n |\n14 | ctx.db.event().insert(Event {\n | ^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `event` found for struct `Local` in the current scope\n --> src\\lib.rs:22:12\n |\n22 | ctx.db.event().insert(Event {\n | ^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "google" - }, - "t_011_helper_function": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "google" - }, - "t_012_spacetime_product_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": false, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n pub left: i32,\n pub right: i32,\n}\n\n#[table(name = result, public)]\npub struct ResultRow {\n #[primary_key]\n pub id: i32,\n pub value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling itoa v1.0.17\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.9\n Compiling zerocopy v0.8.39\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling anyhow v1.0.102\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling ryu v1.0.23\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling bytes v1.11.1\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling bytemuck v1.25.0\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling itertools v0.12.1\n Compiling uuid v1.21.0\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling zmij v1.0.21\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling cc v1.2.56\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling rand v0.8.5\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_012_spacetime_product_type\\rust\\server\\gemini-3-1-pro\\llm)\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = result` with `accessor = result`.\n --> src\\lib.rs:10:9\n |\n10 | #[table(name = result, public)]\n | ^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `ResultRow` in this scope\n --> src\\lib.rs:19:28\n |\n19 | ctx.db.result().insert(ResultRow {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `result` found for struct `Local` in the current scope\n --> src\\lib.rs:19:12\n |\n19 | ctx.db.result().insert(ResultRow {\n | ^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 3 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "google" - }, - "t_013_spacetime_sum_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_014_elementary_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_015_product_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_016_sum_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": false, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing, public)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling itoa v1.0.17\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.9\n Compiling thiserror v1.0.69\n Compiling bitflags v2.11.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling bytes v1.11.1\n Compiling arrayvec v0.7.6\n Compiling ryu v1.0.23\n Compiling heck v0.4.1\n Compiling keccak v0.1.6\n Compiling bytemuck v1.25.0\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling uuid v1.21.0\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling zmij v1.0.21\n Compiling generic-array v0.14.7\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling rand_core v0.6.4\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\rust\\server\\gemini-3-1-pro\\llm)\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = drawing` with `accessor = drawing`.\n --> src\\lib.rs:16:9\n |\n16 | #[table(name = drawing, public)]\n | ^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `Drawing` in this scope\n --> src\\lib.rs:26:29\n |\n26 | ctx.db.drawing().insert(Drawing {\n | ^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `drawing` found for struct `Local` in the current scope\n --> src\\lib.rs:26:12\n |\n26 | ctx.db.drawing().insert(Drawing {\n | ^^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 3 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "google" - }, - "t_017_scheduled_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "google" - }, - "t_018_constraints": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "google" - }, - "t_019_many_to_many": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "google" - }, - "t_020_ecs": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "google" - }, - "t_021_multi_column_index": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "gemini-3.1-pro-preview", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "google" - } - } - }, - { - "name": "Gemini 3 Flash", - "route_api_model": "gemini-3-flash-preview", - "tasks": { - "t_000_empty_reducers": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_001_basic_tables": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_002_scheduled_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_003_struct_in_table": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_004_insert": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "google" - }, - "t_005_update": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "google" - }, - "t_006_delete": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_007_crud": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(accessor = user, public)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n ctx.db.user().id().update(User {\n id: 1,\n name: \"Alice2\".to_string(),\n age: 31,\n active: false,\n });\n\n ctx.db.user().id().delete(2);\n}", - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:65377/v1/database/c20096c5a2babaf26bff2ee7bd720dcc5093a966ac672bda297b2fa31c09128c/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "google" - }, - "t_008_index_lookup": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "google" - }, - "t_009_init": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "google" - }, - "t_010_connect": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "google" - }, - "t_011_helper_function": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "google" - }, - "t_012_spacetime_product_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "google" - }, - "t_013_spacetime_sum_type": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_014_elementary_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_015_product_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_016_sum_type_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "google" - }, - "t_017_scheduled_columns": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "google" - }, - "t_018_constraints": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "google" - }, - "t_019_many_to_many": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "google" - }, - "t_020_ecs": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "google" - }, - "t_021_multi_column_index": { - "hash": "3505a160bff25cbe99eb4235b037d39ade89af05864f8b90b7723b63caaa3781", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "gemini-3-flash-preview", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "google" - } - } - }, - { - "name": "GPT-5.2-Codex", - "route_api_model": "gpt-5.2-codex", - "tasks": { - "t_000_empty_reducers": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext};\n\n#[spacetimedb::table(accessor = Placeholder)]\npub struct Placeholder {\n #[primary_key]\n pub id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling itoa v1.0.17\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.9\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.39\n Compiling anyhow v1.0.102\n Compiling thiserror v1.0.69\n Compiling bitflags v2.11.0\n Compiling nohash-hasher v0.2.0\n Compiling bytes v1.11.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling ryu v1.0.23\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling heck v0.4.1\n Compiling constant_time_eq v0.4.2\n Compiling zmij v1.0.21\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling uuid v1.21.0\n Compiling bytemuck v1.25.0\n Compiling itertools v0.12.1\n Compiling smallvec v1.15.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling arrayref v0.3.9\n Compiling generic-array v0.14.7\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\gpt-5-2-codex\\llm)\nerror[E0428]: the name `Placeholder` is defined multiple times\n --> src\\lib.rs:5:1\n |\n4 | #[spacetimedb::table(accessor = Placeholder)]\n | _________________________________-\n5 | | pub struct Placeholder {\n | | ^^-^^^^^^^^^^^^^^^^^^^\n | |_|_|\n | | previous definition of the trait `Placeholder` here\n | `Placeholder` redefined here\n |\n = note: `Placeholder` must be defined only once in the type namespace of this module\n\nerror[E0574]: expected struct, variant or union type, found trait `Placeholder`\n --> src\\lib.rs:5:12\n |\n5 | pub struct Placeholder {\n | ^^^^^^^^^^^ not a struct, variant or union type\n\nerror[E0782]: expected a type, found a trait\n --> src\\lib.rs:5:12\n |\n5 | pub struct Placeholder {\n | ^^^^^^^^^^^\n |\nhelp: you can add the `dyn` keyword if you want a trait object\n |\n5 | pub struct dyn Placeholder {\n | +++\n\nerror[E0782]: expected a type, found a trait\n --> src\\lib.rs:5:12\n |\n5 | pub struct Placeholder {\n | ^^^^^^^^^^^\n\nerror[E0782]: expected a type, found a trait\n --> src\\lib.rs:5:12\n |\n5 | pub struct Placeholder {\n | ^^^^^^^^^^^\n |\nhelp: use a new generic type parameter, constrained by `Placeholder`\n |\n4 ~ #[spacetimedb::table(accessor = Placeholder)]\n5 ~ pub struct T {\n |\nhelp: you can also use an opaque type, but users won't be able to specify the type parameter when calling the `fn`, having to rely exclusively on type inference\n |\n5 | pub struct impl Placeholder {\n | ++++\nhelp: alternatively, use a trait object to accept any type that implements `Placeholder`, accessing its methods at runtime using dynamic dispatch\n |\n5 | pub struct dyn Placeholder {\n | +++\n\nSome errors have detailed explanations: E0428, E0574, E0782.\nFor more information about an error, try `rustc --explain E0428`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_001_basic_tables": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_002_scheduled_table": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_003_struct_in_table": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_004_insert": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "openai" - }, - "t_005_update": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_006_delete": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_007_crud": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "openai" - }, - "t_008_index_lookup": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_009_init": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "openai" - }, - "t_010_connect": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_011_helper_function": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "openai" - }, - "t_012_spacetime_product_type": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "openai" - }, - "t_013_spacetime_sum_type": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_014_elementary_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_015_product_type_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_016_sum_type_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_017_scheduled_columns": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "openai" - }, - "t_018_constraints": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "openai" - }, - "t_019_many_to_many": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n pub user_id: i32,\n pub name: String,\n}\n\n#[spacetimedb::table(name = group, public)]\npub struct Group {\n #[primary_key]\n pub group_id: i32,\n pub title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n public,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n user_id: 1,\n name: \"Alice\".to_string(),\n });\n ctx.db.user().insert(User {\n user_id: 2,\n name: \"Bob\".to_string(),\n });\n\n ctx.db.group().insert(Group {\n group_id: 10,\n title: \"Admin\".to_string(),\n });\n ctx.db.group().insert(Group {\n group_id: 20,\n title: \"Dev\".to_string(),\n });\n\n ctx.db.membership().insert(Membership {\n id: 1,\n user_id: 1,\n group_id: 10,\n });\n ctx.db.membership().insert(Membership {\n id: 2,\n user_id: 1,\n group_id: 20,\n });\n ctx.db.membership().insert(Membership {\n id: 3,\n user_id: 2,\n group_id: 20,\n });\n}", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.9\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling itoa v1.0.17\n Compiling anyhow v1.0.102\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling zerocopy v0.8.39\n Compiling bitflags v2.11.0\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.1\n Compiling ryu v1.0.23\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling second-stack v0.3.5\n Compiling zmij v1.0.21\n Compiling constant_time_eq v0.4.2\n Compiling itertools v0.12.1\n Compiling hex v0.4.3\n Compiling uuid v1.21.0\n Compiling bytemuck v1.25.0\n Compiling rand_core v0.6.4\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\rust\\server\\gpt-5-2-codex\\llm)\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = user` with `accessor = user`.\n --> src\\lib.rs:4:22\n |\n4 | #[spacetimedb::table(name = user, public)]\n | ^^^^^^\n\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = group` with `accessor = group`.\n --> src\\lib.rs:11:22\n |\n11 | #[spacetimedb::table(name = group, public)]\n | ^^^^^^\n\nerror: Expected a string literal for `name`, but found an identifier. Did you mean to specify an `accessor`?\n \n If you're migrating from SpacetimeDB 1.*, replace `name = membership` with `accessor = membership`.\n --> src\\lib.rs:19:5\n |\n19 | name = membership,\n | ^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `User` in this scope\n --> src\\lib.rs:33:26\n |\n33 | ctx.db.user().insert(User {\n | ^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `User` in this scope\n --> src\\lib.rs:37:26\n |\n37 | ctx.db.user().insert(User {\n | ^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Group` in this scope\n --> src\\lib.rs:42:27\n |\n42 | ctx.db.group().insert(Group {\n | ^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Group` in this scope\n --> src\\lib.rs:46:27\n |\n46 | ctx.db.group().insert(Group {\n | ^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Membership` in this scope\n --> src\\lib.rs:51:32\n |\n51 | ctx.db.membership().insert(Membership {\n | ^^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Membership` in this scope\n --> src\\lib.rs:56:32\n |\n56 | ctx.db.membership().insert(Membership {\n | ^^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Membership` in this scope\n --> src\\lib.rs:61:32\n |\n61 | ctx.db.membership().insert(Membership {\n | ^^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `user` found for struct `Local` in the current scope\n --> src\\lib.rs:33:12\n |\n33 | ctx.db.user().insert(User {\n | ^^^^ method not found in `Local`\n\nerror[E0599]: no method named `user` found for struct `Local` in the current scope\n --> src\\lib.rs:37:12\n |\n37 | ctx.db.user().insert(User {\n | ^^^^ method not found in `Local`\n\nerror[E0599]: no method named `group` found for struct `Local` in the current scope\n --> src\\lib.rs:42:12\n |\n42 | ctx.db.group().insert(Group {\n | ^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `group` found for struct `Local` in the current scope\n --> src\\lib.rs:46:12\n |\n46 | ctx.db.group().insert(Group {\n | ^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `membership` found for struct `Local` in the current scope\n --> src\\lib.rs:51:12\n |\n51 | ctx.db.membership().insert(Membership {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `membership` found for struct `Local` in the current scope\n --> src\\lib.rs:56:12\n |\n56 | ctx.db.membership().insert(Membership {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `membership` found for struct `Local` in the current scope\n --> src\\lib.rs:61:12\n |\n61 | ctx.db.membership().insert(Membership {\n | ^^^^^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 17 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_020_ecs": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "openai" - }, - "t_021_multi_column_index": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "openai" - } - } - }, - { - "name": "GPT-5-mini", - "route_api_model": "gpt-5-mini", - "tasks": { - "t_000_empty_reducers": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_001_basic_tables": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_002_scheduled_table": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_003_struct_in_table": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_004_insert": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "openai" - }, - "t_005_update": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(accessor = user)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n Ok(())\n } else {\n Err(\"User not found\".to_string())\n }\n}", - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: User not found\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62032/v1/database/c2009b55af479168fa8dc45fa8e822846c3e82e1fb2e0ac91f9ffd0da9f1a058/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:62032/v1/database/c2003a3ababf2e2b5374bac162fec14fc8b57142230b968a3bbb889bbd60b560/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_006_delete": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_007_crud": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(accessor = user, public)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) -> Result<(), String> {\n // Insert id=1 Alice\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n // Insert id=2 Bob\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n // Update id=1 -> Alice2, age=31, active=false\n if let Some(mut u) = ctx.db.user().id().find(1) {\n u.name = \"Alice2\".to_string();\n u.age = 31;\n u.active = false;\n ctx.db.user().id().update(u);\n } else {\n return Err(\"User with id=1 not found for update\".to_string());\n }\n\n // Delete id=2\n ctx.db.user().id().delete(2);\n\n Ok(())\n}", - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62032/v1/database/c200b25c6336d7792fc5beed9cc4b084a6e741db8bf7ce1e2484acf8479e91e0/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "openai" - }, - "t_008_index_lookup": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_009_init": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "openai" - }, - "t_010_connect": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_011_helper_function": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "openai" - }, - "t_012_spacetime_product_type": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "openai" - }, - "t_013_spacetime_sum_type": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_014_elementary_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_015_product_type_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_016_sum_type_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_017_scheduled_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "openai" - }, - "t_018_constraints": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "openai" - }, - "t_019_many_to_many": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "openai" - }, - "t_020_ecs": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "openai" - }, - "t_021_multi_column_index": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "openai" - } - } - }, - { - "name": "DeepSeek Chat", - "route_api_model": "deepseek-chat", - "tasks": { - "t_000_empty_reducers": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_001_basic_tables": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_002_scheduled_table": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_003_struct_in_table": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_004_insert": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "deepseek" - }, - "t_005_update": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_006_delete": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_007_crud": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_007_crud", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table, TableWithPrimaryKey};\n\n#[spacetimedb::table(accessor = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n if let Some(mut user) = ctx.db.user().id().find(&1) {\n user.name = \"Alice2\".to_string();\n user.age = 31;\n user.active = false;\n ctx.db.user().id().update(user);\n }\n ctx.db.user().id().delete(&2);\n Ok(())\n}", - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.9\n Compiling itoa v1.0.17\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling thiserror v1.0.69\n Compiling ryu v1.0.23\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling bytes v1.11.1\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling uuid v1.21.0\n Compiling second-stack v0.3.5\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.7\n Compiling itertools v0.12.1\n Compiling smallvec v1.15.1\n Compiling zmij v1.0.21\n Compiling bytemuck v1.25.0\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling cc v1.2.56\n Compiling memchr v2.8.0\n Compiling http v1.4.0\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling blake3 v1.8.3\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling syn v2.0.117\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_007_crud\\rust\\server\\deepseek-chat\\llm)\nerror[E0432]: unresolved import `spacetimedb::TableWithPrimaryKey`\n --> src\\lib.rs:2:42\n |\n2 | use spacetimedb::{ReducerContext, Table, TableWithPrimaryKey};\n | ^^^^^^^^^^^^^^^^^^^ no `TableWithPrimaryKey` in the root\n\nFor more information about this error, try `rustc --explain E0432`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_008_index_lookup": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_009_init": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "deepseek" - }, - "t_010_connect": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_011_helper_function": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "deepseek" - }, - "t_012_spacetime_product_type": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "deepseek" - }, - "t_013_spacetime_sum_type": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_out": "id | value ----+--------------- 1 | (circle = 10)", - "llm_out": "id | value ----+--------------- 1 | (circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle" - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_014_elementary_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_015_product_type_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_016_sum_type_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (circle = 10) | (rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_017_scheduled_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Chat", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt};\n\n#[spacetimedb::table(accessor = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(std::time::Duration::from_micros(50_000).into()),\n });\n}", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling find-msvc-tools v0.1.9\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling itoa v1.0.17\n Compiling bitflags v2.11.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.102\n Compiling nohash-hasher v0.2.0\n Compiling zerocopy v0.8.39\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling bytes v1.11.1\n Compiling convert_case v0.4.0\n Compiling ryu v1.0.23\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling uuid v1.21.0\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.25.0\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling constant_time_eq v0.4.2\n Compiling arrayref v0.3.9\n Compiling hex v0.4.3\n Compiling cc v1.2.56\n Compiling second-stack v0.3.5\n Compiling generic-array v0.14.7\n Compiling zmij v1.0.21\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling blake3 v1.8.3\n Compiling syn v2.0.117\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_017_scheduled_columns\\rust\\server\\deepseek-chat\\llm)\nerror[E0599]: no method named `insert` found for reference `&tick_timer__TableHandle` in the current scope\n --> src\\lib.rs:17:25\n |\n17 | ctx.db.tick_timer().insert(TickTimer {\n | --------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | ctx.db.tick_timer().try_insert(TickTimer {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_018_constraints": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(accessor = account, public)]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::index(accessor = by_name, btree(columns = [name]))]\nimpl Account {}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.account().try_insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n })?;\n ctx.db.account().try_insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n })?;\n Ok(())\n}", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.9\n Compiling itoa v1.0.17\n Compiling shlex v1.3.0\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling anyhow v1.0.102\n Compiling thiserror v1.0.69\n Compiling zerocopy v0.8.39\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling ryu v1.0.23\n Compiling bytes v1.11.1\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.6\n Compiling hex v0.4.3\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling itertools v0.12.1\n Compiling cc v1.2.56\n Compiling second-stack v0.3.5\n Compiling uuid v1.21.0\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling zmij v1.0.21\n Compiling bytemuck v1.25.0\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.4.2\n Compiling generic-array v0.14.7\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling rand_core v0.6.4\n Compiling http v1.4.0\n Compiling blake3 v1.8.3\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_018_constraints\\rust\\server\\deepseek-chat\\llm)\nerror[E0433]: failed to resolve: could not find `index` in `spacetimedb`\n --> src\\lib.rs:13:16\n |\n13 | #[spacetimedb::index(accessor = by_name, btree(columns = [name]))]\n | ^^^^^ could not find `index` in `spacetimedb`\n\nFor more information about this error, try `rustc --explain E0433`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_019_many_to_many": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "deepseek" - }, - "t_020_ecs": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 5, - "passed_tests": 2, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(accessor = entity, public)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(accessor = position, public)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(accessor = velocity, public)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(accessor = next_position, public)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for position in ctx.db.position().iter() {\n if let Some(velocity) = ctx.db.velocity().entity_id().find(&position.entity_id) {\n let x = position.x + velocity.vx;\n let y = position.y + velocity.vy;\n ctx.db.next_position().entity_id().update(NextPosition {\n entity_id: position.entity_id,\n x,\n y,\n });\n }\n }\n}", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:51837/v1/database/c2003ca2cbca9219edc134b65e4f06ddb6ceeec39e53cc91fab5832436194d5e/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:51837/v1/database/c2003ca2cbca9219edc134b65e4f06ddb6ceeec39e53cc91fab5832436194d5e/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:51837/v1/database/c2003ca2cbca9219edc134b65e4f06ddb6ceeec39e53cc91fab5832436194d5e/call/step)\n", - "phase": "call_reducer" - } - } - }, - "vendor": "deepseek" - }, - "t_021_multi_column_index": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "deepseek" - } - } - }, - { - "name": "DeepSeek Reasoner", - "route_api_model": "deepseek-reasoner", - "tasks": { - "t_000_empty_reducers": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::reducer;\n\n#[reducer]\npub fn empty_reducer_no_args() {}\n\n#[reducer]\npub fn empty_reducer_with_int(count: i32) {}\n\n#[reducer]\npub fn empty_reducer_with_string(name: String) {}\n\n#[reducer]\npub fn empty_reducer_with_two_args(count: i32, name: String) {}\n\n#[reducer]\npub fn empty_reducer_with_three_args(active: bool, ratio: f32, label: String) {}", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.9\n Compiling either v1.15.0\n Compiling itoa v1.0.17\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.102\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling zerocopy v0.8.39\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling ryu v1.0.23\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.6\n Compiling bytes v1.11.1\n Compiling uuid v1.21.0\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling zmij v1.0.21\n Compiling cc v1.2.56\n Compiling itertools v0.12.1\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.25.0\n Compiling hex v0.4.3\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling rand_core v0.6.4\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling http v1.4.0\n Compiling blake3 v1.8.3\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling syn v2.0.117\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\deepseek-reasoner\\llm)\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:5:8\n |\n 4 | #[reducer]\n | ---------- required by a bound introduced by this call\n 5 | pub fn empty_reducer_no_args() {}\n | ^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn() {empty_reducer_no_args}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:785:81\n |\n785 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:5:8\n |\n 4 | #[reducer]\n | ---------- required by a bound introduced by this call\n 5 | pub fn empty_reducer_no_args() {}\n | ^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn() {empty_reducer_no_args}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:48:19\n |\n47 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n48 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:8:8\n |\n 7 | #[reducer]\n | ---------- required by a bound introduced by this call\n 8 | pub fn empty_reducer_with_int(count: i32) {}\n | ^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(i32) {empty_reducer_with_int}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:785:81\n |\n785 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: the first argument of a reducer must be `&ReducerContext`\n --> src\\lib.rs:8:38\n |\n 8 | pub fn empty_reducer_with_int(count: i32) {}\n | ^^^ first argument must be `&ReducerContext`\n |\n = help: the trait `ReducerContextArg` is not implemented for `i32`\nhelp: the trait `ReducerContextArg` is implemented for `&ReducerContext`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:249:1\n |\n249 | impl ReducerContextArg for &ReducerContext {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:8:8\n |\n 7 | #[reducer]\n | ---------- required by a bound introduced by this call\n 8 | pub fn empty_reducer_with_int(count: i32) {}\n | ^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(i32) {empty_reducer_with_int}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:48:19\n |\n47 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n48 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:11:8\n |\n 10 | #[reducer]\n | ---------- required by a bound introduced by this call\n 11 | pub fn empty_reducer_with_string(name: String) {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(std::string::String) {empty_reducer_with_string}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:785:81\n |\n785 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: the first argument of a reducer must be `&ReducerContext`\n --> src\\lib.rs:11:40\n |\n 11 | pub fn empty_reducer_with_string(name: String) {}\n | ^^^^^^ first argument must be `&ReducerContext`\n |\n = help: the trait `ReducerContextArg` is not implemented for `std::string::String`\nhelp: the trait `ReducerContextArg` is implemented for `&ReducerContext`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:249:1\n |\n249 | impl ReducerContextArg for &ReducerContext {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:11:8\n |\n10 | #[reducer]\n | ---------- required by a bound introduced by this call\n11 | pub fn empty_reducer_with_string(name: String) {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(std::string::String) {empty_reducer_with_string}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:48:19\n |\n47 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n48 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:14:8\n |\n 13 | #[reducer]\n | ---------- required by a bound introduced by this call\n 14 | pub fn empty_reducer_with_two_args(count: i32, name: String) {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(i32, std::string::String) {empty_reducer_with_two_args}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:785:81\n |\n785 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: the first argument of a reducer must be `&ReducerContext`\n --> src\\lib.rs:14:43\n |\n 14 | pub fn empty_reducer_with_two_args(count: i32, name: String) {}\n | ^^^ first argument must be `&ReducerContext`\n |\n = help: the trait `ReducerContextArg` is not implemented for `i32`\nhelp: the trait `ReducerContextArg` is implemented for `&ReducerContext`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:249:1\n |\n249 | impl ReducerContextArg for &ReducerContext {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:14:8\n |\n13 | #[reducer]\n | ---------- required by a bound introduced by this call\n14 | pub fn empty_reducer_with_two_args(count: i32, name: String) {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(i32, std::string::String) {empty_reducer_with_two_args}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:48:19\n |\n47 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n48 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:17:8\n |\n 16 | #[reducer]\n | ---------- required by a bound introduced by this call\n 17 | pub fn empty_reducer_with_three_args(active: bool, ratio: f32, label: String) {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(bool, f32, std::string::String) {empty_reducer_with_three_args}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:785:81\n |\n785 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: the first argument of a reducer must be `&ReducerContext`\n --> src\\lib.rs:17:46\n |\n 17 | pub fn empty_reducer_with_three_args(active: bool, ratio: f32, label: String) {}\n | ^^^^ first argument must be `&ReducerContext`\n |\n = help: the trait `ReducerContextArg` is not implemented for `bool`\nhelp: the trait `ReducerContextArg` is implemented for `&ReducerContext`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:249:1\n |\n249 | impl ReducerContextArg for &ReducerContext {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\nerror[E0277]: invalid reducer signature\n --> src\\lib.rs:17:8\n |\n16 | #[reducer]\n | ---------- required by a bound introduced by this call\n17 | pub fn empty_reducer_with_three_args(active: bool, ratio: f32, label: String) {}\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `fn(bool, f32, std::string::String) {empty_reducer_with_three_args}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings\\src\\rt.rs:48:19\n |\n47 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n48 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nwarning: unused variable: `count`\n --> src\\lib.rs:8:31\n |\n8 | pub fn empty_reducer_with_int(count: i32) {}\n | ^^^^^ help: if this is intentional, prefix it with an underscore: `_count`\n |\n = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default\n\nwarning: unused variable: `name`\n --> src\\lib.rs:11:34\n |\n11 | pub fn empty_reducer_with_string(name: String) {}\n | ^^^^ help: if this is intentional, prefix it with an underscore: `_name`\n\nwarning: unused variable: `count`\n --> src\\lib.rs:14:36\n |\n14 | pub fn empty_reducer_with_two_args(count: i32, name: String) {}\n | ^^^^^ help: if this is intentional, prefix it with an underscore: `_count`\n\nwarning: unused variable: `name`\n --> src\\lib.rs:14:48\n |\n14 | pub fn empty_reducer_with_two_args(count: i32, name: String) {}\n | ^^^^ help: if this is intentional, prefix it with an underscore: `_name`\n\nwarning: unused variable: `active`\n --> src\\lib.rs:17:38\n |\n17 | pub fn empty_reducer_with_three_args(active: bool, ratio: f32, label: String) {}\n | ^^^^^^ help: if this is intentional, prefix it with an underscore: `_active`\n\nwarning: unused variable: `ratio`\n --> src\\lib.rs:17:52\n |\n17 | pub fn empty_reducer_with_three_args(active: bool, ratio: f32, label: String) {}\n | ^^^^^ help: if this is intentional, prefix it with an underscore: `_ratio`\n\nwarning: unused variable: `label`\n --> src\\lib.rs:17:64\n |\n17 | pub fn empty_reducer_with_three_args(active: bool, ratio: f32, label: String) {}\n | ^^^^^ help: if this is intentional, prefix it with an underscore: `_label`\n\nFor more information about this error, try `rustc --explain E0277`.\nwarning: `spacetime-module` (lib) generated 7 warnings\nerror: could not compile `spacetime-module` (lib) due to 14 previous errors; 7 warnings emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_001_basic_tables": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::*;\n\n#[spacetimedb::table(name = \"user\", public)]\npub struct User {\n #[primary_key]\n pub id: i32,\n pub name: String,\n pub age: i32,\n pub active: bool,\n}\n\n#[spacetimedb::table(name = \"product\", public)]\npub struct Product {\n #[primary_key]\n pub id: i32,\n pub title: String,\n pub price: f32,\n pub in_stock: bool,\n}\n\n#[spacetimedb::table(name = \"note\", public)]\npub struct Note {\n #[primary_key]\n pub id: i32,\n pub body: String,\n pub rating: i64,\n pub pinned: bool,\n}", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling itoa v1.0.17\n Compiling find-msvc-tools v0.1.9\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling bytes v1.11.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling hex v0.4.3\n Compiling zmij v1.0.21\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling uuid v1.21.0\n Compiling itertools v0.12.1\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling generic-array v0.14.7\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.25.0\n Compiling rand_core v0.6.4\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\deepseek-reasoner\\llm)\nerror: Expected an `accessor` in table definition, but got only a `name`.\n Did you mean to specify `accessor` instead?\n `accessor` is required, but `name` is optional.\n \n If you're migrating from SpacetimeDB 1.*, replace `name = \"user\"` with `accessor = user`\n --> src\\lib.rs:4:29\n |\n4 | #[spacetimedb::table(name = \"user\", public)]\n | ^^^^^^\n\nerror: Expected an `accessor` in table definition, but got only a `name`.\n Did you mean to specify `accessor` instead?\n `accessor` is required, but `name` is optional.\n \n If you're migrating from SpacetimeDB 1.*, replace `name = \"product\"` with `accessor = product`\n --> src\\lib.rs:13:29\n |\n13 | #[spacetimedb::table(name = \"product\", public)]\n | ^^^^^^^^^\n\nerror: Expected an `accessor` in table definition, but got only a `name`.\n Did you mean to specify `accessor` instead?\n `accessor` is required, but `name` is optional.\n \n If you're migrating from SpacetimeDB 1.*, replace `name = \"note\"` with `accessor = note`\n --> src\\lib.rs:22:29\n |\n22 | #[spacetimedb::table(name = \"note\", public)]\n | ^^^^^^\n\nwarning: unused import: `spacetimedb::*`\n --> src\\lib.rs:2:5\n |\n2 | use spacetimedb::*;\n | ^^^^^^^^^^^^^^\n |\n = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default\n\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 3 previous errors; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_002_scheduled_table": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt};\nuse std::time::Duration;\n\n#[spacetimedb::table(accessor = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling itoa v1.0.17\n Compiling find-msvc-tools v0.1.9\n Compiling anyhow v1.0.102\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling zerocopy v0.8.39\n Compiling thiserror v1.0.69\n Compiling ryu v1.0.23\n Compiling heck v0.4.1\n Compiling bytes v1.11.1\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling keccak v0.1.6\n Compiling convert_case v0.4.0\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling zmij v1.0.21\n Compiling bytemuck v1.25.0\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling arrayref v0.3.9\n Compiling cc v1.2.56\n Compiling uuid v1.21.0\n Compiling second-stack v0.3.5\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.7\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling blake3 v1.8.3\n Compiling syn v2.0.117\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\deepseek-reasoner\\llm)\nerror[E0599]: no method named `try_insert` found for reference `&tick_timer__TableHandle` in the current scope\n --> src\\lib.rs:20:25\n |\n20 | ctx.db.tick_timer().try_insert(TickTimer {\n | --------------------^^^^^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `try_insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `insert` with a similar name\n |\n20 - ctx.db.tick_timer().try_insert(TickTimer {\n20 + ctx.db.tick_timer().insert(TickTimer {\n |\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_003_struct_in_table": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_004_insert": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "deepseek" - }, - "t_005_update": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, ReducerContext, Table};\n\n#[spacetimedb::table(accessor = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n if let Some(mut user) = ctx.db.user().id().find(&id) {\n user.name = name;\n user.age = age;\n user.active = active;\n ctx.db.user().id().update(user);\n Ok(())\n } else {\n Err(\"User not found\".to_string())\n }\n}", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: User not found\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:51837/v1/database/c2002164b37ce4f96f227fa5bb3cb6da9e53db6e1670422853aeac15b200b7e0/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:51837/v1/database/c2000a90004ed84b414eff7b7f67582479ace69bf4205243a5687144b9b08fa3/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_006_delete": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_007_crud": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "deepseek" - }, - "t_008_index_lookup": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_009_init": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "deepseek" - }, - "t_010_connect": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_011_helper_function": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "deepseek" - }, - "t_012_spacetime_product_type": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "deepseek" - }, - "t_013_spacetime_sum_type": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(accessor = result, public)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling itoa v1.0.17\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.9\n Compiling bitflags v2.11.0\n Compiling nohash-hasher v0.2.0\n Compiling zerocopy v0.8.39\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.102\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling bytes v1.11.1\n Compiling arrayvec v0.7.6\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling humantime v2.3.0\n Compiling smallvec v1.15.1\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling cc v1.2.56\n Compiling generic-array v0.14.7\n Compiling uuid v1.21.0\n Compiling bytemuck v1.25.0\n Compiling hex v0.4.3\n Compiling zmij v1.0.21\n Compiling num-traits v0.2.19\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling rand_core v0.6.4\n Compiling http v1.4.0\n Compiling scoped-tls v1.0.1\n Compiling blake3 v1.8.3\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling rand v0.8.5\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\rust\\server\\deepseek-reasoner\\llm)\nerror[E0599]: no method named `insert` found for reference `&result__TableHandle` in the current scope\n --> src\\lib.rs:25:21\n |\n25 | ctx.db.result().insert(ResultRow {\n | ----------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n25 | ctx.db.result().try_insert(ResultRow {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_014_elementary_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_015_product_type_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(accessor = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".into(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".into(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.9\n Compiling itoa v1.0.17\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling bitflags v2.11.0\n Compiling zerocopy v0.8.39\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.102\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.1\n Compiling bytemuck v1.25.0\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling getrandom v0.2.17\n Compiling zmij v1.0.21\n Compiling arrayref v0.3.9\n Compiling itertools v0.12.1\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.4.2\n Compiling uuid v1.21.0\n Compiling generic-array v0.14.7\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling num-traits v0.2.19\n Compiling memchr v2.8.0\n Compiling http v1.4.0\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.56\n Compiling blake3 v1.8.3\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling syn v2.0.117\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling ethnum v1.5.2\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\deepseek-reasoner\\llm)\nerror[E0599]: no method named `insert` found for reference `&profile__TableHandle` in the current scope\n --> src\\lib.rs:27:22\n |\n27 | ctx.db.profile().insert(Profile {\n | -----------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n27 | ctx.db.profile().try_insert(Profile {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_016_sum_type_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n pub width: i32,\n pub height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(accessor = drawing, public)]\npub struct Drawing {\n #[primary_key]\n pub id: i32,\n pub a: Shape,\n pub b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.drawing().try_insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n })?;\n Ok(())\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling itoa v1.0.17\n Compiling find-msvc-tools v0.1.9\n Compiling serde v1.0.228\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling bitflags v2.11.0\n Compiling thiserror v1.0.69\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling ryu v1.0.23\n Compiling humantime v2.3.0\n Compiling bytes v1.11.1\n Compiling arrayref v0.3.9\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling uuid v1.21.0\n Compiling zmij v1.0.21\n Compiling cc v1.2.56\n Compiling constant_time_eq v0.4.2\n Compiling num-traits v0.2.19\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.25.0\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling rand_core v0.6.4\n Compiling http v1.4.0\n Compiling blake3 v1.8.3\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling ethnum v1.5.2\n Compiling decorum v0.3.1\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_016_sum_type_columns\\rust\\server\\deepseek-reasoner\\llm)\nerror[E0599]: no method named `try_insert` found for reference `&drawing__TableHandle` in the current scope\n --> src\\lib.rs:26:22\n |\n26 | ctx.db.drawing().try_insert(Drawing {\n | -----------------^^^^^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `try_insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `insert` with a similar name\n |\n26 - ctx.db.drawing().try_insert(Drawing {\n26 + ctx.db.drawing().insert(Drawing {\n |\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_017_scheduled_columns": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt};\nuse std::time::Duration;\n\n#[spacetimedb::table(accessor = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n pub scheduled_id: u64,\n pub scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.9\n Compiling itoa v1.0.17\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling nohash-hasher v0.2.0\n Compiling zerocopy v0.8.39\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.102\n Compiling bitflags v2.11.0\n Compiling ryu v1.0.23\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.6\n Compiling heck v0.4.1\n Compiling bytes v1.11.1\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling uuid v1.21.0\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.25.0\n Compiling zmij v1.0.21\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling second-stack v0.3.5\n Compiling num-traits v0.2.19\n Compiling arrayref v0.3.9\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling generic-array v0.14.7\n Compiling cc v1.2.56\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling syn v2.0.117\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_017_scheduled_columns\\rust\\server\\deepseek-reasoner\\llm)\nerror[E0599]: no method named `try_insert` found for reference `&tick_timer__TableHandle` in the current scope\n --> src\\lib.rs:20:25\n |\n20 | ctx.db.tick_timer().try_insert(TickTimer {\n | --------------------^^^^^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `try_insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `insert` with a similar name\n |\n20 - ctx.db.tick_timer().try_insert(TickTimer {\n20 + ctx.db.tick_timer().insert(TickTimer {\n |\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_018_constraints": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(\n accessor = account,\n index(accessor = by_name, btree(name))\n)]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n Ok(())\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling rustversion v1.0.22\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling itoa v1.0.17\n Compiling find-msvc-tools v0.1.9\n Compiling shlex v1.3.0\n Compiling bitflags v2.11.0\n Compiling thiserror v1.0.69\n Compiling zerocopy v0.8.39\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.102\n Compiling ryu v1.0.23\n Compiling heck v0.4.1\n Compiling bytes v1.11.1\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling arrayref v0.3.9\n Compiling uuid v1.21.0\n Compiling getrandom v0.2.17\n Compiling bytemuck v1.25.0\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling zmij v1.0.21\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling generic-array v0.14.7\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling rand v0.8.5\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_018_constraints\\rust\\server\\deepseek-reasoner\\llm)\nerror: expected `columns`\n --> src\\lib.rs:6:37\n |\n6 | index(accessor = by_name, btree(name))\n | ^^^^\n\nerror[E0422]: cannot find struct, variant or union type `Account` in this scope\n --> src\\lib.rs:18:29\n |\n18 | ctx.db.account().insert(Account {\n | ^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Account` in this scope\n --> src\\lib.rs:23:29\n |\n23 | ctx.db.account().insert(Account {\n | ^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `account` found for struct `Local` in the current scope\n --> src\\lib.rs:18:12\n |\n18 | ctx.db.account().insert(Account {\n | ^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `account` found for struct `Local` in the current scope\n --> src\\lib.rs:23:12\n |\n23 | ctx.db.account().insert(Account {\n | ^^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_019_many_to_many": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(accessor = user, public)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(accessor = group, public)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(accessor = membership, public)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n #[index(accessor = by_user, btree(columns = [user_id]))]\n #[index(accessor = by_group, btree(columns = [group_id]))]\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".into() });\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".into() });\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.24\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling itoa v1.0.17\n Compiling find-msvc-tools v0.1.9\n Compiling bitflags v2.11.0\n Compiling nohash-hasher v0.2.0\n Compiling zerocopy v0.8.39\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.102\n Compiling humantime v2.3.0\n Compiling bytes v1.11.1\n Compiling ryu v1.0.23\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.6\n Compiling heck v0.4.1\n Compiling bytemuck v1.25.0\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling uuid v1.21.0\n Compiling zmij v1.0.21\n Compiling arrayref v0.3.9\n Compiling second-stack v0.3.5\n Compiling itertools v0.12.1\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling generic-array v0.14.7\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_019_many_to_many\\rust\\server\\deepseek-reasoner\\llm)\nerror: expected identifier, found `}`\n --> src\\lib.rs:26:1\n |\n19 | pub struct Membership {\n | ---------- while parsing this struct\n...\n26 | }\n | ^ expected identifier\n\nerror: unexpected end of input, expected identifier\n --> src\\lib.rs:26:1\n |\n26 | }\n | ^\n\nerror[E0422]: cannot find struct, variant or union type `Membership` in this scope\n --> src\\lib.rs:34:32\n |\n34 | ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n | ^^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Membership` in this scope\n --> src\\lib.rs:35:32\n |\n35 | ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n | ^^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Membership` in this scope\n --> src\\lib.rs:36:32\n |\n36 | ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n | ^^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `membership` found for struct `Local` in the current scope\n --> src\\lib.rs:34:12\n |\n34 | ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `membership` found for struct `Local` in the current scope\n --> src\\lib.rs:35:12\n |\n35 | ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `membership` found for struct `Local` in the current scope\n --> src\\lib.rs:36:12\n |\n36 | ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n | ^^^^^^^^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_020_ecs": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 5, - "passed_tests": 2, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(accessor = entity, public)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(accessor = position, public)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(accessor = velocity, public)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(accessor = next_position, public)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.entity().insert(Entity { id: 2 });\n\n ctx.db.position().insert(Position {\n entity_id: 1,\n x: 0,\n y: 0,\n });\n ctx.db.velocity().insert(Velocity {\n entity_id: 1,\n vx: 1,\n vy: 0,\n });\n\n ctx.db.position().insert(Position {\n entity_id: 2,\n x: 10,\n y: 0,\n });\n ctx.db.velocity().insert(Velocity {\n entity_id: 2,\n vx: -2,\n vy: 3,\n });\n\n Ok(())\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) -> Result<(), String> {\n for position in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(&position.entity_id) {\n let new_x = position.x + vel.vx;\n let new_y = position.y + vel.vy;\n ctx.db.next_position().entity_id().update(NextPosition {\n entity_id: position.entity_id,\n x: new_x,\n y: new_y,\n });\n }\n }\n Ok(())\n}", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:51837/v1/database/c200850a27f2c7b38bd43bed2c7fbefe5e9fc22392017696d385a87542e5769e/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:51837/v1/database/c200850a27f2c7b38bd43bed2c7fbefe5e9fc22392017696d385a87542e5769e/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:51837/v1/database/c200850a27f2c7b38bd43bed2c7fbefe5e9fc22392017696d385a87542e5769e/call/step)\n", - "phase": "call_reducer" - } - } - }, - "vendor": "deepseek" - }, - "t_021_multi_column_index": { - "hash": "475c22b985e81027333ffa0d043dc9cbbb2ed9171f8083e5747c79c5585722d9", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - } - }, - "vendor": "deepseek" - } - } - } - ] - }, - { - "mode": "llms.md", - "hash": "7bd4918052e02c4b09d50e7037cd2e81add76a4b42468acac54ee8f4073e37cf", - "models": [ - { - "name": "Grok 4", - "route_api_model": "grok-4", - "tasks": { - "t_002_scheduled_table": { - "hash": "7bd4918052e02c4b09d50e7037cd2e81add76a4b42468acac54ee8f4073e37cf", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "```rust\nuse spacetimedb::{table, reducer, ReducerContext, ScheduleAt, TimeDuration};\n\n#[table(accessor = tick_timer, scheduled(tick))]\n#[derive(Clone, Debug)]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n pub scheduled_id: u64,\n pub scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: TimeDuration::from_micros(50_000).into(),\n });\n}\n\n#[reducer]\npub fn tick(ctx: &ReducerContext, args: TickTimer) -> Result<(), String> {\n if ctx.sender() != ctx.identity() {\n return Err(\"Reducer `tick` may not be invoked by clients, only via scheduling.\".into());\n }\n Ok(())\n}\n```", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.9\n Compiling serde v1.0.228\n Compiling itoa v1.0.17\n Compiling anyhow v1.0.102\n Compiling nohash-hasher v0.2.0\n Compiling zerocopy v0.8.39\n Compiling bitflags v2.11.0\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.1\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling constant_time_eq v0.4.2\n Compiling uuid v1.21.0\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.25.0\n Compiling arrayref v0.3.9\n Compiling cc v1.2.56\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling zmij v1.0.21\n Compiling second-stack v0.3.5\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling generic-array v0.14.7\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling blake3 v1.8.3\n Compiling syn v2.0.117\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\basics\\t_002_scheduled_table\\rust\\server\\grok-4\\llm)\nerror[E0599]: no method named `insert` found for reference `&tick_timer__TableHandle` in the current scope\n --> src\\lib.rs:15:25\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | --------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n15 | ctx.db.tick_timer().try_insert(TickTimer {\n | ++++\n\nwarning: unused variable: `args`\n --> src\\lib.rs:22:35\n |\n22 | pub fn tick(ctx: &ReducerContext, args: TickTimer) -> Result<(), String> {\n | ^^^^ help: if this is intentional, prefix it with an underscore: `_args`\n |\n = note: `#[warn(unused_variables)]` (part of `#[warn(unused)]`) on by default\n\nFor more information about this error, try `rustc --explain E0599`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 1 previous error; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "7bd4918052e02c4b09d50e7037cd2e81add76a4b42468acac54ee8f4073e37cf", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "7bd4918052e02c4b09d50e7037cd2e81add76a4b42468acac54ee8f4073e37cf", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "7bd4918052e02c4b09d50e7037cd2e81add76a4b42468acac54ee8f4073e37cf", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, ReducerContext, Table};\n\n#[table(accessor = log, index(accessor = by_user_day, btree(columns = [user_id, day])))]\n#[derive(Clone, Debug)]\npub struct Log {\n #[primary_key]\n pub id: i32,\n pub user_id: i32,\n pub day: i32,\n pub message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.9\n Compiling itoa v1.0.17\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.102\n Compiling zerocopy v0.8.39\n Compiling thiserror v1.0.69\n Compiling bitflags v2.11.0\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling bytes v1.11.1\n Compiling ryu v1.0.23\n Compiling keccak v0.1.6\n Compiling convert_case v0.4.0\n Compiling bytemuck v1.25.0\n Compiling hex v0.4.3\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling arrayref v0.3.9\n Compiling itertools v0.12.1\n Compiling zmij v1.0.21\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling uuid v1.21.0\n Compiling rand_core v0.6.4\n Compiling generic-array v0.14.7\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.56\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_021_multi_column_index\\rust\\server\\grok-4\\llm)\nerror: cannot find attribute `table` in this scope\n --> src\\lib.rs:4:3\n |\n4 | #[table(accessor = log, index(accessor = by_user_day, btree(columns = [user_id, day])))]\n | ^^^^^\n |\nhelp: a built-in attribute with a similar name exists\n |\n4 | #[stable(accessor = log, index(accessor = by_user_day, btree(columns = [user_id, day])))]\n | +\nhelp: consider importing this attribute macro\n |\n2 + use spacetimedb::table;\n |\n\nerror: cannot find attribute `primary_key` in this scope\n --> src\\lib.rs:7:7\n |\n7 | #[primary_key]\n | ^^^^^^^^^^^\n |\n = note: `primary_key` is an attribute that can be used by the derive macro `__TableHelper`, you might be missing a `derive` attribute\n\nerror[E0599]: no method named `log` found for struct `Local` in the current scope\n --> src\\lib.rs:16:12\n |\n16 | ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n | ^^^ method not found in `Local`\n\nerror[E0599]: no method named `log` found for struct `Local` in the current scope\n --> src\\lib.rs:17:12\n |\n17 | ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n | ^^^ method not found in `Local`\n\nerror[E0599]: no method named `log` found for struct `Local` in the current scope\n --> src\\lib.rs:18:12\n |\n18 | ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n | ^^^ method not found in `Local`\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - } - } - } - ] - }, - { - "mode": "rustdoc_json", - "hash": "b0d5bbe755291d8ec65de0a74b4eecfa344670639ca5f1a365ced3ab9717e97c", - "models": [ - { - "name": "Grok 4", - "route_api_model": "grok-4", - "tasks": { - "t_000_empty_reducers": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_000_empty_reducers", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "use spacetimedb::{reducer, ReducerContext};\n\n#[reducer]\npub fn empty_reducer_no_args(ctx: &ReducerContext) {}\n\n#[reducer]\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) {}\n\n#[reducer]\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) {}\n\n#[reducer]\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) {}\n\n#[reducer]\npub fn empty_reducer_with_three_args(ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-000-empty-reducers-golden", - "llm_db": "basics-t-000-empty-reducers-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_000_empty_reducers\\rust\\server\\grok-4\\llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-000-empty-reducers-golden", - "llm_db": "basics-t-000-empty-reducers-grok-4-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "local", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.164175300Z", - "finished_at": "2025-10-22T01:03:40.928800800Z" - }, - "t_001_basic_tables": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_001_basic_tables", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "use spacetimedb::table;\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool\n}\n\n#[table(name = products)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool\n}\n\n#[table(name = notes)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-001-basic-tables-golden", - "llm_db": "basics-t-001-basic-tables-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_001_basic_tables\\rust\\server\\grok-4\\llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-001-basic-tables-golden", - "llm_db": "basics-t-001-basic-tables-grok-4-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "local", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.179899400Z", - "finished_at": "2025-10-22T01:03:50.594608Z" - }, - "t_002_scheduled_table": { - "hash": "b0d5bbe755291d8ec65de0a74b4eecfa344670639ca5f1a365ced3ab9717e97c", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_003_struct_in_table": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_003_struct_in_table", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::spacetimedb;\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entities)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-003-struct-in-table-golden", - "llm_db": "basics-t-003-struct-in-table-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling zerocopy v0.8.27\n Compiling serde v1.0.228\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling keccak v0.1.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.3.1\n Compiling bytes v1.10.1\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.16\n Compiling second-stack v0.3.5\n Compiling cc v1.2.41\n Compiling spacetimedb-lib v1.6.0\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.2\n Compiling spacetimedb-primitives v1.6.0\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling syn v2.0.107\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling ethnum v1.5.2\n Compiling sha3 v0.10.8\n Compiling rand_chacha v0.3.1\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_003_struct_in_table\\rust\\server\\grok-4\\llm)\nerror[E0432]: unresolved import `spacetimedb::spacetimedb`\n --> src\\lib.rs:2:5\n |\n2 | use spacetimedb::spacetimedb;\n | ^^^^^^^^^^^^^^^^^^^^^^^^ no `spacetimedb` in the root\n\nerror: cannot find derive macro `SpacetimeType` in this scope\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n |\nhelp: consider importing this derive macro\n |\n2 + use spacetimedb::SpacetimeType;\n |\n\nFor more information about this error, try `rustc --explain E0432`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.185773400Z", - "finished_at": "2025-10-22T01:03:37.665154500Z" - }, - "t_004_insert": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_004_insert", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.users().insert(User { id, name, age, active });\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_004_insert\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_004_insert\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling serde v1.0.228\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling bytes v1.10.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.16\n Compiling constant_time_eq v0.3.1\n Compiling spacetimedb-lib v1.6.0\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.28\n Compiling generic-array v0.14.9\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.41\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling syn v2.0.107\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling decorum v0.3.1\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_004_insert\\rust\\server\\grok-4\\llm)\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:15:20\n |\n15 | ctx.db.users().insert(User { id, name, age, active });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n15 | ctx.db.users().try_insert(User { id, name, age, active });\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.188490Z", - "finished_at": "2025-10-22T01:02:55.582287600Z" - }, - "t_005_update": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_005_update", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user = User { id, name, age, active };\n ctx.db.users().id().update(user);\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_005_update\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_005_update\\rust\\server\\grok-4\\llm", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-005-update-grok-4-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM users WHERE id=1", - "reducer": "update_user", - "server": "local" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-grok-4-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "local", - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.191882800Z", - "finished_at": "2025-10-22T01:03:51.231015600Z" - }, - "t_006_delete": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_006_delete", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{reducer, ReducerContext, table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.users().id().delete(&id);\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_006_delete\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_006_delete\\rust\\server\\grok-4\\llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-006-delete-golden", - "llm_db": "basics-t-006-delete-grok-4-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "local", - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO users(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.194777800Z", - "finished_at": "2025-10-22T01:03:09.975424400Z" - }, - "t_007_crud": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_007_crud", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, ReducerContext, UniqueColumn};\n\n#[spacetimedb::table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n ctx.db.users().id().update(updated);\n ctx.db.users().id().delete(&2);\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_007_crud\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling heck v0.5.0\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.3.1\n Compiling bytes v1.10.1\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.16\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.6.0\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling cc v1.2.41\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling digest v0.10.7\n Compiling rand_chacha v0.3.1\n Compiling syn v2.0.107\n Compiling ethnum v1.5.2\n Compiling sha3 v0.10.8\n Compiling rand v0.8.5\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_007_crud\\rust\\server\\grok-4\\llm)\nwarning: unused import: `UniqueColumn`\n --> src\\lib.rs:2:44\n |\n2 | use spacetimedb::{reducer, ReducerContext, UniqueColumn};\n | ^^^^^^^^^^^^\n |\n = note: `#[warn(unused_imports)]` on by default\n\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:15:20\n |\n15 | ctx.db.users().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n15 | ctx.db.users().try_insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:16:20\n |\n16 | ctx.db.users().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n16 | ctx.db.users().try_insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.197437100Z", - "finished_at": "2025-10-22T01:03:34.519339400Z" - }, - "t_008_index_lookup": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_008_index_lookup", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool\n}\n\n#[table(name = results)]\npub struct Result {\n #[primary_key]\n id: i32,\n name: String\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.users().id().find(&id) {\n ctx.db.results().insert(Result { id: user.id, name: user.name });\n }\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.27\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling heck v0.5.0\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling bytes v1.10.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling cc v1.2.41\n Compiling itertools v0.12.1\n Compiling getrandom v0.2.16\n Compiling constant_time_eq v0.3.1\n Compiling spacetimedb-lib v1.6.0\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling log v0.4.28\n Compiling num-traits v0.2.19\n Compiling rand_core v0.6.4\n Compiling blake3 v1.8.2\n Compiling spacetimedb-primitives v1.6.0\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling syn v2.0.107\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling ethnum v1.5.2\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling rand_chacha v0.3.1\n Compiling sha3 v0.10.8\n Compiling decorum v0.3.1\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_008_index_lookup\\rust\\server\\grok-4\\llm)\nwarning: unused import: `UniqueColumn`\n --> src\\lib.rs:2:58\n |\n2 | use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n | ^^^^^^^^^^^^\n |\n = note: `#[warn(unused_imports)]` on by default\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:4:1\n |\n 4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:14:12\n |\n14 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&User, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&User, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:13:1\n |\n13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:14:12\n |\n14 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:13:1\n |\n13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Result, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Result, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:4:22\n |\n4 | #[table(name = users)]\n | ---------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[table(name = users)]\n | ^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\nhelp: consider using `Result::expect` to unwrap the `std::result::Result>::Error>` value, panicking if the value is a `Result::Err`\n |\n 13 | #[table(name = results)].expect(\"REASON\")\n | +++++++++++++++++\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:13:24\n |\n13 | #[table(name = results)]\n | -----------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:13:1\n |\n 13 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:14:1\n |\n 14 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nSome errors have detailed explanations: E0053, E0107, E0277, E0308.\nFor more information about an error, try `rustc --explain E0053`.\nwarning: `spacetime-module` (lib) generated 1 warning\nerror: could not compile `spacetime-module` (lib) due to 28 previous errors; 1 warning emitted\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.200443800Z", - "finished_at": "2025-10-22T01:03:49.306266200Z" - }, - "t_009_init": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_009_init", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.users().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.users().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_009_init\\rust\\server\\grok-4\\llm", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM users WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM users" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-grok-4-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "local", - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM users WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.203170700Z", - "finished_at": "2025-10-22T01:03:51.597278500Z" - }, - "t_010_connect": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_010_connect", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = events)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event {\n id: 0,\n kind: \"connected\".into(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.events().insert(Event {\n id: 0,\n kind: \"disconnected\".into(),\n });\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-010-connect-golden", - "llm_db": "basics-t-010-connect-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_010_connect\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_010_connect\\rust\\server\\grok-4\\llm", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-010-connect-golden", - "llm_db": "basics-t-010-connect-grok-4-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "local", - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.205852800Z", - "finished_at": "2025-10-22T01:03:18.971400900Z" - }, - "t_011_helper_function": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_011_helper_function", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, ReducerContext};\n\n#[spacetimedb::table(name = results)]\npub struct Result {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.results().insert(Result { id, sum: add(a, b) });\n}", - "category": "basics", - "route_api_model": "grok-4", - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.27\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling heck v0.5.0\n Compiling keccak v0.1.5\n Compiling bytes v1.10.1\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.3.1\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.6.0\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.16\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling itertools v0.12.1\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling cc v1.2.41\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling syn v2.0.107\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\basics\\t_011_helper_function\\rust\\server\\grok-4\\llm)\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:5:12\n |\n5 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(__ProductVisitor, &_) -> std::result::Result<__ProductFieldIdent, __E>`\n found signature `fn(__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:4:1\n |\n4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Result, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Result, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\nhelp: consider using `Result::expect` to unwrap the `std::result::Result>::Error>` value, panicking if the value is a `Result::Err`\n |\n 4 | #[spacetimedb::table(name = results)].expect(\"REASON\")\n | +++++++++++++++++\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:4:37\n |\n4 | #[spacetimedb::table(name = results)]\n | ------------------------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `spacetimedb::table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:1\n |\n 4 | #[spacetimedb::table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:5:1\n |\n 5 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0599]: no method named `insert` found for reference `&results__TableHandle` in the current scope\n --> src\\lib.rs:17:22\n |\n17 | ctx.db.results().insert(Result { id, sum: add(a, b) });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n17 | ctx.db.results().try_insert(Result { id, sum: add(a, b) });\n | ++++\n\nSome errors have detailed explanations: E0053, E0107, E0277, E0308, E0599.\nFor more information about an error, try `rustc --explain E0053`.\nerror: could not compile `spacetime-module` (lib) due to 15 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.208437500Z", - "finished_at": "2025-10-22T01:03:17.124837500Z" - }, - "t_012_spacetime_product_type": { - "hash": "b0d5bbe755291d8ec65de0a74b4eecfa344670639ca5f1a365ced3ab9717e97c", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_013_spacetime_sum_type": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_013_spacetime_sum_type", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{SpacetimeType, ReducerContext, reducer, table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = results)]\npub struct Result {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.results().insert(Result { id, value: Shape::Circle(radius) });\n}", - "category": "schema", - "route_api_model": "grok-4", - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\schema\\t_013_spacetime_sum_type\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling find-msvc-tools v0.1.4\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling heck v0.5.0\n Compiling humantime v2.3.0\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.6.0\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.3.1\n Compiling hex v0.4.3\n Compiling second-stack v0.3.5\n Compiling getrandom v0.2.16\n Compiling bytes v1.10.1\n Compiling bytemuck v1.24.0\n Compiling itertools v0.12.1\n Compiling scoped-tls v1.0.1\n Compiling log v0.4.28\n Compiling rand_core v0.6.4\n Compiling cc v1.2.41\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling spacetimedb-primitives v1.6.0\n Compiling syn v2.0.107\n Compiling blake3 v1.8.2\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling digest v0.10.7\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_013_spacetime_sum_type\\rust\\server\\grok-4\\llm)\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:17:12\n |\n17 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:4:10\n |\n4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Rect, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Rect, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:10:10\n |\n10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:17:12\n |\n17 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:10:10\n |\n10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_sum` has an incompatible type for trait\n --> src\\lib.rs:10:10\n |\n10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(__SumVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(__SumVisitor, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_tag` has an incompatible type for trait\n --> src\\lib.rs:10:10\n |\n10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `std::result::Result<__Variant, E>`, found `Result`\n |\n = note: expected signature `fn(__SumVisitor, _) -> std::result::Result<__Variant, E>`\n found signature `fn(__SumVisitor, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_name` has an incompatible type for trait\n --> src\\lib.rs:10:10\n |\n10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `std::result::Result<__Variant, E>`, found `Result`\n |\n = note: expected signature `fn(__SumVisitor, &_) -> std::result::Result<__Variant, E>`\n found signature `fn(__SumVisitor, &_) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:10:10\n |\n10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Shape, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Shape, _) -> Result`\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0107]: struct takes 0 generic arguments but 2 generic arguments were supplied\n --> src\\lib.rs:16:1\n |\n16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected 0 generic arguments\n |\nnote: struct defined here, with 0 generic parameters\n --> src\\lib.rs:17:12\n |\n17 | pub struct Result {\n | ^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `deserialize` has an incompatible type for trait\n --> src\\lib.rs:16:1\n |\n16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_) -> std::result::Result>::Error>`\n found signature `fn(_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_seq_product` has an incompatible type for trait\n --> src\\lib.rs:16:1\n |\n16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit_named_product` has an incompatible type for trait\n --> src\\lib.rs:16:1\n |\n16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, _) -> std::result::Result>::Error>`\n found signature `fn(_::__ProductVisitor, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `visit` has an incompatible type for trait\n --> src\\lib.rs:16:1\n |\n16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<__ProductFieldIdent, __E>`, found `Result`\n |\n = note: expected signature `fn(_::__ProductVisitor, &_) -> std::result::Result<_::__ProductFieldIdent, __E>`\n found signature `fn(_::__ProductVisitor, &_) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0053]: method `serialize` has an incompatible type for trait\n --> src\\lib.rs:16:1\n |\n16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^ expected `Result<::Ok, ...>`, found `Result`\n |\n = note: expected signature `fn(&Result, _) -> std::result::Result<::Ok, ::Error>`\n found signature `fn(&Result, _) -> Result`\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:4:22\n |\n4 | #[derive(SpacetimeType)]\n | ------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:4:10\n |\n 4 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:10:10\n |\n 10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:10:22\n |\n10 | #[derive(SpacetimeType)]\n | ------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:10:10\n |\n 10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:10:10\n |\n 10 | #[derive(SpacetimeType)]\n | ^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__Variant, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<__Variant, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<__Variant, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `SpacetimeType` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:12:12\n |\n 10 | #[derive(SpacetimeType)]\n | ------------- expected `Result` because of return type\n 11 | pub enum Shape {\n 12 | Circle(i32),\n | ^^^ expected `Result`, found `Result<::Ok, ...>`\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:16:1\n |\n 16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `Result` and `Result` have similar names, but are actually distinct types\nnote: `Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\nhelp: consider using `Result::expect` to unwrap the `std::result::Result>::Error>` value, panicking if the value is a `Result::Err`\n |\n 16 | #[table(name = results)].expect(\"REASON\")\n | +++++++++++++++++\n\nerror[E0277]: the `?` operator can only be used in a method that returns `Result` or `Option` (or another type that implements `FromResidual`)\n --> src\\lib.rs:16:24\n |\n16 | #[table(name = results)]\n | -----------------------^\n | | |\n | | cannot use the `?` operator in a method that returns `Result`\n | this function should return `Result` or `Option` to accept `?`\n |\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` which comes from the expansion of the attribute macro `table` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:16:1\n |\n 16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result`\n | expected `Result` because of return type\n |\n = note: `std::result::Result` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:16:1\n |\n 16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<_, _>`\n | expected `Result` because of return type\n |\n = note: `std::result::Result<_, _>` and `Result` have similar names, but are actually distinct types\nnote: `std::result::Result<_, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:16:1\n |\n 16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<__ProductFieldIdent, _>`\n | expected `Result` because of return type\n |\n = note: `Result<__ProductFieldIdent, _>` and `Result` have similar names, but are actually distinct types\nnote: `Result<__ProductFieldIdent, _>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0308]: mismatched types\n --> src\\lib.rs:16:1\n |\n 16 | #[table(name = results)]\n | ^^^^^^^^^^^^^^^^^^^^^^^^\n | |\n | expected `Result`, found `Result<::Ok, ...>`\n | expected `Result` because of return type\n |\n = note: `Result<::Ok, ...>` and `Result` have similar names, but are actually distinct types\nnote: `Result<::Ok, ...>` is defined in crate `core`\n --> C:\\Users\\bradl\\.rustup\\toolchains\\1.90.0-x86_64-pc-windows-msvc\\lib/rustlib/src/rust\\library/core/src/result.rs:548:1\n |\n548 | pub enum Result {\n | ^^^^^^^^^^^^^^^^^^^^^\nnote: `Result` is defined in the current crate\n --> src\\lib.rs:17:1\n |\n 17 | pub struct Result {\n | ^^^^^^^^^^^^^^^^^\n = note: this error originates in the derive macro `spacetimedb::__TableHelper` (in Nightly builds, run with -Z macro-backtrace for more info)\n\nerror[E0599]: no method named `insert` found for reference `&results__TableHandle` in the current scope\n --> src\\lib.rs:25:22\n |\n25 | ctx.db.results().insert(Result { id, value: Shape::Circle(radius) });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n25 | ctx.db.results().try_insert(Result { id, value: Shape::Circle(radius) });\n | ++++\n\nSome errors have detailed explanations: E0053, E0107, E0277, E0308, E0599.\nFor more information about an error, try `rustc --explain E0053`.\nerror: could not compile `spacetime-module` (lib) due to 40 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.214065100Z", - "finished_at": "2025-10-22T01:03:53.947923200Z" - }, - "t_014_elementary_columns": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_014_elementary_columns", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[table(name = primitives)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer(init)]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitives().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000i64,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", - "category": "schema", - "route_api_model": "grok-4", - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.27\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling heck v0.5.0\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.3.1\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling bytes v1.10.1\n Compiling spacetimedb-lib v1.6.0\n Compiling itertools v0.12.1\n Compiling cc v1.2.41\n Compiling getrandom v0.2.16\n Compiling generic-array v0.14.9\n Compiling arrayref v0.3.9\n Compiling hex v0.4.3\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling rand_core v0.6.4\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling syn v2.0.107\n Compiling digest v0.10.7\n Compiling ppv-lite86 v0.2.21\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling sha3 v0.10.8\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_014_elementary_columns\\rust\\server\\grok-4\\llm)\nerror[E0599]: no method named `insert` found for reference `&primitives__TableHandle` in the current scope\n --> src\\lib.rs:18:25\n |\n18 | ctx.db.primitives().insert(Primitive {\n | --------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n18 | ctx.db.primitives().try_insert(Primitive {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.216933200Z", - "finished_at": "2025-10-22T01:03:49.010285900Z" - }, - "t_015_product_type_columns": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_015_product_type_columns", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, table, ReducerContext};\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32\n}\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32\n}\n\n#[table(name = profiles)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profiles().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 }\n });\n}", - "category": "schema", - "route_api_model": "grok-4", - "golden_db": "schema-t-015-product-type-columns-golden", - "llm_db": "schema-t-015-product-type-columns-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling unicode-ident v1.0.20\n Compiling quote v1.0.41\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling zerocopy v0.8.27\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling heck v0.5.0\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling bytemuck v1.24.0\n Compiling constant_time_eq v0.3.1\n Compiling spacetimedb-lib v1.6.0\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.16\n Compiling hex v0.4.3\n Compiling bytes v1.10.1\n Compiling scoped-tls v1.0.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.28\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling cc v1.2.41\n Compiling syn v2.0.107\n Compiling blake3 v1.8.2\n Compiling spacetimedb-primitives v1.6.0\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_015_product_type_columns\\rust\\server\\grok-4\\llm)\nerror[E0599]: no method named `insert` found for reference `&profiles__TableHandle` in the current scope\n --> src\\lib.rs:27:23\n |\n27 | ctx.db.profiles().insert(Profile {\n | ------------------^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n27 | ctx.db.profiles().try_insert(Profile {\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.219748600Z", - "finished_at": "2025-10-22T01:04:00.131061Z" - }, - "t_016_sum_type_columns": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{spacetimedb, ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(spacetimedb::SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawings)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer(seed)]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawings().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", - "category": "schema", - "route_api_model": "grok-4", - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\schema\\t_016_sum_type_columns\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\schema\\t_016_sum_type_columns\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling heck v0.5.0\n Compiling arrayvec v0.7.6\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.6.0\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.3.1\n Compiling bytemuck v1.24.0\n Compiling bytes v1.10.1\n Compiling getrandom v0.2.16\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling log v0.4.28\n Compiling generic-array v0.14.9\n Compiling cc v1.2.41\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.2\n Compiling syn v2.0.107\n Compiling spacetimedb-primitives v1.6.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling decorum v0.3.1\n Compiling ppv-lite86 v0.2.21\n Compiling digest v0.10.7\n Compiling ethnum v1.5.2\n Compiling sha3 v0.10.8\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling thiserror-impl v1.0.69\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_016_sum_type_columns\\rust\\server\\grok-4\\llm)\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src\\lib.rs:24:24\n |\n24 | #[spacetimedb::reducer(seed)]\n | ^^^^\n\nerror[E0432]: unresolved import `spacetimedb::spacetimedb`\n --> src\\lib.rs:2:19\n |\n2 | use spacetimedb::{spacetimedb, ReducerContext, Table};\n | ^^^^^^^^^^^ no `spacetimedb` in the root\n\nFor more information about this error, try `rustc --explain E0432`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.222488900Z", - "finished_at": "2025-10-22T01:03:17.896166200Z" - }, - "t_017_scheduled_columns": { - "hash": "b0d5bbe755291d8ec65de0a74b4eecfa344670639ca5f1a365ced3ab9717e97c", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_018_constraints": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_018_constraints", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{reducer, ReducerContext};\n\n#[spacetimedb::table(name = accounts, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.accounts().insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n ctx.db.accounts().insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", - "category": "schema", - "route_api_model": "grok-4", - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\schema\\t_018_constraints\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\schema\\t_018_constraints\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Blocking waiting for file lock on package cache\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 67 packages to latest compatible versions\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.27\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling heck v0.5.0\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.6.0\n Compiling bytemuck v1.24.0\n Compiling bytes v1.10.1\n Compiling second-stack v0.3.5\n Compiling hex v0.4.3\n Compiling getrandom v0.2.16\n Compiling itertools v0.12.1\n Compiling arrayref v0.3.9\n Compiling generic-array v0.14.9\n Compiling cc v1.2.41\n Compiling constant_time_eq v0.3.1\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling spacetimedb-primitives v1.6.0\n Compiling num-traits v0.2.19\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling syn v2.0.107\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling decorum v0.3.1\n Compiling thiserror-impl v1.0.69\n Compiling enum-as-inner v0.6.1\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling derive_more v0.99.20\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_018_constraints\\rust\\server\\grok-4\\llm)\nerror[E0599]: no method named `insert` found for reference `&accounts__TableHandle` in the current scope\n --> src\\lib.rs:15:23\n |\n15 | ctx.db.accounts().insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n15 | ctx.db.accounts().try_insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&accounts__TableHandle` in the current scope\n --> src\\lib.rs:16:23\n |\n16 | ctx.db.accounts().insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n16 | ctx.db.accounts().try_insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 2 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.228601900Z", - "finished_at": "2025-10-22T01:03:49.415425800Z" - }, - "t_019_many_to_many": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_019_many_to_many", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext};\n\n#[table(name = users)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = groups)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(name = memberships, index(name = by_user, btree(columns = [user_id])), index(name = by_group, btree(columns = [group_id])))]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.users().insert(User { user_id: 1, name: \"Alice\".into() });\n ctx.db.users().insert(User { user_id: 2, name: \"Bob\".into() });\n ctx.db.groups().insert(Group { group_id: 10, title: \"Admin\".into() });\n ctx.db.groups().insert(Group { group_id: 20, title: \"Dev\".into() });\n ctx.db.memberships().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.memberships().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.memberships().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", - "category": "schema", - "route_api_model": "grok-4", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\schema\\t_019_many_to_many\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\schema\\t_019_many_to_many\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling cfg-if v1.0.4\n Compiling serde_core v1.0.228\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.27\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling humantime v2.3.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling heck v0.5.0\n Compiling smallvec v1.15.1\n Compiling constant_time_eq v0.3.1\n Compiling arrayref v0.3.9\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling bytes v1.10.1\n Compiling itertools v0.12.1\n Compiling cc v1.2.41\n Compiling getrandom v0.2.16\n Compiling spacetimedb-lib v1.6.0\n Compiling hex v0.4.3\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling rand_core v0.6.4\n Compiling spacetimedb-primitives v1.6.0\n Compiling blake3 v1.8.2\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling syn v2.0.107\n Compiling crypto-common v0.1.6\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling ppv-lite86 v0.2.21\n Compiling decorum v0.3.1\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling thiserror-impl v1.0.69\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_019_many_to_many\\rust\\server\\grok-4\\llm)\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:28:20\n |\n28 | ctx.db.users().insert(User { user_id: 1, name: \"Alice\".into() });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n28 | ctx.db.users().try_insert(User { user_id: 1, name: \"Alice\".into() });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&users__TableHandle` in the current scope\n --> src\\lib.rs:29:20\n |\n29 | ctx.db.users().insert(User { user_id: 2, name: \"Bob\".into() });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n29 | ctx.db.users().try_insert(User { user_id: 2, name: \"Bob\".into() });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&groups__TableHandle` in the current scope\n --> src\\lib.rs:30:21\n |\n30 | ctx.db.groups().insert(Group { group_id: 10, title: \"Admin\".into() });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n30 | ctx.db.groups().try_insert(Group { group_id: 10, title: \"Admin\".into() });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&groups__TableHandle` in the current scope\n --> src\\lib.rs:31:21\n |\n31 | ctx.db.groups().insert(Group { group_id: 20, title: \"Dev\".into() });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n31 | ctx.db.groups().try_insert(Group { group_id: 20, title: \"Dev\".into() });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&memberships__TableHandle` in the current scope\n --> src\\lib.rs:32:26\n |\n32 | ctx.db.memberships().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n32 | ctx.db.memberships().try_insert(Membership { id: 1, user_id: 1, group_id: 10 });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&memberships__TableHandle` in the current scope\n --> src\\lib.rs:33:26\n |\n33 | ctx.db.memberships().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n33 | ctx.db.memberships().try_insert(Membership { id: 2, user_id: 1, group_id: 20 });\n | ++++\n\nerror[E0599]: no method named `insert` found for reference `&memberships__TableHandle` in the current scope\n --> src\\lib.rs:34:26\n |\n34 | ctx.db.memberships().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n34 | ctx.db.memberships().try_insert(Membership { id: 3, user_id: 2, group_id: 20 });\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:05.232020Z", - "finished_at": "2025-10-22T01:03:38.139665400Z" - }, - "t_020_ecs": { - "hash": "d47c11c6f8fd2858029ca4bcb1a9fdc8b79b46dac58a8a1b6ddb16f9f3d7213e", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, spacetimedb};\n\n#[spacetimedb::table(name = entities, public)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = positions, public)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocities, public)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_positions, public)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entities().insert(Entity { id: 1 });\n ctx.db.entities().insert(Entity { id: 2 });\n ctx.db.positions().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.positions().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocities().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n ctx.db.velocities().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.positions().iter() {\n if let Some(vel) = ctx.db.velocities().entity_id().find(&pos.entity_id) {\n let new_x = pos.x + vel.vx;\n let new_y = pos.y + vel.vy;\n let next_pos = NextPosition { entity_id: pos.entity_id, x: new_x, y: new_y };\n if ctx.db.next_positions().entity_id().find(&next_pos.entity_id).is_some() {\n ctx.db.next_positions().entity_id().update(next_pos);\n } else {\n ctx.db.next_positions().insert(next_pos);\n }\n }\n }\n}", - "category": "schema", - "route_api_model": "grok-4", - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-grok-4-llm", - "work_dir_golden": "target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\golden", - "work_dir_llm": "target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\grok-4\\llm", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 67 packages to latest compatible versions\n Compiling proc-macro2 v1.0.101\n Compiling quote v1.0.41\n Compiling unicode-ident v1.0.20\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.27\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.4\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling heck v0.5.0\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling arrayref v0.3.9\n Compiling hex v0.4.3\n Compiling bytes v1.10.1\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.3.1\n Compiling smallvec v1.15.1\n Compiling getrandom v0.2.16\n Compiling spacetimedb-lib v1.6.0\n Compiling log v0.4.28\n Compiling scoped-tls v1.0.1\n Compiling rand_core v0.6.4\n Compiling itertools v0.12.1\n Compiling generic-array v0.14.9\n Compiling num-traits v0.2.19\n Compiling cc v1.2.41\n Compiling syn v2.0.107\n Compiling blake3 v1.8.2\n Compiling spacetimedb-primitives v1.6.0\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.6\n Compiling approx v0.3.2\n Compiling chrono v0.4.42\n Compiling digest v0.10.7\n Compiling decorum v0.3.1\n Compiling sha3 v0.10.8\n Compiling spacetimedb-bindings-sys v1.6.0\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling thiserror-impl v1.0.69\n Compiling enum-as-inner v0.6.1\n Compiling derive_more v0.99.20\n Compiling spacetimedb-bindings-macro v1.6.0\n Compiling spacetimedb-sats v1.6.0\n Compiling spacetimedb v1.6.0\n Compiling spacetime-module v0.1.0 (E:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\private\\target\\llm-runs\\schema\\t_020_ecs\\rust\\server\\grok-4\\llm)\nerror[E0432]: unresolved import `spacetimedb::spacetimedb`\n --> src\\lib.rs:2:35\n |\n2 | use spacetimedb::{ReducerContext, spacetimedb};\n | ^^^^^^^^^^^ no `spacetimedb` in the root\n\nFor more information about this error, try `rustc --explain E0432`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai", - "started_at": "2025-10-22T01:02:55.582569300Z", - "finished_at": "2025-10-22T01:04:24.232996900Z" - }, - "t_021_multi_column_index": { - "hash": "b0d5bbe755291d8ec65de0a74b4eecfa344670639ca5f1a365ced3ab9717e97c", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - } - } - } - ] - }, - { - "mode": "cursor_rules", - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "models": [ - { - "name": "Grok 4", - "route_api_model": "grok-4", - "tasks": { - "t_002_scheduled_table": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_004_insert": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_004_insert", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "xai" - }, - "t_007_crud": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_007_crud", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_016_sum_type_columns": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_016_sum_type_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "llm_out": "id | a | b ----+---------------+--------------------------------------- 1 | (Circle = 10) | (Rectangle = (width = 4, height = 6))", - "query": "SELECT id, a, b FROM drawing WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" - } - } - }, - "vendor": "xai" - }, - "t_020_ecs": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_020_ecs", - "lang": "rust", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM next_position WHERE entity_id=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM next_position" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, Table, ReducerContext};\n\n#[table(accessor = log, public, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n Ok(())\n}", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 104 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling rustversion v1.0.22\n Compiling either v1.15.0\n Compiling itoa v1.0.17\n Compiling find-msvc-tools v0.1.9\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.39\n Compiling anyhow v1.0.102\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.11.0\n Compiling ryu v1.0.23\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.1\n Compiling keccak v0.1.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling hex v0.4.3\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\lib)\n Compiling bytemuck v1.25.0\n Compiling itertools v0.12.1\n Compiling second-stack v0.3.5\n Compiling zmij v1.0.21\n Compiling uuid v1.21.0\n Compiling rand_core v0.6.4\n Compiling log v0.4.29\n Compiling serde_json v1.0.149\n Compiling memchr v2.8.0\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling cc v1.2.56\n Compiling http v1.4.0\n Compiling castaway v0.2.4\n Compiling lean_string v0.5.1\n Compiling syn v2.0.117\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling decorum v0.3.1\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ethnum v1.5.2\n Compiling ppv-lite86 v0.2.21\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling spacetimedb-primitives v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\primitives)\n Compiling spacetimedb-bindings-sys v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-sys)\n Compiling spacetimedb-bindings-macro v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings-macro)\n Compiling spacetimedb-sats v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\sats)\n Compiling spacetimedb-query-builder v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\query-builder)\n Compiling spacetimedb v2.0.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\crates\\bindings)\n Compiling spacetime-module v0.1.0 (D:\\Development\\ClockworkLabs\\SpacetimeDB\\SpacetimeDBPrivate\\public\\target\\llm-runs\\schema\\t_021_multi_column_index\\rust\\server\\grok-4\\llm)\nerror: expected string literal\n --> src\\lib.rs:4:46\n |\n4 | #[table(accessor = log, public, index(name = by_user_day, btree(columns = [user_id, day])))]\n | ^^^^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `Log` in this scope\n --> src\\lib.rs:15:25\n |\n15 | ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n | ^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Log` in this scope\n --> src\\lib.rs:16:25\n |\n16 | ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n | ^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `Log` in this scope\n --> src\\lib.rs:17:25\n |\n17 | ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n | ^^^ not found in this scope\n\nerror[E0599]: no method named `log` found for struct `Local` in the current scope\n --> src\\lib.rs:15:12\n |\n15 | ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n | ^^^ method not found in `Local`\n\nerror[E0599]: no method named `log` found for struct `Local` in the current scope\n --> src\\lib.rs:16:12\n |\n16 | ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n | ^^^ method not found in `Local`\n\nerror[E0599]: no method named `log` found for struct `Local` in the current scope\n --> src\\lib.rs:17:12\n |\n17 | ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n | ^^^ method not found in `Local`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - } - } - } - ] - }, - { - "mode": "none", - "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", - "models": [ - { - "name": "Grok 4", - "route_api_model": "grok-4", - "tasks": { - "t_002_scheduled_table": { - "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", - "task": "t_002_scheduled_table", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", - "task": "t_012_spacetime_product_type", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", - "task": "t_017_scheduled_columns", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", - "task": "t_021_multi_column_index", - "lang": "rust", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "POST https://api.x.ai/v1/chat/completions status 429 Too Many Requests body: {\"code\":\"Some resource has been exhausted\",\"error\":\"Your team 5fc90f6d-3ee0-435e-a2e2-c7d23f7ea90d has either used all available credits or reached its monthly spending limit. To continue making API requests, please purchase more credits or raise your spending limit.\"}", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - } - } - } - ] - } - ], - "golden_answers": { - "basics/t_000_empty_reducers": { - "answer": "use spacetimedb::{reducer, ReducerContext};\r\n\r\n#[reducer]\r\npub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_three_args(\r\n ctx: &ReducerContext,\r\n active: bool,\r\n ratio: f32,\r\n label: String,\r\n) -> Result<(), String> {\r\n Ok(())\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_001_basic_tables": { - "answer": "use spacetimedb::table;\r\n\r\n#[table(name = users)]\r\npub struct Users {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[table(name = products)]\r\npub struct Products {\r\n #[primary_key]\r\n pub id: i32,\r\n pub title: String,\r\n pub price: f32,\r\n pub in_stock: bool,\r\n}\r\n\r\n#[table(name = notes)]\r\npub struct Notes {\r\n #[primary_key]\r\n pub id: i32,\r\n pub body: String,\r\n pub rating: i64,\r\n pub pinned: bool,\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_002_scheduled_table": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\r\nuse std::time::Duration;\r\n\r\n#[table(name = tick_timer, scheduled(tick))]\r\npub struct TickTimer {\r\n #[primary_key]\r\n #[auto_inc]\r\n scheduled_id: u64,\r\n scheduled_at: ScheduleAt,\r\n}\r\n\r\n#[reducer]\r\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer(init)]\r\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\r\n ctx.db.tick_timer().insert(TickTimer {\r\n scheduled_id: 0,\r\n scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()),\r\n });\r\n Ok(())\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_003_struct_in_table": { - "answer": "use spacetimedb::{table, SpacetimeType};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Position {\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[table(name = entities)]\r\npub struct Entity {\r\n #[primary_key]\r\n pub id: i32,\r\n pub pos: Position,\r\n}\r\n\r\n", - "syntax": "rust" - }, - "basics/t_004_insert": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = users)]\r\npub struct Users {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\r\n ctx.db.users().insert(Users { id, name, age, active });\r\n Ok(())\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_005_update": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\r\n\r\n#[table(name = users)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\r\n ctx.db.users().id().update(User { id, name, age, active });\r\n}", - "syntax": "rust" - }, - "basics/t_006_delete": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\r\n\r\n#[table(name = users)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn delete_user(ctx: &ReducerContext, id: i32) {\r\n ctx.db.users().id().delete(id);\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_007_crud": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = users)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn crud(ctx: &ReducerContext) {\r\n ctx.db.users().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\r\n ctx.db.users().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\r\n ctx.db.users().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\r\n ctx.db.users().id().delete(2);\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_008_index_lookup": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = users)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[table(name = results)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n}\r\n\r\n#[reducer]\r\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\r\n if let Some(u) = ctx.db.users().id().find(id) {\r\n ctx.db.results().insert(ResultRow { id: u.id, name: u.name });\r\n }\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_009_init": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = users)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer(init)]\r\npub fn init(ctx: &ReducerContext) {\r\n ctx.db.users().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\r\n ctx.db.users().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_010_connect": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = events)]\r\npub struct Event {\r\n #[primary_key]\r\n #[auto_inc]\r\n pub id: u64,\r\n pub kind: String,\r\n}\r\n\r\n#[reducer(client_connected)]\r\npub fn client_connected(ctx: &ReducerContext) {\r\n ctx.db.events().insert(Event { id: 0, kind: \"connected\".into() });\r\n}\r\n\r\n#[reducer(client_disconnected)]\r\npub fn client_disconnected(ctx: &ReducerContext) {\r\n ctx.db.events().insert(Event { id: 0, kind: \"disconnected\".into() });\r\n}\r\n", - "syntax": "rust" - }, - "basics/t_011_helper_function": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = results)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub sum: i32,\r\n}\r\n\r\nfn add(a: i32, b: i32) -> i32 { a + b }\r\n\r\n#[reducer]\r\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\r\n ctx.db.results().insert(ResultRow { id, sum: add(a, b) });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_012_spacetime_product_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Score {\r\n pub left: i32,\r\n pub right: i32,\r\n}\r\n\r\n#[table(name = results)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub value: Score,\r\n}\r\n\r\n#[reducer]\r\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\r\n ctx.db.results().insert(ResultRow { id, value: Score { left, right } });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_013_spacetime_sum_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Rect {\r\n pub width: i32,\r\n pub height: i32,\r\n}\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub enum Shape {\r\n Circle(i32),\r\n Rectangle(Rect),\r\n}\r\n\r\n#[table(name = results)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub value: Shape,\r\n}\r\n\r\n#[reducer]\r\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\r\n ctx.db.results().insert(ResultRow { id, value: Shape::Circle(radius) });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_014_elementary_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = primitives)]\r\npub struct Primitive {\r\n #[primary_key]\r\n pub id: i32,\r\n pub count: i32,\r\n pub total: i64,\r\n pub price: f32,\r\n pub ratio: f64,\r\n pub active: bool,\r\n pub name: String,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.primitives().insert(Primitive {\r\n id: 1,\r\n count: 2,\r\n total: 3_000_000_000,\r\n price: 1.5,\r\n ratio: 2.25,\r\n active: true,\r\n name: \"Alice\".into(),\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_015_product_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Address {\r\n pub street: String,\r\n pub zip: i32,\r\n}\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Position {\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[table(name = profiles)]\r\npub struct Profile {\r\n #[primary_key]\r\n pub id: i32,\r\n pub home: Address,\r\n pub work: Address,\r\n pub pos: Position,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.profiles().insert(Profile {\r\n id: 1,\r\n home: Address { street: \"1 Main\".into(), zip: 11111 },\r\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\r\n pos: Position { x: 7, y: 9 },\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_016_sum_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Rect {\r\n pub width: i32,\r\n pub height: i32,\r\n}\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub enum Shape {\r\n Circle(i32),\r\n Rectangle(Rect),\r\n}\r\n\r\n#[table(name = drawings)]\r\npub struct Drawing {\r\n #[primary_key]\r\n pub id: i32,\r\n pub a: Shape,\r\n pub b: Shape,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.drawings().insert(Drawing {\r\n id: 1,\r\n a: Shape::Circle(10),\r\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_017_scheduled_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\r\nuse std::time::Duration;\r\n\r\n#[table(name = tick_timer, scheduled(tick))]\r\npub struct TickTimer {\r\n #[primary_key]\r\n #[auto_inc]\r\n pub scheduled_id: u64,\r\n pub scheduled_at: ScheduleAt,\r\n}\r\n\r\n#[reducer]\r\npub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {\r\n}\r\n\r\n#[reducer(init)]\r\npub fn init(ctx: &ReducerContext) {\r\n let every_50ms: ScheduleAt = Duration::from_millis(50).into();\r\n ctx.db.tick_timer().insert(TickTimer {\r\n scheduled_id: 0,\r\n scheduled_at: every_50ms,\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_018_constraints": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(\r\n name = accounts,\r\n index(name = by_name, btree(columns = [name]))\r\n)]\r\npub struct Account {\r\n #[primary_key]\r\n pub id: i32,\r\n #[unique]\r\n pub email: String,\r\n pub name: String,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.accounts().insert(Account { id: 1, email: \"a@example.com\".into(), name: \"Alice\".into() });\r\n ctx.db.accounts().insert(Account { id: 2, email: \"b@example.com\".into(), name: \"Bob\".into() });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_019_many_to_many": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = users)]\r\npub struct User {\r\n #[primary_key]\r\n pub user_id: i32,\r\n pub name: String,\r\n}\r\n\r\n#[table(name = groups)]\r\npub struct Group {\r\n #[primary_key]\r\n pub group_id: i32,\r\n pub title: String,\r\n}\r\n\r\n#[table(\r\n name = memberships,\r\n index(name = by_user, btree(columns = [user_id])),\r\n index(name = by_group, btree(columns = [group_id]))\r\n)]\r\npub struct Membership {\r\n #[primary_key]\r\n pub id: i32,\r\n pub user_id: i32,\r\n pub group_id: i32,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.users().insert(User { user_id: 1, name: \"Alice\".into() });\r\n ctx.db.users().insert(User { user_id: 2, name: \"Bob\".into() });\r\n\r\n ctx.db.groups().insert(Group { group_id: 10, title: \"Admin\".into() });\r\n ctx.db.groups().insert(Group { group_id: 20, title: \"Dev\".into() });\r\n\r\n ctx.db.memberships().insert(Membership { id: 1, user_id: 1, group_id: 10 });\r\n ctx.db.memberships().insert(Membership { id: 2, user_id: 1, group_id: 20 });\r\n ctx.db.memberships().insert(Membership { id: 3, user_id: 2, group_id: 20 });\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_020_ecs": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(name = entities)]\r\npub struct Entity {\r\n #[primary_key]\r\n pub id: i32,\r\n}\r\n\r\n#[table(name = positions)]\r\npub struct Position {\r\n #[primary_key]\r\n pub entity_id: i32,\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[table(name = velocities)]\r\npub struct Velocity {\r\n #[primary_key]\r\n pub entity_id: i32,\r\n pub vx: i32,\r\n pub vy: i32,\r\n}\r\n\r\n#[table(name = next_positions)]\r\npub struct NextPosition {\r\n #[primary_key]\r\n pub entity_id: i32,\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.entities().insert(Entity { id: 1 });\r\n ctx.db.entities().insert(Entity { id: 2 });\r\n\r\n ctx.db.positions().insert(Position {\r\n entity_id: 1,\r\n x: 1,\r\n y: 0,\r\n });\r\n ctx.db.positions().insert(Position {\r\n entity_id: 2,\r\n x: 10,\r\n y: 0,\r\n });\r\n\r\n ctx.db.velocities().insert(Velocity {\r\n entity_id: 1,\r\n vx: 1,\r\n vy: 0,\r\n });\r\n ctx.db.velocities().insert(Velocity {\r\n entity_id: 2,\r\n vx: -2,\r\n vy: 3,\r\n });\r\n}\r\n\r\n#[spacetimedb::reducer]\r\npub fn step(ctx: &ReducerContext) {\r\n for p in ctx.db.positions().iter() {\r\n if let Some(v) = ctx.db.velocities().entity_id().find(p.entity_id) {\r\n let np = NextPosition {\r\n entity_id: p.entity_id,\r\n x: p.x + v.vx,\r\n y: p.y + v.vy,\r\n };\r\n\r\n if ctx.db.next_positions().entity_id().find(p.entity_id).is_some() {\r\n ctx.db.next_positions().entity_id().update(np);\r\n } else {\r\n ctx.db.next_positions().insert(np);\r\n }\r\n }\r\n }\r\n}\r\n", - "syntax": "rust" - }, - "schema/t_021_multi_column_index": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(\r\n name = logs,\r\n index(name = by_user_day, btree(columns = [user_id, day]))\r\n)]\r\npub struct Log {\r\n #[primary_key]\r\n pub id: i32,\r\n pub user_id: i32,\r\n pub day: i32,\r\n pub message: String,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.logs().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\r\n ctx.db.logs().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\r\n ctx.db.logs().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\r\n}\r\n", - "syntax": "rust" - }, - "t_000_empty_reducers": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\r\n\r\n#[table(accessor = empty_table)]\r\npub struct EmptyTable {\r\n #[primary_key]\r\n pub id: i32,\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_int(ctx: &ReducerContext, count: i32) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_string(ctx: &ReducerContext, name: String) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_two_args(ctx: &ReducerContext, count: i32, name: String) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer]\r\npub fn empty_reducer_with_three_args(\r\n ctx: &ReducerContext,\r\n active: bool,\r\n ratio: f32,\r\n label: String,\r\n) -> Result<(), String> {\r\n Ok(())\r\n}\r\n", - "syntax": "rust" - }, - "t_001_basic_tables": { - "answer": "use spacetimedb::table;\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[table(accessor = product)]\r\npub struct Product {\r\n #[primary_key]\r\n pub id: i32,\r\n pub title: String,\r\n pub price: f32,\r\n pub in_stock: bool,\r\n}\r\n\r\n#[table(accessor = note)]\r\npub struct Note {\r\n #[primary_key]\r\n pub id: i32,\r\n pub body: String,\r\n pub rating: i64,\r\n pub pinned: bool,\r\n}\r\n", - "syntax": "rust" - }, - "t_002_scheduled_table": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\r\nuse std::time::Duration;\r\n\r\n#[table(accessor = tick_timer, scheduled(tick))]\r\npub struct TickTimer {\r\n #[primary_key]\r\n #[auto_inc]\r\n scheduled_id: u64,\r\n scheduled_at: ScheduleAt,\r\n}\r\n\r\n#[reducer]\r\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\r\n Ok(())\r\n}\r\n\r\n#[reducer(init)]\r\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\r\n ctx.db.tick_timer().insert(TickTimer {\r\n scheduled_id: 0,\r\n scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()),\r\n });\r\n Ok(())\r\n}\r\n", - "syntax": "rust" - }, - "t_003_struct_in_table": { - "answer": "use spacetimedb::{table, SpacetimeType};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Position {\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[table(accessor = entity)]\r\npub struct Entity {\r\n #[primary_key]\r\n pub id: i32,\r\n pub pos: Position,\r\n}\r\n", - "syntax": "rust" - }, - "t_004_insert": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\r\n ctx.db.user().insert(User { id, name, age, active });\r\n Ok(())\r\n}\r\n", - "syntax": "rust" - }, - "t_005_update": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\r\n ctx.db.user().id().update(User { id, name, age, active });\r\n}\r\n", - "syntax": "rust" - }, - "t_006_delete": { - "answer": "use spacetimedb::{reducer, table, ReducerContext};\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn delete_user(ctx: &ReducerContext, id: i32) {\r\n ctx.db.user().id().delete(id);\r\n}\r\n", - "syntax": "rust" - }, - "t_007_crud": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer]\r\npub fn crud(ctx: &ReducerContext) {\r\n ctx.db.user().insert(User {\r\n id: 1,\r\n name: \"Alice\".into(),\r\n age: 30,\r\n active: true,\r\n });\r\n ctx.db.user().insert(User {\r\n id: 2,\r\n name: \"Bob\".into(),\r\n age: 22,\r\n active: false,\r\n });\r\n ctx.db.user().id().update(User {\r\n id: 1,\r\n name: \"Alice2\".into(),\r\n age: 31,\r\n active: false,\r\n });\r\n ctx.db.user().id().delete(2);\r\n}\r\n", - "syntax": "rust" - }, - "t_008_index_lookup": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[table(accessor = result)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n}\r\n\r\n#[reducer]\r\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\r\n if let Some(u) = ctx.db.user().id().find(id) {\r\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\r\n }\r\n}\r\n", - "syntax": "rust" - }, - "t_009_init": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub id: i32,\r\n pub name: String,\r\n pub age: i32,\r\n pub active: bool,\r\n}\r\n\r\n#[reducer(init)]\r\npub fn init(ctx: &ReducerContext) {\r\n ctx.db.user().insert(User {\r\n id: 1,\r\n name: \"Alice\".into(),\r\n age: 30,\r\n active: true,\r\n });\r\n ctx.db.user().insert(User {\r\n id: 2,\r\n name: \"Bob\".into(),\r\n age: 22,\r\n active: false,\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_010_connect": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = event)]\r\npub struct Event {\r\n #[primary_key]\r\n #[auto_inc]\r\n pub id: u64,\r\n pub kind: String,\r\n}\r\n\r\n#[reducer(client_connected)]\r\npub fn client_connected(ctx: &ReducerContext) {\r\n ctx.db.event().insert(Event {\r\n id: 0,\r\n kind: \"connected\".into(),\r\n });\r\n}\r\n\r\n#[reducer(client_disconnected)]\r\npub fn client_disconnected(ctx: &ReducerContext) {\r\n ctx.db.event().insert(Event {\r\n id: 0,\r\n kind: \"disconnected\".into(),\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_011_helper_function": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = result)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub sum: i32,\r\n}\r\n\r\nfn add(a: i32, b: i32) -> i32 {\r\n a + b\r\n}\r\n\r\n#[reducer]\r\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\r\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\r\n}\r\n", - "syntax": "rust" - }, - "t_012_spacetime_product_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Score {\r\n pub left: i32,\r\n pub right: i32,\r\n}\r\n\r\n#[table(accessor = result)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub value: Score,\r\n}\r\n\r\n#[reducer]\r\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\r\n ctx.db.result().insert(ResultRow {\r\n id,\r\n value: Score { left, right },\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_013_spacetime_sum_type": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Rect {\r\n pub width: i32,\r\n pub height: i32,\r\n}\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub enum Shape {\r\n Circle(i32),\r\n Rectangle(Rect),\r\n}\r\n\r\n#[table(accessor = result)]\r\npub struct ResultRow {\r\n #[primary_key]\r\n pub id: i32,\r\n pub value: Shape,\r\n}\r\n\r\n#[reducer]\r\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\r\n ctx.db.result().insert(ResultRow {\r\n id,\r\n value: Shape::Circle(radius),\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_014_elementary_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = primitive)]\r\npub struct Primitive {\r\n #[primary_key]\r\n pub id: i32,\r\n pub count: i32,\r\n pub total: i64,\r\n pub price: f32,\r\n pub ratio: f64,\r\n pub active: bool,\r\n pub name: String,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.primitive().insert(Primitive {\r\n id: 1,\r\n count: 2,\r\n total: 3_000_000_000,\r\n price: 1.5,\r\n ratio: 2.25,\r\n active: true,\r\n name: \"Alice\".into(),\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_015_product_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Address {\r\n pub street: String,\r\n pub zip: i32,\r\n}\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Position {\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[table(accessor = profile)]\r\npub struct Profile {\r\n #[primary_key]\r\n pub id: i32,\r\n pub home: Address,\r\n pub work: Address,\r\n pub pos: Position,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.profile().insert(Profile {\r\n id: 1,\r\n home: Address {\r\n street: \"1 Main\".into(),\r\n zip: 11111,\r\n },\r\n work: Address {\r\n street: \"2 Broad\".into(),\r\n zip: 22222,\r\n },\r\n pos: Position { x: 7, y: 9 },\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_016_sum_type_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table};\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub struct Rect {\r\n pub width: i32,\r\n pub height: i32,\r\n}\r\n\r\n#[derive(SpacetimeType, Clone, Debug)]\r\npub enum Shape {\r\n Circle(i32),\r\n Rectangle(Rect),\r\n}\r\n\r\n#[table(accessor = drawing)]\r\npub struct Drawing {\r\n #[primary_key]\r\n pub id: i32,\r\n pub a: Shape,\r\n pub b: Shape,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.drawing().insert(Drawing {\r\n id: 1,\r\n a: Shape::Circle(10),\r\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_017_scheduled_columns": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table};\r\nuse std::time::Duration;\r\n\r\n#[table(accessor = tick_timer, scheduled(tick))]\r\npub struct TickTimer {\r\n #[primary_key]\r\n #[auto_inc]\r\n pub scheduled_id: u64,\r\n pub scheduled_at: ScheduleAt,\r\n}\r\n\r\n#[reducer]\r\npub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {}\r\n\r\n#[reducer(init)]\r\npub fn init(ctx: &ReducerContext) {\r\n let every_50ms: ScheduleAt = Duration::from_millis(50).into();\r\n ctx.db.tick_timer().insert(TickTimer {\r\n scheduled_id: 0,\r\n scheduled_at: every_50ms,\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_018_constraints": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(\r\n accessor = account,\r\n index(accessor = by_name, btree(columns = [name]))\r\n)]\r\npub struct Account {\r\n #[primary_key]\r\n pub id: i32,\r\n #[unique]\r\n pub email: String,\r\n pub name: String,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.account().insert(Account {\r\n id: 1,\r\n email: \"a@example.com\".into(),\r\n name: \"Alice\".into(),\r\n });\r\n ctx.db.account().insert(Account {\r\n id: 2,\r\n email: \"b@example.com\".into(),\r\n name: \"Bob\".into(),\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_019_many_to_many": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = user)]\r\npub struct User {\r\n #[primary_key]\r\n pub user_id: i32,\r\n pub name: String,\r\n}\r\n\r\n#[table(accessor = group)]\r\npub struct Group {\r\n #[primary_key]\r\n pub group_id: i32,\r\n pub title: String,\r\n}\r\n\r\n#[table(\r\n accessor = membership,\r\n index(accessor = by_user, btree(columns = [user_id])),\r\n index(accessor = by_group, btree(columns = [group_id]))\r\n)]\r\npub struct Membership {\r\n #[primary_key]\r\n pub id: i32,\r\n pub user_id: i32,\r\n pub group_id: i32,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.user().insert(User {\r\n user_id: 1,\r\n name: \"Alice\".into(),\r\n });\r\n ctx.db.user().insert(User {\r\n user_id: 2,\r\n name: \"Bob\".into(),\r\n });\r\n\r\n ctx.db.group().insert(Group {\r\n group_id: 10,\r\n title: \"Admin\".into(),\r\n });\r\n ctx.db.group().insert(Group {\r\n group_id: 20,\r\n title: \"Dev\".into(),\r\n });\r\n\r\n ctx.db.membership().insert(Membership {\r\n id: 1,\r\n user_id: 1,\r\n group_id: 10,\r\n });\r\n ctx.db.membership().insert(Membership {\r\n id: 2,\r\n user_id: 1,\r\n group_id: 20,\r\n });\r\n ctx.db.membership().insert(Membership {\r\n id: 3,\r\n user_id: 2,\r\n group_id: 20,\r\n });\r\n}\r\n", - "syntax": "rust" - }, - "t_020_ecs": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(accessor = entity)]\r\npub struct Entity {\r\n #[primary_key]\r\n pub id: i32,\r\n}\r\n\r\n#[table(accessor = position)]\r\npub struct Position {\r\n #[primary_key]\r\n pub entity_id: i32,\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[table(accessor = velocity)]\r\npub struct Velocity {\r\n #[primary_key]\r\n pub entity_id: i32,\r\n pub vx: i32,\r\n pub vy: i32,\r\n}\r\n\r\n#[table(accessor = next_position)]\r\npub struct NextPosition {\r\n #[primary_key]\r\n pub entity_id: i32,\r\n pub x: i32,\r\n pub y: i32,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.entity().insert(Entity { id: 1 });\r\n ctx.db.entity().insert(Entity { id: 2 });\r\n\r\n ctx.db.position().insert(Position {\r\n entity_id: 1,\r\n x: 1,\r\n y: 0,\r\n });\r\n ctx.db.position().insert(Position {\r\n entity_id: 2,\r\n x: 10,\r\n y: 0,\r\n });\r\n\r\n ctx.db.velocity().insert(Velocity {\r\n entity_id: 1,\r\n vx: 1,\r\n vy: 0,\r\n });\r\n ctx.db.velocity().insert(Velocity {\r\n entity_id: 2,\r\n vx: -2,\r\n vy: 3,\r\n });\r\n}\r\n\r\n#[spacetimedb::reducer]\r\npub fn step(ctx: &ReducerContext) {\r\n for p in ctx.db.position().iter() {\r\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\r\n let np = NextPosition {\r\n entity_id: p.entity_id,\r\n x: p.x + v.vx,\r\n y: p.y + v.vy,\r\n };\r\n\r\n if ctx.db.next_position().entity_id().find(p.entity_id).is_some() {\r\n ctx.db.next_position().entity_id().update(np);\r\n } else {\r\n ctx.db.next_position().insert(np);\r\n }\r\n }\r\n }\r\n}\r\n", - "syntax": "rust" - }, - "t_021_multi_column_index": { - "answer": "use spacetimedb::{reducer, table, ReducerContext, Table};\r\n\r\n#[table(\r\n accessor = log,\r\n index(accessor = by_user_day, btree(columns = [user_id, day]))\r\n)]\r\npub struct Log {\r\n #[primary_key]\r\n pub id: i32,\r\n pub user_id: i32,\r\n pub day: i32,\r\n pub message: String,\r\n}\r\n\r\n#[reducer]\r\npub fn seed(ctx: &ReducerContext) {\r\n ctx.db.log().insert(Log {\r\n id: 1,\r\n user_id: 7,\r\n day: 1,\r\n message: \"a\".into(),\r\n });\r\n ctx.db.log().insert(Log {\r\n id: 2,\r\n user_id: 7,\r\n day: 2,\r\n message: \"b\".into(),\r\n });\r\n ctx.db.log().insert(Log {\r\n id: 3,\r\n user_id: 9,\r\n day: 1,\r\n message: \"c\".into(),\r\n });\r\n}\r\n", - "syntax": "rust" - } - } - }, - { - "lang": "typescript", - "modes": [ - { - "mode": "docs", - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "models": [ - { - "name": "Grok 4", - "route_api_model": "grok-4", - "tasks": { - "t_000_empty_reducers": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_001_basic_tables": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n {},\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst product = table(\n {},\n {\n id: t.i32().primaryKey(),\n title: t.string(),\n price: t.f32(),\n inStock: t.bool(),\n }\n);\n\nconst note = table(\n {},\n {\n id: t.i32().primaryKey(),\n body: t.string(),\n rating: t.i64(),\n pinned: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user, product, note });\nexport default spacetimedb;", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Errors occurred:\nname `undefined_id_idx_btree` is used for multiple entities\n\n\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:61803/v1/database/basics-t-001-basic-tables-grok-4-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:61803 => http://127.0.0.1:61803\nThis will DESTROY the current basics-t-001-basic-tables-grok-4-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_002_scheduled_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_003_struct_in_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_004_insert": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "xai" - }, - "t_005_update": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_006_delete": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_007_crud": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user });\nexport default spacetimedb;\n\nexport const crud = spacetimedb.reducer((ctx) => {\n ctx.db.user.insert({ id: 1, name: \"Alice\", age: 30, active: true });\n ctx.db.user.insert({ id: 2, name: \"Bob\", age: 22, active: false });\n ctx.db.user.id.update({ id: 1, name: \"Alice2\", age: 31, active: false });\n ctx.db.user.id.delete(2);\n});", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:61803/v1/database/c20068c1ae115e07ee7bd680c6a31b29545b7d0a54dbdbaba4637daaf54dd3ee/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "xai" - }, - "t_008_index_lookup": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { public: true },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst result = table(\n { public: true },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n }\n);\n\nconst spacetimedb = schema({ user, result });\nexport default spacetimedb;\n\nexport const lookupUserName = spacetimedb.reducer({ id: t.i32() }, (ctx, { id }) => {\n const foundUser = ctx.db.user.id.find(id);\n if (foundUser) {\n ctx.db.result.insert({\n id: foundUser.id,\n name: foundUser.name,\n });\n }\n});", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Errors occurred:\nname `undefined_id_idx_btree` is used for multiple entities\n\n\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:61803/v1/database/basics-t-008-index-lookup-grok-4-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:61803 => http://127.0.0.1:61803\nThis will DESTROY the current basics-t-008-index-lookup-grok-4-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_009_init": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "xai" - }, - "t_010_connect": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event' },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected(ctx => {\n ctx.db.event.insert({ id: 0n, kind: \"connected\" });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected(ctx => {\n ctx.db.event.insert({ id: 0n, kind: \"disconnected\" });\n});", - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_011_helper_function": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-4", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_013_spacetime_sum_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\r\n\r\nconst Rect = t.object('Rect', {\r\n width: t.i32(),\r\n height: t.i32(),\r\n});\r\n\r\nconst Shape = t.enum('Shape', {\r\n circle: t.i32(),\r\n rectangle: Rect,\r\n});\r\n\r\nconst result = table(\r\n {},\r\n {\r\n id: t.i32().primaryKey(),\r\n value: Shape,\r\n }\r\n);\r\n\r\nconst spacetimedb = schema({ result });\r\nexport default spacetimedb;\r\n\r\nexport const setCircle = spacetimedb.reducer({ id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => {\r\n ctx.db.result.insert({ id, value: { tag: 'circle', value: radius } });\r\n});", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:61803/v1/database/c200e33d4d652b61ce5d379db48f7b2d1a76dfcbd64d177c3f37c946a561c0db/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_014_elementary_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_015_product_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Address = t.object('Address', {\n street: t.string(),\n zip: t.i32(),\n});\n\nconst Position = t.object('Position', {\n x: t.i32(),\n y: t.i32(),\n});\n\nconst profile = table(\n {},\n {\n id: t.i32().primaryKey(),\n home: Address,\n work: Address,\n pos: Position,\n }\n);\n\nconst spacetimedb = schema({ profile });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.profile.insert({\n id: 0,\n home: { street: \"1 Main\", zip: 11111 },\n work: { street: \"2 Broad\", zip: 22222 },\n pos: { x: 7, y: 9 },\n });\n});", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+------+------+-----", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_016_sum_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\r\n\r\nconst Rect = t.object('Rect', {\r\n width: t.i32(),\r\n height: t.i32(),\r\n});\r\n\r\nconst Shape = t.enum('Shape', {\r\n circle: t.i32(),\r\n rectangle: Rect,\r\n});\r\n\r\nconst drawing = table(\r\n {},\r\n {\r\n id: t.i32().primaryKey(),\r\n a: Shape,\r\n b: Shape,\r\n }\r\n);\r\n\r\nconst spacetimedb = schema({ drawing });\r\nexport default spacetimedb;\r\n\r\nexport const seed = spacetimedb.reducer((ctx) => {\r\n ctx.db.drawing.insert({\r\n id: 0,\r\n a: { tag: 'circle', value: 10 },\r\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\r\n });\r\n});", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:61803/v1/database/c2000788e329b453495afd0fdbb496a34d3af0ea9e8d7984f456fba62dc88ed7/call/seed)\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\r\nimport { ScheduleAt, TimeDuration } from 'spacetimedb';\r\n\r\nconst tickTimer = table(\r\n { scheduled: () => tick },\r\n {\r\n scheduledId: t.u64().primaryKey().autoInc(),\r\n scheduledAt: t.scheduleAt(),\r\n }\r\n);\r\n\r\nconst spacetimedb = schema({ tickTimer });\r\nexport default spacetimedb;\r\n\r\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {});\r\n\r\nexport const init = spacetimedb.init(ctx => {\r\n ctx.db.tickTimer.insert({\r\n scheduledId: 0n,\r\n scheduledAt: ScheduleAt.interval(TimeDuration.fromMillis(50n)),\r\n });\r\n});", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:61803/v1/database/schema-t-017-scheduled-columns-grok-4-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:61803 => http://127.0.0.1:61803\nThis will DESTROY the current schema-t-017-scheduled-columns-grok-4-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_018_constraints": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "xai" - }, - "t_019_many_to_many": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "xai" - }, - "t_020_ecs": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": false, - "model_name": "Grok 4", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst entity = table(\n { public: true },\n {\n id: t.i32().primaryKey().autoInc(),\n }\n);\n\nconst position = table(\n { public: true },\n {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n }\n);\n\nconst velocity = table(\n { public: true },\n {\n entityId: t.i32().primaryKey(),\n vx: t.i32(),\n vy: t.i32(),\n }\n);\n\nconst nextPosition = table(\n { public: true },\n {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n }\n);\n\nconst spacetimedb = schema({ entity, position, velocity, nextPosition });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n const entity1 = ctx.db.entity.insert({ id: 0 });\n const entity2 = ctx.db.entity.insert({ id: 0 });\n ctx.db.position.insert({ entityId: entity1.id, x: 0, y: 0 });\n ctx.db.velocity.insert({ entityId: entity1.id, vx: 1, vy: 0 });\n ctx.db.position.insert({ entityId: entity2.id, x: 10, y: 0 });\n ctx.db.velocity.insert({ entityId: entity2.id, vx: -2, vy: 3 });\n});\n\nexport const step = spacetimedb.reducer((ctx) => {\n for (const pos of ctx.db.position.iter()) {\n const vel = ctx.db.velocity.entityId.find(pos.entityId);\n if (vel) {\n const newX = pos.x + vel.vx;\n const newY = pos.y + vel.vy;\n const existing = ctx.db.nextPosition.entityId.find(pos.entityId);\n if (existing) {\n ctx.db.nextPosition.entityId.update({ ...existing, x: newX, y: newY });\n } else {\n ctx.db.nextPosition.insert({ entityId: pos.entityId, x: newX, y: newY });\n }\n }\n }\n});", - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Errors occurred:\nname `undefined_entityId_idx_btree` is used for multiple entities\n\n\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:61803/v1/database/schema-t-020-ecs-grok-4-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:61803 => http://127.0.0.1:61803\nThis will DESTROY the current schema-t-020-ecs-grok-4-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok 4", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "grok-4", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "xai" - } - } - }, - { - "name": "Claude Opus 4.6", - "route_api_model": "claude-opus-4-6", - "tasks": { - "t_000_empty_reducers": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_001_basic_tables": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_002_scheduled_table": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_003_struct_in_table": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_004_insert": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "anthropic" - }, - "t_005_update": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_006_delete": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_007_crud": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "anthropic" - }, - "t_008_index_lookup": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_009_init": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "anthropic" - }, - "t_010_connect": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event', public: true },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'connected' });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'disconnected' });\n});", - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_011_helper_function": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "anthropic" - }, - "t_012_spacetime_product_type": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "anthropic" - }, - "t_013_spacetime_sum_type": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst result = table(\n { name: 'result', public: true },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer(\n { id: t.i32(), radius: t.i32() },\n (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { tag: 'circle', value: radius },\n });\n }\n);", - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:49945/v1/database/c200be9d999eaedcdb3217b8dd603ef3e0cbae165a407fec27d275fbd2096ac9/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_014_elementary_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_015_product_type_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_016_sum_type_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst drawing = table(\n { name: 'drawing', public: true },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape,\n }\n);\n\nconst spacetimedb = schema({ drawing });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 1,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\n });\n});", - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:49945/v1/database/c20059e92845ad15cf4ad8fe76accef3794cd3e31eb6fe27392e28986ad17fc5/call/seed)\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_017_scheduled_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tickTimer WHERE scheduledId>=0" - } - } - }, - "vendor": "anthropic" - }, - "t_018_constraints": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "anthropic" - }, - "t_019_many_to_many": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "anthropic" - }, - "t_020_ecs": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM nextPosition" - } - } - }, - "vendor": "anthropic" - }, - "t_021_multi_column_index": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Opus 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "claude-opus-4-6", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "anthropic" - } - } - }, - { - "name": "Claude Sonnet 4.6", - "route_api_model": "claude-sonnet-4-6", - "tasks": { - "t_000_empty_reducers": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_001_basic_tables": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_002_scheduled_table": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_003_struct_in_table": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_004_insert": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "anthropic" - }, - "t_005_update": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_006_delete": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_007_crud": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "anthropic" - }, - "t_008_index_lookup": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "anthropic" - }, - "t_009_init": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "anthropic" - }, - "t_010_connect": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event', public: true },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'connected' });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'disconnected' });\n});", - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "anthropic" - }, - "t_011_helper_function": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "anthropic" - }, - "t_012_spacetime_product_type": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "anthropic" - }, - "t_013_spacetime_sum_type": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst result = table(\n { name: 'result', public: true },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer(\n { id: t.i32(), radius: t.i32() },\n (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { tag: 'circle', value: radius },\n });\n }\n);", - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:52577/v1/database/c2006f2817973d324ee8247428b9caa12a024dc31c8ecdc27f11604f3e21ae0b/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_014_elementary_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_015_product_type_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_016_sum_type_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst drawing = table(\n { name: 'drawing', public: true },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape,\n }\n);\n\nconst spacetimedb = schema({ drawing });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 0,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\n });\n});", - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:52577/v1/database/c2007833132462ef336d257f7a96b6e7fbcc3ce713c37259cee85ab13b5a5db4/call/seed)\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "anthropic" - }, - "t_017_scheduled_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tickTimer WHERE scheduledId>=0" - } - } - }, - "vendor": "anthropic" - }, - "t_018_constraints": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "anthropic" - }, - "t_019_many_to_many": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "anthropic" - }, - "t_020_ecs": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM nextPosition" - } - } - }, - "vendor": "anthropic" - }, - "t_021_multi_column_index": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "Claude Sonnet 4.6", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "claude-sonnet-4-6", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "anthropic" - } - } - }, - { - "name": "GPT-5.2-Codex", - "route_api_model": "gpt-5.2-codex", - "tasks": { - "t_000_empty_reducers": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_001_basic_tables": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table({ name: 'user', public: true }, {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n});\n\nconst product = table({ name: 'product', public: true }, {\n id: t.i32().primaryKey(),\n title: t.string(),\n price: t.f32(),\n inStock: t.bool(),\n});\n\nconst note = table({ name: 'note', public: true }, {\n id: t.i32().primaryKey(),\n body: t.string(),\n rating: t.i64(),\n pinned: t.bool(),\n});\n\nconst spacetimedb = schema(user, product, note);\nexport default spacetimedb;", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_001_basic_tables/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_001_basic_tables/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_001_basic_tables/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_001_basic_tables/typescript/server/gpt-5-2-codex/llm/src/index.ts:24:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:63834/v1/database/basics-t-001-basic-tables-gpt-5-2-codex-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:63834 => http://127.0.0.1:63834\nThis will DESTROY the current basics-t-001-basic-tables-gpt-5-2-codex-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_002_scheduled_table": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_003_struct_in_table": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Position = t.object('Position', {\n x: t.i32(),\n y: t.i32(),\n});\n\nconst entity = table(\n { name: 'entity' },\n {\n id: t.i32().primaryKey(),\n pos: Position,\n }\n);\n\nconst spacetimedb = schema(entity);\nexport default spacetimedb;", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_003_struct_in_table/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_003_struct_in_table/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_003_struct_in_table/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_003_struct_in_table/typescript/server/gpt-5-2-codex/llm/src/index.ts:16:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:63834/v1/database/basics-t-003-struct-in-table-gpt-5-2-codex-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:63834 => http://127.0.0.1:63834\nThis will DESTROY the current basics-t-003-struct-in-table-gpt-5-2-codex-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_004_insert": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema(user);\nexport default spacetimedb;\n\nexport const insertUser = spacetimedb.reducer(\n { id: t.i32(), name: t.string(), age: t.i32(), active: t.bool() },\n (ctx, { id, name, age, active }) => {\n ctx.db.user.insert({ id, name, age, active });\n }\n);", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_004_insert/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_004_insert/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_004_insert/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_004_insert/typescript/server/gpt-5-2-codex/llm/src/index.ts:13:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:63834/v1/database/basics-t-004-insert-gpt-5-2-codex-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:63834 => http://127.0.0.1:63834\nThis will DESTROY the current basics-t-004-insert-gpt-5-2-codex-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_005_update": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_006_delete": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_007_crud": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "openai" - }, - "t_008_index_lookup": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_009_init": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_009_init", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema(user);\nexport default spacetimedb;\n\nexport const init = spacetimedb.init(ctx => {\n ctx.db.user.insert({ id: 1, name: \"Alice\", age: 30, active: true });\n ctx.db.user.insert({ id: 2, name: \"Bob\", age: 22, active: false });\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_009_init/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_009_init/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_009_init/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_009_init/typescript/server/gpt-5-2-codex/llm/src/index.ts:13:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:63834/v1/database/basics-t-009-init-gpt-5-2-codex-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:63834 => http://127.0.0.1:63834\nThis will DESTROY the current basics-t-009-init-gpt-5-2-codex-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_010_connect": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event' },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'connected' });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'disconnected' });\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_011_helper_function": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst result = table({ name: 'result' }, {\n id: t.i32().primaryKey(),\n sum: t.i32(),\n});\n\nconst spacetimedb = schema(result);\nexport default spacetimedb;\n\nfunction add(a: number, b: number): number {\n return a + b;\n}\n\nexport const computeSum = spacetimedb.reducer(\n { id: t.i32(), a: t.i32(), b: t.i32() },\n (ctx, { id, a, b }) => {\n ctx.db.result.insert({ id, sum: add(a, b) });\n }\n);", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_011_helper_function/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_011_helper_function/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_011_helper_function/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_011_helper_function/typescript/server/gpt-5-2-codex/llm/src/index.ts:8:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:63834/v1/database/basics-t-011-helper-function-gpt-5-2-codex-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:63834 => http://127.0.0.1:63834\nThis will DESTROY the current basics-t-011-helper-function-gpt-5-2-codex-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_012_spacetime_product_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "openai" - }, - "t_013_spacetime_sum_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst result = table(\n { name: 'result' },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer(\n { id: t.i32(), radius: t.i32() },\n (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { tag: 'circle', value: radius },\n });\n }\n);", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:54010/v1/database/c200acddbfdc9d5afa3818ca12ddf8b0bdff3fed74bb6575dae9acc9c5c48279/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_014_elementary_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_015_product_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_016_sum_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst drawing = table(\n { name: 'drawing' },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape,\n }\n);\n\nconst spacetimedb = schema({ drawing });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 1,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\n });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:54010/v1/database/c2008f58a39bd092f7ca474551f460eaa1669c8aeca334be93da2661ea290e2d/call/seed)\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_017_scheduled_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n { name: 'tickTimer', scheduled: () => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema(tickTimer);\nexport default spacetimedb;\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (_ctx, { arg }) => {\n void arg;\n});\n\nexport const init = spacetimedb.init(ctx => {\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.interval(50_000n),\n });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/gpt-5-2-codex/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/gpt-5-2-codex/llm/src/index.ts:12:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:54010/v1/database/schema-t-017-scheduled-columns-gpt-5-2-codex-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:54010 => http://127.0.0.1:54010\nThis will DESTROY the current schema-t-017-scheduled-columns-gpt-5-2-codex-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_018_constraints": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "openai" - }, - "t_019_many_to_many": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "openai" - }, - "t_020_ecs": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM nextPosition" - } - } - }, - "vendor": "openai" - }, - "t_021_multi_column_index": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "openai" - } - } - }, - { - "name": "GPT-5-mini", - "route_api_model": "gpt-5-mini", - "tasks": { - "t_000_empty_reducers": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_001_basic_tables": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_002_scheduled_table": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n { scheduled: 'tick' },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\nexport default spacetimedb;\n\nexport const tick = spacetimedb.reducer(tickTimer.rowType, (ctx, timer) => {\n // Scheduled tick; perform periodic work here.\n console.info('tick at', ctx.timestamp);\n});\n\nexport const init = spacetimedb.init((ctx) => {\n if (ctx.db.tickTimer.count() === 0) {\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.interval(50_000n),\n });\n }\n});", - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: reducer is not a function\n\tat resolveSchedules (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_002_scheduled_table/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7217:53)\n\tat [SpacetimeDB.moduleHooks.v2] (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_002_scheduled_table/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7248:22)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:54039/v1/database/basics-t-002-scheduled-table-gpt-5-mini-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:54039 => http://127.0.0.1:54039\nThis will DESTROY the current basics-t-002-scheduled-table-gpt-5-mini-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_003_struct_in_table": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_004_insert": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "openai" - }, - "t_005_update": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_006_delete": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_007_crud": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "openai" - }, - "t_008_index_lookup": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "openai" - }, - "t_009_init": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "openai" - }, - "t_010_connect": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { public: true },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected((ctx) => {\n ctx.db.event.insert({\n id: 0n,\n kind: \"connected\",\n });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected((ctx) => {\n ctx.db.event.insert({\n id: 0n,\n kind: \"disconnected\",\n });\n});", - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "openai" - }, - "t_011_helper_function": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "openai" - }, - "t_012_spacetime_product_type": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const Score = t.object('Score', {\n left: t.i32(),\n right: t.i32(),\n});\n\nconst result = table(\n { name: 'result' },\n {\n id: t.i32().primaryKey(),\n value: Score,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setScore = spacetimedb.reducer(\n { id: t.i32(), left: t.i32(), right: t.i32() },\n (ctx, { id, left, right }) => {\n ctx.db.result.insert({\n id,\n value: { left, right },\n });\n }\n);", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: exporting something that is not a spacetime export\n\tat [SpacetimeDB.moduleHooks.v2] (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_012_spacetime_product_type/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7241:15)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:54039/v1/database/schema-t-012-spacetime-product-type-gpt-5-mini-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:54039 => http://127.0.0.1:54039\nThis will DESTROY the current schema-t-012-spacetime-product-type-gpt-5-mini-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_013_spacetime_sum_type": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst result = table(\n { name: 'result', public: true },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer({ id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { circle: radius },\n });\n});", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:54039/v1/database/c200bb2648b6b24a98f1c2c1814fcdf7b12362142249436f950bc3656912d9bd/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_014_elementary_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_015_product_type_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "openai" - }, - "t_016_sum_type_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst drawing = table(\n { name: 'drawing', public: true },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape,\n }\n);\n\nconst spacetimedb = schema({ drawing });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer('seed', (ctx) => {\n ctx.db.drawing.id.insert({\n id: 1,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\n });\n});", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: Cannot read properties of undefined (reading 'tag')\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:1988:64)\n\tat isFixedSizeProduct (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:1987:46)\n\tat makeDeserializer (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:2100:9)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:6380:35)\n\tat ModuleHooksImpl (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:6379:65)\n\tat makeHooks (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:6370:30)\n\tat [SpacetimeDB.moduleHooks.v2] (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7249:12)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:54039/v1/database/schema-t-016-sum-type-columns-gpt-5-mini-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:54039 => http://127.0.0.1:54039\nThis will DESTROY the current schema-t-016-sum-type-columns-gpt-5-mini-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_017_scheduled_columns": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n { scheduled: 'tick' },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\nexport default spacetimedb;\n\nexport const tick = spacetimedb.reducer('tick', tickTimer.rowType, (ctx, timer) => {\n console.info('tick');\n});\n\nexport const init = spacetimedb.init((ctx) => {\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.interval(50_000n),\n });\n});", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: reducer is not a function\n\tat resolveSchedules (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7217:53)\n\tat [SpacetimeDB.moduleHooks.v2] (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7248:22)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:54039/v1/database/schema-t-017-scheduled-columns-gpt-5-mini-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:54039 => http://127.0.0.1:54039\nThis will DESTROY the current schema-t-017-scheduled-columns-gpt-5-mini-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_018_constraints": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "openai" - }, - "t_019_many_to_many": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 5, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user', public: true },\n {\n userId: t.i32().primaryKey(),\n name: t.string(),\n }\n);\n\nconst group = table(\n { name: 'group', public: true },\n {\n groupId: t.i32().primaryKey(),\n title: t.string(),\n }\n);\n\nconst membership = table(\n {\n name: 'membership',\n public: true,\n indexes: [\n { name: 'byUser', algorithm: 'btree', columns: ['userId'] },\n { name: 'byGroup', algorithm: 'btree', columns: ['groupId'] },\n ],\n },\n {\n id: t.i32().primaryKey(),\n userId: t.i32(),\n groupId: t.i32(),\n }\n);\n\nconst spacetimedb = schema({ user, group, membership });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n // Users\n ctx.db.user.userId.insert({ userId: 1, name: \"Alice\" });\n ctx.db.user.userId.insert({ userId: 2, name: \"Bob\" });\n\n // Groups\n ctx.db.group.groupId.insert({ groupId: 10, title: \"Admin\" });\n ctx.db.group.groupId.insert({ groupId: 20, title: \"Dev\" });\n\n // Memberships\n ctx.db.membership.id.insert({ id: 1, userId: 1, groupId: 10 });\n ctx.db.membership.id.insert({ id: 2, userId: 1, groupId: 20 });\n ctx.db.membership.id.insert({ id: 3, userId: 2, groupId: 20 });\n});", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "m2m_has_1_20": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:54039/v1/database/c2008f1f83f5944b0111c371ec9f5d3bb94125e404c1f8044d20b8b2eba4c88c/call/seed)\n", - "phase": "call_reducer" - } - }, - "memberships_three_rows": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "openai" - }, - "t_020_ecs": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5-mini", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst entity = table(\n { },\n {\n id: t.i32().primaryKey(),\n }\n);\n\nconst position = table(\n { },\n {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n }\n);\n\nconst velocity = table(\n { },\n {\n entityId: t.i32().primaryKey(),\n vx: t.i32(),\n vy: t.i32(),\n }\n);\n\nconst nextPosition = table(\n { },\n {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n }\n);\n\nexport const spacetimedb = schema({ entity, position, velocity, nextPosition });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n // Insert Entity 1\n ctx.db.entity.id.insert({ id: 1 });\n ctx.db.position.entityId.insert({ entityId: 1, x: 0, y: 0 });\n ctx.db.velocity.entityId.insert({ entityId: 1, vx: 1, vy: 0 });\n\n // Insert Entity 2\n ctx.db.entity.id.insert({ id: 2 });\n ctx.db.position.entityId.insert({ entityId: 2, x: 10, y: 0 });\n ctx.db.velocity.entityId.insert({ entityId: 2, vx: -2, vy: 3 });\n});\n\nexport const step = spacetimedb.reducer((ctx) => {\n for (const pos of ctx.db.position.iter()) {\n const vel = ctx.db.velocity.entityId.find(pos.entityId);\n if (!vel) continue;\n const nx = pos.x + vel.vx;\n const ny = pos.y + vel.vy;\n const existing = ctx.db.nextPosition.entityId.find(pos.entityId);\n if (existing) {\n ctx.db.nextPosition.entityId.update({ ...existing, x: nx, y: ny });\n } else {\n ctx.db.nextPosition.entityId.insert({ entityId: pos.entityId, x: nx, y: ny });\n }\n }\n});", - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: exporting something that is not a spacetime export\n\tat [SpacetimeDB.moduleHooks.v2] (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_020_ecs/typescript/server/gpt-5-mini/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7241:15)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:54039/v1/database/schema-t-020-ecs-gpt-5-mini-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:54039 => http://127.0.0.1:54039\nThis will DESTROY the current schema-t-020-ecs-gpt-5-mini-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_021_multi_column_index": { - "hash": "c73ba5b8e7438d32e18e96f9ab86c5b8ae2f31d1d614fe8db4abd098176840b0", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "GPT-5-mini", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "gpt-5-mini", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "openai" - } - } - }, - { - "name": "Grok Code", - "route_api_model": "grok-code-fast-1", - "tasks": { - "t_000_empty_reducers": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_001_basic_tables": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_002_scheduled_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_003_struct_in_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_004_insert": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "xai" - }, - "t_005_update": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user', public: true },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user });\nexport default spacetimedb;\n\nexport const updateUser = spacetimedb.reducer({ id: t.i32(), name: t.string(), age: t.i32(), active: t.bool() }, (ctx, { id, name, age, active }) => {\n const userRow = ctx.db.user.id.find(id);\n if (userRow) {\n ctx.db.user.id.update({ ...userRow, name, age, active });\n }\n});", - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+------+-----+--------", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:61803/v1/database/c2008d3a5d3b789a38ac8b7e737fd463d7f7b7de462393e2641dfa765aa7cf55/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_006_delete": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_007_crud": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "xai" - }, - "t_008_index_lookup": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "xai" - }, - "t_009_init": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "xai" - }, - "t_010_connect": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event', public: true },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected(ctx => {\n ctx.db.event.insert({\n id: 0n,\n kind: \"connected\",\n });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected(ctx => {\n ctx.db.event.insert({\n id: 0n,\n kind: \"disconnected\",\n });\n});", - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "xai" - }, - "t_011_helper_function": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "xai" - }, - "t_012_spacetime_product_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "xai" - }, - "t_013_spacetime_sum_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst result = table(\n { name: 'result', public: true },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer({ id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => {\n ctx.db.result.insert({ id, value: { tag: 'circle', value: radius } });\n});", - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:61803/v1/database/c200e33d4d652b61ce5d379db48f7b2d1a76dfcbd64d177c3f37c946a561c0db/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_014_elementary_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_015_product_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_016_sum_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst drawing = table(\n { name: 'drawing', public: true },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape,\n }\n);\n\nconst spacetimedb = schema({ drawing });\n\nexport const seed = spacetimedb.reducer({}, (ctx) => {\n ctx.db.drawing.insert({\n id: 0,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\n });\n});\n\nexport default spacetimedb;", - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:61803/v1/database/c2000788e329b453495afd0fdbb496a34d3af0ea9e8d7984f456fba62dc88ed7/call/seed)\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "xai" - }, - "t_017_scheduled_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "Grok Code", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t, ScheduleAt } from 'spacetimedb/server';\n\nconst tickTimer = table(\n { scheduled: () => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n});\n\nexport const init = spacetimedb.reducer((ctx) => {\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.interval(50000n),\n });\n});\n\nexport default spacetimedb;", - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\n[MISSING_EXPORT] Error: \"ScheduleAt\" is not exported by \"D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/grok-code/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs\".\n ╭─[ D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_017_scheduled_columns/typescript/server/grok-code/llm/src/index.ts:1:28 ]\n │\n 1 │ import { schema, table, t, ScheduleAt } from 'spacetimedb/server';\n │ ─────┬──── \n │ ╰────── Missing export\n───╯\n\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "xai" - }, - "t_018_constraints": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "xai" - }, - "t_019_many_to_many": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "xai" - }, - "t_020_ecs": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM nextPosition" - } - } - }, - "vendor": "xai" - }, - "t_021_multi_column_index": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "Grok Code", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "grok-code-fast-1", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "xai" - } - } - }, - { - "name": "Gemini 3 Flash", - "route_api_model": "google/gemini-3-flash-preview", - "tasks": { - "t_000_empty_reducers": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_001_basic_tables": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_002_scheduled_table": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": false, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t, ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n {\n name: 'tickTimer',\n scheduled: (): any => tick,\n },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\nexport default spacetimedb;\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n // Reducer logic for each 50ms tick\n});\n\nexport const init = spacetimedb.init((ctx) => {\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.interval(50n),\n });\n});", - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Your module doesn't import the `spacetimedb/server` package at all - this is likely a mistake, as your module will not be able to interface with the SpacetimeDB host.\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_003_struct_in_table": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_004_insert": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "meta" - }, - "t_005_update": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_006_delete": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_007_crud": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "meta" - }, - "t_008_index_lookup": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_009_init": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "meta" - }, - "t_010_connect": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event', public: true },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected((ctx) => {\n ctx.db.event.insert({\n id: 0n,\n kind: 'connected',\n });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected((ctx) => {\n ctx.db.event.insert({\n id: 0n,\n kind: 'disconnected',\n });\n});", - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_011_helper_function": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "meta" - }, - "t_012_spacetime_product_type": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "meta" - }, - "t_013_spacetime_sum_type": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": false, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32()\n});\n\nconst Shape = t.enum('Shape', [\n { circle: t.i32() },\n { rectangle: Rect }\n]);\n\nconst result = table(\n { name: 'result', public: true },\n {\n id: t.i32().primaryKey(),\n value: Shape\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer({ id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { tag: 'circle', value: radius }\n });\n});", - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Errors occurred:\nerror at typespace ref `&1`: type ([object Object]: ()) contains identifier error Identifier `[object Object]`'s starting character '[' is neither an underscore ('_') nor a Unicode XID_start character (according to Unicode Standard Annex 31, https://www.unicode.org/reports/tr31/) and cannot be used for entities in a module.\n\n\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:52620/v1/database/schema-t-013-spacetime-sum-type-gemini-3-flash-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:52620 => http://127.0.0.1:52620\nThis will DESTROY the current schema-t-013-spacetime-sum-type-gemini-3-flash-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_014_elementary_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_015_product_type_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_016_sum_type_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32()\n});\n\nconst Shape = t.enum('Shape', [\n { circle: t.i32() },\n { rectangle: Rect }\n]);\n\nconst drawing = table(\n { name: 'drawing', public: true },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape\n }\n);\n\nconst spacetimedb = schema({ drawing });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 1,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } }\n });\n});", - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Errors occurred:\nerror at typespace ref `&1`: type ([object Object]: ()) contains identifier error Identifier `[object Object]`'s starting character '[' is neither an underscore ('_') nor a Unicode XID_start character (according to Unicode Standard Annex 31, https://www.unicode.org/reports/tr31/) and cannot be used for entities in a module.\n\n\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:52620/v1/database/schema-t-016-sum-type-columns-gemini-3-flash-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:52620 => http://127.0.0.1:52620\nThis will DESTROY the current schema-t-016-sum-type-columns-gemini-3-flash-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "meta" - }, - "t_017_scheduled_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tickTimer WHERE scheduledId>=0" - } - } - }, - "vendor": "meta" - }, - "t_018_constraints": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "meta" - }, - "t_019_many_to_many": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "meta" - }, - "t_020_ecs": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM nextPosition" - } - } - }, - "vendor": "meta" - }, - "t_021_multi_column_index": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3 Flash", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "google/gemini-3-flash-preview", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "meta" - } - } - }, - { - "name": "Gemini 3.1 Pro", - "route_api_model": "google/gemini-3.1-pro-preview", - "tasks": { - "t_000_empty_reducers": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_001_basic_tables": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_002_scheduled_table": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_003_struct_in_table": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_004_insert": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "meta" - }, - "t_005_update": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user });\nexport default spacetimedb;\n\nexport const updateUser = spacetimedb.reducer(\n { id: t.i32(), name: t.string(), age: t.i32(), active: t.bool() },\n (ctx, { id, name, age, active }) => {\n const existing = ctx.db.user.id.find(id);\n if (existing) {\n ctx.db.user.id.update({ id, name, age, active });\n }\n }\n);", - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+------+-----+--------", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:52620/v1/database/c200886b2e3f2344ddf070ee0ddd8fedf289888e6ffd6ece476650e341a95371/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_006_delete": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_007_crud": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user });\nexport default spacetimedb;\n\nexport const crud = spacetimedb.reducer((ctx) => {\n ctx.db.user.insert({ id: 1, name: \"Alice\", age: 30, active: true });\n ctx.db.user.insert({ id: 2, name: \"Bob\", age: 22, active: false });\n ctx.db.user.id.update({ id: 1, name: \"Alice2\", age: 31, active: false });\n ctx.db.user.id.delete(2);\n});", - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:52620/v1/database/c200258ae6e48e89f6a3299e7df0959bbd56effb5cffe76a195c6022a7de614e/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "meta" - }, - "t_008_index_lookup": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "meta" - }, - "t_009_init": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "meta" - }, - "t_010_connect": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event' },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected((ctx) => {\n ctx.db.event.insert({\n id: 0n,\n kind: \"connected\",\n });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected((ctx) => {\n ctx.db.event.insert({\n id: 0n,\n kind: \"disconnected\",\n });\n});", - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "meta" - }, - "t_011_helper_function": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "meta" - }, - "t_012_spacetime_product_type": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "meta" - }, - "t_013_spacetime_sum_type": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst result = table({ public: true }, {\n id: t.i32().primaryKey(),\n value: Shape,\n});\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer({ id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { circle: radius }\n });\n});", - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:52620/v1/database/c200f58ab35aeaf0198d81edad605d27351d3848fe734b272362964e0af50880/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_014_elementary_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_015_product_type_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_016_sum_type_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32()\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect\n});\n\nconst drawing = table({ name: 'drawing' }, {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape\n});\n\nconst spacetimedb = schema({ drawing });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 1,\n a: Shape.circle(10),\n b: Shape.rectangle({ width: 4, height: 6 })\n });\n});", - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:52620/v1/database/c2009b00db35748e0584cfc37793d20ac8ac6f98c7a02c234e1a9b9e7c073568/call/seed)\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "meta" - }, - "t_017_scheduled_columns": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 2, - "passed_tests": 2, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "scheduled_seeded_one_row": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM tickTimer WHERE scheduledId>=0" - } - } - }, - "vendor": "meta" - }, - "t_018_constraints": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "meta" - }, - "t_019_many_to_many": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "meta" - }, - "t_020_ecs": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=2 AND x=8 AND y=3" - } - }, - "ecs_next_pos_entity1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM nextPosition WHERE entityId=1 AND x=1 AND y=0" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM nextPosition" - } - } - }, - "vendor": "meta" - }, - "t_021_multi_column_index": { - "hash": "4fc194f370ec8318bb9b257e2e31aa8ead0dec6b437bd32ff93613e39b0c819f", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "Gemini 3.1 Pro", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "google/gemini-3.1-pro-preview", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "meta" - } - } - }, - { - "name": "DeepSeek Reasoner", - "route_api_model": "deepseek-reasoner", - "tasks": { - "t_000_empty_reducers": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_001_basic_tables": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n {},\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst product = table(\n {},\n {\n id: t.i32().primaryKey(),\n title: t.string(),\n price: t.f32(),\n inStock: t.bool(),\n }\n);\n\nconst note = table(\n {},\n {\n id: t.i32().primaryKey(),\n body: t.string(),\n rating: t.i64(),\n pinned: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user, product, note });\nexport default spacetimedb;", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Errors occurred:\nname `undefined_id_idx_btree` is used for multiple entities\n\n\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/basics-t-001-basic-tables-deepseek-reasoner-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current basics-t-001-basic-tables-deepseek-reasoner-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_002_scheduled_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n { name: 'tickTimer', scheduled: (): any => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\nexport default spacetimedb;\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n});\n\nexport const init = spacetimedb.init((ctx) => {\n ctx.db.tickTimer.insert({ scheduledAt: ScheduleAt.interval(50) });\n});", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/basics-t-002-scheduled-table-deepseek-reasoner-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current basics-t-002-scheduled-table-deepseek-reasoner-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_003_struct_in_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_004_insert": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "deepseek" - }, - "t_005_update": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { table, t, schema, SenderError } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user });\nexport default spacetimedb;\n\nexport const updateUser = spacetimedb.reducer(\n { id: t.i32(), name: t.string(), age: t.i32(), active: t.bool() },\n (ctx, { id, name, age, active }) => {\n const row = ctx.db.user.id.find(id);\n if (!row) {\n throw new SenderError(`User with id ${id} not found`);\n }\n ctx.db.user.id.update({ id, name, age, active });\n }\n);", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "data_parity_update_user": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: User with id 1 not found\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c2008b2c1ef07a9b15f2b315ef4565aa0a3801298a2cec74a109a5cf962ea8b7/call/update_user)\n", - "phase": "call_reducer_llm" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: IndexError: Unique constraint violation 'user_id_idx_btree' in table 'user': column(s): '[\"id\"]' value: 1\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:62644/v1/database/c200ed791181a62b587de2774b095e98742d8c8beebb5c72e228770f80c7e4c7/sql)\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_006_delete": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { table, t, schema } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema(user);\nexport default spacetimedb;\n\nexport const deleteUser = spacetimedb.reducer({ id: t.i32() }, (ctx, { id }) => {\n ctx.db.user.id.delete(id);\n});", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_006_delete/typescript/server/deepseek-reasoner/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_006_delete/typescript/server/deepseek-reasoner/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_006_delete/typescript/server/deepseek-reasoner/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/basics/t_006_delete/typescript/server/deepseek-reasoner/llm/src/index.ts:13:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/basics-t-006-delete-deepseek-reasoner-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current basics-t-006-delete-deepseek-reasoner-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_007_crud": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 2, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nconst spacetimedb = schema({ user });\nexport default spacetimedb;\n\nexport const crud = spacetimedb.reducer((ctx) => {\n ctx.db.user.insert({ id: 1, name: 'Alice', age: 30, active: true });\n ctx.db.user.insert({ id: 2, name: 'Bob', age: 22, active: false });\n const alice = ctx.db.user.id.find(1);\n if (alice) {\n ctx.db.user.id.update({ ...alice, name: 'Alice2', age: 31, active: false });\n }\n ctx.db.user.id.delete(2);\n});", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c200a955ecab87ea80b202a6ccd46bb73f8565b76f4c684cad73c8a66d2944cb/call/crud)\n", - "phase": "call_reducer_golden" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "deepseek" - }, - "t_008_index_lookup": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_009_init": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "deepseek" - }, - "t_010_connect": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event', public: true },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected(ctx => {\n ctx.db.event.insert({ id: 0n, kind: 'connected' });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected(ctx => {\n ctx.db.event.insert({ id: 0n, kind: 'disconnected' });\n});", - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_011_helper_function": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - } - }, - "vendor": "deepseek" - }, - "t_012_spacetime_product_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "deepseek" - }, - "t_013_spacetime_sum_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { table, t, schema } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', { width: t.i32(), height: t.i32() });\nconst Shape = t.enum('Shape', { circle: t.i32(), rectangle: Rect });\n\nconst result = table(\n { name: 'result' },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer({ id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => {\n const shapeValue = { circle: radius };\n ctx.db.result.insert({ id, value: shapeValue });\n});", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c200f9c264c6263338e4ae4b5f8f760b46aaf9f47002d4a59e3ccaf61f9b16bc/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_014_elementary_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst primitive = table(\n { name: 'primitive', public: true },\n {\n id: t.i32().primaryKey(),\n count: t.i32(),\n total: t.i64(),\n price: t.f32(),\n ratio: t.f64(),\n active: t.bool(),\n name: t.string(),\n }\n);\n\nconst spacetimedb = schema(primitive);\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.primitive.insert({\n id: 1,\n count: 2,\n total: 3000000000n,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\",\n });\n});", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: table2.tableDef is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_014_elementary_columns/typescript/server/deepseek-reasoner/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7402:31)\n\tat SchemaInner (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_014_elementary_columns/typescript/server/deepseek-reasoner/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7205:23)\n\tat schema (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_014_elementary_columns/typescript/server/deepseek-reasoner/llm/node_modules/.pnpm/spacetimedb@file+..+..+..+._4e6762d0fa0af4732de585440982ec59/node_modules/spacetimedb/dist/server/index.mjs:7399:15)\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_014_elementary_columns/typescript/server/deepseek-reasoner/llm/src/index.ts:16:21)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/schema-t-014-elementary-columns-deepseek-reasoner-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current schema-t-014-elementary-columns-deepseek-reasoner-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_015_product_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_016_sum_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32()\n});\n\nconst Shape = t.union(\n t.object('Circle', { radius: t.i32() }),\n Rect\n);\n\nconst drawing = table(\n { name: 'drawing' },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape\n }\n);\n\nconst spacetimedb = schema({ drawing });\n\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 0,\n a: { radius: 10 },\n b: { width: 4, height: 6 }\n });\n});", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Uncaught TypeError: t.union is not a function\n\tat (D:/Development/ClockworkLabs/SpacetimeDB/SpacetimeDBPrivate/public/target/llm-runs/schema/t_016_sum_type_columns/typescript/server/deepseek-reasoner/llm/src/index.ts:8:17)\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/schema-t-016-sum-type-columns-deepseek-reasoner-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current schema-t-016-sum-type-columns-deepseek-reasoner-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_017_scheduled_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n { name: 'tickTimer', scheduled: (): any => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\nexport default spacetimedb;\n\nexport const init = spacetimedb.init(ctx => {\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.interval(50),\n });\n});\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n ctx.db.tickTimer.scheduledId.update({\n ...arg,\n scheduledAt: ScheduleAt.interval(50),\n });\n});", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/schema-t-017-scheduled-columns-deepseek-reasoner-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current schema-t-017-scheduled-columns-deepseek-reasoner-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_018_constraints": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "deepseek" - }, - "t_019_many_to_many": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Reasoner", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "deepseek" - }, - "t_020_ecs": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst entity = table({}, {\n id: t.i32().primaryKey(),\n});\n\nconst position = table({}, {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n});\n\nconst velocity = table({}, {\n entityId: t.i32().primaryKey(),\n vx: t.i32(),\n vy: t.i32(),\n});\n\nconst nextPosition = table({}, {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n});\n\nconst spacetimedb = schema({ entity, position, velocity, nextPosition });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.entity.insert({ id: 1 });\n ctx.db.entity.insert({ id: 2 });\n ctx.db.position.insert({ entityId: 1, x: 0, y: 0 });\n ctx.db.velocity.insert({ entityId: 1, vx: 1, vy: 0 });\n ctx.db.position.insert({ entityId: 2, x: 10, y: 0 });\n ctx.db.velocity.insert({ entityId: 2, vx: -2, vy: 3 });\n});\n\nexport const step = spacetimedb.reducer((ctx) => {\n for (const pos of ctx.db.position.iter()) {\n const vel = ctx.db.velocity.entityId.find(pos.entityId);\n if (vel) {\n const next = { entityId: pos.entityId, x: pos.x + vel.vx, y: pos.y + vel.vy };\n const existing = ctx.db.nextPosition.entityId.find(pos.entityId);\n if (existing) {\n ctx.db.nextPosition.entityId.update(next);\n } else {\n ctx.db.nextPosition.insert(next);\n }\n }\n }\n});", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Errors occurred:\nname `undefined_entityId_idx_btree` is used for multiple entities\n\n\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/schema-t-020-ecs-deepseek-reasoner-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current schema-t-020-ecs-deepseek-reasoner-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_021_multi_column_index": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Reasoner", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "import { table, t, schema } from 'spacetimedb';\n\nconst log = table(\n {\n name: \"log\",\n indexes: [\n {\n name: \"byUserDay\",\n algorithm: \"btree\",\n columns: [\"userId\", \"day\"]\n }\n ]\n },\n {\n id: t.i32().primaryKey(),\n userId: t.i32(),\n day: t.i32(),\n message: t.string()\n }\n);\n\nconst spacetimedb = schema(log);\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.log.insert({ id: 1, userId: 7, day: 1, message: \"a\" });\n ctx.db.log.insert({ id: 2, userId: 7, day: 2, message: \"b\" });\n ctx.db.log.insert({ id: 3, userId: 9, day: 1, message: \"c\" });\n});", - "category": "schema", - "route_api_model": "deepseek-reasoner", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: Your module doesn't import the `spacetimedb/server` package at all - this is likely a mistake, as your module will not be able to interface with the SpacetimeDB host.\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - } - } - }, - { - "name": "DeepSeek Chat", - "route_api_model": "deepseek-chat", - "tasks": { - "t_000_empty_reducers": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_001_basic_tables": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_002_scheduled_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n { scheduled: (): any => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\nexport default spacetimedb;\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n});\n\nexport const init = spacetimedb.init(ctx => {\n ctx.db.tickTimer.insert({ scheduledId: 0n, scheduledAt: ScheduleAt.interval(50) });\n});", - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/basics-t-002-scheduled-table-deepseek-chat-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current basics-t-002-scheduled-table-deepseek-chat-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_003_struct_in_table": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 1, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_004_insert": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 2, - "passed_tests": 2, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user" - } - } - }, - "vendor": "deepseek" - }, - "t_005_update": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_005_update", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "data_parity_update_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_006_delete": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_007_crud": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud" - } - }, - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - } - }, - "vendor": "deepseek" - }, - "t_008_index_lookup": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1 - ], - "golden_out": "id | name ----+--------- 1 | \"Alice\"", - "llm_out": "id | name ----+--------- 1 | \"Alice\"", - "query": "SELECT id, name FROM result WHERE id=1", - "reducer": "lookup_user_name" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - } - }, - "vendor": "deepseek" - }, - "t_009_init": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_009_init", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 4, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - } - }, - "vendor": "deepseek" - }, - "t_010_connect": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst event = table(\n { name: 'event', public: true },\n {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n }\n);\n\nconst spacetimedb = schema({ event });\nexport default spacetimedb;\n\nexport const clientConnected = spacetimedb.clientConnected(ctx => {\n ctx.db.event.insert({ id: 0n, kind: 'connected' });\n});\n\nexport const clientDisconnected = spacetimedb.clientDisconnected(ctx => {\n ctx.db.event.insert({ id: 0n, kind: 'disconnected' });\n});", - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "reducers_diff": { - "only_golden": [ - "on_connect()", - "on_disconnect()" - ], - "only_llm": [ - "client_connected()", - "client_disconnected()" - ] - }, - "reducers_equal": false, - "tables_diff": null, - "tables_equal": true - } - } - }, - "vendor": "deepseek" - }, - "t_011_helper_function": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "basics", - "route_api_model": "deepseek-chat", - "scorer_details": { - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | sum ----+----- 1 | 5", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum" - } - } - }, - "vendor": "deepseek" - }, - "t_012_spacetime_product_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score" - } - } - }, - "vendor": "deepseek" - }, - "t_013_spacetime_sum_type": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst result = table(\n { name: 'result' },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nconst spacetimedb = schema({ result });\nexport default spacetimedb;\n\nexport const setCircle = spacetimedb.reducer({ id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { tag: 'circle', value: radius },\n });\n});", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c200f9c264c6263338e4ae4b5f8f760b46aaf9f47002d4a59e3ccaf61f9b16bc/call/set_circle)\n", - "phase": "call_reducer_golden" - } - }, - "sum_type_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_014_elementary_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "elementary_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", - "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", - "reducer": "seed" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_015_product_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_016_sum_type_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 1, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nconst Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nconst drawing = table(\n { name: 'drawing' },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape,\n }\n);\n\nconst spacetimedb = schema({ drawing });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 1,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\n });\n});", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c200100766258eabd62fb624fa946e1b530e249ae9b9db139394a06440f0538c/call/seed)\n", - "phase": "call_reducer_golden" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM drawing WHERE id=1" - } - } - }, - "vendor": "deepseek" - }, - "t_017_scheduled_columns": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "DeepSeek Chat", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nconst tickTimer = table(\n { scheduled: (): any => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nconst spacetimedb = schema({ tickTimer });\nexport default spacetimedb;\n\nexport const init = spacetimedb.init(ctx => {\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.interval(50),\n });\n});\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n});", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (500 Internal Server Error) for url (http://127.0.0.1:62644/v1/database/schema-t-017-scheduled-columns-deepseek-chat-llm?clear=true&host_type=Js)\n\n--- stdout ---\nBuild finished successfully.\nUploading to http://127.0.0.1:62644 => http://127.0.0.1:62644\nThis will DESTROY the current schema-t-017-scheduled-columns-deepseek-chat-llm module, and ALL corresponding data.\nSkipping confirmation due to --yes\nPublishing module...\nJavaScript / TypeScript support is currently in BETA.\nThere may be bugs. Please file issues if you encounter any.\n\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "deepseek" - }, - "t_018_constraints": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 3, - "passed_tests": 3, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "constraints_row_parity_after_seed": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - } - }, - "vendor": "deepseek" - }, - "t_019_many_to_many": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 5, - "passed_tests": 5, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=2 AND groupId=20" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE userId=1 AND groupId=10" - } - }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - } - }, - "vendor": "deepseek" - }, - "t_020_ecs": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 5, - "passed_tests": 2, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst entity = table(\n { name: 'entity', public: true },\n {\n id: t.i32().primaryKey(),\n }\n);\n\nconst position = table(\n { name: 'position', public: true },\n {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n }\n);\n\nconst velocity = table(\n { name: 'velocity', public: true },\n {\n entityId: t.i32().primaryKey(),\n vx: t.i32(),\n vy: t.i32(),\n }\n);\n\nconst nextPosition = table(\n { name: 'nextPosition', public: true },\n {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n }\n);\n\nconst spacetimedb = schema({ entity, position, velocity, nextPosition });\nexport default spacetimedb;\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.entity.insert({ id: 1 });\n ctx.db.position.insert({ entityId: 1, x: 0, y: 0 });\n ctx.db.velocity.insert({ entityId: 1, vx: 1, vy: 0 });\n\n ctx.db.entity.insert({ id: 2 });\n ctx.db.position.insert({ entityId: 2, x: 10, y: 0 });\n ctx.db.velocity.insert({ entityId: 2, vx: -2, vy: 3 });\n});\n\nexport const step = spacetimedb.reducer((ctx) => {\n for (const pos of ctx.db.position.iter()) {\n const vel = ctx.db.velocity.entityId.find(pos.entityId);\n if (vel) {\n const nextX = pos.x + vel.vx;\n const nextY = pos.y + vel.vy;\n ctx.db.nextPosition.entityId.update({ entityId: pos.entityId, x: nextX, y: nextY });\n }\n }\n});", - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c200a1edce9df991ad9c3b4c38a078c5d8d969292a7cc2b62ba6d8c83b818cbc/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c200a1edce9df991ad9c3b4c38a078c5d8d969292a7cc2b62ba6d8c83b818cbc/call/step)\n", - "phase": "call_reducer" - } - }, - "ecs_seed_position_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM position" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_step_next_position_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: Response text: The instance encountered a fatal error.\n\nCaused by:\n HTTP status server error (530 ) for url (http://127.0.0.1:62644/v1/database/c200a1edce9df991ad9c3b4c38a078c5d8d969292a7cc2b62ba6d8c83b818cbc/call/step)\n", - "phase": "call_reducer" - } - } - }, - "vendor": "deepseek" - }, - "t_021_multi_column_index": { - "hash": "f0986524ee7bda1802573e7e4794bd56933365fb35e4af7c830014214cb8ec77", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": true, - "model_name": "DeepSeek Chat", - "total_tests": 4, - "passed_tests": 4, - "category": "schema", - "route_api_model": "deepseek-chat", - "scorer_details": { - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "mcindex_lookup_u7_d1": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=1" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "reducers_diff": null, - "reducers_equal": true, - "tables_diff": null, - "tables_equal": true - } - }, - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE userId=7 AND day=2" - } - } - }, - "vendor": "deepseek" - } - } - } - ] - }, - { - "mode": "cursor_rules", - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "models": [ - { - "name": "GPT-5.2-Codex", - "route_api_model": "gpt-5.2-codex", - "tasks": { - "t_000_empty_reducers": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_000_empty_reducers", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const emptyTable = table({ name: 'empty_table' }, {\n id: t.i32().primaryKey(),\n});\n\nexport const spacetimedb = schema(emptyTable);\n\nexport const emptyReducerNoArgs = spacetimedb.reducer('emptyReducerNoArgs', (ctx) => {});\n\nexport const emptyReducerWithInt = spacetimedb.reducer('emptyReducerWithInt', { count: t.i32() }, (ctx, { count }) => {});\n\nexport const emptyReducerWithString = spacetimedb.reducer('emptyReducerWithString', { name: t.string() }, (ctx, { name }) => {});\n\nexport const emptyReducerWithTwoArgs = spacetimedb.reducer('emptyReducerWithTwoArgs', { count: t.i32(), name: t.string() }, (ctx, { count, name }) => {});\n\nexport const emptyReducerWithThreeArgs = spacetimedb.reducer('emptyReducerWithThreeArgs', { active: t.bool(), ratio: t.f32(), label: t.string() }, (ctx, { active, ratio, label }) => {});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_001_basic_tables": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_001_basic_tables", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const user = table({ name: 'user' }, {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n});\n\nexport const product = table({ name: 'product' }, {\n id: t.i32().primaryKey(),\n title: t.string(),\n price: t.f32(),\n inStock: t.bool(),\n});\n\nexport const note = table({ name: 'note' }, {\n id: t.i32().primaryKey(),\n body: t.string(),\n rating: t.i64(),\n pinned: t.bool(),\n});\n\nexport const spacetimedb = schema({ user, product, note });", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_002_scheduled_table": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_002_scheduled_table", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nexport const tickTimer = table(\n { name: 'tick_timer', scheduled: () => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt()\n }\n);\n\nexport const spacetimedb = schema({ tickTimer });\n\nconst INTERVAL_MICROS = 50_000n;\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n const nextAt = ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + INTERVAL_MICROS);\n ctx.db.tickTimer.insert({ scheduledId: 0n, scheduledAt: nextAt });\n});\n\nexport const init = spacetimedb.reducer((ctx) => {\n const firstAt = ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + INTERVAL_MICROS);\n ctx.db.tickTimer.insert({ scheduledId: 0n, scheduledAt: firstAt });\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_003_struct_in_table": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_003_struct_in_table", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const Position = t.object('Position', {\n x: t.i32(),\n y: t.i32(),\n});\n\nexport const entity = table(\n { name: 'entity' },\n {\n id: t.i32().primaryKey(),\n pos: Position,\n }\n);\n\nexport const spacetimedb = schema({ entity });", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_004_insert": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_004_insert", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nexport const spacetimedb = schema({ user });\n\nexport const insertUser = spacetimedb.reducer(\n { id: t.i32(), name: t.string(), age: t.i32(), active: t.bool() },\n (ctx, { id, name, age, active }) => {\n ctx.db.user.insert({ id, name, age, active });\n }\n);", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_005_update": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_005_update", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t, SenderError } from 'spacetimedb/server';\n\nexport const user = table({ name: 'user' }, {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n});\n\nexport const spacetimedb = schema(user);\n\nexport const updateUser = spacetimedb.reducer('updateUser', { id: t.i32(), name: t.string(), age: t.i32(), active: t.bool() }, (ctx, { id, name, age, active }) => {\n const existing = ctx.db.user.id.find(id);\n if (!existing) {\n throw new SenderError('User not found');\n }\n ctx.db.user.id.update({ ...existing, name, age, active });\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_006_delete": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_006_delete", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const user = table(\n { name: 'user' },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n }\n);\n\nexport const spacetimedb = schema({ user });\n\nexport const deleteUser = spacetimedb.reducer({ id: t.i32() }, (ctx, { id }) => {\n ctx.db.user.id.delete(id);\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_007_crud": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_007_crud", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const user = table({ name: 'user' }, {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n});\n\nexport const spacetimedb = schema({ user });\n\nexport const crud = spacetimedb.reducer((ctx) => {\n ctx.db.user.insert({ id: 1, name: \"Alice\", age: 30, active: true });\n ctx.db.user.insert({ id: 2, name: \"Bob\", age: 22, active: false });\n ctx.db.user.id.update({ id: 1, name: \"Alice2\", age: 31, active: false });\n ctx.db.user.id.delete(2);\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_008_index_lookup": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_008_index_lookup", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t, SenderError } from 'spacetimedb/server';\n\nexport const user = table(\n { name: 'user', public: true },\n {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool()\n }\n);\n\nexport const result = table(\n { name: 'result', public: true },\n {\n id: t.i32().primaryKey(),\n name: t.string()\n }\n);\n\nexport const spacetimedb = schema(user, result);\n\nexport const lookupUserName = spacetimedb.reducer(\n 'lookupUserName',\n { id: t.i32() },\n (ctx, { id }) => {\n const u = ctx.db.user.id.find(id);\n if (!u) throw new SenderError('User not found');\n ctx.db.result.insert({ id: u.id, name: u.name });\n }\n);", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_009_init": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_009_init", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const user = table({ name: 'user' }, {\n id: t.i32().primaryKey(),\n name: t.string(),\n age: t.i32(),\n active: t.bool(),\n});\n\nexport const spacetimedb = schema({ user });\n\nexport const init = spacetimedb.reducer((ctx) => {\n ctx.db.user.insert({ id: 1, name: 'Alice', age: 30, active: true });\n ctx.db.user.insert({ id: 2, name: 'Bob', age: 22, active: false });\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_010_connect": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_010_connect", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 1, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const event = table({ name: 'event' }, {\n id: t.u64().primaryKey().autoInc(),\n kind: t.string(),\n});\n\nexport const spacetimedb = schema({ event });\n\nspacetimedb.clientConnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'connected' });\n});\n\nspacetimedb.clientDisconnected((ctx) => {\n ctx.db.event.insert({ id: 0n, kind: 'disconnected' });\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_011_helper_function": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_011_helper_function", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const result = table({ name: 'result' }, {\n id: t.i32().primaryKey(),\n sum: t.i32(),\n});\n\nexport const spacetimedb = schema({ result });\n\nexport function add(a: number, b: number): number {\n return a + b;\n}\n\nexport const computeSum = spacetimedb.reducer({ id: t.i32(), a: t.i32(), b: t.i32() }, (ctx, { id, a, b }) => {\n ctx.db.result.insert({ id, sum: add(a, b) });\n});", - "category": "basics", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_012_spacetime_product_type": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_012_spacetime_product_type", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const Score = t.object('Score', {\n left: t.i32(),\n right: t.i32(),\n});\n\nexport const result = table({ name: 'result' }, {\n id: t.i32().primaryKey(),\n value: Score,\n});\n\nexport const spacetimedb = schema({ result });\n\nexport const setScore = spacetimedb.reducer(\n { id: t.i32(), left: t.i32(), right: t.i32() },\n (ctx, { id, left, right }) => {\n ctx.db.result.insert({ id, value: { left, right } });\n }\n);", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_013_spacetime_sum_type": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_013_spacetime_sum_type", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nexport const Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nexport const result = table(\n { name: 'result' },\n {\n id: t.i32().primaryKey(),\n value: Shape,\n }\n);\n\nexport const spacetimedb = schema(result);\n\nexport const setCircle = spacetimedb.reducer(\n 'setCircle',\n { id: t.i32(), radius: t.i32() },\n (ctx, { id, radius }) => {\n ctx.db.result.insert({\n id,\n value: { tag: 'circle', value: radius },\n });\n }\n);", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_014_elementary_columns": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_014_elementary_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const primitive = table(\n { name: 'primitive' },\n {\n id: t.i32().primaryKey(),\n count: t.i32(),\n total: t.i64(),\n price: t.f32(),\n ratio: t.f64(),\n active: t.bool(),\n name: t.string(),\n }\n);\n\nexport const spacetimedb = schema({ primitive });\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.primitive.insert({\n id: 1,\n count: 2,\n total: 3000000000n,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: 'Alice',\n });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_015_product_type_columns": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_015_product_type_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nconst Address = t.object('Address', {\n street: t.string(),\n zip: t.i32(),\n});\n\nconst Position = t.object('Position', {\n x: t.i32(),\n y: t.i32(),\n});\n\nexport const profile = table(\n { name: 'profile' },\n {\n id: t.i32().primaryKey(),\n home: Address,\n work: Address,\n pos: Position,\n }\n);\n\nexport const spacetimedb = schema({ profile });\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.profile.insert({\n id: 1,\n home: { street: '1 Main', zip: 11111 },\n work: { street: '2 Broad', zip: 22222 },\n pos: { x: 7, y: 9 },\n });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_016_sum_type_columns": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_016_sum_type_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const Rect = t.object('Rect', {\n width: t.i32(),\n height: t.i32(),\n});\n\nexport const Shape = t.enum('Shape', {\n circle: t.i32(),\n rectangle: Rect,\n});\n\nexport const drawing = table(\n { name: 'drawing' },\n {\n id: t.i32().primaryKey(),\n a: Shape,\n b: Shape,\n }\n);\n\nexport const spacetimedb = schema({ drawing });\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.drawing.insert({\n id: 1,\n a: { tag: 'circle', value: 10 },\n b: { tag: 'rectangle', value: { width: 4, height: 6 } },\n });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_017_scheduled_columns": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_017_scheduled_columns", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 2, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\nimport { ScheduleAt } from 'spacetimedb';\n\nexport const tickTimer = table(\n { name: 'tick_timer', scheduled: () => tick },\n {\n scheduledId: t.u64().primaryKey().autoInc(),\n scheduledAt: t.scheduleAt(),\n }\n);\n\nexport const spacetimedb = schema({ tickTimer });\n\nexport const tick = spacetimedb.reducer({ arg: tickTimer.rowType }, (ctx, { arg }) => {\n const nextTime = arg.scheduledAt.tag === 'Time'\n ? arg.scheduledAt.value.microsSinceUnixEpoch + 50_000_000n\n : ctx.timestamp.microsSinceUnixEpoch + 50_000_000n;\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.time(nextTime),\n });\n});\n\nexport const init = spacetimedb.reducer((ctx) => {\n const nextTime = ctx.timestamp.microsSinceUnixEpoch + 50_000_000n;\n ctx.db.tickTimer.insert({\n scheduledId: 0n,\n scheduledAt: ScheduleAt.time(nextTime),\n });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_018_constraints": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_018_constraints", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 3, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const account = table(\n {\n name: 'account',\n indexes: [{ name: 'byName', algorithm: 'btree', columns: ['name'] }],\n },\n {\n id: t.i32().primaryKey(),\n email: t.string().unique(),\n name: t.string(),\n }\n);\n\nexport const spacetimedb = schema({ account });\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.account.insert({ id: 1, email: 'a@example.com', name: 'Alice' });\n ctx.db.account.insert({ id: 2, email: 'b@example.com', name: 'Bob' });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_019_many_to_many": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_019_many_to_many", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const user = table(\n { name: 'user' },\n {\n userId: t.i32().primaryKey(),\n name: t.string(),\n }\n);\n\nexport const group = table(\n { name: 'group' },\n {\n groupId: t.i32().primaryKey(),\n title: t.string(),\n }\n);\n\nexport const membership = table(\n {\n name: 'membership',\n indexes: [\n { name: 'byUser', algorithm: 'btree', columns: ['userId'] },\n { name: 'byGroup', algorithm: 'btree', columns: ['groupId'] },\n ],\n },\n {\n id: t.i32().primaryKey(),\n userId: t.i32(),\n groupId: t.i32(),\n }\n);\n\nexport const spacetimedb = schema({ user, group, membership });\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.user.insert({ userId: 1, name: 'Alice' });\n ctx.db.user.insert({ userId: 2, name: 'Bob' });\n\n ctx.db.group.insert({ groupId: 10, title: 'Admin' });\n ctx.db.group.insert({ groupId: 20, title: 'Dev' });\n\n ctx.db.membership.insert({ id: 1, userId: 1, groupId: 10 });\n ctx.db.membership.insert({ id: 2, userId: 1, groupId: 20 });\n ctx.db.membership.insert({ id: 3, userId: 2, groupId: 20 });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_020_ecs": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_020_ecs", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 5, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const entity = table({ name: 'entity' }, {\n id: t.i32().primaryKey(),\n});\n\nexport const position = table({ name: 'position' }, {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n});\n\nexport const velocity = table({ name: 'velocity' }, {\n entityId: t.i32().primaryKey(),\n vx: t.i32(),\n vy: t.i32(),\n});\n\nexport const nextPosition = table({ name: 'nextPosition' }, {\n entityId: t.i32().primaryKey(),\n x: t.i32(),\n y: t.i32(),\n});\n\nexport const spacetimedb = schema(entity, position, velocity, nextPosition);\n\nexport const seed = spacetimedb.reducer('seed', {}, (ctx) => {\n ctx.db.entity.insert({ id: 1 });\n ctx.db.position.insert({ entityId: 1, x: 0, y: 0 });\n ctx.db.velocity.insert({ entityId: 1, vx: 1, vy: 0 });\n\n ctx.db.entity.insert({ id: 2 });\n ctx.db.position.insert({ entityId: 2, x: 10, y: 0 });\n ctx.db.velocity.insert({ entityId: 2, vx: -2, vy: 3 });\n});\n\nexport const step = spacetimedb.reducer('step', {}, (ctx) => {\n for (const pos of ctx.db.position.iter()) {\n const vel = ctx.db.velocity.entityId.find(pos.entityId);\n if (!vel) continue;\n const x = pos.x + vel.vx;\n const y = pos.y + vel.vy;\n\n const existing = ctx.db.nextPosition.entityId.find(pos.entityId);\n if (existing) {\n ctx.db.nextPosition.entityId.update({ ...existing, x, y });\n } else {\n ctx.db.nextPosition.insert({ entityId: pos.entityId, x, y });\n }\n }\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - }, - "t_021_multi_column_index": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "task": "t_021_multi_column_index", - "lang": "typescript", - "golden_published": false, - "model_name": "GPT-5.2-Codex", - "total_tests": 4, - "passed_tests": 0, - "llm_output": "import { schema, table, t } from 'spacetimedb/server';\n\nexport const log = table(\n {\n name: 'log',\n indexes: [{ name: 'byUserDay', algorithm: 'btree', columns: ['userId', 'day'] }],\n },\n {\n id: t.i32().primaryKey(),\n userId: t.i32(),\n day: t.i32(),\n message: t.string(),\n }\n);\n\nexport const spacetimedb = schema({ log });\n\nexport const seed = spacetimedb.reducer((ctx) => {\n ctx.db.log.insert({ id: 1, userId: 7, day: 1, message: 'a' });\n ctx.db.log.insert({ id: 2, userId: 7, day: 2, message: 'b' });\n ctx.db.log.insert({ id: 3, userId: 9, day: 1, message: 'c' });\n});", - "category": "schema", - "route_api_model": "gpt-5.2-codex", - "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime publish (typescript) failed (exit=1)\n--- stderr ---\ntsc not found in node_modules. Make sure you have the `typescript` package as a dev-dependency and that your dependencies are installed.\nError: It seems like you haven't exported your schema. You must `export default schema(...);`\n\n--- stdout ---\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai" - } - } - } - ] - } - ], - "golden_answers": {} - } - ] -} diff --git a/docs/llms/llm-comparison-summary.json b/docs/llms/llm-comparison-summary.json deleted file mode 100644 index bd86ae3f0bd..00000000000 --- a/docs/llms/llm-comparison-summary.json +++ /dev/null @@ -1,1042 +0,0 @@ -{ - "version": 1, - "generated_at": "2026-02-21T19:00:17.834Z", - "by_language": { - "csharp": { - "modes": { - "docs": { - "hash": "a4ddc223e2035781ae7e86c8c38c0d78212e9a0c4149c6535625b4033878e3a4", - "models": { - "Claude Opus 4.6": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 11.333334, - "task_pass_pct": 94.44445 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 28, - "pass_pct": 82.35294, - "task_pass_equiv": 8.0, - "task_pass_pct": 80.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 53, - "pass_pct": 86.88525, - "task_pass_equiv": 19.333334, - "task_pass_pct": 87.87879 - } - }, - "Claude Sonnet 4.6": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 11.5, - "task_pass_pct": 95.83333 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 33, - "pass_pct": 97.05882, - "task_pass_equiv": 9.666666, - "task_pass_pct": 96.66666 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 58, - "pass_pct": 95.08197, - "task_pass_equiv": 21.166666, - "task_pass_pct": 96.21212 - } - }, - "DeepSeek Chat": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 23, - "pass_pct": 85.18519, - "task_pass_equiv": 10.833334, - "task_pass_pct": 90.27779 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 29, - "pass_pct": 85.29412, - "task_pass_equiv": 8.733334, - "task_pass_pct": 87.333336 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 52, - "pass_pct": 85.2459, - "task_pass_equiv": 19.566668, - "task_pass_pct": 88.9394 - } - }, - "DeepSeek Reasoner": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 10.583334, - "task_pass_pct": 88.19445 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 16, - "pass_pct": 47.058823, - "task_pass_equiv": 5.4, - "task_pass_pct": 54.000004 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 38, - "pass_pct": 62.295082, - "task_pass_equiv": 15.983334, - "task_pass_pct": 72.65152 - } - }, - "GPT-5-mini": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 23, - "pass_pct": 85.18519, - "task_pass_equiv": 10.833334, - "task_pass_pct": 90.27779 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 25, - "pass_pct": 73.52941, - "task_pass_equiv": 7.0, - "task_pass_pct": 70.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 48, - "pass_pct": 78.68852, - "task_pass_equiv": 17.833334, - "task_pass_pct": 81.06061 - } - }, - "GPT-5.2-Codex": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 16, - "pass_pct": 59.25926, - "task_pass_equiv": 6.416667, - "task_pass_pct": 53.472225 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 29, - "pass_pct": 85.29412, - "task_pass_equiv": 8.55, - "task_pass_pct": 85.5 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 45, - "pass_pct": 73.77049, - "task_pass_equiv": 14.966668, - "task_pass_pct": 68.03031 - } - }, - "Gemini 3 Flash": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 23, - "pass_pct": 85.18519, - "task_pass_equiv": 10.833334, - "task_pass_pct": 90.27779 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 22, - "pass_pct": 64.70588, - "task_pass_equiv": 6.666667, - "task_pass_pct": 66.66667 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 45, - "pass_pct": 73.77049, - "task_pass_equiv": 17.5, - "task_pass_pct": 79.545456 - } - }, - "Gemini 3.1 Pro": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 23, - "pass_pct": 85.18519, - "task_pass_equiv": 10.833334, - "task_pass_pct": 90.27779 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 28, - "pass_pct": 82.35294, - "task_pass_equiv": 8.0, - "task_pass_pct": 80.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 51, - "pass_pct": 83.60656, - "task_pass_equiv": 18.833334, - "task_pass_pct": 85.606064 - } - }, - "Grok 4": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 23, - "pass_pct": 85.18519, - "task_pass_equiv": 10.833334, - "task_pass_pct": 90.27779 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 34, - "pass_pct": 100.0, - "task_pass_equiv": 10.0, - "task_pass_pct": 100.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 57, - "pass_pct": 93.44262, - "task_pass_equiv": 20.833334, - "task_pass_pct": 94.696976 - } - }, - "Grok Code": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 10.5, - "task_pass_pct": 87.5 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 28, - "pass_pct": 82.35294, - "task_pass_equiv": 8.8, - "task_pass_pct": 88.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 50, - "pass_pct": 81.96722, - "task_pass_equiv": 19.3, - "task_pass_pct": 87.727264 - } - } - } - }, - "llms.md": { - "hash": "171b01e97d14a2c03356933e7c86331362cc53b87e82cd534cd74273e7bc8ea4", - "models": {} - } - } - }, - "rust": { - "modes": { - "cursor_rules": { - "hash": "edbeadd577ce7acdedf6ce10e46c0aaee75d0cd6c30c8faef266a23ad86d12cf", - "models": { - "Grok 4": { - "categories": { - "basics": { - "tasks": 3, - "total_tests": 7, - "passed_tests": 7, - "pass_pct": 100.0, - "task_pass_equiv": 3.0, - "task_pass_pct": 100.0 - }, - "schema": { - "tasks": 5, - "total_tests": 17, - "passed_tests": 13, - "pass_pct": 76.47059, - "task_pass_equiv": 4.0, - "task_pass_pct": 80.0 - } - }, - "totals": { - "tasks": 8, - "total_tests": 24, - "passed_tests": 20, - "pass_pct": 83.333336, - "task_pass_equiv": 7.0, - "task_pass_pct": 87.5 - } - } - } - }, - "docs": { - "hash": "cb5bf60a6d17e761b5fcec5d8e69bafbaf56602e570522e688d4da57973c7ea8", - "models": { - "Claude Opus 4.6": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 27, - "pass_pct": 100.0, - "task_pass_equiv": 12.0, - "task_pass_pct": 100.0 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 34, - "pass_pct": 100.0, - "task_pass_equiv": 10.0, - "task_pass_pct": 100.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 61, - "pass_pct": 100.0, - "task_pass_equiv": 22.0, - "task_pass_pct": 100.0 - } - }, - "Claude Sonnet 4.6": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 27, - "pass_pct": 100.0, - "task_pass_equiv": 12.0, - "task_pass_pct": 100.0 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 34, - "pass_pct": 100.0, - "task_pass_equiv": 10.0, - "task_pass_pct": 100.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 61, - "pass_pct": 100.0, - "task_pass_equiv": 22.0, - "task_pass_pct": 100.0 - } - }, - "DeepSeek Chat": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 23, - "pass_pct": 85.18519, - "task_pass_equiv": 11.0, - "task_pass_pct": 91.66667 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 26, - "pass_pct": 76.47059, - "task_pass_equiv": 7.4, - "task_pass_pct": 74.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 49, - "pass_pct": 80.327866, - "task_pass_equiv": 18.4, - "task_pass_pct": 83.63636 - } - }, - "DeepSeek Reasoner": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 8.333333, - "task_pass_pct": 69.44444 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 12, - "pass_pct": 35.294117, - "task_pass_equiv": 3.4, - "task_pass_pct": 34.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 34, - "pass_pct": 55.737705, - "task_pass_equiv": 11.733333, - "task_pass_pct": 53.33333 - } - }, - "GPT-5-mini": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 23, - "pass_pct": 85.18519, - "task_pass_equiv": 10.833334, - "task_pass_pct": 90.27779 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 34, - "pass_pct": 100.0, - "task_pass_equiv": 10.0, - "task_pass_pct": 100.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 57, - "pass_pct": 93.44262, - "task_pass_equiv": 20.833334, - "task_pass_pct": 94.696976 - } - }, - "GPT-5.2-Codex": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 26, - "pass_pct": 96.296295, - "task_pass_equiv": 11.0, - "task_pass_pct": 91.66667 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 29, - "pass_pct": 85.29412, - "task_pass_equiv": 9.0, - "task_pass_pct": 90.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 55, - "pass_pct": 90.16393, - "task_pass_equiv": 20.0, - "task_pass_pct": 90.909096 - } - }, - "Gemini 3 Flash": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 11.5, - "task_pass_pct": 95.83333 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 34, - "pass_pct": 100.0, - "task_pass_equiv": 10.0, - "task_pass_pct": 100.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 59, - "pass_pct": 96.72131, - "task_pass_equiv": 21.5, - "task_pass_pct": 97.72727 - } - }, - "Gemini 3.1 Pro": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 9.0, - "task_pass_pct": 75.0 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 28, - "pass_pct": 82.35294, - "task_pass_equiv": 8.0, - "task_pass_pct": 80.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 50, - "pass_pct": 81.96722, - "task_pass_equiv": 17.0, - "task_pass_pct": 77.27273 - } - }, - "Grok 4": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 24, - "pass_pct": 88.888885, - "task_pass_equiv": 11.0, - "task_pass_pct": 91.66667 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 31, - "pass_pct": 91.17647, - "task_pass_equiv": 9.0, - "task_pass_pct": 90.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 55, - "pass_pct": 90.16393, - "task_pass_equiv": 20.0, - "task_pass_pct": 90.909096 - } - }, - "Grok Code": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 9.833334, - "task_pass_pct": 81.94445 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 33, - "pass_pct": 97.05882, - "task_pass_equiv": 9.8, - "task_pass_pct": 98.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 55, - "pass_pct": 90.16393, - "task_pass_equiv": 19.633333, - "task_pass_pct": 89.242424 - } - } - } - }, - "llms.md": { - "hash": "7bd4918052e02c4b09d50e7037cd2e81add76a4b42468acac54ee8f4073e37cf", - "models": { - "Grok 4": { - "categories": { - "basics": { - "tasks": 1, - "total_tests": 1, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - }, - "schema": { - "tasks": 3, - "total_tests": 9, - "passed_tests": 5, - "pass_pct": 55.555557, - "task_pass_equiv": 2.0, - "task_pass_pct": 66.66667 - } - }, - "totals": { - "tasks": 4, - "total_tests": 10, - "passed_tests": 5, - "pass_pct": 50.0, - "task_pass_equiv": 2.0, - "task_pass_pct": 50.0 - } - } - } - }, - "none": { - "hash": "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", - "models": { - "Grok 4": { - "categories": { - "basics": { - "tasks": 1, - "total_tests": 1, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - }, - "schema": { - "tasks": 3, - "total_tests": 3, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - } - }, - "totals": { - "tasks": 4, - "total_tests": 4, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - } - } - } - }, - "rustdoc_json": { - "hash": "b0d5bbe755291d8ec65de0a74b4eecfa344670639ca5f1a365ced3ab9717e97c", - "models": { - "Grok 4": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 13, - "pass_pct": 48.148148, - "task_pass_equiv": 6.0, - "task_pass_pct": 50.0 - }, - "schema": { - "tasks": 10, - "total_tests": 28, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 55, - "passed_tests": 13, - "pass_pct": 23.636364, - "task_pass_equiv": 6.0, - "task_pass_pct": 27.272728 - } - } - } - } - } - }, - "typescript": { - "modes": { - "cursor_rules": { - "hash": "f3691654a6b470fedbee3a465fd09fbf63705ffddbd0454b775385d72e02e2b3", - "models": { - "GPT-5.2-Codex": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 - } - } - } - }, - "docs": { - "hash": "3924da8acd3f980dd3c715b8feeec704c6d1a33c376ebb95ece2d4e0eef81093", - "models": { - "Claude Opus 4.6": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 26, - "pass_pct": 96.296295, - "task_pass_equiv": 11.0, - "task_pass_pct": 91.66667 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 30, - "pass_pct": 88.23529, - "task_pass_equiv": 8.666667, - "task_pass_pct": 86.666664 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 56, - "pass_pct": 91.803276, - "task_pass_equiv": 19.666666, - "task_pass_pct": 89.393936 - } - }, - "Claude Sonnet 4.6": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 26, - "pass_pct": 96.296295, - "task_pass_equiv": 11.0, - "task_pass_pct": 91.66667 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 30, - "pass_pct": 88.23529, - "task_pass_equiv": 8.666667, - "task_pass_pct": 86.666664 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 56, - "pass_pct": 91.803276, - "task_pass_equiv": 19.666666, - "task_pass_pct": 89.393936 - } - }, - "DeepSeek Chat": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 10.0, - "task_pass_pct": 83.33333 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 25, - "pass_pct": 73.52941, - "task_pass_equiv": 7.066667, - "task_pass_pct": 70.66667 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 50, - "pass_pct": 81.96722, - "task_pass_equiv": 17.066666, - "task_pass_pct": 77.57575 - } - }, - "DeepSeek Reasoner": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 17, - "pass_pct": 62.962963, - "task_pass_equiv": 6.833333, - "task_pass_pct": 56.944443 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 15, - "pass_pct": 44.117645, - "task_pass_equiv": 4.3333335, - "task_pass_pct": 43.333332 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 32, - "pass_pct": 52.459015, - "task_pass_equiv": 11.166666, - "task_pass_pct": 50.757576 - } - }, - "GPT-5-mini": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 10.0, - "task_pass_pct": 83.33333 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 15, - "pass_pct": 44.117645, - "task_pass_equiv": 4.533334, - "task_pass_pct": 45.333336 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 40, - "pass_pct": 65.57377, - "task_pass_equiv": 14.533333, - "task_pass_pct": 66.0606 - } - }, - "GPT-5.2-Codex": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 15, - "pass_pct": 55.555557, - "task_pass_equiv": 6.0, - "task_pass_pct": 50.0 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 28, - "pass_pct": 82.35294, - "task_pass_equiv": 7.666667, - "task_pass_pct": 76.66667 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 43, - "pass_pct": 70.491806, - "task_pass_equiv": 13.666667, - "task_pass_pct": 62.121212 - } - }, - "Gemini 3 Flash": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 10.0, - "task_pass_pct": 83.33333 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 28, - "pass_pct": 82.35294, - "task_pass_equiv": 8.0, - "task_pass_pct": 80.0 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 53, - "pass_pct": 86.88525, - "task_pass_equiv": 18.0, - "task_pass_pct": 81.818184 - } - }, - "Gemini 3.1 Pro": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 22, - "pass_pct": 81.48148, - "task_pass_equiv": 9.833334, - "task_pass_pct": 81.94445 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 30, - "pass_pct": 88.23529, - "task_pass_equiv": 8.666667, - "task_pass_pct": 86.666664 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 52, - "pass_pct": 85.2459, - "task_pass_equiv": 18.5, - "task_pass_pct": 84.090904 - } - }, - "Grok 4": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 20, - "pass_pct": 74.07407, - "task_pass_equiv": 8.5, - "task_pass_pct": 70.83333 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 21, - "pass_pct": 61.764706, - "task_pass_equiv": 6.0, - "task_pass_pct": 60.000004 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 41, - "pass_pct": 67.21311, - "task_pass_equiv": 14.499999, - "task_pass_pct": 65.90909 - } - }, - "Grok Code": { - "categories": { - "basics": { - "tasks": 12, - "total_tests": 27, - "passed_tests": 24, - "pass_pct": 88.888885, - "task_pass_equiv": 10.333334, - "task_pass_pct": 86.111115 - }, - "schema": { - "tasks": 10, - "total_tests": 34, - "passed_tests": 28, - "pass_pct": 82.35294, - "task_pass_equiv": 7.666667, - "task_pass_pct": 76.66667 - } - }, - "totals": { - "tasks": 22, - "total_tests": 61, - "passed_tests": 52, - "pass_pct": 85.2459, - "task_pass_equiv": 18.0, - "task_pass_pct": 81.818184 - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/docs/package.json b/docs/package.json index e8a33cd9e6a..64e5d951180 100644 --- a/docs/package.json +++ b/docs/package.json @@ -11,6 +11,7 @@ "clear": "docusaurus clear", "serve": "docusaurus serve", "generate-cli-docs": "node scripts/generate-cli-docs.mjs", + "generate-llms": "docusaurus build && node scripts/generate-llms.mjs", "rewrite-links": "node scripts/rewrite-doc-links.mjs", "rewrite-links:write": "node scripts/rewrite-doc-links.mjs --write", "write-translations": "docusaurus write-translations", @@ -41,6 +42,7 @@ "@docusaurus/module-type-aliases": "3.9.2", "@docusaurus/tsconfig": "3.9.2", "@docusaurus/types": "3.9.2", + "@signalwire/docusaurus-plugin-llms-txt": "^1.2.2", "@types/react": "^18.3.23", "@types/react-dom": "^18.3.0", "typescript": "~5.6.2" diff --git a/docs/scripts/generate-llms.mjs b/docs/scripts/generate-llms.mjs new file mode 100644 index 00000000000..6728d70cb7c --- /dev/null +++ b/docs/scripts/generate-llms.mjs @@ -0,0 +1,48 @@ +#!/usr/bin/env node +/** + * Post-build script: copies the plugin-generated llms.txt to static/llms.md + * so it can be committed to the repo. + * + * Usage: pnpm build && node scripts/generate-llms.mjs + * or: pnpm generate-llms + */ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BUILD_DIR = path.resolve(__dirname, '../build'); +const STATIC_DIR = path.resolve(__dirname, '../static'); + +async function findInBuild(filename) { + for (const candidate of [filename, `docs/${filename}`]) { + const p = path.join(BUILD_DIR, candidate); + try { + await fs.access(p); + return p; + } catch {} + } + return null; +} + +async function main() { + const src = await findInBuild('llms.txt'); + if (!src) { + console.error('Error: llms.txt not found in build output.'); + console.error('Run "pnpm build" first to generate it via the plugin.'); + process.exit(1); + } + + const content = await fs.readFile(src, 'utf8'); + const dest = path.join(STATIC_DIR, 'llms.md'); + await fs.writeFile(dest, content, 'utf8'); + + const lines = content.split('\n').length; + console.log(`${src} -> ${dest}`); + console.log(` ${lines} lines`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/docs/static/ai-guidelines/README.md b/docs/static/ai-guidelines/README.md new file mode 100644 index 00000000000..421ade50343 --- /dev/null +++ b/docs/static/ai-guidelines/README.md @@ -0,0 +1,32 @@ +# AI Guidelines + +Constructive-only "happy path" cheat sheets optimized for **one-shot code generation**. +Each file shows correct SpacetimeDB patterns for a single language — no anti-patterns, +no "don't do this" sections. + +## Files + +| File | Language | +|------|----------| +| `spacetimedb-rust.md` | Rust server modules | +| `spacetimedb-typescript.md` | TypeScript server modules | +| `spacetimedb-csharp.md` | C# server modules | + +## How these are used + +The LLM benchmark (`tools/xtask-llm-benchmark`) injects these as context when running +in `guidelines` mode. The benchmark sends the guidelines as a static prefix along with +task instructions, then validates the generated code by publishing it to SpacetimeDB. + +## Guidelines vs Cursor Rules + +These are **not** the same as the cursor/IDE rules in `docs/static/ai-rules/`. + +- **Guidelines** (this directory): Minimal, constructive-only references for one-shot + generation. Show the correct way to do things and nothing else. +- **Cursor rules** (`ai-rules/`): IDE-oriented `.mdc` rules for Cursor, Windsurf, etc. + Include anti-hallucination guardrails, common mistake tables, client-side patterns, and + migration guidance. These work well in an IDE where the model has project context and + can iterate, but they are not designed for single-shot benchmark generation. + +See the comment in `tools/xtask-llm-benchmark/src/context/constants.rs` for more detail. diff --git a/docs/static/ai-guidelines/spacetimedb-csharp.md b/docs/static/ai-guidelines/spacetimedb-csharp.md new file mode 100644 index 00000000000..18598b1b62f --- /dev/null +++ b/docs/static/ai-guidelines/spacetimedb-csharp.md @@ -0,0 +1,485 @@ +# SpacetimeDB C# Server Module Guidelines + +## Imports + +```csharp +using SpacetimeDB; +``` + +Additional imports when needed: +```csharp +using System.Linq; // For LINQ queries (OrderBy, Take, etc.) +using System.Collections.Generic; // For HashSet, List, etc. +``` + +## Module Structure + +All code goes inside a static partial class: +```csharp +using SpacetimeDB; + +public static partial class Module +{ + // Tables, types, and reducers here +} +``` + +## Table Definitions + +Basic table: +```csharp +[Table(Accessor = "User")] +public partial struct User +{ + [PrimaryKey] + public int Id; + public string Name; + public int Age; + public bool Active; +} +``` + +Public table with auto-increment: +```csharp +[Table(Accessor = "Message", Public = true)] +public partial struct Message +{ + [PrimaryKey] + [AutoInc] + public ulong Id; + public Identity Owner; + public string Text; +} +``` + +Event table (append-only): +```csharp +[Table(Accessor = "DamageEvent", Public = true, Event = true)] +public partial struct DamageEvent +{ + public ulong EntityId; + public uint Damage; + public string Source; +} +``` + +Scheduled table: +```csharp +[Table(Accessor = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))] +public partial struct TickTimer +{ + [PrimaryKey, AutoInc] + public ulong ScheduledId; + public ScheduleAt ScheduledAt; +} +``` + +## Column Types + +| C# Type | Usage | +|---------|-------| +| `int`, `uint` | 32-bit integers | +| `long`, `ulong` | 64-bit integers | +| `float`, `double` | Floating point | +| `bool` | Boolean | +| `string` | Text | +| `Identity` | User identity | +| `Timestamp` | Timestamp | +| `ScheduleAt` | Schedule metadata | +| `T?` (nullable) | Optional field | + +Timestamp arithmetic — use `TimeDuration`: +```csharp +ctx.Timestamp + new TimeDuration { Microseconds = 60_000_000 } // add 60 seconds +ctx.Timestamp.MicrosecondsSinceUnixEpoch // raw long value +``` + +## Constraints and Indexes + +```csharp +[PrimaryKey] public int Id; +[PrimaryKey, AutoInc] public ulong Id; +[Unique] public string Email; +``` + +Single-column index: +```csharp +[SpacetimeDB.Index.BTree] +public Identity Owner; +``` + +Named single-column index: +```csharp +[SpacetimeDB.Index.BTree(Accessor = "by_name", Columns = [nameof(Name)])] +``` + +Multi-column index (on table attribute — MUST include Accessor): +```csharp +[Table(Accessor = "Log")] +[SpacetimeDB.Index.BTree(Accessor = "by_user_day", Columns = new[] { nameof(UserId), nameof(Day) })] +public partial struct Log +{ + [PrimaryKey] public int Id; + public int UserId; + public int Day; +} +``` + +## Product Types (Structs) + +```csharp +[Type] +public partial struct Position +{ + public int X; + public int Y; +} + +[Table(Accessor = "Entity")] +public partial struct Entity +{ + [PrimaryKey] public int Id; + public Position Pos; +} +``` + +## Sum Types (Tagged Unions) + +```csharp +[Type] +public partial struct Circle { public int Radius; } + +[Type] +public partial struct Rectangle { public int Width; public int Height; } + +[Type] +public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> {} + +// Construction: +new Shape.Circle(new Circle { Radius = 10 }) +new Shape.Rectangle(new Rectangle { Width = 5, Height = 3 }) +``` + +## Reducers + +Basic reducer: +```csharp +[Reducer] +public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active) +{ + ctx.Db.User.Insert(new User { Id = id, Name = name, Age = age, Active = active }); +} +``` + +Init reducer (runs once on startup): +```csharp +[Reducer(ReducerKind.Init)] +public static void Init(ReducerContext ctx) +{ + ctx.Db.Config.Insert(new Config { Id = 0, Setting = "default" }); +} +``` + +Scheduled reducer: +```csharp +[Reducer] +public static void Tick(ReducerContext ctx, TickTimer timer) +{ + // Runs when timer fires +} +``` + +## Database Operations + +### Insert +```csharp +ctx.Db.User.Insert(new User { Id = 1, Name = "Alice", Age = 30, Active = true }); + +// Auto-inc: use 0 as placeholder +var row = ctx.Db.Message.Insert(new Message { Id = 0, Text = "Hello" }); +// row.Id now has the assigned value +``` + +### Find (by primary key or unique index) +```csharp +var user = ctx.Db.User.Id.Find(userId); +if (user is not null) { /* use user */ } +if (user is User u) { /* use u */ } + +// With expect pattern: +var msg = ctx.Db.Message.Id.Find(id) ?? throw new Exception("not found"); +``` + +### Filter (by btree index — returns iterator) +```csharp +foreach (var post in ctx.Db.Post.AuthorId.Filter(authorId)) +{ + ctx.Db.Post.Id.Delete(post.Id); +} +``` + +### Iterate all rows +```csharp +foreach (var row in ctx.Db.User.Iter()) +{ + // process row +} +``` + +### Update +```csharp +ctx.Db.User.Id.Update(new User { Id = id, Name = newName, Age = newAge, Active = true }); + +// Using `with` expression: +ctx.Db.RateLimit.Identity.Update(existing with { LastCallUs = now }); +``` + +### Delete +```csharp +ctx.Db.User.Id.Delete(userId); +ctx.Db.OnlinePlayer.Identity.Delete(ctx.Sender); +``` + +## Authentication + +`ctx.Sender` is the authenticated caller's `Identity`: + +```csharp +[Reducer] +public static void SendMessage(ReducerContext ctx, string text) +{ + ctx.Db.Message.Insert(new Message + { + Id = 0, + Owner = ctx.Sender, + Text = text, + }); +} + +[Reducer] +public static void DeleteMessage(ReducerContext ctx, ulong id) +{ + var msg = ctx.Db.Message.Id.Find(id) ?? throw new Exception("not found"); + if (msg.Owner != ctx.Sender) + { + throw new Exception("unauthorized"); + } + ctx.Db.Message.Id.Delete(id); +} +``` + +Identity as primary key: +```csharp +[Table(Accessor = "Player")] +public partial struct Player +{ + [PrimaryKey] + public Identity Identity; + public string Name; +} + +// Check registration +if (ctx.Db.Player.Identity.Find(ctx.Sender) is not null) +{ + throw new Exception("already registered"); +} +``` + +## Lifecycle Hooks + +```csharp +[Reducer(ReducerKind.ClientConnected)] +public static void ClientConnected(ReducerContext ctx) +{ + ctx.Db.OnlinePlayer.Insert(new OnlinePlayer + { + Identity = ctx.Sender, + ConnectedAt = ctx.Timestamp, + }); +} + +[Reducer(ReducerKind.ClientDisconnected)] +public static void ClientDisconnected(ReducerContext ctx) +{ + ctx.Db.OnlinePlayer.Identity.Delete(ctx.Sender); +} +``` + +## Views + +Anonymous view (same result for all clients): +```csharp +[SpacetimeDB.View(Accessor = "ActiveAnnouncements", Public = true)] +public static List ActiveAnnouncements(AnonymousViewContext ctx) +{ + return ctx.Db.Announcement.Active.Filter(true).ToList(); +} +``` + +Per-user view: +```csharp +// Return types: T? for single, List for multiple. Never IEnumerable. +[SpacetimeDB.View(Accessor = "MyProfile", Public = true)] +public static Profile? MyProfile(ViewContext ctx) +{ + return ctx.Db.Profile.Identity.Find(ctx.Sender) as Profile?; +} +``` + +## Scheduled Tables + +```csharp +[Table(Accessor = "Reminder", Scheduled = nameof(SendReminder), ScheduledAt = nameof(Reminder.ScheduledAt))] +public partial struct Reminder +{ + [PrimaryKey, AutoInc] + public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public string Message; +} + +[Reducer] +public static void SendReminder(ReducerContext ctx, Reminder row) +{ + // row.Message available +} + +// Schedule recurring interval +var interval = new TimeDuration { Microseconds = 50_000 }; +ctx.Db.TickTimer.Insert(new TickTimer +{ + ScheduledAt = new ScheduleAt.Interval(interval), +}); + +// Schedule at specific time +var delay = new TimeDuration { Microseconds = 60_000_000 }; +ctx.Db.Reminder.Insert(new Reminder +{ + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + delay), + Message = "Hello!", +}); + +// Cancel a scheduled job +ctx.Db.Reminder.ScheduledId.Delete(scheduledId); +``` + +## Optional Fields + +```csharp +[Table(Accessor = "Player")] +public partial struct Player +{ + [PrimaryKey, AutoInc] + public ulong Id; + public string Name; + public string? Nickname; + public uint? HighScore; +} +``` + +## LINQ Queries + +```csharp +using System.Linq; + +// Sort and limit +var topPlayers = ctx.Db.Player.Iter() + .OrderByDescending(p => p.Score) + .Take((int)limit) + .ToList(); + +// Distinct values +var categories = new HashSet(); +foreach (var o in ctx.Db.Order.Iter()) +{ + categories.Add(o.Category); +} + +// Count +var total = (ulong)ctx.Db.User.Iter().Count(); +``` + +## Helper Functions + +```csharp +static int Add(int a, int b) => a + b; + +[Reducer] +public static void ComputeSum(ReducerContext ctx, int id, int a, int b) +{ + ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) }); +} +``` + +## Complete Module Example + +```csharp +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "User", Public = true)] + public partial struct User + { + [PrimaryKey] + public Identity Identity; + public string Name; + public bool Online; + } + + [Table(Accessor = "Message", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_sender", Columns = [nameof(Message.Sender)])] + public partial struct Message + { + [PrimaryKey, AutoInc] + public ulong Id; + public Identity Sender; + public string Text; + public Timestamp SentAt; + } + + [Reducer(ReducerKind.ClientConnected)] + public static void ClientConnected(ReducerContext ctx) + { + if (ctx.Db.User.Identity.Find(ctx.Sender) is User u) + { + ctx.Db.User.Identity.Update(u with { Online = true }); + } + } + + [Reducer(ReducerKind.ClientDisconnected)] + public static void ClientDisconnected(ReducerContext ctx) + { + if (ctx.Db.User.Identity.Find(ctx.Sender) is User u) + { + ctx.Db.User.Identity.Update(u with { Online = false }); + } + } + + [Reducer] + public static void Register(ReducerContext ctx, string name) + { + if (ctx.Db.User.Identity.Find(ctx.Sender) is not null) + { + throw new Exception("already registered"); + } + ctx.Db.User.Insert(new User { Identity = ctx.Sender, Name = name, Online = true }); + } + + [Reducer] + public static void SendMessage(ReducerContext ctx, string text) + { + if (ctx.Db.User.Identity.Find(ctx.Sender) is null) + { + throw new Exception("not registered"); + } + ctx.Db.Message.Insert(new Message + { + Id = 0, + Sender = ctx.Sender, + Text = text, + SentAt = ctx.Timestamp, + }); + } +} +``` diff --git a/docs/static/ai-guidelines/spacetimedb-rust.md b/docs/static/ai-guidelines/spacetimedb-rust.md new file mode 100644 index 00000000000..3d349a268ef --- /dev/null +++ b/docs/static/ai-guidelines/spacetimedb-rust.md @@ -0,0 +1,456 @@ +# SpacetimeDB Rust Server Module Guidelines + +## Imports + +```rust +use spacetimedb::{reducer, table, ReducerContext, Table}; +``` + +Additional imports when needed: +```rust +use spacetimedb::SpacetimeType; // For custom product/sum types +use spacetimedb::Identity; // For auth identity fields +use spacetimedb::Timestamp; // For timestamp fields +use spacetimedb::ScheduleAt; // For scheduled tables +use spacetimedb::{view, AnonymousViewContext}; // For anonymous views +use spacetimedb::{view, ViewContext}; // For per-user views +use std::time::Duration; // For schedule intervals +``` + +## Table Definitions + +```rust +#[table(accessor = user, public)] +pub struct User { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, + pub active: bool, +} +``` + +Table attribute options: +- `accessor = name` — required, the API name used to access the table +- `public` — makes table visible to clients +- `event` — marks as event table (append-only, no primary key needed) +- `scheduled(reducer_name)` — links table to a scheduled reducer + +## Primary Keys and Constraints + +```rust +#[primary_key] +pub id: i32, + +#[primary_key] +#[auto_inc] +pub id: u64, + +#[unique] +pub email: String, +``` + +## Indexes + +Single-column inline index: +```rust +#[index(btree)] +pub owner: Identity, +``` + +Named index in table attribute: +```rust +#[table( + accessor = order, + index(accessor = by_category, btree(columns = [category])) +)] +pub struct Order { + #[primary_key] + pub id: i32, + pub category: String, + pub amount: u64, +} +``` + +Multi-column index: +```rust +#[table( + accessor = membership, + index(accessor = by_user, btree(columns = [user_id])), + index(accessor = by_group, btree(columns = [group_id])) +)] +pub struct Membership { + #[primary_key] + pub id: i32, + pub user_id: i32, + pub group_id: i32, +} +``` + +## Column Types + +| Rust Type | Usage | +|-----------|-------| +| `i32`, `i64` | Signed integers | +| `u32`, `u64` | Unsigned integers | +| `f32`, `f64` | Floating point | +| `bool` | Boolean | +| `String` | Text | +| `Identity` | User identity | +| `Timestamp` | Timestamp | +| `ScheduleAt` | Schedule metadata | +| `Option` | Nullable field | + +## Product Types (Structs) + +```rust +#[derive(SpacetimeType, Clone, Debug)] +pub struct Position { + pub x: i32, + pub y: i32, +} + +#[table(accessor = entity)] +pub struct Entity { + #[primary_key] + pub id: i32, + pub pos: Position, +} +``` + +## Sum Types (Enums) + +```rust +#[derive(SpacetimeType, Clone, Debug)] +pub struct Rect { + pub width: i32, + pub height: i32, +} + +#[derive(SpacetimeType, Clone, Debug)] +pub enum Shape { + Circle(i32), + Rectangle(Rect), +} + +#[table(accessor = drawing)] +pub struct Drawing { + #[primary_key] + pub id: i32, + pub shape: Shape, +} +``` + +## Reducers + +Basic reducer: +```rust +#[reducer] +pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { + ctx.db.user().insert(User { id, name, age, active }); +} +``` + +Reducer with Result return: +```rust +#[reducer] +pub fn create_item(ctx: &ReducerContext, name: String) -> Result<(), String> { + ctx.db.item().insert(Item { id: 0, name }); + Ok(()) +} +``` + +Reducer with no arguments: +```rust +#[reducer] +pub fn reset_all(ctx: &ReducerContext) -> Result<(), String> { + Ok(()) +} +``` + +## Database Operations + +### Insert +```rust +ctx.db.user().insert(User { id: 1, name: "Alice".into(), age: 30, active: true }); + +// Auto-inc: use 0 as placeholder +ctx.db.message().insert(Message { id: 0, text: "Hello".to_string() }); + +// Insert returns the inserted row +let row = ctx.db.user().insert(User { id: 0, name: "Bob".into() }); +``` + +### Find (by primary key or unique index) +```rust +if let Some(user) = ctx.db.user().id().find(user_id) { + // use user +} + +let user = ctx.db.user().id().find(user_id).expect("not found"); +``` + +### Filter (by btree index — returns iterator) +```rust +for order in ctx.db.order().by_category().filter(&category) { + total += order.amount; +} + +// Collect to vec +let posts: Vec = ctx.db.post().by_author().filter(&author_id).collect(); +``` + +### Iterate all rows +```rust +for row in ctx.db.user().iter() { + // process row +} + +let count = ctx.db.user().iter().count(); +``` + +### Update (by primary key — pass full struct) +```rust +ctx.db.user().id().update(User { id, name: new_name, age: new_age, active: true }); + +// Or modify existing: +let mut user = ctx.db.user().id().find(id).expect("not found"); +user.name = new_name; +ctx.db.user().id().update(user); +``` + +### Delete (by primary key or indexed column) +```rust +ctx.db.user().id().delete(user_id); +ctx.db.online_player().identity().delete(&ctx.sender()); +``` + +## Authentication + +`ctx.sender()` returns the authenticated caller's `Identity`: + +```rust +#[table(accessor = message, public)] +pub struct Message { + #[primary_key] + #[auto_inc] + pub id: u64, + pub owner: Identity, + pub text: String, +} + +#[reducer] +pub fn send_message(ctx: &ReducerContext, text: String) { + ctx.db.message().insert(Message { + id: 0, + owner: ctx.sender(), + text, + }); +} + +#[reducer] +pub fn delete_message(ctx: &ReducerContext, id: u64) { + let msg = ctx.db.message().id().find(id).expect("not found"); + if msg.owner != ctx.sender() { + panic!("unauthorized"); + } + ctx.db.message().id().delete(id); +} +``` + +Identity as primary key: +```rust +#[table(accessor = player)] +pub struct Player { + #[primary_key] + pub identity: Identity, + pub name: String, +} + +// Lookup by identity +if ctx.db.player().identity().find(ctx.sender()).is_some() { + panic!("already registered"); +} +``` + +## Lifecycle Hooks + +```rust +#[reducer(init)] +pub fn init(ctx: &ReducerContext) -> Result<(), String> { + ctx.db.config().insert(Config { id: 0, setting: "default".into() }); + Ok(()) +} + +#[reducer(client_connected)] +pub fn client_connected(ctx: &ReducerContext) { + ctx.db.online_player().insert(OnlinePlayer { + identity: ctx.sender(), + connected_at: ctx.timestamp, + }); +} + +#[reducer(client_disconnected)] +pub fn client_disconnected(ctx: &ReducerContext) { + ctx.db.online_player().identity().delete(&ctx.sender()); +} +``` + +## Views + +Anonymous view (same result for all clients): +```rust +use spacetimedb::{view, AnonymousViewContext}; + +#[view(accessor = active_announcements, public)] +fn active_announcements(ctx: &AnonymousViewContext) -> Vec { + ctx.db.announcement().active().filter(true).collect() +} +``` + +Per-user view (result varies by sender): +```rust +use spacetimedb::{view, ViewContext}; + +#[view(accessor = my_profile, public)] +fn my_profile(ctx: &ViewContext) -> Option { + ctx.db.profile().identity().find(ctx.sender()) +} +``` + +## Scheduled Tables + +```rust +#[table(accessor = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> { + // Runs each time the timer fires + Ok(()) +} + +// Schedule a recurring interval +#[reducer(init)] +pub fn init(ctx: &ReducerContext) -> Result<(), String> { + let interval: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: interval, + }); + Ok(()) +} + +// Schedule at a specific time +let fire_at = ctx.timestamp + Duration::from_secs(60); +ctx.db.reminder().insert(Reminder { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(fire_at), + message: "Hello!".to_string(), +}); + +// Cancel a scheduled job +ctx.db.reminder().scheduled_id().delete(&job_id); +``` + +## Optional Fields + +```rust +#[table(accessor = player)] +pub struct Player { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, + pub nickname: Option, + pub high_score: Option, +} +``` + +## Helper Functions + +Non-reducer utility functions (no `#[reducer]` attribute): +```rust +fn add(a: i32, b: i32) -> i32 { + a + b +} + +#[reducer] +pub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) { + ctx.db.result().insert(ResultRow { id, sum: add(a, b) }); +} +``` + +## Complete Module Example + +```rust +use spacetimedb::{reducer, table, Identity, ReducerContext, Table, Timestamp}; + +#[table(accessor = user, public)] +pub struct User { + #[primary_key] + pub identity: Identity, + pub name: String, + pub online: bool, +} + +#[table( + accessor = message, + public, + index(accessor = by_sender, btree(columns = [sender])) +)] +pub struct Message { + #[primary_key] + #[auto_inc] + pub id: u64, + pub sender: Identity, + pub text: String, + pub sent_at: Timestamp, +} + +#[reducer(client_connected)] +pub fn client_connected(ctx: &ReducerContext) { + if let Some(mut user) = ctx.db.user().identity().find(ctx.sender()) { + user.online = true; + ctx.db.user().identity().update(user); + } +} + +#[reducer(client_disconnected)] +pub fn client_disconnected(ctx: &ReducerContext) { + if let Some(mut user) = ctx.db.user().identity().find(ctx.sender()) { + user.online = false; + ctx.db.user().identity().update(user); + } +} + +#[reducer] +pub fn register(ctx: &ReducerContext, name: String) { + if ctx.db.user().identity().find(ctx.sender()).is_some() { + panic!("already registered"); + } + ctx.db.user().insert(User { + identity: ctx.sender(), + name, + online: true, + }); +} + +#[reducer] +pub fn send_message(ctx: &ReducerContext, text: String) { + if ctx.db.user().identity().find(ctx.sender()).is_none() { + panic!("not registered"); + } + ctx.db.message().insert(Message { + id: 0, + sender: ctx.sender(), + text, + sent_at: ctx.timestamp, + }); +} +``` diff --git a/docs/static/ai-guidelines/spacetimedb-typescript.md b/docs/static/ai-guidelines/spacetimedb-typescript.md new file mode 100644 index 00000000000..3dde2e55d71 --- /dev/null +++ b/docs/static/ai-guidelines/spacetimedb-typescript.md @@ -0,0 +1,330 @@ +# SpacetimeDB TypeScript Module Guidelines + +## Imports + +```typescript +import { schema, table, t } from 'spacetimedb/server'; +import { ScheduleAt } from 'spacetimedb'; // for scheduled tables +``` + +⚠️ CRITICAL: The `name` field in table() MUST be snake_case (e.g. 'order_detail', NOT 'orderDetail'). +This is the single most common mistake. The JS variable can be camelCase, the `name` string cannot. + +## Tables + +`table(OPTIONS, COLUMNS)` — two arguments: + +```typescript +const user = table( + { name: 'user', public: true }, + { + id: t.u64().primaryKey().autoInc(), + email: t.string(), + name: t.string(), + active: t.bool(), + } +); +``` + +**IMPORTANT: The `name` string MUST be snake_case** — it becomes the SQL table name. +The JS variable can be camelCase, but the `name` string is always snake_case: + +```typescript +const orderDetail = table({ name: 'order_detail' }, { ... }); // ✓ snake_case name +const userStats = table({ name: 'user_stats' }, { ... }); // ✓ snake_case name +const eventLog = table({ name: 'event_log' }, { ... }); // ✓ snake_case name +// WRONG: table({ name: 'orderDetail' }, { ... }) // ✗ never camelCase +``` + +**`ctx.db` accessor uses the JS variable name (camelCase), NOT the SQL name:** + +```typescript +// schema({ orderDetail, userStats, eventLog }) → accessors are: +ctx.db.orderDetail.insert({ ... }); +ctx.db.userStats.iter(); +ctx.db.eventLog.id.find(logId); +``` + +Options: +- `name` — required, snake_case SQL name +- `public: true` — visible to clients (default: private) +- `event: true` — event table +- `scheduled: (): any => reducerRef` — scheduled table +- `indexes: [{ name, algorithm: 'btree', columns: [...] }]` + +## Column Types + +| Builder | JS type | Notes | +|---------|---------|-------| +| `t.i32()` | number | | +| `t.i64()` | bigint | Use `0n` literals | +| `t.u32()` | number | | +| `t.u64()` | bigint | Use `0n` literals | +| `t.f32()` | number | | +| `t.f64()` | number | | +| `t.bool()` | boolean | | +| `t.string()` | string | | +| `t.identity()` | Identity | | +| `t.timestamp()` | Timestamp | | +| `t.scheduleAt()` | ScheduleAt | | + +Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.optional()`, `.index('btree')` + +Optional columns: `nickname: t.option(t.string())` + +## Schema Export + +Every module must have exactly this pattern: + +```typescript +const spacetimedb = schema({ user, message }); +export default spacetimedb; +``` + +`schema()` takes one object containing all table references. `export default` is mandatory. + +## Reducers + +Named exports on the schema object. The export name becomes the reducer name: + +```typescript +// No arguments — pass just the callback +export const doReset = spacetimedb.reducer((ctx) => { ... }); + +// With arguments — pass args object, then callback +export const createUser = spacetimedb.reducer( + { name: t.string(), age: t.i32() }, + (ctx, { name, age }) => { + ctx.db.user.insert({ id: 0n, name, age, active: true }); + } +); +``` + +For no-arg reducers, omit the args object entirely — just pass the callback directly. + +## DB Operations + +```typescript +// Insert (pass 0n for autoInc fields) +ctx.db.user.insert({ id: 0n, name: 'Alice', age: 30 }); + +// Find by primary key or unique index → row | undefined +ctx.db.user.id.find(userId); +ctx.db.player.identity.find(ctx.sender); + +// Filter by btree index → iterator (accessor = column name for inline indexes) +for (const post of ctx.db.post.authorId.filter(authorId)) { } +const posts = [...ctx.db.post.authorId.filter(authorId)]; + +// Iterate all rows +for (const row of ctx.db.user.iter()) { } +const allUsers = [...ctx.db.user.iter()]; // spread to Array for .sort(), .filter(), .forEach() +// Note: iter() and filter() return IteratorObject, NOT Array. Use [...spread] first. + +// Update (spread + override) +const existing = ctx.db.user.id.find(userId); +if (existing) ctx.db.user.id.update({ ...existing, name: newName }); + +// Delete by primary key value +ctx.db.user.id.delete(userId); +``` + +## Index Access + +**Prefer inline `.index('btree')` on the column** — it's simpler and the accessor +matches the column name. Only use named indexes in `indexes: [...]` for multi-column indexes. +Do NOT use both inline `.index('btree')` AND a named index on the same column — this causes a duplicate name error. + +```typescript +// Inline btree index (preferred for single-column): +const post = table({ name: 'post' }, { + id: t.u64().primaryKey().autoInc(), + authorId: t.u64().index('btree'), // inline index + title: t.string(), +}); +// Access by column name: +ctx.db.post.authorId.filter(authorId); + +// Multi-column index (must use named index): +const log = table({ + name: 'event_log', + indexes: [{ name: 'by_category_severity', algorithm: 'btree', columns: ['category', 'severity'] }], +}, { ... }); +// Access by index name: +ctx.db.eventLog.by_category_severity.filter(...); + +// Primary key — always accessible by column name +ctx.db.user.id.find(1n); + +// Unique column +ctx.db.player.identity.find(ctx.sender); +``` + +## Lifecycle Hooks + +```typescript +// Init — runs once on first publish +export const init = spacetimedb.init((ctx) => { + ctx.db.config.insert({ id: 0, value: 'default' }); +}); + +// Client connected — must be exported +export const onConnect = spacetimedb.clientConnected((ctx) => { + ctx.db.online.insert({ identity: ctx.sender, connectedAt: ctx.timestamp }); +}); + +// Client disconnected — must be exported +export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { + ctx.db.online.identity.delete(ctx.sender); +}); +``` + +`init` uses `spacetimedb.init()`, NOT `spacetimedb.reducer()`. +`clientConnected`/`clientDisconnected` must be `export const`. + +The EXPORT NAME determines the reducer name visible in the schema: +✓ `export const onConnect = spacetimedb.clientConnected(...)` → reducer "on_connect" +✗ `export const clientConnected = spacetimedb.clientConnected(...)` → WRONG reducer name + +## Authentication + +```typescript +// ctx.sender is the caller's Identity +// Compare identities with .equals(), never === +if (!post.owner.equals(ctx.sender)) throw new Error('unauthorized'); +``` + +## Scheduled Tables + +The scheduled table references a reducer, creating a circular dependency. +Use `(): any =>` return type annotation to break the cycle: + +```typescript +const tickTimer = table({ + name: 'tick_timer', + scheduled: (): any => tick, // (): any => is required +}, { + scheduledId: t.u64().primaryKey().autoInc(), + scheduledAt: t.scheduleAt(), +}); + +const spacetimedb = schema({ tickTimer }); +export default spacetimedb; + +export const tick = spacetimedb.reducer( + { timer: tickTimer.rowType }, + (ctx, { timer }) => { + // timer row is auto-deleted after this reducer runs + } +); + +// Schedule a one-time job (accessor uses JS variable name, not SQL name) +ctx.db.tickTimer.insert({ + scheduledId: 0n, + scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + delayMicros), +}); + +// Schedule a repeating job +ctx.db.tickTimer.insert({ + scheduledId: 0n, + scheduledAt: ScheduleAt.interval(60_000_000n), +}); + +// Cancel a scheduled job +ctx.db.tickTimer.scheduledId.delete(jobId); +``` + +## Product Types (Structs) + +```typescript +const Position = t.object('Position', { x: t.i32(), y: t.i32() }); +const entity = table({ name: 'entity' }, { + id: t.u64().primaryKey().autoInc(), + pos: Position, +}); +``` + +## Sum Types (Tagged Unions) + +```typescript +const Shape = t.enum('Shape', { + circle: t.i32(), + rectangle: t.object('Rect', { w: t.i32(), h: t.i32() }), +}); +// Values: { tag: 'circle', value: 10 } +``` + +## Views + +```typescript +// Anonymous view (same for all clients) +export const activeAnnouncements = spacetimedb.anonymousView( + { name: 'active_announcements', public: true }, + t.array(announcement.rowType), + (ctx) => Array.from(ctx.db.announcement.active.filter(true)) +); + +// Per-user view (varies by ctx.sender) +export const my_profile = spacetimedb.view( + { name: 'my_profile', public: true }, + t.option(profile.rowType), + (ctx) => ctx.db.profile.identity.find(ctx.sender) ?? undefined +); +``` + +## Complete Example + +```typescript +import { schema, table, t } from 'spacetimedb/server'; + +const user = table( + { name: 'user', public: true }, + { + identity: t.identity().primaryKey(), + name: t.string(), + online: t.bool(), + } +); + +const message = table( + { + name: 'message', + public: true, + indexes: [{ name: 'message_sender', algorithm: 'btree', columns: ['sender'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + sender: t.identity(), + text: t.string(), + } +); + +const spacetimedb = schema({ user, message }); +export default spacetimedb; + +export const onConnect = spacetimedb.clientConnected((ctx) => { + const existing = ctx.db.user.identity.find(ctx.sender); + if (existing) ctx.db.user.identity.update({ ...existing, online: true }); +}); + +export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { + const existing = ctx.db.user.identity.find(ctx.sender); + if (existing) ctx.db.user.identity.update({ ...existing, online: false }); +}); + +export const register = spacetimedb.reducer( + { name: t.string() }, + (ctx, { name }) => { + if (ctx.db.user.identity.find(ctx.sender)) throw new Error('already registered'); + ctx.db.user.insert({ identity: ctx.sender, name, online: true }); + } +); + +export const sendMessage = spacetimedb.reducer( + { text: t.string() }, + (ctx, { text }) => { + if (!ctx.db.user.identity.find(ctx.sender)) throw new Error('not registered'); + ctx.db.message.insert({ id: 0n, sender: ctx.sender, text }); + } +); +``` diff --git a/docs/static/ai-rules/spacetimedb-csharp.mdc b/docs/static/ai-rules/spacetimedb-csharp.mdc deleted file mode 100644 index 871be299bd9..00000000000 --- a/docs/static/ai-rules/spacetimedb-csharp.mdc +++ /dev/null @@ -1,741 +0,0 @@ ---- -description: "⛔ MANDATORY: Read this ENTIRE file before writing ANY SpacetimeDB C# code. Contains SDK patterns from official documentation." -globs: **/*.cs -alwaysApply: true ---- - -# SpacetimeDB C# SDK - -## ⛔ COMMON MISTAKES — LLM HALLUCINATIONS - -These are **actual errors** observed when LLMs generate SpacetimeDB C# code: - -### 1. Wrong Package Name for Server Modules -```csharp -// ❌ WRONG — this package doesn't exist - - - -// ✅ CORRECT — use Runtime for server modules - -``` - -### 2. Missing `partial` Keyword -```csharp -// ❌ WRONG — missing partial -public struct MyTable { } -public class Module { } - -// ✅ CORRECT — must be partial -public partial struct MyTable { } -public static partial class Module { } -``` - -### 3. Lifecycle Hook Names Starting with "On" -```csharp -// ❌ WRONG — will cause STDB0010 error -[SpacetimeDB.Reducer(ReducerKind.ClientConnected)] -public static void OnClientConnected(ReducerContext ctx) { } - -// ✅ CORRECT — no "On" prefix -[SpacetimeDB.Reducer(ReducerKind.ClientConnected)] -public static void ClientConnected(ReducerContext ctx) { } -``` - -### 4. Wrong Timestamp Property Name -```csharp -// ❌ WRONG — property doesn't exist -var micros = timestamp.MicrosecondsSinceEpoch; - -// ✅ CORRECT — full name -var micros = timestamp.MicrosecondsSinceUnixEpoch; -``` - -### 5. Wrong ScheduleAt Syntax -```csharp -// ❌ WRONG — ScheduleAt.Time is not a method -ScheduledAt = ScheduleAt.Time(timestamp) - -// ✅ CORRECT — use constructor syntax -ScheduledAt = new ScheduleAt.Time(new Timestamp(microseconds)) -// OR with TimeSpan -ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + TimeSpan.FromSeconds(60)) -``` - -### 6. Table Accessor Casing Is Exact -```csharp -// ✅ CORRECT — match the accessor exactly -[SpacetimeDB.Table(Accessor = "user", Public = true)] -public partial class User { ... } -ctx.Db.user.Insert(...); - -// ❌ WRONG — incorrect casing -ctx.Db.User.Insert(...); // Accessor is "user", so property is ctx.Db.user -``` - -### 7. Client Callback Signature Errors -```csharp -// ❌ WRONG — guessing at signatures -.OnDisconnect((conn, ctx, err) => ...) -.OnConnectError((ctx, err) => ...) - -// ✅ CORRECT — check actual delegate types -.OnDisconnect((conn, err) => ...) // DbConnection, Exception? -.OnConnectError(err => ...) // Exception -``` - -### 8. WithUri Takes String, Not Uri -```csharp -// ❌ WRONG — Uri object not accepted -.WithUri(new Uri("http://localhost:3000")) - -// ✅ CORRECT — use string directly -.WithUri("http://localhost:3000") -``` - -### 9. Missing RuntimeIdentifier for WASM Build -```xml - -net8.0 - - -net8.0 -wasi-wasm -``` - -### 10. Subscribing Before Connected -```csharp -// ❌ WRONG — subscription fails silently -_conn = builder.Build(); -_conn.SubscriptionBuilder().SubscribeToAllTables(); // NOT CONNECTED YET! - -// ✅ CORRECT — subscribe inside OnConnect callback -private void OnConnected(DbConnection conn, Identity identity, string token) -{ - conn.SubscriptionBuilder() - .OnApplied(OnSubscriptionApplied) - .SubscribeToAllTables(); -} -``` - -### 11. Nullable Struct Handling with Find() -```csharp -// ❌ WRONG — accessing properties directly on nullable -if (existing != null) -{ - ctx.Db.User.Id.Update(new User { Id = existing.Id, ... }); -} - -// ✅ CORRECT — use .Value or pattern matching -if (ctx.Db.User.Id.Find(id) is User user) -{ - ctx.Db.User.Id.Update(user with { Name = newName }); -} -``` - -### 12. Sum Type Syntax Errors -```csharp -// ❌ WRONG — struct instead of record -public partial struct Shape : TaggedEnum<(Circle, Rectangle)> { } - -// ❌ WRONG — missing variant names in tuple -public partial record Shape : TaggedEnum<(Circle, Rectangle)> { } - -// ✅ CORRECT — record with named tuple elements -public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { } -``` - -### 13. Index Attribute Ambiguity -```csharp -// ❌ WRONG — ambiguous with System.Index -[Index.BTree(Accessor = "by_name", Columns = new[] { nameof(Name) })] - -// ✅ CORRECT — always use full namespace -[SpacetimeDB.Index.BTree(Accessor = "by_name", Columns = new[] { nameof(Name) })] -``` - ---- - -## 1) Common Mistakes Table - -### Server-side errors - -| Wrong | Right | Error | -|-------|-------|-------| -| Missing `partial` keyword | `public partial struct Table` | Generated code won't compile | -| `ctx.Db.User` when `Accessor = "user"` | `ctx.Db.user` | Accessors are case-sensitive | -| `Optional` | `string?` | Type not found | -| `ctx.Db.Table.Get(id)` | `ctx.Db.Table.Id.Find(id)` | Method not found | -| Wrong .csproj name | `StdbModule.csproj` | Publish fails silently | -| .NET 9 SDK | .NET 8 SDK only | WASI compilation fails | -| Missing WASI workload | `dotnet workload install wasi-experimental` | Build fails | -| `[Procedure]` attribute | Reducers only | Procedures not supported in C# | -| Missing `Public = true` | Add to `[Table]` attribute | Clients can't subscribe | -| Using `Random` | Avoid non-deterministic code | Sandbox violation | -| async/await in reducers | Synchronous only | Not supported | -| `[Index.BTree(...)]` | `[SpacetimeDB.Index.BTree(...)]` | Ambiguous with System.Index | -| `Columns = ["A", "B"]` | `Columns = new[] { "A", "B" }` | Collection expressions invalid in attributes | -| `partial struct` or `partial class : TaggedEnum` | `partial record : TaggedEnum` | Sum types must be record | -| `TaggedEnum<(A, B)>` | `TaggedEnum<(A A, B B)>` | Tuple must include variant names | - ---- - -## 2) Table Definition (CRITICAL) - -**Tables MUST use `partial struct` or `partial class` for code generation.** - -```csharp -using SpacetimeDB; - -// ❌ WRONG — missing partial! -[SpacetimeDB.Table(Accessor = "Player")] -public struct Player { } // Will not generate properly! - -// ✅ CORRECT — with partial keyword -[SpacetimeDB.Table(Accessor = "Player", Public = true)] -public partial struct Player -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - - public Identity OwnerId; - public string Name; - public Timestamp CreatedAt; -} - -// With single-column index -[SpacetimeDB.Table(Accessor = "Task", Public = true)] -public partial struct Task -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - - [SpacetimeDB.Index.BTree] - public Identity OwnerId; - - public string Title; - public bool Completed; -} - -// Multi-column index (use fully-qualified attribute!) -[SpacetimeDB.Table(Accessor = "Score", Public = true)] -[SpacetimeDB.Index.BTree(Accessor = "by_player_game", Columns = new[] { "PlayerId", "GameId" })] -public partial struct Score -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - - public Identity PlayerId; - public string GameId; - public int Points; -} -``` - -### Field attributes - -```csharp -[SpacetimeDB.PrimaryKey] // Exactly one per table (required) -[SpacetimeDB.AutoInc] // Auto-increment (integer fields only) -[SpacetimeDB.Unique] // Unique constraint -[SpacetimeDB.Index.BTree] // Single-column B-tree index -[SpacetimeDB.Default(value)] // Default value for new columns -``` - -### Column types - -```csharp -byte, sbyte, short, ushort // 8/16-bit integers -int, uint, long, ulong // 32/64-bit integers -float, double // Floats -bool // Boolean -string // Text -Identity // User identity -Timestamp // Timestamp -ScheduleAt // For scheduled tables -T? // Nullable (e.g., string?) -List // Arrays -``` - -### Insert with auto-increment - -```csharp -// Insert returns the row with generated ID -var player = ctx.Db.Player.Insert(new Player -{ - Id = 0, // Pass 0 to trigger auto-increment - OwnerId = ctx.Sender, - Name = name, - CreatedAt = ctx.Timestamp -}); -ulong newId = player.Id; // Get actual generated ID -``` - ---- - -## 3) Module and Reducers - -**The Module class MUST be `public static partial class`.** - -```csharp -using SpacetimeDB; - -public static partial class Module -{ - [SpacetimeDB.Reducer] - public static void CreateTask(ReducerContext ctx, string title) - { - // Validate - if (string.IsNullOrEmpty(title)) - { - throw new Exception("Title cannot be empty"); // Rolls back transaction - } - - // Insert - ctx.Db.Task.Insert(new Task - { - Id = 0, - OwnerId = ctx.Sender, - Title = title, - Completed = false - }); - } - - [SpacetimeDB.Reducer] - public static void CompleteTask(ReducerContext ctx, ulong taskId) - { - if (ctx.Db.Task.Id.Find(taskId) is not Task task) - { - throw new Exception("Task not found"); - } - - if (task.OwnerId != ctx.Sender) - { - throw new Exception("Not authorized"); - } - - ctx.Db.Task.Id.Update(task with { Completed = true }); - } - - [SpacetimeDB.Reducer] - public static void DeleteTask(ReducerContext ctx, ulong taskId) - { - ctx.Db.Task.Id.Delete(taskId); - } -} -``` - -### Update Pattern (CRITICAL) - -```csharp -// ✅ CORRECT — use `with` expression or provide complete row -if (ctx.Db.Task.Id.Find(id) is Task task) -{ - ctx.Db.Task.Id.Update(task with { Title = newTitle }); -} - -// ❌ WRONG — partial update nulls out other fields! -ctx.Db.Task.Id.Update(new Task { Id = id, Title = newTitle }); -``` - -### Lifecycle reducers - -```csharp -public static partial class Module -{ - [SpacetimeDB.Reducer(ReducerKind.Init)] - public static void Init(ReducerContext ctx) - { - // Called once when module is first published - Log.Info("Module initialized"); - } - - [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] - public static void ClientConnected(ReducerContext ctx) - { - // ctx.Sender is the connecting client - Log.Info($"Client connected: {ctx.Sender}"); - } - - [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] - public static void ClientDisconnected(ReducerContext ctx) - { - // Clean up client state - Log.Info($"Client disconnected: {ctx.Sender}"); - } -} -``` - -### ReducerContext API - -```csharp -ctx.Sender // Identity of the caller -ctx.Timestamp // Current timestamp -ctx.Db // Database access -ctx.Identity // Module's own identity -ctx.ConnectionId // Connection ID (nullable) -``` - ---- - -## 4) Database Access - -### Primary key operations - -```csharp -// Find by primary key — returns nullable, use pattern matching -if (ctx.Db.Task.Id.Find(taskId) is Task task) -{ - // Use task -} - -// Update by primary key -ctx.Db.Task.Id.Update(updatedTask); - -// Delete by primary key -ctx.Db.Task.Id.Delete(taskId); -``` - -### Index operations - -```csharp -// Find by unique index — returns nullable -if (ctx.Db.Player.Username.Find("alice") is Player player) -{ - // Found player -} - -// Filter by B-tree index — returns iterator -foreach (var task in ctx.Db.Task.OwnerId.Filter(ctx.Sender)) -{ - // Process each task -} -``` - -### Iterate all rows - -```csharp -// Full table scan -foreach (var task in ctx.Db.Task.Iter()) -{ - // Process each task -} -``` - ---- - -## 5) Custom Types - -**Use `[SpacetimeDB.Type]` for custom structs/enums. Must be `partial`.** - -```csharp -using SpacetimeDB; - -[SpacetimeDB.Type] -public partial struct Position -{ - public int X; - public int Y; -} - -[SpacetimeDB.Type] -public partial struct PlayerStats -{ - public int Health; - public int Mana; - public Position Location; -} - -// Use in table -[SpacetimeDB.Table(Accessor = "Player", Public = true)] -public partial struct Player -{ - [SpacetimeDB.PrimaryKey] - public Identity Id; - - public string Name; - public PlayerStats Stats; -} -``` - ---- - -## 6) Sum Types / Tagged Enums (CRITICAL) - -**Sum types MUST use `partial record` (not `partial class`) and inherit from `TaggedEnum`.** - -```csharp -using SpacetimeDB; - -// Step 1: Define variant types as partial structs with [Type] -[SpacetimeDB.Type] -public partial struct Circle { public int Radius; } - -[SpacetimeDB.Type] -public partial struct Rectangle { public int Width; public int Height; } - -// Step 2: Define sum type as partial RECORD (not struct or class!) inheriting TaggedEnum -// The tuple MUST include both the type AND a name for each variant -[SpacetimeDB.Type] -public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { } - -// Step 3: Use in a table -[SpacetimeDB.Table(Accessor = "Drawing", Public = true)] -public partial struct Drawing -{ - [SpacetimeDB.PrimaryKey] - public int Id; - public Shape ShapeA; - public Shape ShapeB; -} -``` - -### Creating sum type values - -```csharp -// Create variant instances using the generated nested types -var circle = new Shape.Circle(new Circle { Radius = 10 }); -var rect = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 }); - -// Insert into table -ctx.Db.Drawing.Insert(new Drawing { Id = 1, ShapeA = circle, ShapeB = rect }); -``` - ---- - -## 7) Client SDK Setup - -```csharp -// Connection pattern -_conn = DbConnection.Builder() - .WithUri("http://localhost:3000") // String, NOT Uri object! - .WithModuleName("my-module") - .WithToken(savedToken) // null for first connection - .OnConnect(OnConnected) - .OnDisconnect((conn, err) => { }) - .OnConnectError(err => { }) - .Build(); - -// Subscribe in OnConnected callback, NOT before! -private void OnConnected(DbConnection conn, Identity identity, string token) -{ - conn.SubscriptionBuilder() - .OnApplied(OnSubscriptionApplied) - .SubscribeToAllTables(); -} -``` - -### ⚠️ CRITICAL: FrameTick - -**You MUST call `FrameTick()` regularly** — without it, no callbacks fire: - -```csharp -while (running) -{ - conn.FrameTick(); - Thread.Sleep(16); // ~60 times per second -} -``` - -### Row Callbacks (Client-side) - -```csharp -// Register callbacks BEFORE subscribing -_conn.Db.Task.OnInsert += (EventContext ctx, Task row) => -{ - // Row was inserted -}; - -_conn.Db.Task.OnUpdate += (EventContext ctx, Task oldRow, Task newRow) => -{ - // Row was updated -}; - -_conn.Db.Task.OnDelete += (EventContext ctx, Task row) => -{ - // Row was deleted -}; -``` - -### Invoking Reducers (Client-side) - -```csharp -// Reducers are called as methods on conn.Reducers -_conn.Reducers.CreateTask("My task"); - -// Register callback for reducer completion -_conn.Reducers.OnCreateTask += (ctx) => -{ - if (ctx.Event.Status is Status.Committed) - { - // Success - } - else if (ctx.Event.Status is Status.Failed failed) - { - // Failed: failed.Item contains error - } -}; -``` - ---- - -## 8) Scheduled Tables - -For C# scheduled tables, `ScheduledAt = ...` must point to a field of type `ScheduleAt` on the scheduled table. - -```csharp -using SpacetimeDB; - -[SpacetimeDB.Table(Accessor = "Reminder", Scheduled = nameof(SendReminder))] -public partial struct Reminder -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - - public string Message; - public ScheduleAt ScheduledAt; -} - -public static partial class Module -{ - // Scheduled reducer receives the full row - [SpacetimeDB.Reducer] - public static void SendReminder(ReducerContext ctx, Reminder reminder) - { - Log.Info($"Reminder: {reminder.Message}"); - // Row is automatically deleted after reducer completes - } - - [SpacetimeDB.Reducer] - public static void CreateReminder(ReducerContext ctx, string message, ulong delaySecs) - { - var futureTime = ctx.Timestamp + TimeSpan.FromSeconds(delaySecs); - ctx.Db.Reminder.Insert(new Reminder - { - Id = 0, - Message = message, - ScheduledAt = new ScheduleAt.Time(futureTime) - }); - } - - [SpacetimeDB.Reducer] - public static void CancelReminder(ReducerContext ctx, ulong reminderId) - { - ctx.Db.Reminder.Id.Delete(reminderId); - } -} -``` - ---- - -## 9) Logging - -```csharp -using SpacetimeDB; - -Log.Debug("Debug message"); -Log.Info("Information"); -Log.Warn("Warning"); -Log.Error("Error occurred"); -Log.Panic("Critical failure"); // Terminates execution -``` - ---- - -## 10) Timestamps - -### Server-side - -```csharp -// Use ctx.Timestamp for current time -ctx.Db.Task.Insert(new Task -{ - // ... - CreatedAt = ctx.Timestamp -}); - -// Never use DateTime.Now - it's non-deterministic! -``` - -### Client-side - -```csharp -// Timestamp has MicrosecondsSinceUnixEpoch property -var dateTime = DateTimeOffset.FromUnixTimeMilliseconds( - row.CreatedAt.MicrosecondsSinceUnixEpoch / 1000 -).LocalDateTime; -``` - ---- - -## 11) Project Setup - -### Required .csproj (MUST be named `StdbModule.csproj`) - -```xml - - - net8.0 - wasi-wasm - Exe - enable - enable - - - - - -``` - -### Prerequisites - -```bash -# Install .NET 8 SDK (required, not .NET 9) -# Download from https://dotnet.microsoft.com/download/dotnet/8.0 - -# Install WASI workload -dotnet workload install wasi-experimental -``` - ---- - -## 12) Commands - -```bash -# Start local server -spacetime start - -# Publish module -spacetime publish --module-path - -# Clear database and republish -spacetime publish --clear-database -y --module-path - -# Generate bindings -spacetime generate --lang csharp --out-dir /SpacetimeDB --module-path - -# View logs -spacetime logs -``` - ---- - -## 13) Hard Requirements - -**C#-specific:** - -1. **Tables and Module MUST be `partial`** — required for code generation -2. **Use PascalCase for table access** — `ctx.Db.TableName`, not `ctx.Db.tableName` -3. **Project file MUST be named `StdbModule.csproj`** — CLI requirement -4. **Requires .NET 8 SDK** — .NET 9 and newer not yet supported -5. **Install WASI workload** — `dotnet workload install wasi-experimental` -6. **C# does NOT support procedures** — use reducers only -7. **Reducers must be deterministic** — no filesystem, network, timers, or `Random` -8. **Add `Public = true`** — if clients need to subscribe to a table -9. **Use `T?` for nullable fields** — not `Optional` -10. **Pass `0` for auto-increment** — to trigger ID generation on insert -11. **MUST call `FrameTick()` regularly** — client callbacks won't fire otherwise -12. **DO NOT edit generated bindings** — regenerate with `spacetime generate` diff --git a/docs/static/ai-rules/spacetimedb-migration-2.0.mdc b/docs/static/ai-rules/spacetimedb-migration-2.0.mdc deleted file mode 100644 index bc2a85e9863..00000000000 --- a/docs/static/ai-rules/spacetimedb-migration-2.0.mdc +++ /dev/null @@ -1,260 +0,0 @@ ---- -description: "When migrating SpacetimeDB 1.0 code to 2.0. Covers breaking changes: reducer callbacks → event tables, name→accessor, sender() method, update-only-via-primary-key." -globs: **/*.ts,**/*.tsx,**/*.js,**/*.jsx,**/*.rs,**/*.cs -alwaysApply: false ---- -# SpacetimeDB 1.0 → 2.0 Migration Rules - -Apply this when migrating existing SpacetimeDB 1.0 modules or clients to 2.0. - -## Overview of Breaking Changes - -1. **Reducer callbacks removed** — use event tables + `on_insert` instead -2. **`light_mode` removed** — no longer necessary -3. **`CallReducerFlags` / `NoSuccessNotify` removed** -4. **Table/index: `name` → `accessor`** — `name` is now for canonical SQL name only -5. **Client: `with_module_name` → `with_database_name`** -6. **Rust: `ctx.sender` → `ctx.sender()`** (method, not field) -7. **Only primary key indexes have `update`** — use primary key accessor for updates, or delete+insert - ---- - -## Reducer Callbacks → Event Tables - -### ❌ 1.0 — REMOVED - -```rust -// Server: reducer args were broadcast (no longer) -// Client: -conn.reducers.on_deal_damage(|ctx, target, amount| { ... }); -``` - -```csharp -conn.Reducers.OnDealDamage += (ctx, target, amount) => { ... }; -``` - -### ✅ 2.0 — Event Table Pattern - -**Server:** Define event table, insert from reducer. - -```rust -use spacetimedb::{table, reducer, ReducerContext, Table, Identity}; - -#[table(accessor = damage_event, public, event)] -pub struct DamageEvent { - pub target: Identity, - pub amount: u32, -} - -#[reducer] -fn deal_damage(ctx: &ReducerContext, target: Identity, amount: u32) { - ctx.db.damage_event().insert(DamageEvent { target, amount }); -} -``` - -```csharp -[SpacetimeDB.Table(Accessor = "DamageEvent", Public = true, Event = true)] -public partial struct DamageEvent { public Identity Target; public uint Amount; } - -[SpacetimeDB.Reducer] -public static void DealDamage(ReducerContext ctx, Identity target, uint amount) { - ctx.Db.DamageEvent.Insert(new DamageEvent { Target = target, Amount = amount }); -} -``` - -**Client:** Subscribe to event table, use `on_insert`. - -```rust -conn.db.damage_event().on_insert(|ctx, event| { - play_damage_animation(event.target, event.amount); -}); -``` - -```csharp -conn.Db.DamageEvent.OnInsert += (ctx, damageEvent) => { - PlayDamageAnimation(damageEvent.Target, damageEvent.Amount); -}; -``` - -**Event tables must be subscribed explicitly** — they are excluded from `subscribe_to_all_tables()`. - -**TypeScript:** Use `reducer(params, fn)` with `export const name = ...` — never `reducer('name', params, fn)`. - -```typescript -// ❌ 1.0 — NO LONGER VALID -spacetimedb.reducer('deal_damage', { target: t.identity(), amount: t.u32() }, (ctx, { target, amount }) => { ... }); - -// ✅ 2.0 — name from export -export const deal_damage = spacetimedb.reducer({ target: t.identity(), amount: t.u32() }, (ctx, { target, amount }) => { - ctx.db.damage_event.insert({ target, amount }); -}); -``` - ---- - -## TypeScript: schema, reducer, scheduled - -### schema() — object only -```typescript -// ❌ 1.0 — NO LONGER VALID -const spacetimedb = schema(myTable); -const spacetimedb = schema(t1, t2, t3); - -// ✅ 2.0 — single object argument -const moduleSchema = schema({ myTable }); -export default moduleSchema; -// or: const moduleSchema = schema({ t1, t2, t3 }); -``` - -### scheduled tables -```typescript -// ❌ 1.0 — string reducer name -scheduled: 'run_cleanup' - -// ✅ 2.0 — function returning exported reducer -export const run_cleanup = spacetimedb.reducer({ arg: CleanupJob.rowType }, (ctx, { arg }) => { ... }); -// In table options: -scheduled: () => run_cleanup -``` - ---- - -## Table/Index: `name` → `accessor` - -### ❌ 1.0 — NO LONGER VALID - -```rust -#[table(name = my_table, public, index(name = position, btree(columns = [x, y])))] -``` - -```csharp -[SpacetimeDB.Table(Name = "MyTable", Public = true)] -[SpacetimeDB.Index.BTree(Name = "Position", Columns = new[] { nameof(X), nameof(Y) })] -``` - -```typescript -// schema(myTable) or schema(t1, t2) — NO LONGER VALID -``` - -### ✅ 2.0 - -```rust -#[table(accessor = my_table, public, index(accessor = position, btree(columns = [x, y])))] -``` - -```csharp -[SpacetimeDB.Table(Accessor = "MyTable", Public = true)] -[SpacetimeDB.Index.BTree(Accessor = "Position", Columns = new[] { nameof(X), nameof(Y) })] -``` - -```typescript -// schema({ myTable }) — object keys become accessors -const spacetimedb = schema({ myTable }); -export default spacetimedb; -``` - -- `accessor` = API names in module/client code -- `name` = optional override for canonical SQL name (for existing DB compatibility) - ---- - -## Client Connection: `with_module_name` → `with_database_name` - -### ❌ 1.0 - -```rust -.with_module_name("my-database") -``` - -```csharp -.WithModuleName("my-database") -``` - -### ✅ 2.0 - -```rust -.with_database_name("my-database") -``` - -```csharp -.WithDatabaseName("my-database") -``` - ---- - -## Rust: `ctx.sender` → `ctx.sender()` - -### ❌ 1.0 - -```rust -let sender_identity = ctx.sender; -``` - -### ✅ 2.0 - -```rust -let sender_identity = ctx.sender(); -``` - ---- - -## Only Primary Key Has `update` - -### ❌ 1.0 — REMOVED (unique index update) - -```rust -ctx.db.user().name().update(User { ... }); -``` - -```csharp -ctx.Db.User.Name.Update(new User { ... }); -``` - -### ✅ 2.0 — Update via primary key accessor - -```rust -ctx.db.user().identity().update(User { ... }); -``` - -```csharp -ctx.Db.User.Identity.Update(new User { ... }); -``` - -If the primary key value changes, use **delete + insert** instead of update. - ---- - -## Per-Call Result Callbacks - -If you only need the result of a reducer **you** called, use `_then()`: - -```rust -ctx.reducers.deal_damage_then(target, amount, |ctx, result| { - match result { - Ok(Ok(())) => println!("succeeded"), - Ok(Err(err)) => println!("failed: {err}"), - Err(internal) => println!("error: {internal:?}"), - } -}).unwrap(); -``` - ---- - -## Existing 1.0 Database Compatibility - -To avoid table name changes when republishing to an existing DB, set case conversion to `None`: - -```rust -use spacetimedb::CaseConversionPolicy; - -#[spacetimedb::settings] -const CASE_CONVERSION_POLICY: CaseConversionPolicy = CaseConversionPolicy::None; -``` - -```csharp -[SpacetimeDB.Settings] -public const SpacetimeDB.CaseConversionPolicy CASE_CONVERSION_POLICY = - SpacetimeDB.CaseConversionPolicy.None; -``` - -Or specify `name = "ExactTableName"` on each table to preserve canonical names. diff --git a/docs/static/ai-rules/spacetimedb-rust.mdc b/docs/static/ai-rules/spacetimedb-rust.mdc deleted file mode 100644 index f0ebeafbcf1..00000000000 --- a/docs/static/ai-rules/spacetimedb-rust.mdc +++ /dev/null @@ -1,663 +0,0 @@ ---- -description: "⛔ MANDATORY: Read this ENTIRE file before writing ANY SpacetimeDB Rust code. Contains SDK patterns from official documentation." -globs: **/*.rs -alwaysApply: true ---- - -# SpacetimeDB Rust SDK - -## ⛔ COMMON MISTAKES — LLM HALLUCINATIONS - -These are **actual errors** observed when LLMs generate SpacetimeDB Rust code: - -### 1. Wrong Crate for Server vs Client - -```rust -// ❌ WRONG — using client crate for server module -use spacetimedb_sdk::*; // This is for CLIENTS only! - -// ✅ CORRECT — use spacetimedb for server modules -use spacetimedb::{table, reducer, Table, ReducerContext, Identity, Timestamp}; -``` - -### 2. Wrong Table Macro Syntax - -```rust -// ❌ WRONG — using attribute-style like C# -#[spacetimedb::table] -#[primary_key] -pub struct User { ... } - -// ❌ WRONG — SpacetimeType on tables (causes conflicts!) -#[derive(SpacetimeType)] -#[table(accessor = my_table)] -pub struct MyTable { ... } - -// ✅ CORRECT — use #[table(...)] macro with options, NO SpacetimeType -#[table(accessor = user, public)] -pub struct User { - #[primary_key] - identity: Identity, - name: Option, -} -``` - -### 3. Wrong Table Access Pattern - -```rust -// ❌ WRONG — using ctx.Db or ctx.db() method or field access -ctx.Db.user.Insert(...); -ctx.db().user().insert(...); -ctx.db.player; // Field access - -// ✅ CORRECT — ctx.db is a field, table names are methods with parentheses -ctx.db.user().insert(User { ... }); -ctx.db.user().identity().find(ctx.sender); -ctx.db.player().id().find(&player_id); -``` - -### 4. Wrong Update Pattern - -```rust -// ❌ WRONG — partial update or using .update() directly on table -ctx.db.user().update(User { name: Some("new".into()), ..Default::default() }); - -// ✅ CORRECT — find existing, spread it, update via primary key accessor -if let Some(user) = ctx.db.user().identity().find(ctx.sender) { - ctx.db.user().identity().update(User { name: Some("new".into()), ..user }); -} -``` - -### 5. Wrong Reducer Return Type - -```rust -// ❌ WRONG — returning data from reducer -#[reducer] -pub fn get_user(ctx: &ReducerContext, id: Identity) -> Option { ... } - -// ❌ WRONG — mutable context -pub fn my_reducer(ctx: &mut ReducerContext, ...) { } - -// ✅ CORRECT — reducers return Result<(), String> or nothing, immutable context -#[reducer] -pub fn do_something(ctx: &ReducerContext, value: String) -> Result<(), String> { - if value.is_empty() { - return Err("Value cannot be empty".to_string()); - } - Ok(()) -} -``` - -### 6. Wrong Client Connection Pattern - -```rust -// ❌ WRONG — subscribing before connected -let conn = DbConnection::builder().build()?; -conn.subscription_builder().subscribe_to_all_tables(); // NOT CONNECTED YET! - -// ✅ CORRECT — subscribe in on_connect callback -DbConnection::builder() - .on_connect(|conn, identity, token| { - conn.subscription_builder() - .on_applied(|ctx| println!("Ready!")) - .subscribe_to_all_tables(); - }) - .build()?; -``` - -### 7. Forgetting to Advance the Connection - -```rust -// ❌ WRONG — connection never processes messages -let conn = DbConnection::builder().build()?; -// ... callbacks never fire ... - -// ✅ CORRECT — must call one of these to process messages -conn.run_threaded(); // Spawn background thread -// OR -conn.run_async().await; // Async task -// OR (in game loop) -conn.frame_tick()?; // Manual polling -``` - -### 8. Missing Table Trait Import - -```rust -// ❌ WRONG — "no method named `insert` found" -use spacetimedb::{table, reducer, ReducerContext}; -ctx.db.user().insert(...); // ERROR! - -// ✅ CORRECT — import Table trait for table methods -use spacetimedb::{table, reducer, Table, ReducerContext}; -ctx.db.user().insert(...); // Works! -``` - -### 9. Wrong ScheduleAt Variant - -```rust -// ❌ WRONG — At variant doesn't exist -scheduled_at: ScheduleAt::At(future_time), - -// ✅ CORRECT — use Time variant -scheduled_at: ScheduleAt::Time(future_time), -``` - -### 10. Identity to String Conversion - -```rust -// ❌ WRONG — to_hex() returns HexString<32>, not String -let id: String = identity.to_hex(); // Type mismatch! - -// ✅ CORRECT — chain .to_string() -let id: String = identity.to_hex().to_string(); -``` - -### 11. Timestamp Duration Extraction - -```rust -// ❌ WRONG — returns Result, not Duration directly -let micros = ctx.timestamp.to_duration_since_unix_epoch().as_micros(); - -// ✅ CORRECT — unwrap the Result -let micros = ctx.timestamp.to_duration_since_unix_epoch() - .unwrap_or_default() - .as_micros(); -``` - -### 12. Borrow After Move - -```rust -// ❌ WRONG — `tool` moved into struct, then borrowed -ctx.db.stroke().insert(Stroke { tool, color, ... }); -if tool == "eraser" { ... } // ERROR: value moved! - -// ✅ CORRECT — check before move, or use clone -let is_eraser = tool == "eraser"; -ctx.db.stroke().insert(Stroke { tool, color, ... }); -if is_eraser { ... } -``` - -### 13. Client SDK Uses Blocking I/O - -The SpacetimeDB Rust client SDK uses blocking I/O. If mixing with async runtimes (Tokio, async-std), use `spawn_blocking` or run the SDK on a dedicated thread to avoid blocking the async executor. - -### 14. Wrong Schedule Syntax -```rust -// ❌ WRONG — `schedule` is not a valid table type -#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - -// ✅ CORRECT — `scheduled` is a valid table type -#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] -``` ---- - -## 1) Common Mistakes Table - -### Server-side errors - -| Wrong | Right | Error | -|-------|-------|-------| -| `#[derive(SpacetimeType)]` on `#[table]` | Remove it — macro handles this | Conflicting derive macros | -| `ctx.db.player` (field access) | `ctx.db.player()` (method) | "no field `player` on type" | -| `ctx.db.player().find(id)` | `ctx.db.player().id().find(&id)` | Must access via index | -| `&mut ReducerContext` | `&ReducerContext` | Wrong context type | -| Missing `use spacetimedb::Table;` | Add import | "no method named `insert`" | -| `#[table(accessor = "my_table")]` | `#[table(accessor = my_table)]` | String literals not allowed | -| Missing `public` on table | Add `public` flag | Clients can't subscribe | -| `#[spacetimedb::reducer]` | `#[reducer]` after import | Wrong attribute path | -| Network/filesystem in reducer | Use procedures instead | Sandbox violation | -| Panic for expected errors | Return `Result<(), String>` | WASM instance destroyed | - ---- - -## 2) Table Definition (CRITICAL) - -**Tables use `#[table(...)]` macro on `pub struct`. DO NOT derive `SpacetimeType` on tables!** - -> ⚠️ **CRITICAL:** Always import `Table` trait — required for `.insert()`, `.iter()`, `.find()`, etc. - -```rust -use spacetimedb::{table, reducer, Table, ReducerContext, Identity, Timestamp}; - -// ❌ WRONG — DO NOT derive SpacetimeType on tables! -#[derive(SpacetimeType)] // REMOVE THIS! -#[table(accessor = task)] -pub struct Task { ... } - -// ✅ CORRECT — just the #[table] attribute -#[table(accessor = user, public)] -pub struct User { - #[primary_key] - identity: Identity, - - #[unique] - username: Option, - - online: bool, -} - -#[table(accessor = message, public)] -pub struct Message { - #[primary_key] - #[auto_inc] - id: u64, - - sender: Identity, - text: String, - sent: Timestamp, -} - -// With multi-column index -#[table(accessor = task, public, index(name = by_owner, btree(columns = [owner_id])))] -pub struct Task { - #[primary_key] - #[auto_inc] - pub id: u64, - pub owner_id: Identity, - pub title: String, -} -``` - -### Table Options - -```rust -#[table(accessor = my_table)] // Private table (default) -#[table(accessor = my_table, public)] // Public table - clients can subscribe -``` - -### Column Attributes - -```rust -#[primary_key] // Primary key (auto-indexed, enables .find()) -#[auto_inc] // Auto-increment (use with #[primary_key]) -#[unique] // Unique constraint (auto-indexed) -#[index(btree)] // B-Tree index for queries -``` - -### Insert returns ROW, not ID - -```rust -let row = ctx.db.task().insert(Task { - id: 0, // auto-inc placeholder - owner_id: ctx.sender, - title: "New task".to_string(), - created_at: ctx.timestamp, -}); -let new_id = row.id; // Get the actual ID -``` - ---- - -## 3) Reducers - -### Definition Syntax - -```rust -use spacetimedb::{reducer, ReducerContext, Table}; - -#[reducer] -pub fn send_message(ctx: &ReducerContext, text: String) -> Result<(), String> { - // Validate input - if text.is_empty() { - return Err("Message cannot be empty".to_string()); - } - - // Insert returns the inserted row - let row = ctx.db.message().insert(Message { - id: 0, // auto-inc placeholder - sender: ctx.sender, - text, - sent: ctx.timestamp, - }); - - log::info!("Message {} sent by {:?}", row.id, ctx.sender); - Ok(()) -} -``` - -### Update Pattern (CRITICAL) - -```rust -#[reducer] -pub fn set_name(ctx: &ReducerContext, name: String) -> Result<(), String> { - // Find existing row - let user = ctx.db.user().identity().find(ctx.sender) - .ok_or("User not found")?; - - // ✅ CORRECT — spread existing row, override specific fields - ctx.db.user().identity().update(User { - name: Some(name), - ..user // Preserves identity, online, etc. - }); - - Ok(()) -} - -// ❌ WRONG — partial update nulls out other fields! -// ctx.db.user().identity().update(User { identity: ctx.sender, name: Some(name), ..Default::default() }); -``` - -### Delete Pattern - -```rust -#[reducer] -pub fn delete_message(ctx: &ReducerContext, message_id: u64) -> Result<(), String> { - ctx.db.message().id().delete(&message_id); - Ok(()) -} -``` - -### Lifecycle Hooks - -```rust -#[reducer(init)] -pub fn init(ctx: &ReducerContext) { - // Called when module is first published -} - -#[reducer(client_connected)] -pub fn client_connected(ctx: &ReducerContext) { - // ctx.sender is the connecting identity - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { - ctx.db.user().identity().update(User { online: true, ..user }); - } else { - ctx.db.user().insert(User { - identity: ctx.sender, - username: None, - online: true, - }); - } -} - -#[reducer(client_disconnected)] -pub fn client_disconnected(ctx: &ReducerContext) { - if let Some(user) = ctx.db.user().identity().find(ctx.sender) { - ctx.db.user().identity().update(User { online: false, ..user }); - } -} -``` - -### ReducerContext fields - -```rust -ctx.sender // Identity of the caller -ctx.timestamp // Current timestamp -ctx.db // Database access -ctx.rng // Deterministic RNG (use instead of rand) -``` - ---- - -## 4) Index Access - -### Primary Key / Unique — `.find()` returns `Option` - -```rust -// Primary key lookup -let user = ctx.db.user().identity().find(ctx.sender); - -// Unique column lookup -let user = ctx.db.user().username().find(&"alice".to_string()); - -if let Some(user) = user { - // Found -} -``` - -### BTree Index — `.filter()` returns iterator - -```rust -#[table(accessor = message, public)] -pub struct Message { - #[primary_key] - #[auto_inc] - id: u64, - - #[index(btree)] - room_id: u64, - - text: String, -} - -// Filter by indexed column -for msg in ctx.db.message().room_id().filter(&room_id) { - // Process each message in room -} -``` - -### No Index — `.iter()` + manual filter - -```rust -// Full table scan -for user in ctx.db.user().iter() { - if user.online { - // Process online users - } -} -``` - ---- - -## 5) Custom Types - -**Use `#[derive(SpacetimeType)]` ONLY for custom structs/enums used as fields or parameters.** - -```rust -use spacetimedb::SpacetimeType; - -// Custom struct for table fields -#[derive(SpacetimeType, Clone, Debug, PartialEq)] -pub struct Position { - pub x: i32, - pub y: i32, -} - -// Custom enum -#[derive(SpacetimeType, Clone, Debug, PartialEq)] -pub enum PlayerStatus { - Idle, - Walking(Position), - Fighting(Identity), -} - -// Use in table (DO NOT derive SpacetimeType on the table!) -#[table(accessor = player, public)] -pub struct Player { - #[primary_key] - pub id: Identity, - pub position: Position, - pub status: PlayerStatus, -} -``` - ---- - -## 6) Scheduled Tables - -```rust -use spacetimedb::{table, reducer, ReducerContext, ScheduleAt, Timestamp}; - -#[table(accessor = cleanup_job, scheduled(cleanup_expired))] -pub struct CleanupJob { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - - scheduled_at: ScheduleAt, - target_id: u64, -} - -#[reducer] -pub fn cleanup_expired(ctx: &ReducerContext, job: CleanupJob) { - // Job row is auto-deleted after reducer completes - log::info!("Cleaning up: {}", job.target_id); -} - -// Schedule a job -#[reducer] -pub fn schedule_cleanup(ctx: &ReducerContext, target_id: u64, delay_ms: u64) { - let future_time = ctx.timestamp + std::time::Duration::from_millis(delay_ms); - ctx.db.cleanup_job().insert(CleanupJob { - scheduled_id: 0, // auto-inc placeholder - scheduled_at: ScheduleAt::Time(future_time), - target_id, - }); -} - -// Cancel by deleting the row -#[reducer] -pub fn cancel_cleanup(ctx: &ReducerContext, job_id: u64) { - ctx.db.cleanup_job().scheduled_id().delete(&job_id); -} -``` - ---- - -## 7) Client SDK - -```rust -// Connection pattern -let conn = DbConnection::builder() - .with_uri("http://localhost:3000") - .with_module_name("my-module") - .with_token(load_saved_token()) // None for first connection - .on_connect(on_connected) - .build() - .expect("Failed to connect"); - -// Subscribe in on_connect callback, NOT before! -fn on_connected(conn: &DbConnection, identity: Identity, token: &str) { - conn.subscription_builder() - .on_applied(|ctx| println!("Ready!")) - .subscribe_to_all_tables(); -} -``` - -### ⚠️ CRITICAL: Advance the Connection - -**You MUST call one of these** — without it, no callbacks fire: - -```rust -conn.run_threaded(); // Background thread (simplest) -conn.run_async().await; // Async task -conn.frame_tick()?; // Manual polling (game loops) -``` - -### Table Access & Callbacks - -```rust -// Iterate -for user in ctx.db.user().iter() { ... } - -// Find by primary key -if let Some(user) = ctx.db.user().identity().find(&identity) { ... } - -// Row callbacks -ctx.db.user().on_insert(|ctx, user| { ... }); -ctx.db.user().on_update(|ctx, old, new| { ... }); -ctx.db.user().on_delete(|ctx, user| { ... }); - -// Call reducers -ctx.reducers.set_name("Alice".to_string()).unwrap(); -``` - ---- - -## 8) Procedures (Beta) - -**Procedures are for side effects (HTTP, filesystem) that reducers can't do.** - -⚠️ Procedures are currently in beta. API may change. - -```rust -use spacetimedb::{procedure, ProcedureContext}; - -// Simple procedure -#[procedure] -fn add_numbers(_ctx: &mut ProcedureContext, a: u32, b: u32) -> u64 { - a as u64 + b as u64 -} - -// Procedure with database access -#[procedure] -fn save_external_data(ctx: &mut ProcedureContext, url: String) -> Result<(), String> { - // HTTP request (allowed in procedures, not reducers) - let data = fetch_from_url(&url)?; - - // Database access requires explicit transaction - ctx.try_with_tx(|tx| { - tx.db.external_data().insert(ExternalData { - id: 0, - content: data, - }); - Ok(()) - })?; - - Ok(()) -} -``` - -### Key differences from reducers - -| Reducers | Procedures | -|----------|------------| -| `&ReducerContext` (immutable) | `&mut ProcedureContext` (mutable) | -| Direct `ctx.db` access | Must use `ctx.with_tx()` | -| No HTTP/network | HTTP allowed | -| No return values | Can return data | - ---- - -## 9) Logging - -```rust -use spacetimedb::log; - -log::trace!("Detailed trace"); -log::debug!("Debug info"); -log::info!("Information"); -log::warn!("Warning"); -log::error!("Error occurred"); -``` - ---- - -## 10) Commands - -```bash -# Start local server -spacetime start - -# Publish module -spacetime publish --module-path - -# Clear database and republish -spacetime publish --clear-database -y --module-path - -# Generate bindings -spacetime generate --lang rust --out-dir /src/module_bindings --module-path - -# View logs -spacetime logs -``` - ---- - -## 11) Hard Requirements - -**Rust-specific:** - -1. **DO NOT derive `SpacetimeType` on `#[table]` structs** — the macro handles this -2. **Import `Table` trait** — `use spacetimedb::Table;` required for `.insert()`, `.iter()`, etc. -3. **Use `&ReducerContext`** — not `&mut ReducerContext` -4. **Tables are methods** — `ctx.db.table()` not `ctx.db.table` -5. **Server modules use `spacetimedb` crate** — clients use `spacetimedb-sdk` -6. **Reducers must be deterministic** — no filesystem, network, timers, or external RNG -7. **Use `ctx.rng`** — not `rand` crate for random numbers -8. **Use `ctx.timestamp`** — never `std::time::SystemTime::now()` in reducers -9. **Client MUST advance connection** — call `run_threaded()`, `run_async()`, or `frame_tick()` -10. **Subscribe in `on_connect` callback** — not before connection is established -11. **Update requires full row** — spread existing row with `..existing` -12. **DO NOT edit generated bindings** — regenerate with `spacetime generate` -13. **Identity to String needs `.to_string()`** — `identity.to_hex().to_string()` -14. **Client SDK is blocking** — use `spawn_blocking` or dedicated thread if mixing with async runtimes diff --git a/docs/static/ai-rules/spacetimedb-typescript.mdc b/docs/static/ai-rules/spacetimedb-typescript.mdc deleted file mode 100644 index 22503fbefba..00000000000 --- a/docs/static/ai-rules/spacetimedb-typescript.mdc +++ /dev/null @@ -1,659 +0,0 @@ ---- -description: "⛔ MANDATORY: Read this ENTIRE file before writing ANY SpacetimeDB TypeScript code. Contains critical SDK patterns and HALLUCINATED APIs to avoid." -globs: **/*.ts,**/*.tsx,**/*.js,**/*.jsx -alwaysApply: true ---- - -# SpacetimeDB TypeScript SDK - -## ⛔ HALLUCINATED APIs — DO NOT USE - -**These APIs DO NOT EXIST. LLMs frequently hallucinate them.** - -```typescript -// ❌ WRONG PACKAGE — does not exist -import { SpacetimeDBClient } from "@clockworklabs/spacetimedb-sdk"; - -// ❌ WRONG — these methods don't exist -SpacetimeDBClient.connect(...); -SpacetimeDBClient.call("reducer_name", [...]); -connection.call("reducer_name", [arg1, arg2]); - -// ❌ WRONG — positional reducer arguments -conn.reducers.doSomething("value"); // WRONG! - -// ❌ WRONG — static methods on generated types don't exist -User.filterByName('alice'); -Message.findById(123n); -tables.user.filter(u => u.name === 'alice'); // No .filter() on tables object! -``` - -### ✅ CORRECT PATTERNS: - -```typescript -// ✅ CORRECT IMPORTS -import { DbConnection, tables } from './module_bindings'; // Generated! -import { SpacetimeDBProvider, useTable, Identity } from 'spacetimedb/react'; - -// ✅ CORRECT REDUCER CALLS — object syntax, not positional! -conn.reducers.doSomething({ value: 'test' }); -conn.reducers.updateItem({ itemId: 1n, newValue: 42 }); - -// ✅ CORRECT DATA ACCESS — useTable returns [rows, isLoading] -const [items, isLoading] = useTable(tables.item); -``` - -### ⛔ DO NOT: -- **Invent hooks** like `useItems()`, `useData()` — use `useTable(tables.tableName)` -- **Import from fake packages** — only `spacetimedb`, `spacetimedb/react`, `./module_bindings` - ---- - -## 1) Common Mistakes Table - -### Server-side errors - -| Wrong | Right | Error | -|-------|-------|-------| -| Missing `package.json` | Create `package.json` | "could not detect language" | -| Missing `tsconfig.json` | Create `tsconfig.json` | "TsconfigNotFound" | -| Entrypoint not at `src/index.ts` | Use `src/index.ts` | Module won't bundle | -| `indexes` in COLUMNS (2nd arg) | `indexes` in OPTIONS (1st arg) | "reading 'tag'" error | -| Index without `algorithm` | `algorithm: 'btree'` | "reading 'tag'" error | -| `filter({ ownerId })` | `filter(ownerId)` | "does not exist in type 'Range'" | -| `.filter()` on unique column | `.find()` on unique column | TypeError | -| `insert({ ...without id })` | `insert({ id: 0n, ... })` | "Property 'id' is missing" | -| `const id = table.insert(...)` | `const row = table.insert(...)` | `.insert()` returns ROW, not ID | -| `.unique()` + explicit index | Just use `.unique()` | "name is used for multiple entities" | -| Index on `.primaryKey()` column | Don't — already indexed | "name is used for multiple entities" | -| Same index name in multiple tables | Prefix with table name | "name is used for multiple entities" | -| `.indexName.filter()` after removing index | Use `.iter()` + manual filter | "Cannot read properties of undefined" | -| Import spacetimedb from index.ts | Import from schema.ts | "Cannot access before initialization" | -| Multi-column index `.filter()` | **⚠️ BROKEN** — use single-column | PANIC or silent empty results | -| `JSON.stringify({ id: row.id })` | Convert BigInt first: `{ id: row.id.toString() }` | "Do not know how to serialize a BigInt" | -| `ScheduleAt.Time(timestamp)` | `ScheduleAt.time(timestamp)` (lowercase) | "ScheduleAt.Time is not a function" | -| `ctx.db.foo.myIndexName.filter()` | Use exact name: `ctx.db.foo.my_index_name.filter()` | "Cannot read properties of undefined" | -| `.iter()` in views | Use index lookups | Severe performance issues (re-evaluates on any change) | -| `ctx.db` in procedures | `ctx.withTx(tx => tx.db...)` | Procedures need explicit transactions | -| `ctx.myTable` in procedure tx | `tx.db.myTable` | Wrong context variable | - -### Client-side errors - -| Wrong | Right | Error | -|-------|-------|-------| -| `@spacetimedb/sdk` | `spacetimedb` | 404 / missing subpath | -| `conn.reducers.foo("val")` | `conn.reducers.foo({ param: "val" })` | Wrong reducer syntax | -| Inline `connectionBuilder` | `useMemo(() => ..., [])` | Reconnects every render | -| `const rows = useTable(table)` | `const [rows, isLoading] = useTable(table)` | Tuple destructuring | -| Optimistic UI updates | Let subscriptions drive state | Desync issues | -| `` | `connectionBuilder={...}` | Wrong prop name | - ---- - -## 2) Table Definition (CRITICAL) - -**`table()` takes TWO arguments: `table(OPTIONS, COLUMNS)`** - -```typescript -import { schema, table, t } from 'spacetimedb/server'; - -// ❌ WRONG — indexes in COLUMNS causes "reading 'tag'" error -export const Task = table({ name: 'task' }, { - id: t.u64().primaryKey().autoInc(), - ownerId: t.identity(), - indexes: [{ name: 'by_owner', algorithm: 'btree', columns: ['ownerId'] }] // ❌ WRONG! -}); - -// ✅ RIGHT — indexes in OPTIONS (first argument) -export const Task = table({ - name: 'task', - public: true, - indexes: [{ name: 'by_owner', algorithm: 'btree', columns: ['ownerId'] }] -}, { - id: t.u64().primaryKey().autoInc(), - ownerId: t.identity(), - title: t.string(), - createdAt: t.timestamp(), -}); -``` - -### Column types -```typescript -t.identity() // User identity (primary key for per-user tables) -t.u64() // Unsigned 64-bit integer (use for IDs) -t.string() // Text -t.bool() // Boolean -t.timestamp() // Timestamp (use ctx.timestamp for current time) -t.scheduleAt() // For scheduled tables only - -// Product types (nested objects) — use t.object, NOT t.struct -const Point = t.object('Point', { x: t.i32(), y: t.i32() }); - -// Sum types (tagged unions) — use t.enum, NOT t.sum -const Shape = t.enum('Shape', { circle: t.i32(), rectangle: Point }); -// Values use { tag: 'circle', value: 10 } or { tag: 'rectangle', value: { x: 1, y: 2 } } - -// Modifiers -t.string().optional() // Nullable -t.u64().primaryKey() // Primary key -t.u64().primaryKey().autoInc() // Auto-increment primary key -``` - -> ⚠️ **BIGINT SYNTAX:** All `u64`, `i64`, and ID fields use JavaScript BigInt. -> - Literals: `0n`, `1n`, `100n` (NOT `0`, `1`, `100`) -> - Comparisons: `row.id === 5n` (NOT `row.id === 5`) -> - Arithmetic: `row.count + 1n` (NOT `row.count + 1`) - -### Auto-increment placeholder -```typescript -// ✅ MUST provide 0n placeholder for auto-inc fields -ctx.db.task.insert({ id: 0n, ownerId: ctx.sender, title: 'New', createdAt: ctx.timestamp }); -``` - -### Insert returns ROW, not ID -```typescript -// ❌ WRONG -const id = ctx.db.task.insert({ ... }); - -// ✅ RIGHT -const row = ctx.db.task.insert({ ... }); -const newId = row.id; // Extract .id from returned row -``` - -### Schema export (CRITICAL) -```typescript -// At end of schema.ts — schema() takes exactly ONE argument: an object -const spacetimedb = schema({ table1, table2, table3 }); -export default spacetimedb; - -// ❌ WRONG — never pass tables directly or as multiple args -schema(myTable); // WRONG! -schema(t1, t2, t3); // WRONG! -``` - ---- - -## 3) Index Access - -### TypeScript Query Patterns - -```typescript -// 1. PRIMARY KEY — use .pkColumn.find() -const user = ctx.db.user.identity.find(ctx.sender); -const msg = ctx.db.message.id.find(messageId); - -// 2. EXPLICIT INDEX — use .indexName.filter(value) -const msgs = [...ctx.db.message.message_room_id.filter(roomId)]; - -// 3. NO INDEX — use .iter() + manual filter -for (const m of ctx.db.roomMember.iter()) { - if (m.roomId === roomId) { /* ... */ } -} -``` - -### Index Definition Syntax - -```typescript -// In table OPTIONS (first argument), not columns -export const Message = table({ - name: 'message', - public: true, - indexes: [{ name: 'message_room_id', algorithm: 'btree', columns: ['roomId'] }] -}, { - id: t.u64().primaryKey().autoInc(), - roomId: t.u64(), - // ... -}); -``` - -### Naming conventions - -**Table names — automatic transformation:** -- Schema: `table({ name: 'my_messages' })` -- Access: `ctx.db.myMessages` (automatic snake_case → camelCase) - -**Index names — NO transformation, use EXACTLY as defined:** -```typescript -// Schema definition -indexes: [{ name: 'canvas_member_canvas_id', algorithm: 'btree', columns: ['canvasId'] }] - -// ❌ WRONG — don't assume camelCase transformation -ctx.db.canvasMember.canvasMember_canvas_id.filter(...) // WRONG! -ctx.db.canvasMember.canvasMemberCanvasId.filter(...) // WRONG! - -// ✅ RIGHT — use exact name from schema -ctx.db.canvasMember.canvas_member_canvas_id.filter(...) -``` - -> ⚠️ **Index names are used VERBATIM** — pick a convention (snake_case or camelCase) and stick with it. - -**Index naming pattern — use `{tableName}_{columnName}`:** -```typescript -// ✅ GOOD — unique names across entire module -indexes: [{ name: 'message_room_id', algorithm: 'btree', columns: ['roomId'] }] -indexes: [{ name: 'reaction_message_id', algorithm: 'btree', columns: ['messageId'] }] - -// ❌ BAD — will collide if multiple tables use same index name -indexes: [{ name: 'by_owner', ... }] // in Task table -indexes: [{ name: 'by_owner', ... }] // in Note table — CONFLICT! -``` - -**Client-side table names:** -- Check generated `module_bindings/index.ts` for exact export names -- Usage: `useTable(tables.MyMessages)` or `tables.myMessages` (varies by SDK version) - -### Filter vs Find -```typescript -// Filter takes VALUE directly, not object — returns iterator -const rows = [...ctx.db.task.by_owner.filter(ownerId)]; - -// Unique columns use .find() — returns single row or undefined -const row = ctx.db.player.identity.find(ctx.sender); -``` - -### ⚠️ Multi-column indexes are BROKEN -```typescript -// ❌ DON'T — causes PANIC -ctx.db.scores.by_player_level.filter(playerId); - -// ✅ DO — use single-column index + manual filter -for (const row of ctx.db.scores.by_player.filter(playerId)) { - if (row.level === targetLevel) { /* ... */ } -} -``` - ---- - -## 4) Reducers - -### Definition syntax (CRITICAL) -**Reducer name comes from the export — NOT from a string argument.** Use `reducer(params, fn)` or `reducer(fn)`. - -```typescript -import spacetimedb from './schema'; -import { t, SenderError } from 'spacetimedb/server'; - -// ✅ CORRECT — export const name = spacetimedb.reducer(params, fn) -export const reducer_name = spacetimedb.reducer({ param1: t.string(), param2: t.u64() }, (ctx, { param1, param2 }) => { - // Validation - if (!param1) throw new SenderError('param1 required'); - - // Access tables via ctx.db - const row = ctx.db.myTable.primaryKey.find(param2); - - // Mutations - ctx.db.myTable.insert({ ... }); - ctx.db.myTable.primaryKey.update({ ...row, newField: value }); - ctx.db.myTable.primaryKey.delete(param2); -}); - -// No params: export const init = spacetimedb.reducer((ctx) => { ... }); -``` - -```typescript -// ❌ WRONG — reducer('name', params, fn) does NOT exist -spacetimedb.reducer('reducer_name', { param1: t.string() }, (ctx, { param1 }) => { ... }); -``` - -### Update pattern (CRITICAL) -```typescript -// ✅ CORRECT — spread existing row, override specific fields -const existing = ctx.db.task.id.find(taskId); -if (!existing) throw new SenderError('Task not found'); -ctx.db.task.id.update({ ...existing, title: newTitle, updatedAt: ctx.timestamp }); - -// ❌ WRONG — partial update nulls out other fields! -ctx.db.task.id.update({ id: taskId, title: newTitle }); -``` - -### Delete pattern -```typescript -// Delete by primary key VALUE (not row object) -ctx.db.task.id.delete(taskId); // taskId is the u64 value -ctx.db.player.identity.delete(ctx.sender); // delete by identity -``` - -### Lifecycle hooks -```typescript -spacetimedb.clientConnected((ctx) => { - // ctx.sender is the connecting identity - // Create/update user record, set online status, etc. -}); - -spacetimedb.clientDisconnected((ctx) => { - // Clean up: set offline status, remove ephemeral data, etc. -}); -``` - -### Snake_case to camelCase conversion -- Server: `export const do_something = spacetimedb.reducer(...)` — name from export -- Client: `conn.reducers.doSomething({ ... })` - -### Object syntax required -```typescript -// ❌ WRONG - positional -conn.reducers.doSomething('value'); - -// ✅ RIGHT - object -conn.reducers.doSomething({ param: 'value' }); -``` - ---- - -## 5) Scheduled Tables - -```typescript -// 1. Define table first (scheduled: () => reducer — pass the exported reducer) -export const CleanupJob = table({ - name: 'cleanup_job', - scheduled: () => run_cleanup // reducer defined below -}, { - scheduledId: t.u64().primaryKey().autoInc(), - scheduledAt: t.scheduleAt(), - targetId: t.u64(), // Your custom data -}); - -// 2. Define scheduled reducer (receives full row as arg) -export const run_cleanup = spacetimedb.reducer({ arg: CleanupJob.rowType }, (ctx, { arg }) => { - // arg.scheduledId, arg.targetId available - // Row is auto-deleted after reducer completes -}); - -// Schedule a job -import { ScheduleAt } from 'spacetimedb'; -const futureTime = ctx.timestamp.microsSinceUnixEpoch + 60_000_000n; // 60 seconds -ctx.db.cleanupJob.insert({ - scheduledId: 0n, - scheduledAt: ScheduleAt.time(futureTime), - targetId: someId -}); - -// Cancel a job by deleting the row -ctx.db.cleanupJob.scheduledId.delete(jobId); -``` - ---- - -## 6) Timestamps - -### Server-side -```typescript -import { Timestamp, ScheduleAt } from 'spacetimedb'; - -// Current time -ctx.db.item.insert({ id: 0n, createdAt: ctx.timestamp }); - -// Future time (add microseconds) -const future = ctx.timestamp.microsSinceUnixEpoch + 300_000_000n; // 5 minutes -``` - -### Client-side (CRITICAL) -**Timestamps are objects, not numbers:** -```typescript -// ❌ WRONG -const date = new Date(row.createdAt); -const date = new Date(Number(row.createdAt / 1000n)); - -// ✅ RIGHT -const date = new Date(Number(row.createdAt.microsSinceUnixEpoch / 1000n)); -``` - -### ScheduleAt on client -```typescript -// ScheduleAt is a tagged union -if (scheduleAt.tag === 'Time') { - const date = new Date(Number(scheduleAt.value.microsSinceUnixEpoch / 1000n)); -} -``` - ---- - -## 7) Data Visibility & Subscriptions - -**`public: true` exposes ALL rows to ALL clients.** - -| Scenario | Pattern | -|----------|---------| -| Everyone sees all rows | `public: true` | -| Users see only their data | Private table + filtered subscription | - -### Subscription patterns (client-side) -```typescript -// Subscribe to ALL public tables (simplest) -conn.subscriptionBuilder().subscribeToAll(); - -// Subscribe to specific tables with SQL -conn.subscriptionBuilder().subscribe([ - 'SELECT * FROM message', - 'SELECT * FROM room WHERE is_public = true', -]); - -// Handle subscription lifecycle -conn.subscriptionBuilder() - .onApplied(() => console.log('Initial data loaded')) - .onError((e) => console.error('Subscription failed:', e)) - .subscribeToAll(); -``` - -### Private table + view pattern (RECOMMENDED) - -**Views are the recommended approach** for controlling data visibility. They provide: -- Server-side filtering (reduces network traffic) -- Real-time updates when underlying data changes -- Full control over what data clients can access - -> ⚠️ **Do NOT use Row Level Security (RLS)** — it is deprecated. - -> ⚠️ **CRITICAL:** Procedural views (views that compute results in code) can ONLY access data via index lookups, NOT `.iter()`. -> If you need a view that scans/filters across many rows (including the entire table), return a **query** built with the query builder (`ctx.from...`). - -```typescript -// Private table with index on ownerId -export const PrivateData = table( - { name: 'private_data', - indexes: [{ name: 'by_owner', algorithm: 'btree', columns: ['ownerId'] }] - }, - { - id: t.u64().primaryKey().autoInc(), - ownerId: t.identity(), - secret: t.string() - } -); - -// ❌ BAD — .iter() causes performance issues (re-evaluates on ANY row change) -spacetimedb.view( - { name: 'my_data_slow', public: true }, - t.array(PrivateData.rowType), - (ctx) => [...ctx.db.privateData.iter()] // Works but VERY slow at scale -); - -// ✅ GOOD — index lookup enables targeted invalidation -spacetimedb.view( - { name: 'my_data', public: true }, - t.array(PrivateData.rowType), - (ctx) => [...ctx.db.privateData.by_owner.filter(ctx.sender)] -); -``` - -### Query builder view pattern (can scan) - -```typescript -// Query-builder views return a query; the SQL engine maintains the result incrementally. -// This can scan the whole table if needed (e.g. leaderboard-style queries). -spacetimedb.anonymousView( - { name: 'top_players', public: true }, - t.array(Player.rowType), - (ctx) => - ctx.from.player - .where(p => p.score.gt(1000)) -); -``` - -### ViewContext vs AnonymousViewContext -```typescript -// ViewContext — has ctx.sender, result varies per user (computed per-subscriber) -spacetimedb.view({ name: 'my_items', public: true }, t.array(Item.rowType), (ctx) => { - return [...ctx.db.item.by_owner.filter(ctx.sender)]; -}); - -// AnonymousViewContext — no ctx.sender, same result for everyone (shared, better perf) -spacetimedb.anonymousView({ name: 'leaderboard', public: true }, t.array(LeaderboardRow), (ctx) => { - return [...ctx.db.player.by_score.filter(/* top scores */)]; -}); -``` - -**Views require explicit subscription:** -```typescript -conn.subscriptionBuilder().subscribe([ - 'SELECT * FROM public_table', - 'SELECT * FROM my_data', // Views need explicit SQL! -]); -``` - ---- - -## 8) React Integration - -### Key patterns -```typescript -// Memoize connectionBuilder to prevent reconnects on re-render -const builder = useMemo(() => - DbConnection.builder() - .withUri(SPACETIMEDB_URI) - .withDatabaseName(MODULE_NAME) - .withToken(localStorage.getItem('auth_token') || undefined) - .onConnect(onConnect) - .onConnectError(onConnectError), - [] // Empty deps - only create once -); - -// useTable returns tuple [rows, isLoading] -const [rows, isLoading] = useTable(tables.myTable); - -// Compare identities using toHexString() -const isOwner = row.ownerId.toHexString() === myIdentity.toHexString(); -``` - ---- - -## 9) Procedures (Beta) - -**Procedures are for side effects (HTTP requests, etc.) that reducers can't do.** - -⚠️ Procedures are currently in beta. API may change. - -### Defining a procedure -**Procedure name comes from the export — NOT from a string argument.** Use `procedure(params, ret, fn)` or `procedure(ret, fn)`. - -```typescript -// ✅ CORRECT — export const name = spacetimedb.procedure(params, ret, fn) -export const fetch_external_data = spacetimedb.procedure( - { url: t.string() }, - t.string(), // return type - (ctx, { url }) => { - const response = ctx.http.fetch(url); - return response.text(); - } -); -``` - -### Database access in procedures - -⚠️ **CRITICAL: Procedures don't have `ctx.db`. Use `ctx.withTx()` for database access.** - -```typescript -spacetimedb.procedure({ url: t.string() }, t.unit(), (ctx, { url }) => { - // Fetch external data (outside transaction) - const response = ctx.http.fetch(url); - const data = response.text(); - - // ❌ WRONG — ctx.db doesn't exist in procedures - ctx.db.myTable.insert({ ... }); - - // ✅ RIGHT — use ctx.withTx() for database access - ctx.withTx(tx => { - tx.db.myTable.insert({ - id: 0n, - content: data, - fetchedAt: tx.timestamp, - fetchedBy: tx.sender, - }); - }); - - return {}; -}); -``` - -### Key differences from reducers -| Reducers | Procedures | -|----------|------------| -| `ctx.db` available directly | Must use `ctx.withTx(tx => tx.db...)` | -| Automatic transaction | Manual transaction management | -| No HTTP/network | `ctx.http.fetch()` available | -| No return values to caller | Can return data to caller | - ---- - -## 10) Project Structure - -### Server (`backend/spacetimedb/`) -``` -src/schema.ts → Tables, export spacetimedb -src/index.ts → Reducers, lifecycle, import schema -package.json → { "type": "module", "dependencies": { "spacetimedb": "^1.11.0" } } -tsconfig.json → Standard config -``` - -### Avoiding circular imports -``` -schema.ts → defines tables AND exports spacetimedb -index.ts → imports spacetimedb from ./schema, defines reducers -``` - -### Client (`client/`) -``` -src/module_bindings/ → Generated (spacetime generate) -src/main.tsx → Provider, connection setup -src/App.tsx → UI components -src/config.ts → MODULE_NAME, SPACETIMEDB_URI -``` - ---- - -## 11) Commands - -```bash -# Start local server -spacetime start - -# Publish module -spacetime publish --module-path - -# Clear database and republish -spacetime publish --clear-database -y --module-path - -# Generate bindings -spacetime generate --lang typescript --out-dir /src/module_bindings --module-path - -# View logs -spacetime logs -``` - ---- - -## 12) Hard Requirements - -**TypeScript-specific:** - -1. **`schema({ table })`** — takes exactly one object; never `schema(table)` or `schema(t1, t2, t3)` -2. **Reducer/procedure names from exports** — `export const name = spacetimedb.reducer(params, fn)`; never `reducer('name', ...)` -3. **Reducer calls use object syntax** — `{ param: 'value' }` not positional args -4. **Import `DbConnection` from `./module_bindings`** — not from `spacetimedb` -5. **DO NOT edit generated bindings** — regenerate with `spacetime generate` -6. **Indexes go in OPTIONS (1st arg)** — not in COLUMNS (2nd arg) of `table()` -7. **Use BigInt for u64/i64 fields** — `0n`, `1n`, not `0`, `1` -8. **Reducers are transactional** — they do not return data -9. **Reducers must be deterministic** — no filesystem, network, timers, random -10. **Views should use index lookups** — `.iter()` causes severe performance issues -11. **Procedures need `ctx.withTx()`** — `ctx.db` doesn't exist in procedures -12. **Sum type values** — use `{ tag: 'variant', value: payload }` not `{ variant: payload }` diff --git a/docs/static/ai-rules/spacetimedb.mdc b/docs/static/ai-rules/spacetimedb.mdc deleted file mode 100644 index 31cb45694cb..00000000000 --- a/docs/static/ai-rules/spacetimedb.mdc +++ /dev/null @@ -1,116 +0,0 @@ ---- -description: "⛔ MANDATORY: Core SpacetimeDB concepts (all languages)." -globs: **/*.ts,**/*.tsx,**/*.js,**/*.jsx,**/*.rs,**/*.cs -alwaysApply: true ---- -# SpacetimeDB Rules (All Languages) - -## Migrating from 1.0 to 2.0? - -**If you are migrating existing SpacetimeDB 1.0 code to 2.0, apply `spacetimedb-migration-2.0.mdc` first.** It documents breaking changes (reducer callbacks → event tables, `name`→`accessor`, `sender()` method, etc.) and should be considered before other rules. - ---- - -## Language-Specific Rules - -| Language | Rule File | -|----------|-----------| -| **TypeScript/React** | `spacetimedb-typescript.mdc` (MANDATORY) | -| **Rust** | `spacetimedb-rust.mdc` (MANDATORY) | -| **C#** | `spacetimedb-csharp.mdc` (MANDATORY) | -| **Migrating 1.0 → 2.0** | `spacetimedb-migration-2.0.mdc` | - ---- - -## Core Concepts - -1. **Reducers are transactional** — they do not return data to callers -2. **Reducers must be deterministic** — no filesystem, network, timers, or random -3. **Read data via tables/subscriptions** — not reducer return values -4. **Auto-increment IDs are not sequential** — gaps are normal, don't use for ordering -5. **`ctx.sender` is the authenticated principal** — never trust identity args - ---- - -## Feature Implementation Checklist - -When implementing a feature that spans backend and client: - -1. **Backend:** Define table(s) to store the data -2. **Backend:** Define reducer(s) to mutate the data -3. **Client:** Subscribe to the table(s) -4. **Client:** Call the reducer(s) from UI — **don't forget this step!** -5. **Client:** Render the data from the table(s) - -**Common mistake:** Building backend tables/reducers but forgetting to wire up the client to call them. - ---- - -## Index System - -SpacetimeDB automatically creates indexes for: -- Primary key columns -- Columns marked as unique - -You can add explicit indexes on non-unique columns for query performance. - -**Index names must be unique across your entire module (all tables).** If two tables have indexes with the same declared name → conflict error. - -**Schema ↔ Code coupling:** -- Your query code references indexes by name -- If you add/remove/rename an index in the schema, update all code that uses it -- Removing an index without updating queries causes runtime errors - ---- - -## Commands - -```bash -# Login to allow remote database deployment e.g. to maincloud -spacetime login - -# Start local SpacetimeDB -spacetime start - -# Publish module -spacetime publish --module-path - -# Clear and republish -spacetime publish --clear-database -y --module-path - -# Generate client bindings -spacetime generate --lang --out-dir --module-path - -# View logs -spacetime logs -``` - ---- - -## Deployment - -- Maincloud is the spacetimedb hosted cloud and the default location for module publishing -- The default server marked by *** in `spacetime server list` should be used when publishing -- If the default server is maincloud you should publish to maincloud -- Publishing to maincloud is free of charge -- When publishing to maincloud the database dashboard will be at the url: https://spacetimedb.com/@/ -- The database owner can view utilization and performance metrics on the dashboard - ---- - -## Debugging Checklist - -1. Is SpacetimeDB server running? (`spacetime start`) -2. Is the module published? (`spacetime publish`) -3. Are client bindings generated? (`spacetime generate`) -4. Check server logs for errors (`spacetime logs `) -5. **Is the reducer actually being called from the client?** - ---- - -## Editing Behavior - -- Make the smallest change necessary -- Do NOT touch unrelated files, configs, or dependencies -- Do NOT invent new SpacetimeDB APIs — use only what exists in docs or this repo -- Do NOT add restrictions the prompt didn't ask for — if "users can do X", implement X for all users diff --git a/docs/static/llms.md b/docs/static/llms.md index c605e2b6bcd..2f17b8fd21a 100644 --- a/docs/static/llms.md +++ b/docs/static/llms.md @@ -1,2946 +1,179 @@ # SpacetimeDB -> SpacetimeDB is a fully-featured relational database system that integrates -> application logic directly within the database, eliminating the need for -> separate web or game servers. It supports multiple programming languages, -> including C# and Rust, allowing developers to write and deploy entire -> applications as a single binary. It is optimized for high-throughput and low -> latency multiplayer applications like multiplayer games. +> SpacetimeDB is a database that lets you write your entire application as a database module. Server logic runs inside the database as WebAssembly. Clients subscribe to queries and get real-time updates over WebSocket. No separate server needed. -Users upload their application logic to run inside SpacetimeDB as a WebAssembly -module. There are three main features of SpacetimeDB: tables, reducers, and -subscription queries. Tables are relational database tables like you would find -in a database like Postgres. Reducers are atomic, transactional, RPC functions -that are defined in the WebAssembly module which can be called by clients. -Subscription queries are SQL queries which are made over a WebSocket connection -which are initially evaluated by SpacetimeDB and then incrementally evaluated -sending changes to the query result over the WebSocket. -All data in the tables are stored in memory, but are persisted to the disk via a -Write-Ahead Log (WAL) called the Commitlog. All tables are persistent in -SpacetimeDB. +## docs -SpacetimeDB allows users to code generate type-safe client libraries based on -the tables, types, and reducers defined in their module. Subscription queries -allows the client SDK to store a partial, live updating, replica of the servers -state. This makes reading database state on the client extremely low-latency. +Installation -Authentication is implemented in SpacetimeDB using the OpenID Connect protocol. -An OpenID Connect token with a valid `iss`/`sub` pair constitutes a unique and -authenticable SpacetimeDB identity. SpacetimeDB uses the `Identity` type as an -identifier for all such identities. `Identity` is computed from the `iss`/`sub` -pair using the following algorithm: +- [Getting Started](/docs/): Installation -1. Concatenate the issuer and subject with a pipe symbol (`|`). -2. Perform the first BLAKE3 hash on the concatenated string. -3. Get the first 26 bytes of the hash (let's call this `idHash`). -4. Create a 28-byte sequence by concatenating the bytes `0xc2`, `0x00`, and `idHash`. -5. Compute the BLAKE3 hash of the 28-byte sequence from step 4 (let's call this `checksumHash`). -6. Construct the final 32-byte `Identity` by concatenating: the two prefix bytes (`0xc2`, `0x00`), the first 4 bytes of `checksumHash`, and the 26-byte `idHash`. -7. This final 32-byte value is typically represented as a hexadecimal string. +### bsatn -```ascii -Byte Index: | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | ... | 31 | - +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ -Contents: | 0xc2| 0x00| Checksum Hash (4 bytes) | ID Hash (26 bytes) | - +-----+-----+-------------------------+---------------------+ - (First 4 bytes of (First 26 bytes of - BLAKE3(0xc200 || idHash)) BLAKE3(iss|sub)) -``` +The Spacetime Algebraic Type Notation binary (BSATN) format defines -This allows SpacetimeDB to easily integrate with OIDC authentication -providers like FirebaseAuth, Auth0, or SuperTokens. +- [BSATN Data Format](/docs/bsatn): The Spacetime Algebraic Type Notation binary (BSATN) format defines -Clockwork Labs, the developers of SpacetimeDB, offers three products: +### cli-reference -1. SpacetimeDB Standalone: a source available (Business Source License), single node, self-hosted version -2. SpacetimeDB Maincloud: a hosted, managed-service, serverless cluster -3. SpacetimeDB Enterprise: a closed-source, clusterized version of SpacetimeDB which can be licensed for on-prem hosting or dedicated hosting +This document contains the help content for the spacetime command-line program. -## Documentation Directory +- [Command-Line Help for spacetime](/docs/cli-reference): This document contains the help content for the spacetime command-line program. +- [spacetime.json Configuration File](/docs/cli-reference/spacetime-json): The spacetime.json file defines project-level configuration for the SpacetimeDB CLI. It eliminates repetitive CLI flags and enables multi-target workflows such as publishing multiple databases or generating bindings for multiple languages from a single project. +- [Standalone Configuration](/docs/cli-reference/standalone-config): A local database instance (as started by spacetime start) can be configured in /config.toml, where {data-dir} is the database's data directory. This directory is printed when you run spacetime start: -### Getting Started -- [What is SpacetimeDB](/intro/what-is-spacetimedb) - Overview and core concepts -- [Key Architecture](/intro/key-architecture) - How SpacetimeDB works -- [Language Support](/intro/language-support) - Supported languages and SDKs -- [FAQ](/intro/faq) - Frequently asked questions +### clients -### Quickstarts -- [React Quickstart](/quickstarts/react) - Get started with React + TypeScript -- [TypeScript Quickstart](/quickstarts/typescript) - TypeScript server module -- [Rust Quickstart](/quickstarts/rust) - Rust server module -- [C# Quickstart](/quickstarts/c-sharp) - C# server module +The SpacetimeDB Client SDKs provide a comprehensive interface for building applications that connect to SpacetimeDB databases. Client applications can query data, invoke server-side functions, and receive real-time updates as the database state changes. -### Core Concepts -- [Databases](/databases) - Database modules overview -- [Tables](/tables) - Defining and working with tables - - [Columns](/tables/columns) - Column types and definitions - - [Indexes](/tables/indexes) - Creating and using indexes - - [Scheduled Tables](/tables/scheduled-tables) - Time-based scheduling - - [Access Permissions](/tables/access-permissions) - Table visibility (public/private) -- [Functions](/functions) - Server-side logic - - [Reducers](/functions/reducers) - Transactional RPC functions - - [Reducer Context](/functions/reducers/reducer-context) - ctx.db, ctx.sender, etc. - - [Lifecycle Reducers](/functions/reducers/lifecycle) - init, client_connected, client_disconnected - - [Error Handling](/functions/reducers/error-handling) - Handling errors in reducers - - [Procedures](/functions/procedures) - Non-transactional functions with side effects - - [Views](/functions/views) - Computed data views -- [Subscriptions](/subscriptions) - Real-time data synchronization - - [Subscription Semantics](/subscriptions/semantics) - How subscriptions work -- [Client SDKs](/sdks) - Client-side integration - - [Code Generation](/sdks/codegen) - Generating type-safe bindings +- [Clients](/docs/clients): The SpacetimeDB Client SDKs provide a comprehensive interface for building applications that connect to SpacetimeDB databases. Client applications can query data, invoke server-side functions, and receive real-time updates as the database state changes. +- [SDK API Overview](/docs/clients/api): The SpacetimeDB client SDKs provide a comprehensive API for interacting with your database. After generating client bindings and establishing a connection, you can query data, invoke server functions, and observe real-time changes. +- [C# Reference](/docs/clients/c-sharp): The SpacetimeDB client for C# contains all the tools you need to build native clients for SpacetimeDB modules using C#. +- [Generating Client Bindings](/docs/clients/codegen): Before you can interact with a SpacetimeDB database from a client application, you must generate client bindings for your module. These bindings create type-safe interfaces that allow your client to query tables, invoke reducers, call procedures, and subscribe to tables, and/or views. +- [Connecting to SpacetimeDB](/docs/clients/connection): After generating client bindings for your module, you can establish a connection to your SpacetimeDB database from your client application. The DbConnection type provides a persistent WebSocket connection that enables real-time communication with the server. +- [Rust Reference](/docs/clients/rust): The SpacetimeDB client SDK for Rust contains all the tools you need to build native clients for SpacetimeDB modules using Rust. +- [Subscriptions](/docs/clients/subscriptions): Subscriptions replicate database rows to your client in real-time. When you subscribe to a query, SpacetimeDB sends you the matching rows immediately and then pushes updates whenever those rows change. +- [Subscription Semantics](/docs/clients/subscriptions/semantics): This document describes the subscription semantics maintained by the SpacetimeDB host over WebSocket connections. These semantics outline message ordering guarantees, subscription handling, transaction updates, and client cache consistency. +- [TypeScript Reference](/docs/clients/typescript): The SpacetimeDB client SDK for TypeScript contains all the tools you need to build clients for SpacetimeDB modules using Typescript, either in the browser or with NodeJS. +- [Unreal Reference](/docs/clients/unreal): The SpacetimeDB client for Unreal Engine contains all the tools you need to build native clients for SpacetimeDB modules using C++ and Blueprint. -### Development -- [spacetime dev](/databases/developing) - Interactive development mode -- [Building & Publishing](/databases/building-publishing) - Deploying modules -- [Cheat Sheet](/databases/cheat-sheet) - Quick reference for common operations -- [Automatic Migrations](/databases/automatic-migrations) - Schema migration handling +### core-concepts -### Deployment -- [Deploy to Maincloud](/how-to/deploy/maincloud) - Hosted deployment -- [Self-Hosting](/how-to/deploy/self-hosting) - Run your own server +This section covers the fundamental concepts you need to understand to build applications with SpacetimeDB. -### Reference -- [CLI Reference](/cli-reference) - Command-line tool documentation -- [SQL Reference](/reference/sql) - SQL query syntax -- [HTTP API](/http/database) - REST API reference +- [Core Concepts](/docs/core-concepts): This section covers the fundamental concepts you need to understand to build applications with SpacetimeDB. +- [Authentication](/docs/core-concepts/authentication): SpacetimeDB modules are exposed to the open internet and anyone can connect to +- [Auth0](/docs/core-concepts/authentication/Auth0): This guilde will walk you through integrating Auth0 authentication with your SpacetimeDB application. +- [Clerk](/docs/core-concepts/authentication/Clerk): This guide will walk you through integrating Clerk authentication with your SpacetimeDB React application. You will configure a Clerk application, obtain a JWT from Clerk, and pass it to your SpacetimeDB connection as the authentication token. +- [Overview](/docs/core-concepts/authentication/spacetimeauth/): SpacetimeAuth is currently in beta, some features may not be available yet or +- [Configuring your project](/docs/core-concepts/authentication/spacetimeauth/configuring-a-project): SpacetimeAuth is currently in beta, some features may not be available yet or may change in the future. You might encounter bugs or issues while using the service. Please report any problems you encounter to help us improve SpacetimeAuth. +- [Creating a project](/docs/core-concepts/authentication/spacetimeauth/creating-a-project): SpacetimeAuth is currently in beta, some features may not be available yet or may change in the future. You might encounter bugs or issues while using the service. Please report any problems you encounter to help us improve SpacetimeAuth. +- [React Integration](/docs/core-concepts/authentication/spacetimeauth/react-integration): SpacetimeAuth is currently in beta, some features may not be available yet or may change in the future. You might encounter bugs or issues while using the service. Please report any problems you encounter to help us improve SpacetimeAuth. +- [Testing](/docs/core-concepts/authentication/spacetimeauth/testing): SpacetimeAuth is currently in beta, some features may not be available yet or may change in the future. You might encounter bugs or issues while using the service. Please report any problems you encounter to help us improve SpacetimeAuth. +- [Using Auth Claims](/docs/core-concepts/authentication/usage): SpacetimeDB allows you to easily access authentication (auth) claims embedded in -### AI Assistant Rules +### databases -**IMPORTANT:** Before writing SpacetimeDB code, consult the language-specific rules files. These contain critical information about hallucinated APIs, common mistakes, and correct patterns: +A module is a collection of functions and schema definitions, which can be written in TypeScript, C#, Rust, or C++. Modules define the structure of your database and the server-side logic that processes and handles client requests. -| Language | Rules | -|----------|-------| -| All Languages | [spacetimedb.mdc](https://spacetimedb.com/docs/ai-rules/spacetimedb.mdc) | -| TypeScript | [spacetimedb-typescript.mdc](https://spacetimedb.com/docs/ai-rules/spacetimedb-typescript.mdc) | -| Rust | [spacetimedb-rust.mdc](https://spacetimedb.com/docs/ai-rules/spacetimedb-rust.mdc) | -| C# | [spacetimedb-csharp.mdc](https://spacetimedb.com/docs/ai-rules/spacetimedb-csharp.mdc) | -## Basic Project Workflow +- [The Database Module](/docs/databases): A module is a collection of functions and schema definitions, which can be written in TypeScript, C#, Rust, or C++. Modules define the structure of your database and the server-side logic that processes and handles client requests. +- [Automatic Migrations](/docs/databases/automatic-migrations): When you republish a module to an existing database using spacetime publish , SpacetimeDB attempts to automatically migrate your database schema to match the new module definition. This allows you to update your module code and redeploy without losing existing data, as long as the changes are compatible. +- [spacetime publish](/docs/databases/building-publishing): This guide covers how to build and publish your SpacetimeDB module. +- [Cheat Sheet](/docs/databases/cheat-sheet): Quick reference for SpacetimeDB module syntax across Rust, C#, and TypeScript. +- [spacetime dev](/docs/databases/developing): This guide covers how to create a new SpacetimeDB database module project. +- [Incremental Migrations](/docs/databases/incremental-migrations): SpacetimeDB does not provide built-in support for general schema-modifying migrations. It does, however, allow adding new tables, and changing reducers' definitions in arbitrary ways. It's possible to run general migrations using an external tool, but this is tedious, necessitates downtime, and imposes the requirement that you update all your clients at the same time as publishing your new module version. +- [Transactions and Atomicity](/docs/databases/transactions-atomicity): SpacetimeDB provides strong transactional guarantees for all database operations. Every reducer runs inside a database transaction, ensuring your data remains consistent and reliable even under concurrent load. -Getting started with SpacetimeDB involves a few key steps: +### functions -1. **Install SpacetimeDB:** Install the `spacetime` CLI tool for your operating system. This tool is used for managing modules, databases, and local instances. - - **macOS:** - ```bash - curl -sSf https://install.spacetimedb.com | sh - ``` - - **Windows (PowerShell):** - ```powershell - iwr https://windows.spacetimedb.com -useb | iex - ``` - - **Linux:** - ```bash - curl -sSf https://install.spacetimedb.com | sh - ``` - - **Docker (to run the server):** - ```bash - # This command starts a SpacetimeDB server instance in Docker - docker run --rm --pull always -p 3000:3000 clockworklabs/spacetime start - # Note: While the CLI can be installed separately (see above), you can also execute - # CLI commands *within* the running Docker container (e.g., using `docker exec`) - # or use the image as a base for a custom image containing your module management tools. - ``` - - **Docker (to execute CLI commands directly):** - You can also use the Docker image to run `spacetime` CLI commands without installing the CLI locally. For commands that operate on local files (like `build`, `publish`, `generate`), this involves mounting your project directory into the container. For commands that only interact with a database instance (like `sql`, `status`), mounting is typically not required, but network access to the database is. +| Property / Characteristic | Reducers | Procedures | Views | - ```bash - # Example: Build a module located in the current directory (.) - # Mount current dir to /module inside container, set working dir to /module - docker run --rm -v "$(pwd):/module" -w /module clockworklabs/spacetime build --module-path . +- [Functions](/docs/functions): | Property / Characteristic | Reducers | Procedures | Views | +- [Procedures](/docs/functions/procedures): A procedure is a function exported by a database, similar to a reducer. +- [Overview](/docs/functions/reducers): Reducers are functions that modify database state in response to client requests or system events. They are the only way to mutate tables in SpacetimeDB - all database changes must go through reducers. +- [Error Handling](/docs/functions/reducers/error-handling): Error Handling +- [Lifecycle Reducers](/docs/functions/reducers/lifecycle): Special reducers handle system events during the database lifecycle. +- [Reducer Context](/docs/functions/reducers/reducer-context): Every reducer receives a special context parameter as its first argument. This context provides read-write access to the database, information about the caller, and additional utilities like random number generation. +- [Views](/docs/functions/views): Views are read-only functions that compute and return results from your tables. Unlike reducers, views do not modify database state - they only query and return data. Views are useful for computing derived data, aggregations, or joining multiple tables before sending results to clients. - # Example: Publish the module after building - # Assumes a local server is running (or use --host for Maincloud/other) - docker run --rm -v "$(pwd):/module" -w /module --network host clockworklabs/spacetime publish --module-path . my-database-name - # Note: `--network host` is often needed to connect to a local server from the container. - ``` +### how-to - - For more details or troubleshooting, see the official [Getting Started Guide](https://spacetimedb.com/docs/getting-started) and [Installation Page](https://spacetimedb.com/install). +- [Maincloud](/docs/how-to/deploy/maincloud): Maincloud is SpacetimeDB's fully managed serverless platform. It handles infrastructure, scaling, replication, and backups so you can focus on building your application. Maincloud scales to zero when your database is idle, so you only pay for what you use. +- [Self-hosting](/docs/how-to/deploy/self-hosting): This tutorial will guide you through setting up SpacetimeDB on an Ubuntu 24.04 server, securing it with HTTPS using Nginx and Let's Encrypt, and configuring a systemd service to keep it running. +- [Logging](/docs/how-to/logging): SpacetimeDB provides logging capabilities for debugging and monitoring your modules. Log messages are private to the database owner and are not visible to clients. +- [PostgreSQL Wire Protocol (PGWire)](/docs/how-to/pg-wire): SpacetimeDB supports the PostgreSQL wire protocol (PGWire), +- [Reject Client Connections](/docs/how-to/reject-client-connections): SpacetimeDB provides a way to disconnect a client during a client connection attempt. +- [Row Level Security](/docs/how-to/rls): Row Level Security is an experimental, unstable feature. The API may change or be removed in future releases. +- [Azure Self-Hosted VMs + Key Rotation & Key Vault](/docs/how-to/self-hosted-key-rotation): This guide explains how JWT signing key rotation works in self-hosted SpacetimeDB and how to avoid breaking spacetime publish during rotation. -1.b **Log In (If Necessary):** If you plan to publish to a server that requires authentication (like the public Maincloud at `maincloud.spacetimedb.com`), you generally need to log in first using `spacetime login`. This associates your actions with your global SpacetimeDB identity (e.g., linked to your spacetimedb.com account). -`bash - spacetime login - # Follow the prompts to authenticate via web browser - ` -If you attempt commands like `publish` against an authenticated server without being logged in, the CLI will prompt you: `You are not logged in. Would you like to log in with spacetimedb.com? [y/N]`. -_ Choosing `y` initiates the standard browser login flow. -_ Choosing `n` proceeds without a global login for this operation. The CLI will confirm `We have logged in directly to your target server. WARNING: This login will NOT work for any other servers.` This uses or creates a server-issued identity specific to that server (see Step 5). +### http - In general, using `spacetime login` (which authenticates via spacetimedb.com) is recommended, as the resulting identities are portable across different SpacetimeDB servers. +- [Authorization](/docs/http/authorization): Generating identities and tokens +- [/v1/database](/docs/http/database): The HTTP endpoints in /v1/database allow clients to interact with Spacetime databases in a variety of ways, including retrieving information, creating and deleting databases, invoking reducers and evaluating SQL queries. +- [/v1/identity](/docs/http/identity): The HTTP endpoints in /v1/identity allow clients to generate and manage Spacetime public identities and private tokens. -2. **Initialize Server Module:** Create a new directory for your project and use the CLI to initialize the server module structure: - ```bash - # For Rust - spacetime init --lang rust --project-path my_server_module my-server-module - # For C# - spacetime init --lang csharp --project-path my_server_module my-server-module - ``` - :::note C# Project Filename Convention (SpacetimeDB CLI) - The `spacetime` CLI tool (particularly `publish` and `build`) follows a convention and often expects the C# project file (`.csproj`) to be named `StdbModule.csproj`, matching the default generated by `spacetime init`. This **is** a requirement of the SpacetimeDB tool itself (due to how it locates build artifacts), not the underlying .NET build system. This is a known issue tracked [here](https://github.com/clockworklabs/SpacetimeDB/issues/2475). If you encounter issues where the build succeeds but publishing fails (e.g., "couldn't find the output file" or silent failures after build), ensure your `.csproj` file is named `StdbModule.csproj` within your module's directory. - ::: -3. **Define Schema & Logic:** Edit the generated module code (`lib.rs` for Rust, `Lib.cs` for C#) to define your custom types (`[SpacetimeType]`/`[Type]`), database tables (`#[table]`/`[Table]`), and reducers (`#[reducer]`/`[Reducer]`). -4. **Build Module:** Compile your module code into WebAssembly using the CLI: - ```bash - # Run from the directory containing your module folder - spacetime build --module-path my_server_module - ``` - :::note C# Build Prerequisite (.NET SDK) - Building a **C# module** (on any platform: Windows, macOS, Linux) requires the .NET SDK to be installed. If the build fails with an error mentioning `dotnet workload list` or `No .NET SDKs were found`, you need to install the SDK first. Download and install the **.NET 8 SDK** specifically from the official Microsoft website: [https://dotnet.microsoft.com/download](https://dotnet.microsoft.com/download). Newer versions (like .NET 9) are not currently supported for building SpacetimeDB modules, although they can be installed alongside .NET 8 without conflicting. - ::: -5. **Publish Module:** Deploy your compiled module to a SpacetimeDB instance (either a local one started with `spacetime start` or the managed Maincloud). Publishing creates or updates a database associated with your module. - - Providing a `[name|identity]` for the database is **optional**. If omitted, a nameless database will be created and assigned a unique `Identity` automatically. If providing a _name_, it must match the regex `^[a-z0-9]+(-[a-z0-9]+)*$`. - - By default (`--module-path`), it builds the module before publishing. Use `--bin-path ` to publish a pre-compiled WASM instead. - - Use `-s, --server ` to specify the target instance (e.g., `maincloud.spacetimedb.com` or the nickname `maincloud`). If omitted, it targets a local instance or uses your configured default (check with `spacetime server list`). - - Use `-c, --delete-data` when updating an existing database identity to destroy all existing data first. +### intro - :::note Server-Issued Identities - If you publish without being logged in (and choose to proceed without a global login when prompted), the SpacetimeDB server instance will generate or use a unique "server-issued identity" for the database operation. This identity is specific to that server instance. Its issuer (`iss`) is specifically `http://localhost`, and its subject (`sub`) will be a generated UUIDv4. This differs from the global identities derived from OIDC providers (like spacetimedb.com) when you use `spacetime login`. The token associated with this identity is signed by the issuing server, and the signature will be considered invalid if the token is presented to any other SpacetimeDB server instance. - ::: +- [FAQ](/docs/intro/faq): General +- [Key Architecture](/docs/intro/key-architecture): Host +- [Language Support](/docs/intro/language-support): Server Database Modules +- [What is SpacetimeDB?](/docs/intro/what-is-spacetimedb): SpacetimeDB is a database that is also a server. +- [The Zen of SpacetimeDB](/docs/intro/zen): SpacetimeDB is built on 5 core principles. As you embrace these simple principles, you will find your troubles simply melt away. These principles guide both how we develop SpacetimeDB and how you should think about building applications with it. - ```bash - # Build and publish from source to 'my-database-name' on the default server - spacetime publish --module-path my_server_module my-database-name +### quickstarts - # Example: Publish a pre-compiled wasm to Maincloud using its nickname, clearing existing data - spacetime publish --bin-path ./my_module/target/wasm32-wasi/debug/my_module.wasm -s maincloud -c my-cloud-db-identity - ``` +- [Angular Quickstart](/docs/quickstarts/angular): Get a SpacetimeDB Angular app running in under 5 minutes. +- [Browser Quickstart](/docs/quickstarts/browser): Get a SpacetimeDB app running in the browser with inline JavaScript. +- [Bun Quickstart](/docs/quickstarts/bun): Get a SpacetimeDB Bun app running in under 5 minutes. +- [C++ Quickstart](/docs/quickstarts/c-plus-plus): Get a SpacetimeDB C++ app running in under 5 minutes. +- [C# Quickstart](/docs/quickstarts/c-sharp): Get a SpacetimeDB C# app running in under 5 minutes. +- [Deno Quickstart](/docs/quickstarts/deno): Get a SpacetimeDB Deno app running in under 5 minutes. +- [Next.js Quickstart](/docs/quickstarts/nextjs): Get a SpacetimeDB Next.js app running in under 5 minutes. +- [Node.js Quickstart](/docs/quickstarts/nodejs): Get a SpacetimeDB Node.js app running in under 5 minutes. +- [Nuxt Quickstart](/docs/quickstarts/nuxt): Get a SpacetimeDB Nuxt app running in under 5 minutes. +- [React Quickstart](/docs/quickstarts/react): Get a SpacetimeDB React app running in under 5 minutes. +- [Remix Quickstart](/docs/quickstarts/remix): Get a SpacetimeDB Remix app running in under 5 minutes. +- [Rust Quickstart](/docs/quickstarts/rust): Get a SpacetimeDB Rust app running in under 5 minutes. +- [Svelte Quickstart](/docs/quickstarts/svelte): Get a SpacetimeDB Svelte app running in under 5 minutes. +- [TanStack Start Quickstart](/docs/quickstarts/tanstack): Get a SpacetimeDB app with TanStack Start running in under 5 minutes. +- [TypeScript Quickstart](/docs/quickstarts/typescript): Get a SpacetimeDB TypeScript app running in under 5 minutes. +- [Vue Quickstart](/docs/quickstarts/vue): Get a SpacetimeDB Vue app running in under 5 minutes. -6. **List Databases (Optional):** Use `spacetime list` to see the databases associated with your logged-in identity on the target server (defaults to your configured server). This is helpful to find the `Identity` of databases, especially unnamed ones. +### reference - ```bash - # List databases on the default server - spacetime list +- [SQL Reference](/docs/reference/sql): SpacetimeDB supports two subsets of SQL: - # List databases on Maincloud - # spacetime list -s maincloud - ``` +### resources + +Guides, references, and tools to help you build with SpacetimeDB. + +- [Developer Resources](/docs/resources): Guides, references, and tools to help you build with SpacetimeDB. + +### sats-json -7. **Generate Client Bindings:** Create type-safe client code based on your module's definitions. - This command inspects your compiled module's schema (tables, types, reducers) and generates corresponding code (classes, structs, functions) for your target client language. This allows you to interact with your SpacetimeDB module in a type-safe way on the client. - ```bash - # For Rust client (output to src/module_bindings) - spacetime generate --lang rust --out-dir path/to/client/src/module_bindings --module-path my_server_module - # For C# client (output to module_bindings directory) - spacetime generate --lang csharp --out-dir path/to/client/module_bindings --module-path my_server_module - ``` -8. **Develop Client:** Create your client application (e.g., Rust binary, C# console app, Unity game). Use the generated bindings and the appropriate client SDK to: - - Connect to the database (`my-database-name`). - - Subscribe to data in public tables. - - Register callbacks to react to data changes. - - Call reducers defined in your module. -9. **Run:** Start your SpacetimeDB instance (if local or Docker), then run your client application. +The Spacetime Algebraic Type System JSON format defines how Spacetime AlgebraicTypes and AlgebraicValues are encoded as JSON. Algebraic types and values are JSON-encoded for transport via the HTTP Databases API and the WebSocket text protocol. Note that SATS-JSON is not self-describing, and so a SATS value represented in JSON requires knowing the value's schema to meaningfully understand it - for example, it's not possible to tell whether a JSON object with a single field is a ProductValue with one element or a SumValue. -10. **Inspect Data (Optional):** Use the `spacetime sql` command to run SQL queries directly against your database to view or verify data. +- [SATS-JSON Data Format](/docs/sats-json): The Spacetime Algebraic Type System JSON format defines how Spacetime AlgebraicTypes and AlgebraicValues are encoded as JSON. Algebraic types and values are JSON-encoded for transport via the HTTP Databases API and the WebSocket text protocol. Note that SATS-JSON is not self-describing, and so a SATS value represented in JSON requires knowing the value's schema to meaningfully understand it - for example, it's not possible to tell whether a JSON object with a single field is a ProductValue with one element or a SumValue. - ```bash - # Query all data from the 'player_state' table in 'my-database-name' - # Note: Table names are case-sensitive (match your definition) - spacetime sql my-database-name "SELECT * FROM PlayerState" +### tables - # Use --interactive for a SQL prompt - # spacetime sql --interactive my-database-name - ``` +Tables are the way to store data in SpacetimeDB. All data in SpacetimeDB is stored in memory for extremely low latency and high throughput access. SpacetimeDB also automatically persists all data to disk. -11. **View Logs (Optional):** Use the `spacetime logs` command to view logs generated by your module's reducers (e.g., using `log::info!` in Rust or `Log.Info()` in C#). +- [Tables](/docs/tables): Tables are the way to store data in SpacetimeDB. All data in SpacetimeDB is stored in memory for extremely low latency and high throughput access. SpacetimeDB also automatically persists all data to disk. +- [Table Access Permissions](/docs/tables/access-permissions): SpacetimeDB controls data access through table visibility and context-based permissions. Tables can be public or private, and different execution contexts (reducers, views, clients) have different levels of access. +- [Auto-Increment](/docs/tables/auto-increment): Auto-increment columns automatically generate unique integer values for new rows. When you insert a row with a zero value in an auto-increment column, SpacetimeDB assigns the next value from an internal sequence. +- [Column Types](/docs/tables/column-types): Columns define the structure of your tables. SpacetimeDB supports primitive types, composite types for complex data, and special types for database-specific functionality. +- [Constraints](/docs/tables/constraints): Constraints enforce data integrity rules on your tables. SpacetimeDB supports primary key and unique constraints. +- [Default Values](/docs/tables/default-values): Default values allow you to add new columns to existing tables during automatic migrations. When you republish a module with a new column that has a default value, existing rows are automatically populated with that default. +- [Event Tables](/docs/tables/event-tables): In many applications, particularly games and real-time systems, modules need to notify clients about things that happened without storing that information permanently. A combat system might need to tell clients "entity X took 50 damage" so they can display a floating damage number, but there is no reason to keep that record in the database after the moment has passed. +- [File Storage](/docs/tables/file-storage): SpacetimeDB can store binary data directly in table columns, making it suitable for files, images, and other blobs that need to participate in transactions and subscriptions. +- [Indexes](/docs/tables/indexes): Indexes accelerate queries by maintaining sorted data structures alongside your tables. Without an index, finding rows that match a condition requires scanning every row. With an index, the database locates matching rows directly. +- [Performance Best Practices](/docs/tables/performance): Follow these guidelines to optimize table performance in your SpacetimeDB modules. +- [Schedule Tables](/docs/tables/schedule-tables): Tables can trigger reducers or procedures at specific times by including a special scheduling column. This allows you to schedule future actions like sending reminders, expiring items, or running periodic maintenance tasks. - ```bash - # Show all logs for 'my-database-name' - spacetime logs my-database-name +### tutorials - # Follow the logs in real-time (like tail -f) - # spacetime logs -f my-database-name +- [Chat App Tutorial](/docs/tutorials/chat-app): In this tutorial, we'll implement a simple chat server as a SpacetimeDB module. You can write your module in TypeScript, C#, or Rust - use the tabs throughout this guide to see code examples in your preferred language. +- [Unity Tutorial](/docs/tutorials/unity): Need help with the tutorial or CLI commands? Join our Discord server! +- [1 - Setup](/docs/tutorials/unity/part-1): Unity Tutorial Hero Image +- [2 - Connecting to SpacetimeDB](/docs/tutorials/unity/part-2): Need help with the tutorial? Join our Discord server! +- [3 - Gameplay](/docs/tutorials/unity/part-3): Need help with the tutorial? Join our Discord server! +- [4 - Moving and Colliding](/docs/tutorials/unity/part-4): Need help with the tutorial? Join our Discord server! +- [Unreal Tutorial](/docs/tutorials/unreal): Need help with the tutorial or CLI commands? Join our Discord server! +- [1 - Setup](/docs/tutorials/unreal/part-1): Need help with the tutorial? Join our Discord server! +- [2 - Connecting to SpacetimeDB](/docs/tutorials/unreal/part-2): Need help with the tutorial? Join our Discord server! +- [3 - Gameplay](/docs/tutorials/unreal/part-3): Need help with the tutorial? Join our Discord server! +- [4 - Moving and Colliding](/docs/tutorials/unreal/part-4): Need help with the tutorial? Join our Discord server! - # Show the last 50 log lines - # spacetime logs -n 50 my-database-name - ``` +### upgrade -12. **Delete Database (Optional):** When you no longer need a database (e.g., after testing), you can delete it using `spacetime delete` with its name or identity. +This guide covers the breaking changes between SpacetimeDB 1.0 and 2.0 and how to update your code. - ```bash - # Delete the database named 'my-database-name' - spacetime delete my-database-name +- [Migrating from SpacetimeDB 1.0 to 2.0](/docs/upgrade): This guide covers the breaking changes between SpacetimeDB 1.0 and 2.0 and how to update your code. - # Delete a database by its identity (replace with actual identity) - # spacetime delete 0x123abc... - ``` +### webassembly-abi -## Core Concepts and Syntax Examples +This document specifies the low level details of module-host interactions ("Module ABI"). **Most users** looking to interact with the host will want to use derived and higher level functionality like bindings], #[spacetimedb(table)], and #[derive(SpacetimeType)] rather than this low level ABI. For more on those, read the [Rust module quick start guide and the Rust module reference. -### Reducer Context: Understanding Identities and Execution Information - -When a reducer function executes, it is provided with a **Reducer Context**. This context contains vital information about the call's origin and environment, crucial for logic, especially security checks. Key pieces of information typically available within the context include: - -- **Sender Identity**: The authenticated [`Identity`](#identity) of the entity that invoked the reducer. This could be: - - A client application connected to the database. - - The module itself, if the reducer was triggered by the internal scheduler (for scheduled reducers). - - The module itself, if the reducer was called internally by another reducer function within the same module. -- **Module Identity**: The authenticated [`Identity`](#identity) representing the database (module) itself. This is useful for checks where an action should only be performed by the module (e.g., in scheduled reducers). -- **Database Access**: Handles or interfaces for interacting with the database tables defined in the module. This allows the reducer to perform operations like inserting, updating, deleting, and querying rows based on primary keys or indexes. -- **Timestamp**: A [`Timestamp`](#timestamp) indicating precisely when the current reducer execution began. -- **Connection ID**: A [`ConnectionId`](#connectionid) representing the specific network connection instance (like a WebSocket session or a stateless HTTP request) that invoked the reducer. This is a unique, server-assigned identifier that persists only for the duration of that connection (from connection start to disconnect). - - **Important Distinction**: Unlike the **Sender Identity** (which represents the _authenticated user or module_), the **Connection ID** solely identifies the _transient network session_. It is assigned by the server and is not based on client-provided authentication credentials. Use the Connection ID for logic tied to a specific connection instance (e.g., tracking session state, rate limiting per connection), and use the Sender Identity for logic related to the persistent, authenticated user or the module itself. - -Understanding the difference between the **Sender Identity** and the **Module Identity** is particularly important for security. For example, when writing scheduled reducers, you often need to verify that the **Sender Identity** matches the **Module Identity** to ensure the action wasn't improperly triggered by an external client. - -### Server Module (Rust) - -#### Defining Types - -Custom structs or enums intended for use as fields within database tables or as parameters/return types in reducers must derive `SpacetimeType`. This derivation enables SpacetimeDB to handle the serialization and deserialization of these types. - -- **Basic Usage:** Apply `#[derive(SpacetimeType, ...)]` to your structs and enums. Other common derives like `Clone`, `Debug`, `PartialEq` are often useful. -- **Cross-Language Naming:** Use the `#[sats(name = "Namespace.TypeName")]` attribute _on the type definition_ to explicitly control the name exposed in generated client bindings (e.g., for C# or TypeScript). This helps prevent naming collisions and provides better organization. You can also use `#[sats(name = "VariantName")]` _on enum variants_ to control their generated names. -- **Type Aliases:** Standard Rust `pub type` aliases can be used for clarity (e.g., `pub type PlayerScore = u32;`). The underlying primitive type must still be serializable by SpacetimeDB. -- **Advanced Deserialization:** For types with complex requirements (like lifetimes or custom binary representations), you might need manual implementation using `spacetimedb::Deserialize` and the `bsatn` crate (available via `spacetimedb::spacetimedb_lib`), though this is uncommon for typical application types. - -```rust -use spacetimedb::{SpacetimeType, Identity, Timestamp}; - -// Example Struct -#[derive(SpacetimeType, Clone, Debug, PartialEq)] -pub struct Position { - pub x: i32, - pub y: i32, -} - -// Example Enum -#[derive(SpacetimeType, Clone, Debug, PartialEq)] -pub enum PlayerStatus { - Idle, - Walking(Position), - Fighting(Identity), // Store the identity of the opponent -} - -// Example Enum with Cross-Language Naming Control -// This enum will appear as `Game.ItemType` in C# bindings. -#[derive(SpacetimeType, Clone, Debug, PartialEq)] -#[sats(name = "Game.ItemType")] -pub enum ItemType { - Weapon, - Armor, - // This specific variant will be `ConsumableItem` in C# bindings. - #[sats(name = "ConsumableItem")] - Potion, -} - -// Example Type Alias -pub type PlayerScore = u32; - -// Advanced: For types with lifetimes or custom binary representations, -// you can derive `spacetimedb::Deserialize` and use the `bsatn` crate -// (provided by spacetimedb::spacetimedb_lib) for manual deserialization if needed. -``` - -:::info Rust `crate-type = ["cdylib"]` -The `[lib]` section in your module's `Cargo.toml` must contain `crate-type = ["cdylib"]`. This tells the Rust compiler to produce a dynamic system library compatible with the C ABI, which allows the SpacetimeDB host (written in Rust) to load and interact with your compiled WebAssembly module. -::: - -#### Defining Tables - -Database tables store the application's persistent state. They are defined using Rust structs annotated with the `#[table]` macro. - -- **Core Attribute:** `#[table(accessor = my_table_name, ...)]` marks a struct as a database table definition. The specified `name` (an identifier, _not_ a string literal) is how the table will be referenced in SQL queries and generated APIs. -- **Derivations:** The `#[table]` macro automatically handles deriving necessary traits like `SpacetimeType`, `Serialize`, `Deserialize`, and `Debug`. **Do not** manually add `#[derive(SpacetimeType)]` to a `#[table]` struct, as it will cause compilation conflicts. -- **Public vs. Private:** By default, tables are **private**, accessible only by server-side reducer code. To allow clients to read or subscribe to a table's data, mark it as `public` using `#[table(..., public)]`. This is a common source of errors if forgotten. -- **Primary Keys:** Designate a single field as the primary key using `#[primary_key]`. This ensures uniqueness, creates an efficient index, and allows clients to track row updates. -- **Auto-Increment:** Mark an integer-typed primary key field with `#[auto_inc]` to have SpacetimeDB automatically assign unique, sequentially increasing values upon insertion. Provide `0` as the value for this field when inserting a new row to trigger the auto-increment mechanism. -- **Unique Constraints:** Enforce uniqueness on non-primary key fields using `#[unique]`. Attempts to insert or update rows violating this constraint will fail. -- **Indexes:** Create B-tree indexes for faster lookups on specific fields or combinations of fields. Use `#[index(btree)]` on a single field for a simple index, or `#[table(index(accessor = my_index_name, btree(columns = [col_a, col_b])))])` within the `#[table(...)]` attribute for named, multi-column indexes. -- **Nullable Fields:** Use standard Rust `Option` for fields that can hold null values. -- **Instances vs. Database:** Remember that table struct instances (e.g., `let player = PlayerState { ... };`) are just data. Modifying an instance does **not** automatically update the database. Interaction happens through generated handles accessed via the `ReducerContext` (e.g., `ctx.db.player_state().insert(...)`). -- **Case Sensitivity:** Table names specified via `name = ...` are case-sensitive and must be matched exactly in SQL queries. -- **Pitfalls:** - - Avoid manually inserting values into `#[auto_inc]` fields that are also `#[unique]`, especially values larger than the current sequence counter, as this can lead to future unique constraint violations when the counter catches up. - - Ensure `public` is set if clients need access. - - Do not manually derive `SpacetimeType`. - - Define indexes _within_ the main `#[table(accessor=..., index=...)]` attribute. Each `#[table]` macro invocation defines a _distinct_ table and requires an `accessor`; separate `#[table]` attributes cannot be used solely to add indexes to a previously named table. - -```rust -use spacetimedb::{table, Identity, Timestamp, SpacetimeType, Table}; // Added Table import - -// Assume Position, PlayerStatus, ItemType are defined as types - -// Example Table Definition -#[table( - accessor = player_state, - public, - // Index definition is included here - index(accessor = idx_level_btree, btree(columns = [level])) -)] -#[derive(Clone, Debug)] // No SpacetimeType needed here -pub struct PlayerState { - #[primary_key] - player_id: Identity, - #[unique] // Player names must be unique - name: String, - conn_id: Option, // Nullable field - health: u32, - level: u16, - position: Position, // Custom type field - status: PlayerStatus, // Custom enum field - last_login: Option, // Nullable timestamp -} - -#[table(accessor = inventory_item, public)] -#[derive(Clone, Debug)] -pub struct InventoryItem { - #[primary_key] - #[auto_inc] // Automatically generate IDs - item_id: u64, - owner_id: Identity, - #[index(btree)] // Simple index on this field - item_type: ItemType, - quantity: u32, -} - -// Example of a private table -#[table(accessor = internal_game_data)] // No `public` flag -#[derive(Clone, Debug)] -struct InternalGameData { - #[primary_key] - key: String, - value: String, -} -``` - -##### Multiple Tables from One Struct - -:::caution Wrapper Struct Pattern Not Supported for This Use Case -Defining multiple tables using wrapper tuple structs (e.g., `struct ActiveCharacter(CharacterInfo);`) where field attributes like `#[primary_key]`, `#[unique]`, etc., are defined only on fields inside the inner struct (`CharacterInfo` in this example) is **not supported**. This pattern can lead to macro expansion issues and compilation errors because the `#[table]` macro applied to the wrapper struct cannot correctly process attributes defined within the inner type. -::: - -**Recommended Pattern:** Apply multiple `#[table(...)]` attributes directly to the single struct definition that contains the necessary fields and field-level attributes (like `#[primary_key]`). This maps the same underlying type definition to multiple distinct tables reliably: - -```rust -use spacetimedb::{table, Identity, Timestamp, Table}; // Added Table import - -// Define the core data structure once -// Note: #[table] automatically derives SpacetimeType, Serialize, Deserialize -// Do NOT add #[derive(SpacetimeType)] here. -#[derive(Clone, Debug)] -#[table(accessor = logged_in_players, public)] // Identifier name -#[table(accessor = players_in_lobby, public)] // Identifier name -pub struct PlayerSessionData { - #[primary_key] - player_id: Identity, - #[unique] - #[auto_inc] - session_id: u64, - last_activity: Timestamp, -} - -// Example Reducer demonstrating interaction -#[spacetimedb::reducer] -fn example_reducer(ctx: &spacetimedb::ReducerContext) { - // Reducers interact with the specific table handles: - let session = PlayerSessionData { - player_id: ctx.sender(), // Example: Use sender identity - session_id: 0, // Assuming auto_inc - last_activity: ctx.timestamp, - }; - - // Insert into the 'logged_in_players' table - match ctx.db.logged_in_players().try_insert(session.clone()) { - Ok(inserted) => spacetimedb::log::info!("Player {} logged in, session {}", inserted.player_id, inserted.session_id), - Err(e) => spacetimedb::log::error!("Failed to insert into logged_in_players: {}", e), - } - - // Find a player in the 'players_in_lobby' table by primary key - if let Some(lobby_player) = ctx.db.players_in_lobby().player_id().find(&ctx.sender()) { - spacetimedb::log::info!("Player {} found in lobby.", lobby_player.player_id); - } - - // Delete from the 'logged_in_players' table using the PK index - ctx.db.logged_in_players().player_id().delete(&ctx.sender()); -} -``` - -##### Browsing Generated Table APIs - -The `#[table]` macro generates specific accessor methods based on your table definition (name, fields, indexes, constraints). To see the exact API generated for your tables: - -1. Run `cargo doc --open` in your module project directory. -2. This compiles your code and opens the generated documentation in your web browser. -3. Navigate to your module's documentation. You will find: - - The struct you defined (e.g., `PlayerState`). - - A generated struct representing the table handle (e.g., `player_state__TableHandle`), which implements `spacetimedb::Table` and contains methods for accessing indexes and unique columns. - - A generated trait (e.g., `player_state`) used to access the table handle via `ctx.db.{table_name}()`. - -Reviewing this generated documentation is the best way to understand the specific methods available for interacting with your defined tables and their indexes. - -#### Defining Reducers - -Reducers are the functions within your server module responsible for atomically modifying the database state in response to client requests or internal events (like lifecycle triggers or schedules). - -- **Core Attribute:** Reducers are defined as standard Rust functions annotated with `#[reducer]`. -- **Signature:** Every reducer function must accept `&ReducerContext` as its first argument. Subsequent arguments represent data passed from the client caller or scheduler, and their types must derive `SpacetimeType`. -- **Return Type:** Reducers typically return `()` for success or `Result<(), E>` (where `E: Display`) to signal recoverable errors. -- **Necessary Imports:** To perform table operations (insert, update, delete, query indexes), the `spacetimedb::Table` trait must be in scope. Add `use spacetimedb::Table;` to the top of your `lib.rs`. -- **Reducer Context:** The `ReducerContext` (`ctx`) provides access to: - - `ctx.db`: Handles for interacting with database tables. - - `ctx.sender`: The `Identity` of the caller. - - `ctx.identity`: The `Identity` of the module itself. - - `ctx.timestamp`: The `Timestamp` of the invocation. - - `ctx.connection_id`: The optional `ConnectionId` of the caller. - - `ctx.rng`: A source for deterministic random number generation (if needed). -- **Transactionality:** Each reducer call executes within a single, atomic database transaction. If the function returns `()` or `Ok(())`, all database changes are committed. If it returns `Err(...)` or panics, the transaction is aborted, and **all changes are rolled back**, preserving data integrity. -- **Execution Environment:** Reducers run in a sandbox and **cannot** directly perform network I/O (`std::net`) or filesystem operations (`std::fs`, `std::io`). External interaction primarily occurs through database table modifications (observed by clients) and logging (`spacetimedb::log`). -- **Calling Other Reducers:** A reducer can directly call another reducer defined in the same module. This is a standard function call and executes within the _same_ transaction; it does not create a sub-transaction. - -```rust -use spacetimedb::{reducer, ReducerContext, Table, Identity, Timestamp, log}; - -// Assume User and Message tables are defined as previously -#[table(accessor = user, public)] -#[derive(Clone, Debug)] pub struct User { #[primary_key] identity: Identity, name: Option, online: bool } -#[table(accessor = message, public)] -#[derive(Clone, Debug)] pub struct Message { #[primary_key] #[auto_inc] id: u64, sender: Identity, text: String, sent: Timestamp } - -// Example: Basic reducer to set a user's name -#[reducer] -pub fn set_name(ctx: &ReducerContext, name: String) -> Result<(), String> { - let sender_id = ctx.sender(); - let name = validate_name(name)?; // Use helper for validation - - // Find the user row by primary key - if let Some(mut user) = ctx.db.user().identity().find(&sender_id) { - // Update the field - user.name = Some(name); - // Persist the change using the PK index update method - ctx.db.user().identity().update(user); - log::info!("User {} set name", sender_id); - Ok(()) - } else { - Err(format!("User not found: {}", sender_id)) - } -} - -// Example: Basic reducer to send a message -#[reducer] -pub fn send_message(ctx: &ReducerContext, text: String) -> Result<(), String> { - let text = validate_message(text)?; // Use helper for validation - log::info!("User {} sent message: {}", ctx.sender(), text); - - // Insert a new row into the Message table - // Note: id is auto_inc, so we provide 0. insert() panics on constraint violation. - let new_message = Message { - id: 0, - sender: ctx.sender(), - text, - sent: ctx.timestamp, - }; - ctx.db.message().insert(new_message); - // For Result-based error handling on insert, use try_insert() - see below - - Ok(()) -} - -// Helper validation functions (example) -fn validate_name(name: String) -> Result { - if name.is_empty() { Err("Name cannot be empty".to_string()) } else { Ok(name) } -} - -fn validate_message(text: String) -> Result { - if text.is_empty() { Err("Message cannot be empty".to_string()) } else { Ok(text) } -} -``` - -##### Error Handling: `Result` vs. Panic - -Reducers can indicate failure either by returning `Err` from a function with a `Result` return type or by panicking (e.g., using `panic!`, `unwrap`, `expect`). Both methods trigger a transaction rollback, ensuring atomicity. - -- **Returning `Err(E)**:\*\* - - This is generally preferred for handling _expected_ or recoverable failures (e.g., invalid input, failed validation checks). - - The error value `E` (which must implement `Display`) is propagated back to the calling client and can be observed in the `ReducerEventContext` status. - - Crucially, returning `Err` does **not** destroy the underlying WebAssembly (WASM) instance. - -- **Panicking:** - - This typically represents an _unexpected_ bug, violated invariant, or unrecoverable state (e.g., assertion failure, unexpected `None` value). - - The client **will** receive an error message derived from the panic payload (the argument provided to `panic!`, or the messages from `unwrap`/`expect`). - - Panicking does **not** cause the client to be disconnected. - - However, a panic **destroys the current WASM instance**. This means the _next_ reducer call (from any client) that runs on this module will incur additional latency as SpacetimeDB needs to create and initialize a fresh WASM instance. - -**Choosing between them:** While both ensure data consistency via rollback, returning `Result::Err` is generally better for predictable error conditions as it avoids the performance penalty associated with WASM instance recreation caused by panics. Use `panic!` for truly exceptional circumstances where state is considered unrecoverable or an unhandled bug is detected. - -##### Lifecycle Reducers - -Special reducers handle specific events: - -- `#[reducer(init)]`: Runs once when the module is first published **and** any time the database is manually cleared (e.g., via `spacetime publish -c` or `spacetime server clear`). Failure prevents publishing or clearing. Often used for initial data setup. -- `#[reducer(client_connected)]`: Runs when any distinct client connection (e.g., WebSocket, HTTP call) is established. Failure disconnects the client. `ctx.connection_id()` is guaranteed to be `Some(...)` within this reducer. -- `#[reducer(client_disconnected)]`: Runs when any distinct client connection terminates. Failure is logged but does not prevent disconnection. `ctx.connection_id()` is guaranteed to be `Some(...)` within this reducer. - -These reducers cannot take arguments beyond `&ReducerContext`. - -```rust -use spacetimedb::{reducer, table, ReducerContext, Table, log}; - -#[table(accessor = settings)] -#[derive(Clone, Debug)] -pub struct Settings { - #[primary_key] - key: String, - value: String, -} - -// Example init reducer: Insert default settings if the table is empty -#[reducer(init)] -pub fn initialize_database(ctx: &ReducerContext) { - log::info!( - "Database Initializing! Module Identity: {}, Timestamp: {}", - ctx.identity(), - ctx.timestamp - ); - // Check if settings table is empty - if ctx.db.settings().count() == 0 { - log::info!("Settings table is empty, inserting default values..."); - // Insert default settings - ctx.db.settings().insert(Settings { - key: "welcome_message".to_string(), - value: "Hello from SpacetimeDB!".to_string(), - }); - ctx.db.settings().insert(Settings { - key: "default_score".to_string(), - value: "0".to_string(), - }); - } else { - log::info!("Settings table already contains data."); - } -} - -// Example client_connected reducer -#[reducer(client_connected)] -pub fn handle_connect(ctx: &ReducerContext) { - log::info!("Client connected: {}, Connection ID: {:?}", ctx.sender(), ctx.connection_id()); - // ... setup initial state for ctx.sender ... -} - -// Example client_disconnected reducer -#[reducer(client_disconnected)] -pub fn handle_disconnect(ctx: &ReducerContext) { - log::info!("Client disconnected: {}, Connection ID: {:?}", ctx.sender(), ctx.connection_id()); - // ... cleanup state for ctx.sender ... -} -``` - -##### Filtering and Deleting with Indexes - -SpacetimeDB provides powerful ways to filter and delete table rows using B-tree indexes. The generated accessor methods accept various argument types: - -- **Single Value (Equality):** - - For columns of type `String`, you can pass `&String` or `&str`. - - For columns of a type `T` that implements `Copy`, you can pass `&T` or an owned `T`. - - For other column types `T`, pass a reference `&T`. -- **Ranges:** Use Rust's range syntax (`start..end`, `start..=end`, `..end`, `..=end`, `start..`). Values within the range can typically be owned or references. -- **Multi-Column Indexes:** - - To filter on an exact match for a _prefix_ of the index columns, provide a tuple containing single values (following the rules above) for that prefix (e.g., `filter((val_a, val_b))` for an index on `[a, b, c]`). - - To filter using a range, you **must** provide single values for all preceding columns in the index, and the range can **only** be applied to the _last_ column in your filter tuple (e.g., `filter((val_a, val_b, range_c))` is valid, but `filter((val_a, range_b, val_c))` or `filter((range_a, val_b))` are **not** valid tuple filters). - - Filtering or deleting using a range on _only the first column_ of the index (without using a tuple) remains valid (e.g., `filter(range_a)`). - -```rust -use spacetimedb::{table, reducer, ReducerContext, Table, log}; - -#[table(accessor = points, index(accessor = idx_xy, btree(columns = [x, y])))] -#[derive(Clone, Debug)] -pub struct Point { #[primary_key] id: u64, x: i64, y: i64 } -#[table(accessor = items, index(btree(columns = [name])))] -#[derive(Clone, Debug)] // No SpacetimeType derive -pub struct Item { #[primary_key] item_key: u32, name: String } - -#[reducer] -fn index_operations(ctx: &ReducerContext) { - // Example: Find items named "Sword" using the generated 'name' index handle - // Passing &str for a String column is allowed. - for item in ctx.db.items().name().filter("Sword") { - // ... - } - - // Example: Delete points where x is between 5 (inclusive) and 10 (exclusive) - // using the multi-column index 'idx_xy' - filtering on first column range is OK. - let num_deleted = ctx.db.points().idx_xy().delete(5i64..10i64); - log::info!("Deleted {} points", num_deleted); - - // Example: Find points where x = 3 and y >= 0 - // using the multi-column index 'idx_xy' - (value, range) is OK. - // Note: x is i64 which is Copy, so passing owned 3i64 is allowed. - for point in ctx.db.points().idx_xy().filter((3i64, 0i64..)) { - // ... - } - - // Example: Find points where x > 5 and y = 1 - // This is INVALID: Cannot use range on non-last element of tuple filter. - // for point in ctx.db.points().idx_xy().filter((5i64.., 1i64)) { ... } - - // Example: Delete all points where x = 7 (filtering on index prefix with single value) - // using the multi-column index 'idx_xy'. Passing owned 7i64 is allowed (Copy type). - ctx.db.points().idx_xy().delete(7i64); - - // Example: Delete a single item by its primary key 'item_key' - // Use the PK field name as the method to get the PK index handle, then call delete. - // item_key is u32 (Copy), passing owned value is allowed. - let item_id_to_delete = 101u32; - ctx.db.items().item_key().delete(item_id_to_delete); - - // Using references for a range filter on the first column - OK - let min_x = 100i64; - let max_x = 200i64; - for point in ctx.db.points().idx_xy().filter(&min_x..=&max_x) { - // ... - } -} -``` - -##### Using `try_insert()` - -Instead of `insert()`, which panics or throws if a constraint (like a primary key or unique index violation) occurs, Rust modules can use `try_insert()`. This method returns a `Result>`, allowing you to gracefully handle potential insertion failures without aborting the entire reducer transaction due to a panic. - -The `TryInsertError` enum provides specific variants detailing the cause of failure, such as `UniqueConstraintViolation` or `AutoIncOverflow`. These variants contain associated types specific to the table's constraints (e.g., `TableHandleType::UniqueConstraintViolation`). If a table lacks a certain constraint (like a unique index), the corresponding associated type might be uninhabited. - -````rust -use spacetimedb::{table, reducer, ReducerContext, Table, log, TryInsertError}; - -#[table(accessor = items)] -#[derive(Clone, Debug)] -pub struct Item { - #[primary_key] #[auto_inc] id: u64, - #[unique] name: String -} - -#[reducer] -pub fn try_add_item(ctx: &ReducerContext, name: String) -> Result<(), String> { - // Assume Item has an auto-incrementing primary key 'id' and a unique 'name' - let new_item = Item { id: 0, name }; // Provide 0 for auto_inc - - // try_insert returns Result> - match ctx.db.items().try_insert(new_item) { - Ok(inserted_item) => { - // try_insert returns the inserted row (with assigned PK if auto_inc) on success - log::info!("Successfully inserted item with ID: {}", inserted_item.id); - Ok(()) - } - Err(e) => { - // Match on the specific TryInsertError variant - match e { - TryInsertError::UniqueConstraintViolation(constraint_error) => { - // constraint_error is of type items__TableHandle::UniqueConstraintViolation - // This type often provides details about the violated constraint. - // For simplicity, we just log a generic message here. - let error_msg = format!("Failed to insert item: Name '{}' already exists.", name); - log::error!("{}", error_msg); - // Return an error to the calling client - Err(error_msg) - } - TryInsertError::AutoIncOverflow(_) => { - // Handle potential overflow of the auto-incrementing key - let error_msg = "Failed to insert item: Auto-increment counter overflow.".to_string(); - log::error!("{}", error_msg); - Err(error_msg) - } - // Use a wildcard for other potential errors or uninhabited variants - _ => { - let error_msg = format!("Failed to insert item: Unknown constraint violation."); - log::error!("{}", error_msg); - Err(error_msg) - } - } - } - } -} - -#### Scheduled Reducers (Rust) - -In addition to lifecycle annotations, reducers can be scheduled. This allows calling the reducers at a particular time, or in a loop. This can be used for game loops. - -The scheduling information for a reducer is stored in a table. This table has two mandatory fields: - -* A primary key that identifies scheduled reducer calls (often using `#[auto_inc]`). -* A field of type `spacetimedb::ScheduleAt` that says when to call the reducer. - -The table definition itself links to the reducer function using the `scheduled(reducer_function_name)` parameter within the `#[table(...)]` attribute. - -Managing timers with a scheduled table is as simple as inserting or deleting rows from the table. This makes scheduling transactional in SpacetimeDB. If a reducer A first schedules B but then errors for some other reason, B will not be scheduled to run. - -A `ScheduleAt` value can be created using `.into()` from: - -* A `spacetimedb::Timestamp`: Schedules the reducer to run **once** at that specific time. -* A `spacetimedb::TimeDuration` or `std::time::Duration`: Schedules the reducer to run **periodically** with that duration as the interval. - -The scheduled reducer function itself is defined like a normal reducer (`#[reducer]`), taking `&ReducerContext` and an instance of the schedule table struct as arguments. - -```rust -use spacetimedb::{table, reducer, ReducerContext, Timestamp, TimeDuration, ScheduleAt, Table}; -use log::debug; - -// 1. Declare the table with scheduling information, linking it to `send_message`. -#[table(accessor = send_message_schedule, scheduled(send_message))] -struct SendMessageSchedule { - // Mandatory fields: - // ============================ - - /// An identifier for the scheduled reducer call. - #[primary_key] - #[auto_inc] - scheduled_id: u64, - - /// Information about when the reducer should be called. - scheduled_at: ScheduleAt, - - // In addition to the mandatory fields, any number of fields can be added. - // These can be used to provide extra information to the scheduled reducer. - - // Custom fields: - // ============================ - - /// The text of the scheduled message to send. - text: String, -} - -// 2. Declare the scheduled reducer. -// The second argument is a row of the scheduling information table. -#[reducer] -fn send_message(ctx: &ReducerContext, args: SendMessageSchedule) -> Result<(), String> { - // Security check is important! - if ctx.sender() != ctx.identity() { - return Err("Reducer `send_message` may not be invoked by clients, only via scheduling.".into()); - } - - let message_to_send = &args.text; - log::info!("Scheduled SendMessage: {}", message_to_send); - - // ... potentially send the message or perform other actions ... - - Ok(()) -} - -// 3. Example of scheduling reducers (e.g., in init) -#[reducer(init)] -fn init(ctx: &ReducerContext) -> Result<(), String> { - - let current_time = ctx.timestamp; - let ten_seconds = TimeDuration::from_micros(10_000_000); - let future_timestamp: Timestamp = ctx.timestamp + ten_seconds; - - // Schedule a one-off message - ctx.db.send_message_schedule().insert(SendMessageSchedule { - scheduled_id: 0, // Use 0 for auto_inc - text: "I'm a bot sending a message one time".to_string(), - // Creating a `ScheduleAt` from a `Timestamp` results in the reducer - // being called once, at exactly the time `future_timestamp`. - scheduled_at: future_timestamp.into() - }); - log::info!("Scheduled one-off message."); - - // Schedule a periodic message (every 10 seconds) - let loop_duration: TimeDuration = ten_seconds; - ctx.db.send_message_schedule().insert(SendMessageSchedule { - scheduled_id: 0, // Use 0 for auto_inc - text: "I'm a bot sending a message every 10 seconds".to_string(), - // Creating a `ScheduleAt` from a `Duration`/`TimeDuration` results in the reducer - // being called in a loop, once every `loop_duration`. - scheduled_at: loop_duration.into() - }); - log::info!("Scheduled periodic message."); - - Ok(()) -} -```` - -Refer to the [official Rust Module SDK documentation on docs.rs](https://docs.rs/spacetimedb/latest/spacetimedb/attr.reducer.html#scheduled-reducers) for more detailed syntax and alternative scheduling approaches (like using `schedule::periodic`). - -##### Scheduled Reducer Details - -- **Best-Effort Scheduling:** Scheduled reducers are called on a best-effort basis and may be slightly delayed in their execution when a database is under heavy load. - -- **Restricting Access (Security):** Scheduled reducers are normal reducers and _can_ still be called directly by clients. If a scheduled reducer should _only_ be called by the scheduler, it is crucial to begin the reducer with a check comparing the caller's identity (`ctx.sender()`) to the module's own identity (`ctx.identity()`). - - ```rust - use spacetimedb::{reducer, ReducerContext}; - // Assuming MyScheduleArgs table is defined - struct MyScheduleArgs {/*...*/} - - #[reducer] - fn my_scheduled_reducer(ctx: &ReducerContext, args: MyScheduleArgs) -> Result<(), String> { - if ctx.sender() != ctx.identity() { - return Err("Reducer `my_scheduled_reducer` may not be invoked by clients, only via scheduling.".into()); - } - // ... Reducer body proceeds only if called by scheduler ... - Ok(()) - } - ``` - -:::info Scheduled Reducers and Connections -Scheduled reducer calls originate from the SpacetimeDB scheduler itself, not from an external client connection. Therefore, within a scheduled reducer, `ctx.sender()` will be the module's own identity, and `ctx.connection_id()` will be `None`. -::: - -#### View Functions - -Views are read-only functions that compute and return results from your tables. Unlike reducers, views do not modify database state - they only query and return data. Views are useful for: - -- **Computing derived data**: Join multiple tables or aggregate data before sending to clients -- **User-specific queries**: Return data specific to the requesting client (e.g., "my player") -- **Performance**: Compute results server-side, reducing data sent to clients -- **Encapsulation**: Hide complex queries behind simple interfaces - -Views can be subscribed to just like tables and automatically update when underlying data changes. - -**Defining Views** - -Views are defined using the `#[view]` macro and must specify a `name` and `public` attribute: - -```rust -use spacetimedb::{view, ViewContext, AnonymousViewContext, table, SpacetimeType}; -use spacetimedb_lib::Identity; - -#[spacetimedb::table(accessor = player, public)] -pub struct Player { - #[primary_key] - #[auto_inc] - id: u64, - #[unique] - identity: Identity, - name: String, -} - -#[spacetimedb::table(accessor = player_level, public)] -pub struct PlayerLevel { - #[unique] - player_id: u64, - #[index(btree)] - level: u64, -} - -// Custom type for joined results -#[derive(SpacetimeType)] -pub struct PlayerAndLevel { - id: u64, - identity: Identity, - name: String, - level: u64, -} - -// View that returns the caller's player (user-specific) -// Returns Option for at-most-one row -#[view(accessor = my_player, public)] -fn my_player(ctx: &ViewContext) -> Option { - ctx.db.player().identity().find(ctx.sender()) -} - -// View that returns all players at a specific level (same for all callers) -// Returns Vec for multiple rows -#[view(accessor = players_for_level, public)] -fn players_for_level(ctx: &AnonymousViewContext) -> Vec { - ctx.db - .player_level() - .level() - .filter(2u64) - .flat_map(|player_level| { - ctx.db - .player() - .id() - .find(player_level.player_id) - .map(|p| PlayerAndLevel { - id: p.id, - identity: p.identity, - name: p.name, - level: player_level.level, - }) - }) - .collect() -} -``` - -**ViewContext vs AnonymousViewContext** - -Views use one of two context types: - -- **`ViewContext`**: Provides access to the caller's `Identity` through `ctx.sender()`. Use this when the view depends on who is querying it (e.g., "get my player"). -- **`AnonymousViewContext`**: Does not provide caller information. Use this when the view produces the same results regardless of who queries it (e.g., "get top 10 players"). - -Both contexts provide read-only access to tables and indexes through `ctx.db`. - -**Performance Note**: Because `AnonymousViewContext` is guaranteed not to access the caller's identity, SpacetimeDB can share the computed view results across multiple connected clients. This provides significant performance benefits for views that return the same data to all clients. Prefer `AnonymousViewContext` when possible. - -**Return Types** - -Views can return: -- `Option` - For at-most-one row (e.g., looking up a specific player) -- `Vec` - For multiple rows (e.g., listing all players at a level) -- `impl IQuery` - A typed SQL query that behaves like the deprecated RLS (Row-Level Security) feature - -Where `T` can be a table type or any custom type derived with `SpacetimeType`. - -**impl IQuery Return Type** - -When a view returns `impl IQuery`, SpacetimeDB computes results incrementally as the underlying data changes. This enables efficient table scanning because query results are maintained incrementally rather than recomputed from scratch. Without `impl IQuery`, you must use indexed column lookups to access tables inside view functions. - -The query builder provides a fluent API for constructing type-safe SQL queries: - -```rust -use spacetimedb::{view, ViewContext, Query}; - -// This view can scan the whole table efficiently because -// impl IQuery results are computed incrementally -#[view(accessor = my_messages, public)] -fn my_messages(ctx: &ViewContext) -> impl Query { - // Return a typed query builder directly - ctx.db.message().filter(|cols| cols.sender.eq(ctx.sender())) -} - -// Query builder supports various operations: -// - .filter(|cols| cols.field.eq(value)) - equality -// - .filter(|cols| cols.field.ne(value)) - not equal -// - .filter(|cols| cols.field.gt(value)) - greater than -// - .filter(|cols| cols.field.lt(value)) - less than -// - .filter(|cols| cols.field.gte(value)) - greater than or equal -// - .filter(|cols| cols.field.lte(value)) - less than or equal -// - .filter(|cols| expr1.or(expr2)) - logical OR -// - .left_semijoin(other_table, |a, b| a.field.eq(b.field)) - joins -``` - -**Querying Views** - -Views can be queried and subscribed to using SQL, just like tables: - -```sql -SELECT * FROM my_player; -SELECT * FROM players_for_level; -``` - -When subscribed to, views automatically update when their underlying tables change. - -**Best Practices** - -1. Use `ViewContext` when results depend on the caller's identity. -2. Use `AnonymousViewContext` when results are the same for all callers. -3. Keep views simple - complex joins can be expensive to recompute. -4. Views are recomputed when underlying tables change, so minimize dependencies on frequently-changing tables. -5. Use indexes on columns you filter or join on for better performance. - -### Client SDK (Rust) - -This section details how to build native Rust client applications that interact with a SpacetimeDB module. - -#### 1. Project Setup - -Start by creating a standard Rust binary project and adding the `spacetimedb_sdk` crate as a dependency: - -```bash -cargo new my_rust_client -cd my_rust_client -cargo add spacetimedb_sdk # Ensure version matches your SpacetimeDB installation -``` - -#### 2. Generate Module Bindings - -Client code relies on generated bindings specific to your server module. Use the `spacetime generate` command, pointing it to your server module project: - -```bash -# From your client project directory -mkdir -p src/module_bindings -spacetime generate --lang rust \ - --out-dir src/module_bindings \ - --module-path ../path/to/your/server_module -``` - -Then, declare the generated module in your `main.rs` or `lib.rs`: - -```rust -mod module_bindings; -// Optional: bring generated types into scope -// use module_bindings::*; -``` - -#### 3. Connecting to the Database - -The core type for managing a connection is `module_bindings::DbConnection`. You configure and establish a connection using a builder pattern. - -- **Builder:** Start with `DbConnection::builder()`. -- **URI & Name:** Specify the SpacetimeDB instance URI (`.with_uri("http://localhost:3000")`) and the database name or identity (`.with_database_name("my_database")`). -- **Authentication:** Provide an identity token using `.with_token(Option)`. If `None` or omitted for the first connection, the server issues a new identity and token (retrieved via the `on_connect` callback). -- **Callbacks:** Register callbacks for connection lifecycle events: - - `.on_connect(|conn, identity, token| { ... })`: Runs on successful connection. Often used to store the `token` for future connections. - - `.on_connect_error(|err_ctx, error| { ... })`: Runs if connection fails. - - `.on_disconnect(|err_ctx, maybe_error| { ... })`: Runs when the connection closes, either gracefully or due to an error. -- **Build:** Call `.build()` to initiate the connection attempt. - -```rust -use spacetimedb_sdk::{identity, DbContext, Identity, credentials}; -use crate::module_bindings::{DbConnection, connect_event_callbacks, table_update_callbacks}; - -const HOST: &str = "http://localhost:3000"; -const DB_NAME: &str = "my_database"; // Or your specific DB name/identity - -fn connect_to_db() -> DbConnection { - // Helper for storing/loading auth token - fn creds_store() -> credentials::File { - credentials::File::new(".my_client_creds") // Unique filename - } - - DbConnection::builder() - .with_uri(HOST) - .with_database_name(DB_NAME) - .with_token(creds_store().load().ok()) // Load token if exists - .on_connect(|conn, identity, auth_token| { - println!("Connected. Identity: {}", identity.to_hex()); - // Save the token for future connections - if let Err(e) = creds_store().save(auth_token) { - eprintln!("Failed to save auth token: {}", e); - } - // Register other callbacks *after* successful connection - connect_event_callbacks(conn); - table_update_callbacks(conn); - // Initiate subscriptions - subscribe_to_tables(conn); - }) - .on_connect_error(|err_ctx, err| { - eprintln!("Connection Error: {}", err); - std::process::exit(1); - }) - .on_disconnect(|err_ctx, maybe_err| { - println!("Disconnected. Reason: {:?}", maybe_err); - std::process::exit(0); - }) - .build() - .expect("Failed to connect") -} -``` - -#### 4. Managing the Connection Loop - -After establishing the connection, you need to continuously process incoming messages and trigger callbacks. The SDK offers several ways: - -- **Threaded:** `connection.run_threaded()`: Spawns a dedicated background thread that automatically handles message processing. -- **Async:** `async connection.run_async()`: Integrates with async runtimes like Tokio or async-std. -- **Manual Tick:** `connection.frame_tick()`: Processes pending messages without blocking. Suitable for integrating into game loops or other manual polling scenarios. You must call this repeatedly. - -```rust -// Example using run_threaded -fn main() { - let connection = connect_to_db(); - let handle = connection.run_threaded(); // Spawns background thread - - // Main thread can now do other work, like handling user input - // handle_user_input(&connection); - - handle.join().expect("Connection thread panicked"); -} -``` - -#### 5. Subscribing to Data - -Clients receive data by subscribing to SQL queries against the database's public tables. - -- **Builder:** Start with `connection.subscription_builder()`. -- **Callbacks:** - - `.on_applied(|sub_ctx| { ... })`: Runs when the initial data for the subscription arrives. - - `.on_error(|err_ctx, error| { ... })`: Runs if the subscription fails (e.g., invalid SQL). -- **Subscribe:** Call `.subscribe(vec!["SELECT * FROM table_a", "SELECT * FROM table_b WHERE some_col > 10"])` with a list of query strings. This returns a `SubscriptionHandle`. -- **All Tables:** `.subscribe_to_all_tables()` is a convenience for simple clients but cannot be easily unsubscribed. -- **Unsubscribing:** Use `handle.unsubscribe()` or `handle.unsubscribe_then(|sub_ctx| { ... })` to stop receiving updates for specific queries. - -```rust -use crate::module_bindings::{SubscriptionEventContext, ErrorContext}; - -fn subscribe_to_tables(conn: &DbConnection) { - println!("Subscribing to tables..."); - conn.subscription_builder() - .on_applied(on_subscription_applied) - .on_error(|err_ctx, err| { - eprintln!("Subscription failed: {}", err); - }) - // Example: Subscribe to all rows from 'player' and 'message' tables - .subscribe(vec!["SELECT * FROM player", "SELECT * FROM message"]); -} - -fn on_subscription_applied(ctx: &SubscriptionEventContext) { - println!("Subscription applied! Initial data received."); - // Example: Print initial messages sorted by time - let mut messages: Vec<_> = ctx.db().message().iter().collect(); - messages.sort_by_key(|m| m.sent); - for msg in messages { - // print_message(ctx.db(), &msg); // Assuming a print_message helper - } -} -``` - -#### 6. Accessing Cached Data & Handling Row Callbacks - -Subscribed data is stored locally in the client cache, accessible via `ctx.db()` (where `ctx` can be a `DbConnection` or any event context). - -- **Accessing Tables:** Use `ctx.db().table_name()` to get a handle to a table. -- **Iterating:** `table_handle.iter()` returns an iterator over all cached rows. -- **Filtering/Finding:** Use index accessors like `table_handle.primary_key_field().find(&pk_value)` or `table_handle.indexed_field().filter(value_or_range)` for efficient lookups (similar to server-side). -- **Row Callbacks:** Register callbacks to react to changes in the cache: - - `table_handle.on_insert(|event_ctx, inserted_row| { ... })` - - `table_handle.on_delete(|event_ctx, deleted_row| { ... })` - - `table_handle.on_update(|event_ctx, old_row, new_row| { ... })` (Only for tables with a `#[primary_key]`) - -```rust -use crate::module_bindings::{Player, Message, EventContext, Event, DbView}; - -// Placeholder for where other callbacks are registered -fn table_update_callbacks(conn: &DbConnection) { - conn.db().player().on_insert(handle_player_insert); - conn.db().player().on_update(handle_player_update); - conn.db().message().on_insert(handle_message_insert); -} - -fn handle_player_insert(ctx: &EventContext, player: &Player) { - // Only react to updates caused by reducers, not initial subscription load - if let Event::Reducer(_) = ctx.event { - println!("Player joined: {}", player.name.as_deref().unwrap_or("Unknown")); - } -} - -fn handle_player_update(ctx: &EventContext, old: &Player, new: &Player) { - if old.name != new.name { - println!("Player renamed: {} -> {}", - old.name.as_deref().unwrap_or("??"), - new.name.as_deref().unwrap_or("??") - ); - } - // ... handle other changes like online status ... -} - -fn handle_message_insert(ctx: &EventContext, message: &Message) { - if let Event::Reducer(_) = ctx.event { - // Find sender name from cache - let sender_name = ctx.db().player().identity().find(&message.sender) - .map_or("Unknown".to_string(), |p| p.name.clone().unwrap_or("??".to_string())); - println!("{}: {}", sender_name, message.text); - } -} -``` - -:::info Handling Initial Data vs. Live Updates in Callbacks -Callbacks like `on_insert` and `on_update` are triggered for both the initial data received when a subscription is first applied _and_ for subsequent live changes caused by reducers. If you need to differentiate (e.g., only react to _new_ messages, not the backlog), you can inspect the `ctx.event` type. For example, `if let Event::Reducer(_) = ctx.event { ... }` checks if the change came from a reducer call. -::: - -#### 7. Invoking Reducers & Handling Reducer Callbacks - -Clients trigger state changes by calling reducers defined in the server module. - -- **Invoking:** Access generated reducer functions via `ctx.reducers().reducer_name(arg1, arg2, ...)`. -- **Reducer Callbacks:** Register callbacks to react to the _outcome_ of reducer calls (especially useful for handling failures or confirming success if not directly observing table changes): - - `ctx.reducers().on_reducer_name(|reducer_event_ctx, arg1, ...| { ... })` - - The `reducer_event_ctx.event` contains: - - `reducer`: The specific reducer variant and its arguments. - - `status`: `Status::Committed`, `Status::Failed(reason)`, or `Status::OutOfEnergy`. - - `caller_identity`, `timestamp`, etc. - -```rust -use crate::module_bindings::{ReducerEventContext, Status}; - -// Placeholder for where other callbacks are registered -fn connect_event_callbacks(conn: &DbConnection) { - conn.reducers().on_set_name(handle_set_name_result); - conn.reducers().on_send_message(handle_send_message_result); -} - -fn handle_set_name_result(ctx: &ReducerContext, name: &String) { - if let Status::Failed(reason) = &ctx.event.status { - // Check if the failure was for *our* call (important in multi-user contexts) - if ctx.event.caller_identity == ctx.identity() { - eprintln!("Error setting name to '{}': {}", name, reason); - } - } -} - -fn handle_send_message_result(ctx: &ReducerContext, text: &String) { - if let Status::Failed(reason) = &ctx.event.status { - if ctx.event.caller_identity == ctx.identity() { // Our call failed - eprintln!("[Error] Failed to send message '{}': {}", text, reason); - } - } -} - -// Example of calling a reducer (e.g., from user input handler) -fn send_chat_message(conn: &DbConnection, message: String) { - if !message.is_empty() { - conn.reducers().send_message(message); // Fire-and-forget style - } -} -``` - -// ... (Keep the second info box about C# callbacks, it will be moved later) ... -:::info Handling Initial Data vs. Live Updates in Callbacks -Callbacks like `OnInsert` and `OnUpdate` are triggered for both the initial data received when a subscription is first applied _and_ for subsequent live changes caused by reducers. If you need to differentiate (e.g., only react to _new_ messages, not the backlog), you can inspect the `ctx.Event` type. For example, checking `if (ctx.Event is not Event.SubscribeApplied) { ... }` ensures the code only runs for events triggered by reducers, not the initial subscription data load. -::: - -### Server Module (C#) - -#### Defining Types - -Custom classes, structs, or records intended for use as fields within database tables or as parameters/return types in reducers must be marked with the `[Type]` attribute. This attribute enables SpacetimeDB to handle the serialization and deserialization of these types. - -- **Basic Usage:** Apply `[Type]` to your classes, structs, or records. Use the `partial` modifier to allow SpacetimeDB's source generators to augment the type definition. -- **Cross-Language Naming:** Currently, the C# module SDK does **not** provide a direct equivalent to Rust's `#[sats(name = "...")]` attribute for controlling the generated names in _other_ client languages (like TypeScript). The C# type name itself (including its namespace) is typically used. Standard C# namespacing (`namespace MyGame.SharedTypes { ... }`) is the primary way to organize and avoid collisions. -- **Enums:** Standard C# enums can be marked with `[Type]`. For "tagged unions" or "discriminated unions" (like Rust enums with associated data), use the pattern of an abstract base record with `[Type]`, derived records for each variant, and a final `[Type]` **partial record** that inherits from `TaggedEnum<(...)>`. The TaggedEnum type must be `partial record`, not `partial class`. -- **Type Aliases:** Use standard C# `using` aliases for clarity (e.g., `using PlayerScore = System.UInt32;`). The underlying primitive type must still be serializable by SpacetimeDB. - -```csharp -using SpacetimeDB; -using System; // Required for System.UInt32 if using aliases like below - -// Example Struct -[Type] -public partial struct Position { public int X; public int Y; } - -// Example Tagged Union (Enum with Data) Pattern: -// 1. Base abstract record -[Type] public abstract partial record PlayerStatusBase { } -// 2. Derived records for variants -[Type] public partial record IdleStatus : PlayerStatusBase { } -[Type] public partial record WalkingStatus : PlayerStatusBase { public Position Target; } -[Type] public partial record FightingStatus : PlayerStatusBase { public Identity OpponentId; } -// 3. Final type inheriting from TaggedEnum -[Type] -public partial record PlayerStatus : TaggedEnum<( - IdleStatus Idle, - WalkingStatus Walking, - FightingStatus Fighting -)> { } - -// Example Standard Enum -[Type] -public enum ItemType { Weapon, Armor, Potion } - -// Example Type Alias -using PlayerScore = System.UInt32; - -``` - -:::info C# `partial` Keyword -Table and Type definitions in C# should use the `partial` keyword (e.g., `public partial class MyTable`). This allows the SpacetimeDB source generator to add necessary internal methods and serialization logic to your types without requiring you to write boilerplate code. -::: - -#### Defining Tables - -Database tables store the application's persistent state. They are defined using C# classes or structs marked with the `[Table]` attribute. - -- **Core Attribute:** `[Table(Accessor = "TableName", ...)]` marks a class or struct as a database table definition. `Accessor` controls generated API names, while canonical SQL names are derived unless `Name` is explicitly set. -- **Partial Modifier:** Use the `partial` keyword (e.g., `public partial class MyTable`) to allow SpacetimeDB's source generators to add necessary methods and logic to your definition. -- **Public vs. Private:** By default, tables are **private**, accessible only by server-side reducer code. To allow clients to read or subscribe to a table's data, set `Public = true` within the attribute: `[Table(..., Public = true)]`. This is a common source of errors if forgotten. -- **Primary Keys:** Designate a single **public field** as the primary key using `[PrimaryKey]`. This ensures uniqueness, creates an efficient index, and allows clients to track row updates. -- **Auto-Increment:** Mark an integer-typed primary key **public field** with `[AutoInc]` to have SpacetimeDB automatically assign unique, sequentially increasing values upon insertion. Provide `0` as the value for this field when inserting a new row to trigger the auto-increment mechanism. -- **Unique Constraints:** Enforce uniqueness on non-primary key **public fields** using `[Unique]`. Attempts to insert or update rows violating this constraint will fail (throw an exception). -- **Indexes:** Create B-tree indexes for faster lookups on specific **public fields** or combinations of fields. Use `[SpacetimeDB.Index.BTree]` on a single field (never bare `Index`), or define indexes at the class/struct level using `[SpacetimeDB.Index.BTree(Accessor = "MyIndexName", Columns = new[] { nameof(ColA), nameof(ColB) })]`. -- **Nullable Fields:** Use standard C# nullable reference types (`string?`) or nullable value types (`int?`, `Timestamp?`) for fields that can hold null values. -- **Instances vs. Database:** Remember that table class/struct instances (e.g., `var player = new PlayerState { ... };`) are just data objects. Modifying an instance does **not** automatically update the database. Interaction happens through generated handles accessed via the `ReducerContext` (e.g., `ctx.Db.PlayerState.Insert(...)`). -- **Case Sensitivity:** Table names specified via `Accessor = "..."` are case-sensitive and must be matched exactly in SQL queries. -- **Pitfalls:** - - SpacetimeDB attributes (`[PrimaryKey]`, `[AutoInc]`, `[Unique]`, `[SpacetimeDB.Index.BTree]`) **must** be applied to **public fields**, not properties (`{ get; set; }`). Using properties can cause build errors or runtime issues. - - Avoid manually inserting values into `[AutoInc]` fields that are also `[Unique]`, especially values larger than the current sequence counter, as this can lead to future unique constraint violations when the counter catches up. - - Ensure `Public = true` is set if clients need access. - - Always use the `partial` keyword on table definitions. - - Define indexes _within_ the main `#[table(accessor=..., index=...)]` attribute. Each `#[table]` macro invocation defines a _distinct_ table and requires an `accessor`; separate `#[table]` attributes cannot be used solely to add indexes to a previously named table. - -```csharp -using SpacetimeDB; -using System; // For Nullable types if needed - -// Assume Position, PlayerStatus, ItemType are defined as types - -// Example Table Definition -[Table(Accessor = "PlayerState", Public = true)] -[SpacetimeDB.Index.BTree(Accessor = "idx_level", Columns = new[] { nameof(Level) })] // Table-level index -public partial class PlayerState -{ - [PrimaryKey] - public Identity PlayerId; // Public field - [Unique] - public string Name = ""; // Public field (initialize to avoid null warnings if needed) - public uint Health; // Public field - public ushort Level; // Public field - public Position Position; // Public field (custom struct type) - public PlayerStatus Status; // Public field (custom record type) - public Timestamp? LastLogin; // Public field, nullable struct -} - -[Table(Accessor = "InventoryItem", Public = true)] -public partial class InventoryItem -{ - [PrimaryKey] - [AutoInc] // Automatically generate IDs - public ulong ItemId; // Public field - public Identity OwnerId; // Public field - [SpacetimeDB.Index.BTree] // Simple index on this field - public ItemType ItemType; // Public field - public uint Quantity; // Public field -} - -// Example of a private table -[Table(Accessor = "InternalGameData")] // Public = false is default -public partial class InternalGameData -{ - [PrimaryKey] - public string Key = ""; // Public field - public string Value = ""; // Public field -} -``` - -##### Multiple Tables from One Class - -You can use the same underlying data class for multiple tables, often using inheritance. Ensure SpacetimeDB attributes like `[PrimaryKey]` are applied to **public fields**, not properties. - -```csharp -using SpacetimeDB; - -// Define the core data structure (must be [Type] if used elsewhere) -[Type] -public partial class CharacterInfo -{ - [PrimaryKey] - public ulong CharacterId; // Use public field - public string Name = ""; // Use public field - public ushort Level; // Use public field -} - -// Define derived classes, each with its own table attribute -[Table(Accessor = "ActiveCharacter")] -public partial class ActiveCharacter : CharacterInfo { - // Can add specific public fields if needed - public bool IsOnline; -} - -[Table(Accessor = "DeletedCharacter")] -public partial class DeletedCharacter : CharacterInfo { - // Can add specific public fields if needed - public Timestamp DeletionTime; -} - -// Reducers would interact with ActiveCharacter or DeletedCharacter tables -// E.g., ctx.Db.ActiveCharacter.Insert(new ActiveCharacter { CharacterId = 1, Name = "Hero", Level = 10, IsOnline = true }); -``` - -Alternatively, you can define multiple `[Table]` attributes directly on a single class or struct. This maps the same underlying type to multiple distinct tables: - -```csharp -using SpacetimeDB; - -// Define the core data structure once -// Apply multiple [Table] attributes to map it to different tables -[Type] // Mark as a type if used elsewhere (e.g., reducer args) -[Table(Accessor = "LoggedInPlayer", Public = true)] -[Table(Accessor = "PlayerInLobby", Public = true)] -public partial class PlayerSessionData -{ - [PrimaryKey] - public Identity PlayerId; // Use public field - [Unique] - [AutoInc] - public ulong SessionId; // Use public field - public Timestamp LastActivity; -} - -// Reducers would interact with the specific table handles: -// E.g., ctx.Db.logged_in_players.Insert(new PlayerSessionData { ... }); -// E.g., var lobbyPlayer = ctx.Db.players_in_lobby.PlayerId.Find(someId); -``` - -#### Defining Reducers - -Reducers are the functions within your server module responsible for atomically modifying the database state in response to client requests or internal events (like lifecycle triggers or schedules). - -- **Core Attribute:** Reducers are defined as `static` methods within a (typically `static partial`) class, annotated with `[SpacetimeDB.Reducer]`. -- **Signature:** Every reducer method must accept `ReducerContext` as its first argument. Subsequent arguments represent data passed from the client caller or scheduler, and their types must be marked with `[Type]`. -- **Return Type:** Reducers should typically return `void`. Errors are signaled by throwing exceptions. -- **Reducer Context:** The `ReducerContext` (`ctx`) provides access to: - - `ctx.Db`: Handles for interacting with database tables. - - `ctx.Sender`: The `Identity` of the caller. - - `ctx.Identity`: The `Identity` of the module itself. - - `ctx.Timestamp`: The `Timestamp` of the invocation. - - `ctx.ConnectionId`: The nullable `ConnectionId` of the caller. - - `ctx.Rng`: A `System.Random` instance for deterministic random number generation (if needed). -- **Transactionality:** Each reducer call executes within a single, atomic database transaction. If the method completes without an unhandled exception, all database changes are committed. If an exception is thrown, the transaction is aborted, and **all changes are rolled back**, preserving data integrity. -- **Execution Environment:** Reducers run in a sandbox and **cannot** directly perform network I/O (`System.Net`) or filesystem operations (`System.IO`). External interaction primarily occurs through database table modifications (observed by clients) and logging (`SpacetimeDB.Log`). -- **Calling Other Reducers:** A reducer can directly call another static reducer method defined in the same module. This is a standard method call and executes within the _same_ transaction; it does not create a sub-transaction. - -```csharp -using SpacetimeDB; -using System; -using System.Linq; // Used in more complex examples later - -public static partial class Module -{ - // Assume PlayerState and InventoryItem tables are defined as previously - [Table(Accessor = "PlayerState", Public = true)] public partial class PlayerState { - [PrimaryKey] public Identity PlayerId; - [Unique] public string Name = ""; - public uint Health; public ushort Level; /* ... other fields */ } - [Table(Accessor = "InventoryItem", Public = true)] public partial class InventoryItem { - [PrimaryKey] #[AutoInc] public ulong ItemId; - public Identity OwnerId; /* ... other fields */ } - - // Example: Basic reducer to update player data - [Reducer] - public static void UpdatePlayerData(ReducerContext ctx, string? newName) - { - var playerId = ctx.Sender; - - // Find player by primary key - var player = ctx.Db.player_state.PlayerId.Find(playerId); - if (player == null) - { - throw new Exception($"Player not found: {playerId}"); - } - - // Update fields conditionally - bool requiresUpdate = false; - if (!string.IsNullOrWhiteSpace(newName)) - { - // Basic check for name uniqueness (simplified) - var existing = ctx.Db.player_state.Name.Find(newName); - if(existing != null && !existing.PlayerId.Equals(playerId)) { - throw new Exception($"Name '{newName}' already taken."); - } - if (player.Name != newName) { - player.Name = newName; - requiresUpdate = true; - } - } - - if (player.Level < 100) { // Example simple update - player.Level += 1; - requiresUpdate = true; - } - - // Persist changes if any were made - if (requiresUpdate) { - ctx.Db.player_state.PlayerId.Update(player); - Log.Info($"Updated player data for {playerId}"); - } - } - - // Example: Basic reducer to register a player - [Reducer] - public static void RegisterPlayer(ReducerContext ctx, string name) - { - if (string.IsNullOrWhiteSpace(name)) { - throw new ArgumentException("Name cannot be empty."); - } - Log.Info($"Attempting to register player: {name} ({ctx.Sender})"); - - // Check if player identity or name already exists - if (ctx.Db.player_state.PlayerId.Find(ctx.Sender) != null || ctx.Db.player_state.Name.Find(name) != null) - { - throw new Exception("Player already registered or name taken."); - } - - // Create new player instance - var newPlayer = new PlayerState - { - PlayerId = ctx.Sender, - Name = name, - Health = 100, - Level = 1, - // Initialize other fields as needed... - }; - - // Insert the new player. This will throw on constraint violation. - ctx.Db.player_state.Insert(newPlayer); - Log.Info($"Player registered successfully: {ctx.Sender}"); - } - - // Example: Basic reducer showing deletion - [Reducer] - public static void DeleteMyItems(ReducerContext ctx) - { - var ownerId = ctx.Sender; - int deletedCount = 0; - - // Find items by owner (Requires an index on OwnerId for efficiency) - // This example iterates if no index exists. - var itemsToDelete = ctx.Db.inventory_item.Iter() - .Where(item => item.OwnerId.Equals(ownerId)) - .ToList(); // Collect IDs to avoid modification during iteration - - foreach(var item in itemsToDelete) - { - // Delete using the primary key index - if (ctx.Db.inventory_item.ItemId.Delete(item.ItemId)) { - deletedCount++; - } - } - Log.Info($"Deleted {deletedCount} items for player {ownerId}."); - } -} -``` - -##### Handling Insert Constraint Violations - -Unlike Rust's `try_insert` which returns a `Result`, the C# `Insert` method throws an exception if a constraint (like a primary key or unique index violation) occurs. There are two main ways to handle this in C# reducers: - -1. **Pre-checking:** Before calling `Insert`, explicitly query the database using the relevant indexes to check if the insertion would violate any constraints (e.g., check if a user with the same ID or unique name already exists). This is often cleaner if the checks are straightforward. The `RegisterPlayer` example above demonstrates this pattern. - -2. **Using `try-catch`:** Wrap the `Insert` call in a `try-catch` block. This allows you to catch the specific exception (often a `SpacetimeDB.ConstraintViolationException` or potentially a more general `Exception` depending on the SDK version and error type) and handle the failure gracefully (e.g., log an error, return a specific error message to the client via a different mechanism if applicable, or simply allow the transaction to roll back cleanly without crashing the reducer unexpectedly). - -```csharp -using SpacetimeDB; -using System; - -public static partial class Module -{ - [Table(Accessor = "UniqueItem")] - public partial class UniqueItem { - [PrimaryKey] public string ItemName; - public int Value; - } - - // Example using try-catch for insertion - [Reducer] - public static void AddUniqueItemWithCatch(ReducerContext ctx, string name, int value) - { - var newItem = new UniqueItem { ItemName = name, Value = value }; - try - { - // Attempt to insert - ctx.Db.unique_items.Insert(newItem); - Log.Info($"Successfully inserted item: {name}"); - } - catch (Exception ex) // Catch a general exception or a more specific one if available - { - // Log the specific error - Log.Error($"Failed to insert item '{name}': Constraint violation or other error. Details: {ex.Message}"); - // Optionally, re-throw a custom exception or handle differently - // Throwing ensures the transaction is rolled back - throw new Exception($"Item name '{name}' might already exist."); - } - } -} -``` - -Choosing between pre-checking and `try-catch` depends on the complexity of the constraints and the desired flow. Pre-checking can avoid the overhead of exception handling for predictable violations, while `try-catch` provides a direct way to handle unexpected insertion failures. - -:::note C# `Insert` vs Rust `try_insert` -Unlike Rust, the C# SDK does not currently provide a `TryInsert` method that returns a result. The standard `Insert` method will throw an exception if a constraint (primary key, unique index) is violated. Therefore, C# reducers should typically check for potential constraint violations _before_ calling `Insert`, or be prepared to handle the exception (which will likely roll back the transaction). -::: - -##### Lifecycle Reducers - -Special reducers handle specific events: - -- `[Reducer(ReducerKind.Init)]`: Runs once when the module is first published **and** any time the database is manually cleared (e.g., via `spacetime publish -c` or `spacetime server clear`). Failure prevents publishing or clearing. Often used for initial data setup. -- `[Reducer(ReducerKind.ClientConnected)]`: Runs when any distinct client connection (e.g., WebSocket, HTTP call) is established. Failure disconnects the client. `ctx.connection_id` is guaranteed to have a value within this reducer. -- `[Reducer(ReducerKind.ClientDisconnected)]`: Runs when any distinct client connection terminates. Failure is logged but does not prevent disconnection. `ctx.connection_id` is guaranteed to have a value within this reducer. - -These reducers cannot take arguments beyond `&ReducerContext`. - -```csharp -// Example init reducer is shown in Scheduled Reducers section -[Reducer(ReducerKind.ClientConnected)] -public static void HandleConnect(ReducerContext ctx) { - Log.Info($"Client connected: {ctx.Sender}"); - // ... setup initial state for ctx.sender ... -} - -[Reducer(ReducerKind.ClientDisconnected)] -public static void HandleDisconnect(ReducerContext ctx) { - Log.Info($"Client disconnected: {ctx.Sender}"); - // ... cleanup state for ctx.sender ... -} -``` - -#### Scheduled Reducers (C#) - -In addition to lifecycle annotations, reducers can be scheduled. This allows calling the reducers at a particular time, or periodically for loops (e.g., game loops). - -The scheduling information for a reducer is stored in a table. This table links to the reducer function and has specific mandatory fields: - -1. **Define the Schedule Table:** Create a table class/struct using `[Table(Accessor = ..., Scheduled = "YourReducerName", ScheduledAt = "ScheduledAt")]`. - - The `Scheduled` parameter links this table to the static reducer method `YourReducerName`. - - The `ScheduledAt` parameter specifies the name of the field within this table that holds the scheduling information. This field **must** be of type `SpacetimeDB.ScheduleAt`. - - The table **must** also have a primary key field (often `[AutoInc] ulong Id`). - - Additional fields can be included to pass arguments to the scheduled reducer. -2. **Define the Scheduled Reducer:** Create the `static` reducer method (`YourReducerName`) specified in the table attribute. It takes `ReducerContext` and an instance of the schedule table class/struct as arguments. -3. **Schedule an Invocation:** Inside another reducer, create an instance of your schedule table struct. - - Set the `ScheduleAt` field (using the name specified in the `ScheduledAt` parameter) to either: - - `new ScheduleAt.Time(timestamp)`: Schedules the reducer to run **once** at the specified `Timestamp`. - - `new ScheduleAt.Interval(timeDuration)`: Schedules the reducer to run **periodically** with the specified `TimeDuration` interval. - - Set the primary key (e.g., to `0` if using `[AutoInc]`) and any other argument fields. - - Insert this instance into the schedule table using `ctx.Db.your_schedule_table_name.Insert(...)`. - -Managing timers with a scheduled table is as simple as inserting or deleting rows. This makes scheduling transactional in SpacetimeDB. If a reducer A schedules B but then throws an exception, B will not be scheduled. - -```csharp -using SpacetimeDB; -using System; - -public static partial class Module -{ - // 1. Define the table with scheduling information, linking to `SendMessage` reducer. - // Specifies that the `ScheduledAt` field holds the schedule info. - [Table(Accessor = "SendMessageSchedule", Scheduled = "SendMessage", ScheduledAt = "ScheduledAt")] - public partial struct SendMessageSchedule - { - // Mandatory fields: - [PrimaryKey] - [AutoInc] - public ulong Id; // Identifier for the scheduled call - - public ScheduleAt ScheduledAt; // Holds the schedule timing - - // Custom fields (arguments for the reducer): - public string Message; - } - - // 2. Define the scheduled reducer. - // It takes the schedule table struct as its second argument. - [Reducer] - public static void SendMessage(ReducerContext ctx, SendMessageSchedule scheduleArgs) - { - // Security check is important! - if (!ctx.Sender.Equals(ctx.Identity)) - { - throw new Exception("Reducer SendMessage may not be invoked by clients, only via scheduling."); - } - - Log.Info($"Scheduled SendMessage: {scheduleArgs.Message}"); - // ... perform action with scheduleArgs.Message ... - } - - // 3. Example of scheduling reducers (e.g., in Init) - [Reducer(ReducerKind.Init)] - public static void Init(ReducerContext ctx) - { - // Avoid rescheduling if Init runs again - if (ctx.Db.SendMessageSchedule.Count > 0) { - return; - } - - var tenSeconds = new TimeDuration { Microseconds = 10_000_000 }; - var futureTimestamp = ctx.Timestamp + tenSeconds; - - // Schedule a one-off message - ctx.Db.SendMessageSchedule.Insert(new SendMessageSchedule - { - Id = 0, // Let AutoInc assign ID - // Use ScheduleAt.Time for one-off execution at a specific Timestamp - ScheduledAt = new ScheduleAt.Time(futureTimestamp), - Message = "I'm a bot sending a message one time!" - }); - Log.Info("Scheduled one-off message."); - - // Schedule a periodic message (every 10 seconds) - ctx.Db.SendMessageSchedule.Insert(new SendMessageSchedule - { - Id = 0, // Let AutoInc assign ID - // Use ScheduleAt.Interval for periodic execution with a TimeDuration - ScheduledAt = new ScheduleAt.Interval(tenSeconds), - Message = "I'm a bot sending a message every 10 seconds!" - }); - Log.Info("Scheduled periodic message."); - } -} -``` - -##### Scheduled Reducer Details - -- **Best-Effort Scheduling:** Scheduled reducers are called on a best-effort basis and may be slightly delayed in their execution when a database is under heavy load. - -- **Restricting Access (Security):** Scheduled reducers are normal reducers and _can_ still be called directly by clients. If a scheduled reducer should _only_ be called by the scheduler, it is crucial to begin the reducer with a check comparing the caller's identity (`ctx.Sender`) to the module's own identity (`ctx.Identity`). - ```csharp - [Reducer] // Assuming linked via [Table(Scheduled=...)] - public static void MyScheduledTask(ReducerContext ctx, MyScheduleArgs args) - { - if (!ctx.Sender.Equals(ctx.Identity)) - { - throw new Exception("Reducer MyScheduledTask may not be invoked by clients, only via scheduling."); - } - // ... Reducer body proceeds only if called by scheduler ... - Log.Info("Executing scheduled task..."); - } - // Define MyScheduleArgs table elsewhere with [Table(Scheduled="MyScheduledTask", ...)] - public partial struct MyScheduleArgs { /* ... fields including ScheduleAt ... */ } - ``` - -:::info Scheduled Reducers and Connections -Scheduled reducer calls originate from the SpacetimeDB scheduler itself, not from an external client connection. Therefore, within a scheduled reducer, `ctx.Sender` will be the module's own identity, and `ctx.ConnectionId` will be `null`. -::: - -##### Error Handling: Exceptions - -Throwing an unhandled exception within a C# reducer will cause the transaction to roll back. - -- **Expected Failures:** For predictable errors (e.g., invalid arguments, state violations), explicitly `throw` an `Exception`. The exception message can be observed by the client in the `ReducerEventContext` status. -- **Unexpected Errors:** Unhandled runtime exceptions (e.g., `NullReferenceException`) also cause rollbacks but might provide less informative feedback to the client, potentially just indicating a general failure. - -It's generally good practice to validate input and state early in the reducer and `throw` specific exceptions for handled error conditions. - -#### View Functions - -Views are read-only functions that compute and return results from your tables. Unlike reducers, views do not modify database state - they only query and return data. Views are useful for: - -- **Computing derived data**: Join multiple tables or aggregate data before sending to clients -- **User-specific queries**: Return data specific to the requesting client (e.g., "my player") -- **Performance**: Compute results server-side, reducing data sent to clients -- **Encapsulation**: Hide complex queries behind simple interfaces - -Views can be subscribed to just like tables and automatically update when underlying data changes. - -**Defining Views** - -Views are defined using the `[SpacetimeDB.View]` attribute on static methods: - -```cs -using SpacetimeDB; - -public static partial class Module -{ - [SpacetimeDB.Table] - public partial struct Player - { - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - - [SpacetimeDB.Unique] - public Identity Identity; - - public string Name; - } - - [SpacetimeDB.Table] - public partial struct PlayerLevel - { - [SpacetimeDB.Unique] - public ulong PlayerId; - - [SpacetimeDB.Index.BTree] - public ulong Level; - } - - // Custom type for joined results - [SpacetimeDB.Type] - public partial struct PlayerAndLevel - { - public ulong Id; - public Identity Identity; - public string Name; - public ulong Level; - } - - // View that returns the caller's player (user-specific) - // Returns T? for at-most-one row - [SpacetimeDB.View(Accessor = "MyPlayer", Public = true)] - public static Player? MyPlayer(ViewContext ctx) - { - return ctx.Db.Player.Identity.Find(ctx.Sender); - } - - // View that returns all players at a specific level (same for all callers) - // Returns List for multiple rows - [SpacetimeDB.View(Accessor = "PlayersForLevel", Public = true)] - public static List PlayersForLevel(AnonymousViewContext ctx) - { - var rows = new List(); - foreach (var playerLevel in ctx.Db.PlayerLevel.Level.Filter(2)) - { - if (ctx.Db.Player.Id.Find(playerLevel.PlayerId) is Player p) - { - rows.Add(new PlayerAndLevel - { - Id = p.Id, - Identity = p.Identity, - Name = p.Name, - Level = playerLevel.Level - }); - } - } - return rows; - } -} -``` - -**ViewContext vs AnonymousViewContext** - -Views use one of two context types: - -- **`ViewContext`**: Provides access to the caller's `Identity` through `ctx.Sender`. Use this when the view depends on who is querying it (e.g., "get my player"). -- **`AnonymousViewContext`**: Does not provide caller information. Use this when the view produces the same results regardless of who queries it (e.g., "get top 10 players"). - -Both contexts provide read-only access to tables and indexes through `ctx.Db`. - -**Performance Note**: Because `AnonymousViewContext` is guaranteed not to access the caller's identity, SpacetimeDB can share the computed view results across multiple connected clients. This provides significant performance benefits for views that return the same data to all clients. Prefer `AnonymousViewContext` when possible. - -**Return Types** - -Views can return: -- `T?` (nullable) - For at-most-one row (e.g., looking up a specific player) -- `List` or `T[]` - For multiple rows (e.g., listing all players at a level) -- `IQuery` - A typed SQL query that behaves like the deprecated RLS (Row-Level Security) feature - -Where `T` can be a table type or any custom type marked with `[SpacetimeDB.Type]`. - -**IQuery Return Type** - -When a view returns `IQuery`, SpacetimeDB computes results incrementally as the underlying data changes. This enables efficient table scanning because query results are maintained incrementally rather than recomputed from scratch. Without `IQuery`, you must use indexed column lookups to access tables inside view functions. - -The query builder provides a fluent API for constructing type-safe SQL queries: - -```csharp -// This view can scan the whole table efficiently because -// IQuery results are computed incrementally -[SpacetimeDB.View(Accessor = "MyMessages", Public = true)] -public static IQuery MyMessages(ViewContext ctx) -{ - return ctx.Db.Message.Filter(msg => msg.Sender == ctx.Sender); -} - -// Query builder supports various operations: -// - .Filter(x => x.Field == value) - equality -// - .Filter(x => x.Field != value) - not equal -// - .Filter(x => x.Field > value) - greater than -// - .Filter(x => x.Field < value) - less than -// - .Filter(x => expr1 || expr2) - logical OR -``` - -**Querying Views** - -Views can be queried and subscribed to using SQL, just like tables: - -```sql -SELECT * FROM MyPlayer; -SELECT * FROM PlayersForLevel; -``` - -When subscribed to, views automatically update when their underlying tables change. - -**Best Practices** - -1. Use `ViewContext` when results depend on the caller's identity. -2. Use `AnonymousViewContext` when results are the same for all callers. -3. Keep views simple - complex joins can be expensive to recompute. -4. Views are recomputed when underlying tables change, so minimize dependencies on frequently-changing tables. -5. Use indexes on columns you filter or join on for better performance. - -### Client SDK (C#) - -This section details how to build native C# client applications (including Unity games) that interact with a SpacetimeDB module. - -#### 1. Project Setup - -- **For .NET Console/Desktop Apps:** Create a new project and add the `SpacetimeDB.ClientSDK` NuGet package: - ```bash - dotnet new console -o my_csharp_client - cd my_csharp_client - dotnet add package SpacetimeDB.ClientSDK - ``` -- **For Unity:** Add the SDK to the Unity package manager by the URL: https://github.com/clockworklabs/com.clockworklabs.spacetimedbsdk. - -#### 2. Generate Module Bindings - -Client code relies on generated bindings specific to your server module. Use the `spacetime generate` command, pointing it to your server module project: - -```bash -# From your client project directory -mkdir -p module_bindings # Or your preferred output location -spacetime generate --lang csharp \ - --out-dir module_bindings \ - --module-path ../path/to/your/server_module -``` - -Include the generated `.cs` files in your C# project or Unity Assets folder. - -#### 3. Connecting to the Database - -The core type for managing a connection is `SpacetimeDB.Types.DbConnection` (this type name comes from the generated bindings). You configure and establish a connection using a builder pattern. - -- **Builder:** Start with `DbConnection.Builder()`. -- **URI & Name:** Specify the SpacetimeDB instance URI (`.WithUri("http://localhost:3000")`) and the database name or identity (`.WithDatabaseName("my_database")`). -- **Authentication:** Provide an identity token using `.WithToken(string?)`. The SDK provides a helper `AuthToken.Token` which loads a token from a local file (initialized via `AuthToken.Init(".credentials_filename")`). If `null` or omitted for the first connection, the server issues a new identity and token (retrieved via the `OnConnect` callback). -- **Callbacks:** Register callbacks (as delegates or lambda expressions) for connection lifecycle events: - - `.OnConnect((conn, identity, token) => { ... })`: Runs on successful connection. Often used to save the `token` using `AuthToken.SaveToken(token)`. - - `.OnConnectError((exception) => { ... })`: Runs if connection fails. - - `.OnDisconnect((conn, maybeException) => { ... })`: Runs when the connection closes, either gracefully (`maybeException` is null) or due to an error. -- **Build:** Call `.Build()` to initiate the connection attempt. - -```csharp -using SpacetimeDB; -using SpacetimeDB.Types; -using System; - -public class ClientManager // Example class -{ - const string HOST = "http://localhost:3000"; - const string DB_NAME = "my_database"; // Or your specific DB name/identity - private DbConnection connection; - - public void StartConnecting() - { - // Initialize token storage (e.g., in AppData) - AuthToken.Init(".my_client_creds"); - - connection = DbConnection.Builder() - .WithUri(HOST) - .WithDatabaseName(DB_NAME) - .WithToken(AuthToken.Token) // Load token if exists - .OnConnect(HandleConnect) - .OnConnectError(HandleConnectError) - .OnDisconnect(HandleDisconnect) - .Build(); - - // Need to call FrameTick regularly - see next section - } - - private void HandleConnect(DbConnection conn, Identity identity, string authToken) - { - Console.WriteLine($"Connected. Identity: {identity}"); - AuthToken.SaveToken(authToken); // Save token for future connections - - // Register other callbacks after connecting - RegisterEventCallbacks(conn); - - // Subscribe to data - SubscribeToTables(conn); - } - - private void HandleConnectError(Exception e) - { - Console.WriteLine($"Connection Error: {e.Message}"); - // Handle error, e.g., retry or exit - } - - private void HandleDisconnect(DbConnection conn, Exception? e) - { - Console.WriteLine($"Disconnected. Reason: {(e == null ? "Requested" : e.Message)}"); - // Handle disconnection - } - - // Placeholder methods - implementations shown in later sections - private void RegisterEventCallbacks(DbConnection conn) { /* ... */ } - private void SubscribeToTables(DbConnection conn) { /* ... */ } -} -``` - -#### 4. Managing the Connection Loop - -Unlike the Rust SDK's `run_threaded` or `run_async`, the C# SDK primarily uses a manual update loop. You **must** call `connection.FrameTick()` regularly (e.g., every frame in Unity's `Update`, or in a loop in a console app) to process incoming messages and trigger callbacks. - -- **`FrameTick()`:** Processes all pending network messages, updates the local cache, and invokes registered callbacks. -- **Threading:** It is generally **not recommended** to call `FrameTick()` on a background thread if your main thread also accesses the connection's data (`connection.Db`), as this can lead to race conditions. Handle computationally intensive logic triggered by callbacks separately if needed. - -```csharp -// Example in a simple console app loop: -public void RunUpdateLoop() -{ - Console.WriteLine("Running update loop..."); - bool isRunning = true; - while(isRunning && connection != null && connection.IsConnected) - { - connection.FrameTick(); // Process messages - - // Check for user input or other app logic... - if (Console.KeyAvailable) { - var key = Console.ReadKey(true).Key; - if (key == ConsoleKey.Escape) isRunning = false; - // Handle other input... - } - - System.Threading.Thread.Sleep(16); // Avoid busy-waiting - } - connection?.Disconnect(); - Console.WriteLine("Update loop stopped."); -} -``` - -#### 5. Subscribing to Data - -Clients receive data by subscribing to SQL queries against the database's public tables. - -- **Builder:** Start with `connection.SubscriptionBuilder()`. -- **Callbacks:** - - `.OnApplied((subCtx) => { ... })`: Runs when the initial data for the subscription arrives. - - `.OnError((errCtx, exception) => { ... })`: Runs if the subscription fails (e.g., invalid SQL). -- **Subscribe:** Call `.Subscribe(new string[] {"SELECT * FROM table_a", "SELECT * FROM table_b WHERE some_col > 10"})` with a list of query strings. This returns a `SubscriptionHandle`. -- **All Tables:** `.SubscribeToAllTables()` is a convenience for simple clients but cannot be easily unsubscribed. -- **Unsubscribing:** Use `handle.Unsubscribe()` or `handle.UnsubscribeThen((subCtx) => { ... })` to stop receiving updates for specific queries. - -```csharp -using SpacetimeDB.Types; // For SubscriptionEventContext, ErrorContext -using System.Linq; - -// In ClientManager or similar class... -private void SubscribeToTables(DbConnection conn) -{ - Console.WriteLine("Subscribing to tables..."); - conn.SubscriptionBuilder() - .OnApplied(on_subscription_applied) - .OnError((errCtx, err) => { - Console.WriteLine($"Subscription failed: {err.Message}"); - }) - // Example: Subscribe to all rows from 'player' and 'message' tables - .Subscribe(new string[] { "SELECT * FROM Player", "SELECT * FROM Message" }); -} - -private void OnSubscriptionApplied(SubscriptionEventContext ctx) -{ - Console.WriteLine("Subscription applied! Initial data received."); - // Example: Print initial messages sorted by time - var messages = ctx.Db.Message.Iter().ToList(); - messages.Sort((a, b) => a.Sent.CompareTo(b.Sent)); - foreach (var msg in messages) - { - // PrintMessage(ctx.Db, msg); // Assuming a PrintMessage helper - } -} -``` - -#### 6. Accessing Cached Data & Handling Row Callbacks - -Subscribed data is stored locally in the client cache, accessible via `ctx.Db` (where `ctx` can be a `DbConnection` or any event context like `EventContext`, `SubscriptionEventContext`). - -- **Accessing Tables:** Use `ctx.Db.TableName` (e.g., `ctx.Db.Player`) to get a handle to a table's cache. -- **Iterating:** `tableHandle.Iter()` returns an `IEnumerable` over all cached rows. -- **Filtering/Finding:** Use LINQ methods (`.Where()`, `.FirstOrDefault()`, etc.) on the result of `Iter()`, or use generated index accessors like `tableHandle.FindByPrimaryKeyField(pkValue)` or `tableHandle.FilterByIndexField(value)` for efficient lookups. -- **Row Callbacks:** Register callbacks using C# events to react to changes in the cache: - - `tableHandle.OnInsert += (eventCtx, insertedRow) => { ... };` - - `tableHandle.OnDelete += (eventCtx, deletedRow) => { ... };` - - `tableHandle.OnUpdate += (eventCtx, oldRow, newRow) => { ... };` (Only for tables with a `[PrimaryKey]`) - -```csharp -using SpacetimeDB.Types; // For EventContext, Event, Reducer -using System.Linq; - -// In ClientManager or similar class... -private void RegisterEventCallbacks(DbConnection conn) -{ - conn.Db.Player.OnInsert += HandlePlayerInsert; - conn.Db.Player.OnUpdate += HandlePlayerUpdate; - conn.Db.Message.OnInsert += HandleMessageInsert; - // Remember to unregister callbacks on disconnect/cleanup: -= HandlePlayerInsert; -} - -private void HandlePlayerInsert(EventContext ctx, Player insertedPlayer) -{ - // Only react to updates caused by reducers, not initial subscription load - if (ctx.Event is not Event.SubscribeApplied) - { - Console.WriteLine($"Player joined: {insertedPlayer.Name ?? "Unknown"}"); - } -} - -private void HandlePlayerUpdate(EventContext ctx, Player oldPlayer, Player newPlayer) -{ - if (oldPlayer.Name != newPlayer.Name) - { - Console.WriteLine($"Player renamed: {oldPlayer.Name ?? "??"} -> {newPlayer.Name ?? "??"}"); - } - // ... handle other changes like online status ... -} - -private void HandleMessageInsert(EventContext ctx, Message insertedMessage) -{ - if (ctx.Event is not Event.SubscribeApplied) - { - // Find sender name from cache - var sender = ctx.Db.Player.FindByPlayerId(insertedMessage.Sender); - string senderName = sender?.Name ?? "Unknown"; - Console.WriteLine($"{senderName}: {insertedMessage.Text}"); - } -} -``` - -:::info Handling Initial Data vs. Live Updates in Callbacks -Callbacks like `OnInsert` and `OnUpdate` are triggered for both the initial data received when a subscription is first applied _and_ for subsequent live changes caused by reducers. If you need to differentiate (e.g., only react to _new_ messages, not the backlog), you can inspect the `ctx.Event` type. For example, checking `if (ctx.Event is not Event.SubscribeApplied) { ... }` ensures the code only runs for events triggered by reducers, not the initial subscription data load. -::: - -#### 7. Invoking Reducers & Handling Reducer Callbacks - -Clients trigger state changes by calling reducers defined in the server module. - -- **Invoking:** Access generated static reducer methods via `SpacetimeDB.Types.Reducer.ReducerName(arg1, arg2, ...)`. -- **Reducer Callbacks:** Register callbacks using C# events to react to the _outcome_ of reducer calls: - - `Reducer.OnReducerName += (reducerEventCtx, arg1, ...) => { ... };` - - The `reducerEventCtx.Event` contains: - - `Reducer`: The specific reducer variant record and its arguments. - - `Status`: A tagged union record: `Status.Committed`, `Status.Failed(reason)`, or `Status.OutOfEnergy`. - - `CallerIdentity`, `Timestamp`, etc. - -```csharp -using SpacetimeDB.Types; - -// In ClientManager or similar class, likely where HandleConnect is... -private void RegisterEventCallbacks(DbConnection conn) // Updated registration point -{ - // Table callbacks (from previous section) - conn.Db.Player.OnInsert += HandlePlayerInsert; - conn.Db.Player.OnUpdate += HandlePlayerUpdate; - conn.Db.Message.OnInsert += HandleMessageInsert; - - // Reducer callbacks - Reducer.OnSetName += HandleSetNameResult; - Reducer.OnSendMessage += HandleSendMessageResult; -} - -private void HandleSetNameResult(ReducerEventContext ctx, string name) -{ - // Check if the status is Failed - if (ctx.Event.Status is Status.Failed failedStatus) - { - // Check if the failure was for *our* call - if (ctx.Event.CallerIdentity == ctx.Identity) { - Console.WriteLine($"Error setting name to '{name}': {failedStatus.Reason}"); - } - } -} - -private void HandleSendMessageResult(ReducerEventContext ctx, string text) -{ - if (ctx.Event.Status is Status.Failed failedStatus) - { - if (ctx.Event.CallerIdentity == ctx.Identity) { // Our call failed - Console.WriteLine($"[Error] Failed to send message '{text}': {failedStatus.Reason}"); - } - } -} - -// Example of calling a reducer (e.g., from user input handler) -public void SendChatMessage(string message) -{ - if (!string.IsNullOrEmpty(message)) - { - Reducer.SendMessage(message); // Static method call - } -} - -``` - -### Server Module (TypeScript) - -SpacetimeDB supports TypeScript as a first-class language for writing server modules. TypeScript modules run in a WebAssembly environment and have full access to SpacetimeDB's features including tables, reducers, views, and scheduling. - -#### 1. Project Setup - -Initialize a new SpacetimeDB TypeScript module project: - -```bash -spacetime init --lang typescript my_module -cd my_module -npm install -``` - -This creates a project with the following structure: -- `src/lib.ts` - Main module file -- `package.json` - Node.js package configuration -- `tsconfig.json` - TypeScript configuration - -#### 2. Defining Tables - -Tables are defined using the `table()` function. You define the schema using type builders from the `t` object. Tables are then composed into a schema using the `schema()` function, which returns the `spacetimedb` object used for defining reducers. - -```typescript -import { schema, t, table, SenderError } from 'spacetimedb/server'; - -// Define a User table with columns -// table() takes two objects: options first, then columns -const user = table( - { name: 'user', public: true }, - { - identity: t.identity().primaryKey(), - name: t.string().optional(), - online: t.bool(), - } -); - -// Define a Message table -const message = table( - { name: 'message', public: true }, - { - sender: t.identity(), - sent: t.timestamp(), - text: t.string(), - } -); - -// Compose the schema - this gives us ctx.db.user and ctx.db.message -const spacetimedb = schema({ user, message }); -export default spacetimedb; -``` - -**Type Builders** - -SpacetimeDB TypeScript uses type builders to define column schemas. Note that type builders are called as functions (e.g., `t.string()` not `t.string`): - -- **Primitives**: `t.bool()`, `t.u8()`, `t.u16()`, `t.u32()`, `t.u64()`, `t.u128()`, `t.u256()`, `t.i8()`, `t.i16()`, `t.i32()`, `t.i64()`, `t.i128()`, `t.i256()`, `t.f32()`, `t.f64()`, `t.string()`, `t.bytes()` -- **Identity**: `t.identity()` for SpacetimeDB identity values -- **ConnectionId**: `t.connectionId()` for connection identifiers -- **Timestamp**: `t.timestamp()` for timestamps -- **Product types (structs)**: Use `t.object('TypeName', { ... })` or `t.row({ ... })` for inline row definitions -- **Sum types (enums)**: Use `t.enum('TypeName', { variant1: type1, variant2: type2 })` -- **Optional**: Use `.optional()` method on any type -- **Arrays**: Use `t.array(elementType)` - -**Column Modifiers** - -- `.primaryKey()` - Marks a column as the primary key -- `.autoInc()` - Auto-increment for numeric primary keys -- `.unique()` - Creates a unique constraint and index -- `.index()` - Creates a non-unique B-tree index -- `.optional()` - Makes the column nullable - -**Custom Types** - -For complex nested structures, define reusable types: - -```typescript -import { schema, table, t } from 'spacetimedb/server'; - -// Product type (struct) using t.object() -const PlayerStats = t.object('PlayerStats', { - health: t.u32(), - mana: t.u32(), - level: t.u32(), -}); - -// Sum type (enum) using t.enum() -const GameState = t.enum('GameState', { - Waiting: t.unit(), - Playing: t.object('Playing', { round: t.u32() }), - Finished: t.object('Finished', { winner: t.identity() }), -}); - -// Use in a table - note table() takes options object first, then columns -const player = table( - { name: 'player', public: true }, - { - id: t.u64().primaryKey().autoInc(), - identity: t.identity().unique(), - stats: PlayerStats, - gameState: GameState, - } -); - -const spacetimedb = schema({ player }); -export default spacetimedb; -``` - -#### 3. Writing Reducers - -Reducers are functions that modify database state. In TypeScript, reducer names come from exports (not string arguments): use `export const my_reducer = spacetimedb.reducer({ ... }, (ctx, args) => { ... })` or `spacetimedb.reducer((ctx) => { ... })`. - -```typescript -import { schema, table, t, SenderError } from 'spacetimedb/server'; - -const user = table( - { name: 'user', public: true }, - { - identity: t.identity().primaryKey(), - name: t.string().optional(), - online: t.bool(), - } -); - -const message = table( - { name: 'message', public: true }, - { - sender: t.identity(), - sent: t.timestamp(), - text: t.string(), - } -); - -const spacetimedb = schema({ user, message }); -export default spacetimedb; - -// Helper function for validation -function validateName(name: string) { - if (!name) { - throw new SenderError('Names must not be empty'); - } -} - -// Set user's name -// Arguments: argument types object and callback -export const set_name = spacetimedb.reducer({ name: t.string() }, (ctx, { name }) => { - validateName(name); - const user = ctx.db.user.identity.find(ctx.sender); - if (!user) { - throw new SenderError('Cannot set name for unknown user'); - } - ctx.db.user.identity.update({ ...user, name }); -}); - -// Send a message -export const send_message = spacetimedb.reducer({ text: t.string() }, (ctx, { text }) => { - if (!text) { - throw new SenderError('Messages must not be empty'); - } - ctx.db.message.insert({ - sender: ctx.sender, - text, - sent: ctx.timestamp, - }); -}); -``` - -**Reducer Context** - -The first parameter of every reducer callback is the context (`ctx`), which provides: -- `ctx.sender` - The `Identity` of the client that called the reducer -- `ctx.timestamp` - The current timestamp -- `ctx.db` - Access to database tables (e.g., `ctx.db.user`, `ctx.db.message`) - -**SenderError** - -Use `SenderError` to throw user-visible errors that will abort the transaction and be reported to the client. - -#### 4. Lifecycle Reducers - -SpacetimeDB provides special lifecycle methods on the `spacetimedb` object: - -```typescript -// Called once when the module is first published -export const init = spacetimedb.init(ctx => { - console.log('Module initialized'); - // Seed initial data, set up schedules, etc. -}); - -// Called when a client connects -export const onConnect = spacetimedb.clientConnected(ctx => { - const user = ctx.db.user.identity.find(ctx.sender); - if (user) { - ctx.db.user.identity.update({ ...user, online: true }); - } else { - ctx.db.user.insert({ - identity: ctx.sender, - name: undefined, - online: true, - }); - } -}); - -// Called when a client disconnects -export const onDisconnect = spacetimedb.clientDisconnected(ctx => { - const user = ctx.db.user.identity.find(ctx.sender); - if (user) { - ctx.db.user.identity.update({ ...user, online: false }); - } -}); -``` - -#### 5. View Functions - -Views are read-only functions that return computed data from tables. Define views using `spacetimedb.view()` or `spacetimedb.anonymousView()`: - -```typescript -// View that returns the caller's user (user-specific) -// Uses spacetimedb.view() which provides ctx.sender -export const my_user = spacetimedb.view({ name: 'my_user' }, ctx => { - return ctx.db.user.identity.find(ctx.sender); -}); - -// View that returns all online users (same for all callers) -// Uses spacetimedb.anonymousView() - no access to ctx.sender -export const online_users = spacetimedb.anonymousView({ name: 'online_users' }, ctx => { - return ctx.db.user.filter(u => u.online); -}); -``` - -**view() vs anonymousView()** - -- **`spacetimedb.view()`**: Provides access to the caller's `Identity` through `ctx.sender`. Use when results depend on who is querying. -- **`spacetimedb.anonymousView()`**: Does not provide caller information. Use when results are the same for all callers. - -**Performance Note**: Because anonymous views are guaranteed not to access the caller's identity, SpacetimeDB can share the computed view results across multiple connected clients. This provides significant performance benefits. Prefer `anonymousView()` when possible. - -#### 6. Table Operations - -All table operations are performed via `ctx.db`: - -**Insert** -```typescript -ctx.db.user.insert({ identity: ctx.sender, name: 'Alice', online: true }); -``` - -**Find by unique/primary key** -```typescript -const user = ctx.db.user.identity.find(ctx.sender); // Returns row or undefined -``` - -**Filter by indexed column** -```typescript -const onlineUsers = ctx.db.user.filter(u => u.online === true); -``` - -**Update (for tables with primary key)** -```typescript -const user = ctx.db.user.identity.find(ctx.sender); -if (user) { - ctx.db.user.identity.update({ ...user, online: false }); -} -``` - -**Delete** -```typescript -ctx.db.user.identity.delete(ctx.sender); -``` - -#### 7. Building and Publishing - -Build the module to WebAssembly: - -```bash -npm run build -``` - -Publish to SpacetimeDB: - -```bash -# Local development (from the project root, spacetimedb/ is the module directory) -spacetime publish --server local --module-path spacetimedb my_module - -# Or to SpacetimeDB cloud -spacetime publish --server maincloud --module-path spacetimedb my_module -``` - -### Client SDK (TypeScript) - -This section details how to build TypeScript/JavaScript client applications (for web browsers or Node.js) that interact with a SpacetimeDB module, using a framework-agnostic approach. - -#### 1. Project Setup - -Install the SDK package into your project: - -```bash -# Using npm -npm install spacetimedb - -# Or using pnpm -pnpm add spacetimedb - -# Or using yarn -yarn add spacetimedb -``` - -#### 2. Generate Module Bindings - -Generate the module-specific bindings using the `spacetime generate` command: - -```bash -mkdir -p src/module_bindings -spacetime generate --lang typescript \ - --out-dir src/module_bindings \ - --module-path ../path/to/your/server_module -``` - -Import the necessary generated types and SDK components: - -```typescript -// Import SDK core types -import { Identity, Status } from 'spacetimedb'; -// Import generated connection class, event contexts, and table types -import { - DbConnection, - EventContext, - ReducerEventContext, - Message, - User, -} from './module_bindings'; -// Reducer functions are accessed via conn.reducers -``` - -#### 3. Connecting to the Database - -Use the generated `DbConnection` class and its builder pattern to establish a connection. - -```typescript -import { - DbConnection, - EventContext, - ReducerEventContext, - Message, - User, -} from './module_bindings'; -import { Identity, Status } from 'spacetimedb'; - -const HOST = 'ws://localhost:3000'; -const DB_NAME = 'quickstart-chat'; -const CREDS_KEY = 'auth_token'; - -class ChatClient { - public conn: DbConnection | null = null; - public identity: Identity | null = null; - public connected: boolean = false; - // Client-side cache for user lookups - private userMap: Map = new Map(); - - constructor() { - // Bind methods to ensure `this` is correct in callbacks - this.handleConnect = this.handleConnect.bind(this); - this.handleDisconnect = this.handleDisconnect.bind(this); - this.handleConnectError = this.handleConnectError.bind(this); - this.registerTableCallbacks = this.registerTableCallbacks.bind(this); - this.registerReducerCallbacks = this.registerReducerCallbacks.bind(this); - this.subscribeToTables = this.subscribeToTables.bind(this); - this.handleMessageInsert = this.handleMessageInsert.bind(this); - this.handleUserInsert = this.handleUserInsert.bind(this); - this.handleUserUpdate = this.handleUserUpdate.bind(this); - this.handleUserDelete = this.handleUserDelete.bind(this); - this.handleSendMessageResult = this.handleSendMessageResult.bind(this); - } - - public connect() { - console.log('Attempting to connect...'); - const token = localStorage.getItem(CREDS_KEY) || null; - - const connectionInstance = DbConnection.builder() - .withUri(HOST) - .withDatabaseName(DB_NAME) - .withToken(token) - .onConnect(this.handleConnect) - .onDisconnect(this.handleDisconnect) - .onConnectError(this.handleConnectError) - .build(); - - this.conn = connectionInstance; - } - - private handleConnect(conn: DbConnection, identity: Identity, token: string) { - this.identity = identity; - this.connected = true; - localStorage.setItem(CREDS_KEY, token); // Save new/refreshed token - console.log('Connected with identity:', identity.toHexString()); - - // Register callbacks and subscribe now that we are connected - this.registerTableCallbacks(); - this.registerReducerCallbacks(); - this.subscribeToTables(); - } - - private handleDisconnect() { - console.log('Disconnected'); - this.connected = false; - this.identity = null; - this.conn = null; - this.userMap.clear(); // Clear local cache on disconnect - } - - private handleConnectError(err: Error) { - console.error('Connection Error:', err); - localStorage.removeItem(CREDS_KEY); // Clear potentially invalid token - this.conn = null; // Ensure connection is marked as unusable - } - - // Placeholder implementations for callback registration and subscription - private registerTableCallbacks() { - /* See Section 6 */ - } - private registerReducerCallbacks() { - /* See Section 7 */ - } - private subscribeToTables() { - /* See Section 5 */ - } - - // Placeholder implementations for table callbacks - private handleMessageInsert(ctx: EventContext | undefined, message: Message) { - /* See Section 6 */ - } - private handleUserInsert(ctx: EventContext | undefined, user: User) { - /* See Section 6 */ - } - private handleUserUpdate( - ctx: EventContext | undefined, - oldUser: User, - newUser: User - ) { - /* See Section 6 */ - } - private handleUserDelete(ctx: EventContext, user: User) { - /* See Section 6 */ - } - - // Placeholder for reducer callback - private handleSendMessageResult( - ctx: ReducerEventContext, - messageText: string - ) { - /* See Section 7 */ - } - - // Public methods for interaction - public sendChatMessage(message: string) { - /* See Section 7 */ - } - public setPlayerName(newName: string) { - /* See Section 7 */ - } -} - -// Example Usage: -// const client = new ChatClient(); -// client.connect(); -``` - -#### 4. Managing the Connection Loop - -The TypeScript SDK is event-driven. No manual `FrameTick()` is needed. - -#### 5. Subscribing to Data - -Subscribe to SQL queries to receive data. - -```typescript -// Part of the ChatClient class -private subscribeToTables() { - if (!this.conn) return; - - const queries = ["SELECT * FROM message", "SELECT * FROM user"]; - - console.log("Subscribing..."); - this.conn - .subscriptionBuilder() - .onApplied(() => { - console.log(`Subscription applied for: ${queries}`); - // Initial cache is now populated, process initial data if needed - this.processInitialCache(); - }) - .onError((error: Error) => { - console.error(`Subscription error:`, error); - }) - .subscribe(queries); -} - -private processInitialCache() { - if (!this.conn) return; - console.log("Processing initial cache..."); - // Populate userMap from initial cache - this.userMap.clear(); - for (const user of this.conn.db.User.iter()) { - this.handleUserInsert(undefined, user); // Pass undefined context for initial load - } - // Process initial messages, e.g., sort and display - const initialMessages = Array.from(this.conn.db.Message.iter()); - initialMessages.sort((a, b) => a.sent.getTime() - b.sent.getTime()); - for (const message of initialMessages) { - this.handleMessageInsert(undefined, message); // Pass undefined context - } -} -``` - -#### 6. Accessing Cached Data & Handling Row Callbacks - -Maintain your own collections (e.g., `Map`) updated via table callbacks for efficient lookups. - -```typescript -// Part of the ChatClient class -private registerTableCallbacks() { - if (!this.conn) return; - - this.conn.db.Message.onInsert(this.handleMessageInsert); - - // User table callbacks update the local userMap - this.conn.db.User.onInsert(this.handleUserInsert); - this.conn.db.User.onUpdate(this.handleUserUpdate); - this.conn.db.User.onDelete(this.handleUserDelete); - - // Note: In a real app, you might return a cleanup function - // to unregister these if the ChatClient is destroyed. - // e.g., return () => { this.conn?.db.Message.removeOnInsert(...) }; -} - -private handleMessageInsert(ctx: EventContext | undefined, message: Message) { - const identityStr = message.sender.toHexString(); - // Look up sender in our local map - const sender = this.userMap.get(identityStr); - const senderName = sender?.name ?? identityStr.substring(0, 8); - - if (ctx) { // Live update - console.log(`LIVE MSG: ${senderName}: ${message.text}`); - // TODO: Update UI (e.g., add to message list) - } else { // Initial load (handled in processInitialCache) - // console.log(`Initial MSG loaded: ${message.text} from ${senderName}`); - } -} - -private handleUserInsert(ctx: EventContext | undefined, user: User) { - const identityStr = user.identity.toHexString(); - this.userMap.set(identityStr, user); - const name = user.name ?? identityStr.substring(0, 8); - if (ctx) { // Live update - if (user.online) console.log(`${name} connected.`); - } else { // Initial load - // console.log(`Loaded user: ${name} (Online: ${user.online})`); - } - // TODO: Update UI (e.g., user list) -} - -private handleUserUpdate(ctx: EventContext | undefined, oldUser: User, newUser: User) { - const oldIdentityStr = oldUser.identity.toHexString(); - const newIdentityStr = newUser.identity.toHexString(); - if(oldIdentityStr !== newIdentityStr) { - this.userMap.delete(oldIdentityStr); - } - this.userMap.set(newIdentityStr, newUser); - - const name = newUser.name ?? newIdentityStr.substring(0, 8); - if (ctx) { // Live update - if (!oldUser.online && newUser.online) console.log(`${name} connected.`); - else if (oldUser.online && !newUser.online) console.log(`${name} disconnected.`); - else if (oldUser.name !== newUser.name) console.log(`Rename: ${oldUser.name ?? '...'} -> ${name}.`); - } - // TODO: Update UI (e.g., user list, messages from this user) -} - -private handleUserDelete(ctx: EventContext, user: User) { - const identityStr = user.identity.toHexString(); - const name = user.name ?? identityStr.substring(0, 8); - this.userMap.delete(identityStr); - console.log(`${name} left/deleted.`); - // TODO: Update UI -} -``` - -:::info Handling Initial Data vs. Live Updates in Callbacks -In TypeScript, the first argument (`ctx: EventContext | undefined`) to row callbacks indicates the cause. If `ctx` is defined, it's a live update. If `undefined`, it's part of the initial subscription load. -::: - -#### 7. Invoking Reducers & Handling Reducer Callbacks - -Call reducers via `conn.reducers`. Register callbacks via `conn.reducers.onReducerName(...)` to observe outcomes. - -- **Invoking:** Access generated reducer functions via `conn.reducers.reducerName(arg1, arg2, ...)`. Calling these functions sends the request to the server. -- **Reducer Callbacks:** Register callbacks using `conn.reducers.onReducerName((ctx: ReducerEventContext, arg1, ...) => { ... })` to react to the _outcome_ of reducer calls initiated by _any_ client (including your own). -- **ReducerEventContext (`ctx`)**: Contains information about the completed reducer call: - - `ctx.event.reducer`: The specific reducer variant record and its arguments. - - `ctx.event.status`: An object indicating the outcome. Check `ctx.event.status.tag` which will be a string like `"Committed"` or `"Failed"`. If failed, the reason is typically in `ctx.event.status.value`. - - `ctx.event.callerIdentity`: The `Identity` of the client that originally invoked the reducer. - - `ctx.event.message`: Contains the failure message if `ctx.event.status.tag === "Failed"`. - - `ctx.event.timestamp`, etc. - -```typescript -// Part of the ChatClient class -private registerReducerCallbacks() { - if (!this.conn) return; - - this.conn.reducers.onSendMessage(this.handleSendMessageResult); - // Register other reducer callbacks if needed - // this.conn.reducers.onSetName(handleSetNameResult); - - // Note: Consider returning a cleanup function to unregister -} - -private handleSendMessageResult(ctx: ReducerEventContext, messageText: string) { - // Check if this callback corresponds to a call made by this client instance - const wasOurCall = ctx.event.callerIdentity.isEqual(this.identity); - if (!wasOurCall) return; // Optional: Only react to your own calls - - switch(ctx.event.status.tag) { - case "Committed": - console.log(`Our message "${messageText}" sent successfully.`); - break; - case "Failed": - // Access the error message via status.value or event.message - const errorMessage = ctx.event.status.value || ctx.event.message || "Unknown error"; - console.error(`Failed to send "${messageText}": ${errorMessage}`); - break; - case "OutOfEnergy": - console.error(`Failed to send "${messageText}": Out of Energy!`); - break; - } -} - -// Public methods to be called from application logic -public sendChatMessage(message: string) { - if (this.conn && this.connected && message.trim()) { - this.conn.reducers.sendMessage(message); - } -} - -public setPlayerName(newName: string) { - if (this.conn && this.connected && newName.trim()) { - this.conn.reducers.setName(newName); - } -} -``` - -## SpacetimeDB Subscription Semantics - -This document describes the subscription semantics maintained by the SpacetimeDB host over WebSocket connections. These semantics outline message ordering guarantees, subscription handling, transaction updates, and client cache consistency. - -### WebSocket Communication Channels - -A single WebSocket connection between a client and the SpacetimeDB host consists of two distinct message channels: - -- **Client → Server:** Sends requests such as reducer invocations and subscription queries. -- **Server → Client:** Sends responses to client requests and database transaction updates. - -#### Ordering Guarantees - -The server maintains the following guarantees: - -1. **Sequential Response Ordering:** - - Responses to client requests are always sent back in the same order the requests were received. If request A precedes request B, the response to A will always precede the response to B, even if A takes longer to process. - -2. **Atomic Transaction Updates:** - - Each database transaction (e.g., reducer invocation, INSERT, UPDATE, DELETE queries) generates exactly zero or one update message sent to clients. These updates are atomic and reflect the exact order of committed transactions. - -3. **Atomic Subscription Initialization:** - - When subscriptions are established, clients receive exactly one response containing all initially matching rows from a consistent database state snapshot taken between two transactions. - - The state snapshot reflects a committed database state that includes all previous transaction updates received and excludes all future transaction updates. - -### Subscription Workflow - -When invoking `SubscriptionBuilder::subscribe(QUERIES)` from the client SDK: - -1. **Client SDK → Host:** - - Sends a `Subscribe` message containing the specified QUERIES. - -2. **Host Processing:** - - Captures a snapshot of the committed database state. - - Evaluates QUERIES against this snapshot to determine matching rows. - -3. **Host → Client SDK:** - - Sends a `SubscribeApplied` message containing the matching rows. - -4. **Client SDK Processing:** - - Receives and processes the message. - - Locks the client cache and inserts all rows atomically. - - Invokes relevant callbacks: - - `on_insert` callback for each row. - - `on_applied` callback for the subscription. - -> **Note:** No relative ordering guarantees are made regarding the invocation order of these callbacks. - -### Transaction Update Workflow - -Upon committing a database transaction: - -1. **Host Evaluates State Delta:** - - Calculates the state delta (inserts and deletes) resulting from the transaction. - -2. **Host Evaluates Queries:** - - Computes the incremental query updates relevant to subscribed clients. - -3. **Host → Client SDK:** - - Sends a `TransactionUpdate` message if relevant updates exist, containing affected rows and transaction metadata. - -4. **Client SDK Processing:** - - Receives and processes the message. - - Locks the client cache, applying deletions and insertions atomically. - - Invokes relevant callbacks: - - `on_insert`, `on_delete`, `on_update`, and `on_reducer` as necessary. - -> **Note:** - -- No relative ordering guarantees are made regarding the invocation order of these callbacks. -- Delete and insert operations within a `TransactionUpdate` have no internal order guarantees and are grouped into operation maps. - -#### Client Updates and Compute Processing - -Client SDKs must explicitly request processing time (e.g., `conn.FrameTick()` in C# or `conn.run_threaded()` in Rust) to receive and process messages. Until such a processing call is made, messages remain queued on the server-to-client channel. - -### Multiple Subscription Sets - -If multiple subscription sets are active, updates across these sets are bundled together into a single `TransactionUpdate` message. - -### Client Cache Guarantees - -- The client cache always maintains a consistent and correct subset of the committed database state. -- Callback functions invoked due to events have guaranteed visibility into a fully updated cache state. -- Reads from the client cache are effectively free as they access locally cached data. -- During callback execution, the client cache accurately reflects the database state immediately following the event-triggering transaction. - -#### Pending Callbacks and Cache Consistency - -Callbacks (`pendingCallbacks`) are queued and deferred until the cache updates (inserts/deletes) from a transaction are fully applied. This ensures all callbacks see the fully consistent state of the cache, preventing callbacks from observing an inconsistent intermediate state. +- [Module ABI Reference](/docs/webassembly-abi): This document specifies the low level details of module-host interactions ("Module ABI"). **Most users** looking to interact with the host will want to use derived and higher level functionality like bindings], #[spacetimedb(table)], and #[derive(SpacetimeType)] rather than this low level ABI. For more on those, read the [Rust module quick start guide and the Rust module reference. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 384072b8616..126a3cd14aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -248,6 +248,9 @@ importers: '@docusaurus/types': specifier: 3.9.2 version: 3.9.2(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@signalwire/docusaurus-plugin-llms-txt': + specifier: ^1.2.2 + version: 1.2.2(@docusaurus/core@3.9.2(@mdx-js/react@3.1.1(@types/react@18.3.23)(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.6.3)) '@types/react': specifier: ^18.3.23 version: 18.3.23 @@ -5470,6 +5473,12 @@ packages: '@sideway/pinpoint@2.0.0': resolution: {integrity: sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==} + '@signalwire/docusaurus-plugin-llms-txt@1.2.2': + resolution: {integrity: sha512-Qo5ZBDZpyXFlcrWwise77vs6B0R3m3/qjIIm1IHf4VzW6sVYgaR/y46BJwoAxMrV3WJkn0CVGpyYC+pMASFBZw==} + engines: {node: '>=18.0.0'} + peerDependencies: + '@docusaurus/core': ^3.0.0 + '@sigstore/bundle@4.0.0': resolution: {integrity: sha512-NwCl5Y0V6Di0NexvkTqdoVfmjTaQwoLM236r89KEojGmq/jMls8S+zb7yOwAPdXvbwfKDlP+lmXgAL4vKSQT+A==} engines: {node: ^20.17.0 || >=22.9.0} @@ -7190,6 +7199,9 @@ packages: batch@0.6.1: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} + bcp-47-match@2.0.3: + resolution: {integrity: sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==} + beasties@0.3.5: resolution: {integrity: sha512-NaWu+f4YrJxEttJSm16AzMIFtVldCvaJ68b1L098KpqXmxt9xOLtKoLkKxb8ekhOrLqEJAbvT6n6SEvB/sac7A==} engines: {node: '>=14.0.0'} @@ -7859,6 +7871,9 @@ packages: css-select@6.0.0: resolution: {integrity: sha512-rZZVSLle8v0+EY8QAkDWrKhpgt6SA5OtHsgBnsj6ZaLb5dmDVOWUDtQitd9ydxxvEjhewNudS6eTVU7uOyzvXw==} + css-selector-parser@3.3.0: + resolution: {integrity: sha512-Y2asgMGFqJKF4fq4xHDSlFYIkeVfRsm69lQC1q9kbEsH5XtnINTMrweLkjYMeaUgiXBy/uvKeO/a1JHTNnmB2g==} + css-tree@2.2.1: resolution: {integrity: sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} @@ -8137,6 +8152,10 @@ packages: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} + direction@2.0.1: + resolution: {integrity: sha512-9S6m9Sukh1cZNknO1CWAr2QAWsbKLafQiyM5gZ7VgXHeuaoUwffKN4q6NC4A/Mf9iiPlOXQEKW/Mv/mh9/3YFA==} + hasBin: true + dns-packet@5.6.1: resolution: {integrity: sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==} engines: {node: '>=6'} @@ -9129,24 +9148,48 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hast-util-embedded@3.0.0: + resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} + + hast-util-from-html@2.0.3: + resolution: {integrity: sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==} + hast-util-from-parse5@7.1.2: resolution: {integrity: sha512-Nz7FfPBuljzsN3tCQ4kCBKqdNhQE2l0Tn+X1ubgKBPRoiDIu1mL08Cfw4k7q71+Duyaw7DXDN+VTAp4Vh3oCOw==} hast-util-from-parse5@8.0.3: resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} + hast-util-has-property@3.0.0: + resolution: {integrity: sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==} + + hast-util-is-body-ok-link@3.0.1: + resolution: {integrity: sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==} + + hast-util-is-element@3.0.0: + resolution: {integrity: sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==} + + hast-util-minify-whitespace@1.0.1: + resolution: {integrity: sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==} + hast-util-parse-selector@3.1.1: resolution: {integrity: sha512-jdlwBjEexy1oGz0aJ2f4GKMaVKkA9jwjr4MjAAI22E5fM/TXVZHuS5OpONtdeIkRKqAaryQ2E9xNQxijoThSZA==} hast-util-parse-selector@4.0.0: resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + hast-util-phrasing@3.0.1: + resolution: {integrity: sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==} + hast-util-raw@7.2.3: resolution: {integrity: sha512-RujVQfVsOrxzPOPSzZFiwofMArbQke6DJjnFfceiEbFh7S05CbPt0cYN+A5YeD3pso0JQk6O1aHBnx9+Pm2uqg==} hast-util-raw@9.1.0: resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==} + hast-util-select@6.0.4: + resolution: {integrity: sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==} + hast-util-to-estree@2.3.3: resolution: {integrity: sha512-ihhPIUPxN0v0w6M5+IiAZZrn0LH2uZomeWwhn7uP7avZC6TE7lIiEh2yBMPr5+zi1aUCXq6VoYRgs2Bw9xmycQ==} @@ -9159,6 +9202,9 @@ packages: hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} + hast-util-to-mdast@10.1.2: + resolution: {integrity: sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==} + hast-util-to-parse5@7.1.0: resolution: {integrity: sha512-YNRgAJkH2Jky5ySkIqFXTQiaqcAtJyVE+D5lkN6CdtOqrnkLfGYYrEcKuHOJZlp+MwjSwuD3fZuawI+sic/RBw==} @@ -9168,6 +9214,9 @@ packages: hast-util-to-string@3.0.1: resolution: {integrity: sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==} + hast-util-to-text@4.0.2: + resolution: {integrity: sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==} + hast-util-whitespace@2.0.1: resolution: {integrity: sha512-nAxA0v8+vXSBDt3AnRUNjyRIQ0rD+ntpbAp4LnPkumc5M9yUbSMa4XDU9Q6etY4f1Wp4bNgvc1yjiZtsTTrSng==} @@ -10526,10 +10575,6 @@ packages: minimalistic-assert@1.0.1: resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} - minimatch@10.0.3: - resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==} - engines: {node: 20 || >=22} - minimatch@10.1.1: resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==} engines: {node: 20 || >=22} @@ -12388,6 +12433,12 @@ packages: react: optional: true + rehype-minify-whitespace@6.0.2: + resolution: {integrity: sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==} + + rehype-parse@9.0.1: + resolution: {integrity: sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==} + rehype-raw@6.1.1: resolution: {integrity: sha512-d6AKtisSRtDRX4aSPsJGTfnzrX2ZkHQLE5kiUuGOeEoLpbEulFF4hj0mLPbsa+7vmguDKOVVEQdHKDSwoaIDsQ==} @@ -12397,6 +12448,9 @@ packages: rehype-recma@1.0.0: resolution: {integrity: sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==} + rehype-remark@10.0.1: + resolution: {integrity: sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==} + relateurl@0.2.7: resolution: {integrity: sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==} engines: {node: '>= 0.10'} @@ -13331,6 +13385,9 @@ packages: trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + trim-trailing-lines@2.1.0: + resolution: {integrity: sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==} + trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} @@ -13595,6 +13652,9 @@ packages: resolution: {integrity: sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ==} engines: {node: '>=12'} + unist-util-find-after@5.0.0: + resolution: {integrity: sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==} + unist-util-generated@2.0.1: resolution: {integrity: sha512-qF72kLmPxAw0oN2fwpWIqbXAVyEqUzDHMsbtPvOudIlUzXYFIeQIuxXQCRCFh22B7cixvU0MG7m3MW8FTq/S+A==} @@ -18920,7 +18980,7 @@ snapshots: dependencies: string-width: 5.1.2 string-width-cjs: string-width@4.2.3 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 strip-ansi-cjs: strip-ansi@6.0.1 wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 @@ -21040,6 +21100,24 @@ snapshots: '@sideway/pinpoint@2.0.0': {} + '@signalwire/docusaurus-plugin-llms-txt@1.2.2(@docusaurus/core@3.9.2(@mdx-js/react@3.1.1(@types/react@18.3.23)(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.6.3))': + dependencies: + '@docusaurus/core': 3.9.2(@mdx-js/react@3.1.1(@types/react@18.3.23)(react@18.3.1))(debug@4.4.3)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(typescript@5.6.3) + fs-extra: 11.3.2 + hast-util-select: 6.0.4 + hast-util-to-html: 9.0.5 + hast-util-to-string: 3.0.1 + p-map: 7.0.4 + rehype-parse: 9.0.1 + rehype-remark: 10.0.1 + remark-gfm: 4.0.1 + remark-stringify: 11.0.0 + string-width: 5.1.2 + unified: 11.0.5 + unist-util-visit: 5.0.0 + transitivePeerDependencies: + - supports-color + '@sigstore/bundle@4.0.0': dependencies: '@sigstore/protobuf-specs': 0.5.0 @@ -23797,6 +23875,8 @@ snapshots: batch@0.6.1: {} + bcp-47-match@2.0.3: {} + beasties@0.3.5: dependencies: css-select: 6.0.0 @@ -24256,7 +24336,7 @@ snapshots: cliui@9.0.1: dependencies: string-width: 7.2.0 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 wrap-ansi: 9.0.2 clone-deep@4.0.1: @@ -24445,7 +24525,7 @@ snapshots: cosmiconfig@8.3.6(typescript@5.6.3): dependencies: import-fresh: 3.3.1 - js-yaml: 4.1.0 + js-yaml: 4.1.1 parse-json: 5.2.0 path-type: 4.0.0 optionalDependencies: @@ -24549,6 +24629,8 @@ snapshots: domutils: 3.2.2 nth-check: 2.1.1 + css-selector-parser@3.3.0: {} + css-tree@2.2.1: dependencies: mdn-data: 2.0.28 @@ -24808,6 +24890,8 @@ snapshots: dependencies: path-type: 4.0.0 + direction@2.0.1: {} + dns-packet@5.6.1: dependencies: '@leichtgewicht/ip-codec': 2.0.5 @@ -25828,7 +25912,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 4.1.1 - minimatch: 10.0.3 + minimatch: 10.1.1 minipass: 7.1.2 package-json-from-dist: 1.0.1 path-scurry: 2.0.0 @@ -25986,6 +26070,20 @@ snapshots: dependencies: function-bind: 1.1.2 + hast-util-embedded@3.0.0: + dependencies: + '@types/hast': 3.0.4 + hast-util-is-element: 3.0.0 + + hast-util-from-html@2.0.3: + dependencies: + '@types/hast': 3.0.4 + devlop: 1.1.0 + hast-util-from-parse5: 8.0.3 + parse5: 7.3.0 + vfile: 6.0.3 + vfile-message: 4.0.3 + hast-util-from-parse5@7.1.2: dependencies: '@types/hast': 2.3.10 @@ -26007,6 +26105,26 @@ snapshots: vfile-location: 5.0.3 web-namespaces: 2.0.1 + hast-util-has-property@3.0.0: + dependencies: + '@types/hast': 3.0.4 + + hast-util-is-body-ok-link@3.0.1: + dependencies: + '@types/hast': 3.0.4 + + hast-util-is-element@3.0.0: + dependencies: + '@types/hast': 3.0.4 + + hast-util-minify-whitespace@1.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-embedded: 3.0.0 + hast-util-is-element: 3.0.0 + hast-util-whitespace: 3.0.0 + unist-util-is: 6.0.0 + hast-util-parse-selector@3.1.1: dependencies: '@types/hast': 2.3.10 @@ -26015,6 +26133,14 @@ snapshots: dependencies: '@types/hast': 3.0.4 + hast-util-phrasing@3.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-embedded: 3.0.0 + hast-util-has-property: 3.0.0 + hast-util-is-body-ok-link: 3.0.1 + hast-util-is-element: 3.0.0 + hast-util-raw@7.2.3: dependencies: '@types/hast': 2.3.10 @@ -26045,6 +26171,24 @@ snapshots: web-namespaces: 2.0.1 zwitch: 2.0.4 + hast-util-select@6.0.4: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 + bcp-47-match: 2.0.3 + comma-separated-tokens: 2.0.3 + css-selector-parser: 3.3.0 + devlop: 1.1.0 + direction: 2.0.1 + hast-util-has-property: 3.0.0 + hast-util-to-string: 3.0.1 + hast-util-whitespace: 3.0.0 + nth-check: 2.1.1 + property-information: 7.1.0 + space-separated-tokens: 2.0.2 + unist-util-visit: 5.0.0 + zwitch: 2.0.4 + hast-util-to-estree@2.3.3: dependencies: '@types/estree': 1.0.8 @@ -26120,6 +26264,23 @@ snapshots: transitivePeerDependencies: - supports-color + hast-util-to-mdast@10.1.2: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.0 + hast-util-phrasing: 3.0.1 + hast-util-to-html: 9.0.5 + hast-util-to-text: 4.0.2 + hast-util-whitespace: 3.0.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-hast: 13.2.0 + mdast-util-to-string: 4.0.0 + rehype-minify-whitespace: 6.0.2 + trim-trailing-lines: 2.1.0 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + hast-util-to-parse5@7.1.0: dependencies: '@types/hast': 2.3.10 @@ -26143,6 +26304,13 @@ snapshots: dependencies: '@types/hast': 3.0.4 + hast-util-to-text@4.0.2: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.3 + hast-util-is-element: 3.0.0 + unist-util-find-after: 5.0.0 + hast-util-whitespace@2.0.1: {} hast-util-whitespace@3.0.0: @@ -26977,7 +27145,7 @@ snapshots: ansi-escapes: 7.3.0 cli-cursor: 5.0.0 slice-ansi: 7.1.2 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 wrap-ansi: 9.0.2 longest-streak@3.1.0: {} @@ -27957,10 +28125,6 @@ snapshots: minimalistic-assert@1.0.1: {} - minimatch@10.0.3: - dependencies: - '@isaacs/brace-expansion': 5.0.0 - minimatch@10.1.1: dependencies: '@isaacs/brace-expansion': 5.0.0 @@ -30142,6 +30306,17 @@ snapshots: '@types/react': 18.3.23 react: 18.3.1 + rehype-minify-whitespace@6.0.2: + dependencies: + '@types/hast': 3.0.4 + hast-util-minify-whitespace: 1.0.1 + + rehype-parse@9.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-from-html: 2.0.3 + unified: 11.0.5 + rehype-raw@6.1.1: dependencies: '@types/hast': 2.3.10 @@ -30162,6 +30337,14 @@ snapshots: transitivePeerDependencies: - supports-color + rehype-remark@10.0.1: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + hast-util-to-mdast: 10.1.2 + unified: 11.0.5 + vfile: 6.0.3 + relateurl@0.2.7: {} remark-directive@3.0.1: @@ -30930,13 +31113,13 @@ snapshots: dependencies: eastasianwidth: 0.2.0 emoji-regex: 9.2.2 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 string-width@7.2.0: dependencies: emoji-regex: 10.6.0 get-east-asian-width: 1.4.0 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 string-width@8.1.1: dependencies: @@ -31289,6 +31472,8 @@ snapshots: trim-lines@3.0.1: {} + trim-trailing-lines@2.1.0: {} + trough@2.2.0: {} ts-api-utils@2.1.0(typescript@5.6.3): @@ -31606,6 +31791,11 @@ snapshots: dependencies: crypto-random-string: 4.0.0 + unist-util-find-after@5.0.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.0 + unist-util-generated@2.0.1: {} unist-util-is@5.2.1: @@ -32536,7 +32726,7 @@ snapshots: dependencies: ansi-styles: 6.2.1 string-width: 7.2.0 - strip-ansi: 7.1.0 + strip-ansi: 7.1.2 wrappy@1.0.2: {} @@ -32553,7 +32743,6 @@ snapshots: ws@8.18.3: {} - wsl-utils@0.1.0: dependencies: is-wsl: 3.1.0 diff --git a/skills/spacetimedb-cli/SKILL.md b/skills/cli/SKILL.md similarity index 75% rename from skills/spacetimedb-cli/SKILL.md rename to skills/cli/SKILL.md index 0aa0d2ac9f5..b984f44459a 100644 --- a/skills/spacetimedb-cli/SKILL.md +++ b/skills/cli/SKILL.md @@ -1,6 +1,14 @@ --- -name: spacetimedb-cli +name: cli description: SpacetimeDB CLI reference for initializing projects, building modules, publishing databases, querying data, and managing servers +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: shared + language: all + cursor_globs: "**/*" + cursor_always_apply: false triggers: - spacetime init - spacetime build @@ -51,7 +59,7 @@ spacetime publish my-database --yes spacetime publish my-database --server local --yes # Clear database and republish -spacetime publish my-database --clear-database --yes +spacetime publish my-database --delete-data always --yes ``` ### Database Interaction @@ -61,8 +69,8 @@ spacetime publish my-database --clear-database --yes spacetime sql my-database "SELECT * FROM users" spacetime sql my-database --interactive # REPL mode -# Call reducers -spacetime call my-database my_reducer '{"arg1": "value", "arg2": 123}' +# Call reducers (each argument is a separate positional arg) +spacetime call my-database my_reducer '"value"' '123' # Subscribe to changes spacetime subscribe my-database "SELECT * FROM users" --num-updates 10 @@ -136,46 +144,6 @@ spacetime logout | `maincloud` | `https://maincloud.spacetimedb.com` | Production cloud (default) | | `local` | `http://127.0.0.1:3000` | Local development server | -## Common Workflows - -### New Project Setup - -```bash -# 1. Login -spacetime login - -# 2. Create project -spacetime init my-game --lang rust -cd my-game - -# 3. Start dev mode (auto-rebuilds and publishes) -spacetime dev -``` - -### Local Development - -```bash -# Start local server (in separate terminal) -spacetime start - -# Publish to local -spacetime publish my-db --server local --clear-database --yes - -# Query local database -spacetime sql my-db --server local "SELECT * FROM players" -``` - -### Generate Client Bindings - -```bash -# After building module -spacetime build -spacetime generate --lang typescript --out-dir ./client/src/bindings --module-path . - -# Or use dev mode which auto-generates -spacetime dev --client-lang typescript --module-bindings-path ./client/src/bindings -``` - ## Common Flags | Flag | Short | Description | @@ -202,7 +170,7 @@ spacetime server ping ### "Schema conflict" ```bash # Clear data and republish -spacetime publish my-db --clear-database --yes +spacetime publish my-db --delete-data always --yes ``` ### "Build failed" @@ -216,12 +184,6 @@ rustup target add wasm32-unknown-unknown ## Module Languages **Server-side (modules):** Rust, C#, TypeScript, C++ -**Client SDKs:** TypeScript, C#, Rust, Python, Unreal Engine +**Client SDKs:** TypeScript, C#, Rust, Unreal Engine **CLI `generate` targets:** TypeScript, C#, Rust, Unreal C++ -## Notes - -- Many commands are marked UNSTABLE and may change -- Default server is `maincloud` unless configured otherwise -- Use `--yes` flag in scripts to avoid interactive prompts -- Dev mode watches files and auto-rebuilds on changes diff --git a/skills/concepts/SKILL.md b/skills/concepts/SKILL.md new file mode 100644 index 00000000000..2452fe862d6 --- /dev/null +++ b/skills/concepts/SKILL.md @@ -0,0 +1,108 @@ +--- +name: concepts +description: Understand SpacetimeDB architecture and core concepts. Use when learning SpacetimeDB or making architectural decisions. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: shared + language: all + cursor_globs: "**/*" + cursor_always_apply: true +--- + +# SpacetimeDB Core Concepts + +SpacetimeDB is a relational database that is also a server. It lets you upload application logic directly into the database via WebAssembly modules, eliminating the traditional web/game server layer entirely. + +--- + +## Critical Rules + +1. **Reducers are transactional** — they do not return data to callers. Use subscriptions to read data. +2. **Reducers must be deterministic** — no filesystem, network, timers, or random. All state must come from tables. +3. **Read data via tables/subscriptions** — not reducer return values. Clients get data through subscribed queries. +4. **Auto-increment IDs are not sequential** — gaps are normal, do not use for ordering. Use timestamps or explicit sequence columns. +5. **`ctx.sender` is the authenticated principal** — never trust identity passed as arguments. + +--- + +## Feature Implementation Checklist + +1. **Backend:** Define table(s) to store the data +2. **Backend:** Define reducer(s) to mutate the data +3. **Client:** Subscribe to the table(s) +4. **Client:** Call the reducer(s) from UI +5. **Client:** Render the data from the table(s) + +--- + +## Debugging Checklist + +1. Is SpacetimeDB server running? (`spacetime start`) +2. Is the module published? (`spacetime publish`) +3. Are client bindings generated? (`spacetime generate`) +4. Check server logs for errors (`spacetime logs `) +5. Is the reducer actually being called from the client? + +--- + +## Tables + +- **Private tables** (default): Only accessible by reducers and the database owner. +- **Public tables**: Exposed for client read access through subscriptions. Writes still require reducers. + +Organize data by access pattern, not by entity: + +``` +Player PlayerState PlayerStats +id <-- player_id player_id +name position_x total_kills + position_y total_deaths + velocity_x play_time +``` + +## Reducers + +Reducers are transactional functions that modify database state. They run atomically, cannot interact with the outside world, and do not return data to callers. See the language-specific server skills for syntax. + +## Event Tables + +Event tables broadcast reducer-specific data to clients. Rows are never stored in the client cache (`count()` returns 0, `iter()` yields nothing) — only `onInsert` callbacks fire. + +## Subscriptions + +Subscriptions replicate database rows to clients in real-time. + +1. **Subscribe**: Register SQL queries describing needed data +2. **Receive initial data**: All matching rows are sent immediately +3. **Receive updates**: Real-time updates when subscribed rows change +4. **React to changes**: Use callbacks (`onInsert`, `onDelete`, `onUpdate`) + +Best practices: +- Group subscriptions by lifetime +- Subscribe before unsubscribing when updating subscriptions +- Avoid overlapping queries +- Use indexes for efficient queries + +## Modules + +Modules are WebAssembly bundles containing application logic that runs inside the database. + +- **Tables**: Define the data schema +- **Reducers**: Define callable functions that modify state +- **Event Tables**: Broadcast reducer-specific data to clients +- **Views**: Read-only functions that expose computed subsets of data to clients +- **Procedures**: (Unstable) Functions that can have side effects (HTTP requests, `ctx.withTx`) + +Server-side modules can be written in: Rust, C#, TypeScript, C++ + +Lifecycle: Write → Compile → Publish (`spacetime publish`) → Hot-swap (republish without disconnecting clients) + +## Identity + +- **Identity**: A long-lived, globally unique identifier for a user. +- **ConnectionId**: Identifies a specific client connection. +- Always use `ctx.sender` / `ctx.Sender` / `ctx.sender()` for authorization. + +SpacetimeDB works with many OIDC providers, including SpacetimeAuth (built-in), Auth0, Clerk, Keycloak, Google, and GitHub. diff --git a/skills/cpp-server/SKILL.md b/skills/cpp-server/SKILL.md new file mode 100644 index 00000000000..9984eeed54d --- /dev/null +++ b/skills/cpp-server/SKILL.md @@ -0,0 +1,273 @@ +--- +name: cpp-server +description: SpacetimeDB C++ server module SDK reference. Use when writing tables, reducers, or module logic in C++. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: server + language: cpp + cursor_globs: "**/*.cpp,**/*.h,**/*.hpp" + cursor_always_apply: true +--- + +# SpacetimeDB C++ SDK Reference + +## Imports + +```cpp +#include +using namespace SpacetimeDB; +``` + +## Tables + +Register structs with macros, then declare as tables: + +```cpp +struct Entity { + uint64_t id; + Identity owner; + std::string name; + bool active; +}; +SPACETIMEDB_STRUCT(Entity, id, owner, name, active) +SPACETIMEDB_TABLE(Entity, entity, Public) +FIELD_PrimaryKeyAutoInc(entity, id) +FIELD_Index(entity, name) +``` + +Options: `SPACETIMEDB_TABLE(Type, accessor, Public|Private)` + +Field constraints: +- `FIELD_PrimaryKey(accessor, field)` — primary key +- `FIELD_PrimaryKeyAutoInc(accessor, field)` — primary key with auto-increment (use 0 on insert) +- `FIELD_Unique(accessor, field)` — unique constraint +- `FIELD_Index(accessor, field)` — btree index (enables `.filter()`) + +## Column Types + +| C++ type | Notes | +|----------|-------| +| `uint8_t` / `uint16_t` / `uint32_t` / `uint64_t` | unsigned integers | +| `SpacetimeDB::u128` / `SpacetimeDB::u256` | large unsigned integers | +| `int8_t` / `int16_t` / `int32_t` / `int64_t` | signed integers | +| `SpacetimeDB::i128` / `SpacetimeDB::i256` | large signed integers | +| `float` / `double` | floats | +| `bool` | boolean | +| `std::string` | text | +| `std::vector` | list/array | +| `std::optional` | nullable column | +| `Identity` | user identity | +| `ConnectionId` | connection handle | +| `Timestamp` | server timestamp (microseconds since epoch) | +| `TimeDuration` | duration in microseconds | +| `ScheduleAt` | for scheduled tables | + +## Indexes + +```cpp +// Single-column: +FIELD_Index(entity, name) +// Access: ctx.db[entity_name].filter("Alice") + +// Multi-column: +FIELD_NamedMultiColumnIndex(score, by_player_and_level, player_id, level) +``` + +Range queries (requires `#include `): +```cpp +ctx.db[user_age].filter(range_inclusive(uint8_t(18), uint8_t(65))); +ctx.db[user_age].filter(range_from(uint8_t(18))); +``` + +## Reducers + +All reducers return `ReducerResult` — use `Ok()` or `Err(message)`: + +```cpp +SPACETIMEDB_REDUCER(create_entity, ReducerContext ctx, std::string name) { + if (name.empty()) { + return Err("Name cannot be empty"); + } + ctx.db[entity].insert(Entity{0, ctx.sender(), name, true}); + return Ok(); +} +``` + +## DB Operations + +```cpp +ctx.db[entity].insert(Entity{0, owner, "Sample", true}); // Insert (0 for autoInc) +ctx.db[entity_id].find(entityId); // Find by PK → std::optional +ctx.db[entity_identity].find(ctx.sender()); // Find by unique column +ctx.db[entity_name].filter("Alice"); // Filter by index → iterable +ctx.db[entity]; // All rows → iterable (range-for) +ctx.db[entity].count(); // Count rows + +// Update: find, mutate, update +if (auto e = ctx.db[entity_id].find(entityId)) { + e->name = "New Name"; + ctx.db[entity_id].update(*e); +} + +// Delete by primary key +ctx.db[entity_id].delete_by_key(entityId); +``` + +Note: Bracket notation `ctx.db[accessor]` is used for all table access. The accessor name comes from `SPACETIMEDB_TABLE` and `FIELD_*` macros. + +## Lifecycle Hooks + +```cpp +SPACETIMEDB_INIT(init, ReducerContext ctx) { + LOG_INFO("Database initializing..."); + return Ok(); +} + +SPACETIMEDB_CLIENT_CONNECTED(on_connect, ReducerContext ctx) { + LOG_INFO("Connected: " + ctx.sender().to_string()); + return Ok(); +} + +SPACETIMEDB_CLIENT_DISCONNECTED(on_disconnect, ReducerContext ctx) { + LOG_INFO("Disconnected: " + ctx.sender().to_string()); + return Ok(); +} +``` + +## Authentication & Timestamps + +```cpp +// Auth: ctx.sender() is the caller's Identity +if (row.owner != ctx.sender()) { + return Err("unauthorized"); +} + +// Server timestamps +ctx.db[item].insert(Item{0, ctx.sender(), ctx.timestamp}); + +// Timestamp arithmetic +Timestamp later = ctx.timestamp + TimeDuration::from_seconds(10); +``` + +## Reducer Context + +```cpp +ctx.db[table] // Table access (bracket notation) +ctx.sender() // Caller's Identity +ctx.timestamp // Invocation timestamp +ctx.connection_id // std::optional +ctx.identity() // Module's own identity +ctx.rng() // Deterministic RNG +ctx.sender_auth() // AuthCtx with JWT claims +``` + +## Scheduled Tables + +```cpp +struct Reminder { + uint64_t scheduled_id; + ScheduleAt scheduled_at; + std::string message; +}; +SPACETIMEDB_STRUCT(Reminder, scheduled_id, scheduled_at, message) +SPACETIMEDB_TABLE(Reminder, reminder, Public) +FIELD_PrimaryKeyAutoInc(reminder, scheduled_id) +SPACETIMEDB_SCHEDULE(reminder, 1, send_reminder) // 1 = scheduled_at field index (0-based) + +SPACETIMEDB_REDUCER(send_reminder, ReducerContext ctx, Reminder arg) { + LOG_INFO("Reminder: " + arg.message); + return Ok(); +} + +// One-time: fires at a specific time +ctx.db[reminder].insert(Reminder{0, ScheduleAt::time(ctx.timestamp + TimeDuration::from_seconds(10)), "msg"}); +// Repeating: fires on an interval +ctx.db[reminder].insert(Reminder{0, ScheduleAt::interval(TimeDuration::from_seconds(5)), "msg"}); +``` + +## Custom Types + +```cpp +// Struct (product type): +struct Point { float x; float y; }; +SPACETIMEDB_STRUCT(Point, x, y) + +// Enum (sum type): +SPACETIMEDB_UNIT_TYPE(Active) +SPACETIMEDB_UNIT_TYPE(Inactive) +SPACETIMEDB_ENUM(PlayerStatus, + (Active, Active), + (Inactive, Inactive), + (Suspended, std::string) +) +``` + +## Logging + +```cpp +LOG_INFO("Message: " + msg); +LOG_WARN("Warning: " + msg); +LOG_ERROR("Error: " + msg); +LOG_DEBUG("Debug: " + msg); +LOG_PANIC("Fatal: " + msg); // terminates reducer +``` + +## Complete Example + +```cpp +#include +using namespace SpacetimeDB; + +struct Entity { + Identity identity; + std::string name; + bool active; +}; +SPACETIMEDB_STRUCT(Entity, identity, name, active) +SPACETIMEDB_TABLE(Entity, entity, Public) +FIELD_PrimaryKey(entity, identity) + +struct Record { + uint64_t id; + Identity owner; + uint32_t value; + Timestamp created_at; +}; +SPACETIMEDB_STRUCT(Record, id, owner, value, created_at) +SPACETIMEDB_TABLE(Record, record, Public) +FIELD_PrimaryKeyAutoInc(record, id) + +SPACETIMEDB_CLIENT_CONNECTED(on_connect, ReducerContext ctx) { + if (auto existing = ctx.db[entity_identity].find(ctx.sender())) { + existing->active = true; + ctx.db[entity_identity].update(*existing); + } + return Ok(); +} + +SPACETIMEDB_CLIENT_DISCONNECTED(on_disconnect, ReducerContext ctx) { + if (auto existing = ctx.db[entity_identity].find(ctx.sender())) { + existing->active = false; + ctx.db[entity_identity].update(*existing); + } + return Ok(); +} + +SPACETIMEDB_REDUCER(create_entity, ReducerContext ctx, std::string name) { + if (ctx.db[entity_identity].find(ctx.sender())) { + return Err("already exists"); + } + ctx.db[entity].insert(Entity{ctx.sender(), name, true}); + return Ok(); +} + +SPACETIMEDB_REDUCER(add_record, ReducerContext ctx, uint32_t value) { + if (!ctx.db[entity_identity].find(ctx.sender())) { + return Err("not found"); + } + ctx.db[record].insert(Record{0, ctx.sender(), value, ctx.timestamp}); + return Ok(); +} +``` diff --git a/skills/csharp-client/SKILL.md b/skills/csharp-client/SKILL.md new file mode 100644 index 00000000000..93a44e55f7f --- /dev/null +++ b/skills/csharp-client/SKILL.md @@ -0,0 +1,163 @@ +--- +name: csharp-client +description: SpacetimeDB C#/.NET client SDK reference. Use when building C# clients that connect to SpacetimeDB (console, desktop, or any .NET app). +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: client + language: csharp + cursor_globs: "**/*.cs" + cursor_always_apply: true +--- + +# SpacetimeDB C# Client + +Install: `dotnet add package SpacetimeDB.ClientSDK` + +## Connection + +```csharp +using SpacetimeDB; +using SpacetimeDB.Types; + +var conn = DbConnection.Builder() + .WithUri("http://localhost:3000") + .WithDatabaseName("my-database") + .WithToken(savedToken) + .OnConnect((conn, identity, token) => + { + Console.WriteLine($"Connected as: {identity}"); + // Save token for reconnection + File.WriteAllText("auth_token.txt", token); + + conn.SubscriptionBuilder() + .OnApplied(OnSubscriptionApplied) + .SubscribeToAllTables(); + }) + .OnConnectError(err => Console.Error.WriteLine($"Connection failed: {err}")) + .OnDisconnect((conn, err) => + { + if (err != null) Console.Error.WriteLine($"Disconnected: {err}"); + }) + .Build(); +``` + +## Event Loop — Critical + +**`FrameTick()` must be called in your main loop.** The SDK queues all network messages and only processes them when you call `FrameTick()`. Without it, no callbacks fire. + +```csharp +while (running) +{ + conn.FrameTick(); + // Your application logic... + Thread.Sleep(16); // ~60fps +} +``` + +**Thread safety**: `FrameTick()` processes messages on the calling thread. Do NOT call it from a background thread. Do NOT access `conn.Db` from background threads. + +## Subscriptions + +```csharp +// Subscribe to all tables +conn.SubscriptionBuilder() + .OnApplied(ctx => Console.WriteLine("Subscription ready")) + .SubscribeToAllTables(); + +// Subscribe with typed query builder (recommended) +conn.SubscriptionBuilder() + .OnApplied(OnSubscriptionApplied) + .AddQuery(q => q.From.Player().Where(p => p.Level.Gte(5u))) + .AddQuery(q => q.From.GameState()) + .Subscribe(); + +// Or with raw SQL strings +conn.SubscriptionBuilder() + .OnApplied(OnSubscriptionApplied) + .Subscribe(new[] { + "SELECT * FROM player WHERE level >= 5", + "SELECT * FROM game_state" + }); +``` + +## Row Callbacks + +```csharp +conn.Db.Player.OnInsert += (EventContext ctx, Player player) => +{ + Console.WriteLine($"Player joined: {player.Name}"); +}; + +conn.Db.Player.OnDelete += (EventContext ctx, Player player) => +{ + Console.WriteLine($"Player left: {player.Name}"); +}; + +conn.Db.Player.OnUpdate += (EventContext ctx, Player oldPlayer, Player newPlayer) => +{ + Console.WriteLine($"Player updated: {newPlayer.Name}"); +}; +``` + +## Reading the Client Cache + +```csharp +// Find by primary key +if (conn.Db.Player.Id.Find(playerId) is Player player) +{ + Console.WriteLine($"Player: {player.Name}"); +} + +// Find by unique column +var me = conn.Db.Player.Identity.Find(myIdentity); + +// Filter by indexed column +foreach (var p in conn.Db.Player.Level.Filter(5)) +{ + Console.WriteLine($"Level 5: {p.Name}"); +} + +// Iterate all rows +foreach (var p in conn.Db.Player.Iter()) +{ + Console.WriteLine(p.Name); +} + +// Count +int total = conn.Db.Player.Count; +``` + +## Calling Reducers + +```csharp +conn.Reducers.CreatePlayer("Alice"); +conn.Reducers.MovePlayer(10.0f, 20.0f); +conn.Reducers.SendMessage("Hello!"); +``` + +## Reducer Callbacks + +```csharp +conn.Reducers.OnSendMessage += (ReducerEventContext ctx, string text) => +{ + if (ctx.Event.Status is Status.Committed) + Console.WriteLine($"Message sent: {text}"); + else if (ctx.Event.Status is Status.Failed(var reason)) + Console.Error.WriteLine($"Send failed: {reason}"); +}; +``` + +## Identity + +```csharp +// Identities from OnConnect callback +Identity myIdentity; + +// Compare identities +if (player.Owner == myIdentity) { /* it's me */ } + +// Display +Console.WriteLine($"Identity: {identity}"); +``` diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md new file mode 100644 index 00000000000..7695d6f88e6 --- /dev/null +++ b/skills/csharp-server/SKILL.md @@ -0,0 +1,303 @@ +--- +name: csharp-server +description: SpacetimeDB C# server module SDK reference. Use when writing tables, reducers, or module logic in C#. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: server + language: csharp + cursor_globs: "**/*.cs" + cursor_always_apply: true +--- + +# SpacetimeDB C# SDK Reference + +## Imports + +```csharp +using SpacetimeDB; +``` + +## Module Structure + +All tables, types, and reducers go inside a static partial class: + +```csharp +using SpacetimeDB; + +public static partial class Module +{ + // Tables, types, and reducers here +} +``` + +## Tables + +`[SpacetimeDB.Table(...)]` on a `public partial struct` — `Accessor` should be PascalCase: + +```csharp +[SpacetimeDB.Table(Accessor = "Entity", Public = true)] +public partial struct Entity +{ + [PrimaryKey] + [AutoInc] + public ulong Id; + public Identity Owner; + public string Name; + public bool Active; +} +``` + +Options: `Accessor = "PascalCase"` (recommended), `Public = true`, `Scheduled = nameof(ReducerFn)`, `ScheduledAt = nameof(field)`, `Event = true` + +`ctx.Db` accessors use the `Accessor` name: `ctx.Db.Entity`, `ctx.Db.Record`. + +## Column Types + +| C# type | Notes | +|---------|-------| +| `byte` / `ushort` / `uint` / `ulong` | unsigned integers | +| `U128` / `U256` | large unsigned integers (SpacetimeDB types) | +| `sbyte` / `short` / `int` / `long` | signed integers | +| `I128` / `I256` | large signed integers (SpacetimeDB types) | +| `float` / `double` | floats | +| `bool` | boolean | +| `string` | text | +| `List` | list/array | +| `Identity` | user identity | +| `ConnectionId` | connection handle | +| `Timestamp` | server timestamp (microseconds since epoch) | +| `TimeDuration` | duration in microseconds | +| `Uuid` | UUID | + +## Column Attributes + +```csharp +[PrimaryKey] // primary key +[AutoInc] // auto-increment (use 0 as placeholder on insert) +[Unique] // unique constraint +[SpacetimeDB.Index.BTree] // btree index (enables .Filter() on this column) +``` + +## Indexes + +Prefer `[SpacetimeDB.Index.BTree]` inline for single-column. Multi-column uses struct-level: + +```csharp +// Inline (preferred for single-column): +[SpacetimeDB.Index.BTree] +public ulong AuthorId; +// Access: ctx.Db.Post.AuthorId.Filter(authorId) + +// Multi-column (struct-level): +[SpacetimeDB.Table(Accessor = "Membership")] +[SpacetimeDB.Index.BTree(Accessor = "ByGroupUser", Columns = ["GroupId", "UserId"])] +public partial struct Membership { public ulong GroupId; public Identity UserId; ... } +``` + +When you frequently look up rows by multiple columns, prefer a multi-column index over filtering by one column and looping over the results. + +## Reducers + +```csharp +[SpacetimeDB.Reducer] +public static void CreateEntity(ReducerContext ctx, string name, int age) +{ + ctx.Db.Entity.Insert(new Entity { Owner = ctx.Sender, Name = name, Age = age, Active = true }); +} + +// No arguments: +[SpacetimeDB.Reducer] +public static void DoReset(ReducerContext ctx) { ... } +``` + +## DB Operations + +```csharp +ctx.Db.Entity.Insert(new Entity { Name = "Sample" }); // Insert +ctx.Db.Entity.Id.Find(entityId); // Find by PK → Entity? (nullable) +ctx.Db.Entity.Identity.Find(ctx.Sender); // Find by unique column → Entity? +ctx.Db.Item.AuthorId.Filter(authorId); // Filter by index → IEnumerable +ctx.Db.Entity.Iter(); // All rows → IEnumerable +ctx.Db.Entity.Count; // Count rows +ctx.Db.Entity.Id.Update(existing with { Name = newName }); // Update by PK +ctx.Db.Entity.Id.Delete(entityId); // Delete by PK +``` + +Note: Filter/Iter return enumerables. Use `.ToList()` if you need to sort or mutate. + +The pattern is `ctx.Db.{Accessor}.{ColumnName}.{Method}(value)` for all indexed column operations. + +## Lifecycle Hooks + +```csharp +[SpacetimeDB.Reducer(ReducerKind.Init)] +public static void OnInit(ReducerContext ctx) { ... } + +[SpacetimeDB.Reducer(ReducerKind.ClientConnected)] +public static void OnConnect(ReducerContext ctx) { ... } + +[SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] +public static void OnDisconnect(ReducerContext ctx) { ... } +``` + +## Views + +```csharp +// Anonymous view (same result for all clients): +[SpacetimeDB.View(Accessor = "ActiveUsers", Public = true)] +public static List ActiveUsers(AnonymousViewContext ctx) +{ + return ctx.Db.Entity.Iter().Where(e => e.Active).ToList(); +} + +// Per-user view: +[SpacetimeDB.View(Accessor = "MyProfile", Public = true)] +public static Entity? MyProfile(ViewContext ctx) +{ + return ctx.Db.Entity.Identity.Find(ctx.Sender) as Entity?; +} +``` + +## Authentication & Timestamps + +```csharp +// Auth: ctx.Sender is the caller's Identity +if (row.Owner != ctx.Sender) + throw new Exception("unauthorized"); + +// Server timestamps +ctx.Db.Item.Insert(new Item { CreatedAt = ctx.Timestamp, .. }); + +// Timestamp arithmetic +var expiry = ctx.Timestamp + new TimeDuration(delayMicros); + +// Client: Timestamp → milliseconds since epoch +timestamp.MicrosecondsSinceUnixEpoch / 1000 +``` + +## Scheduled Tables + +```csharp +[SpacetimeDB.Table( + Accessor = "TickTimer", + Scheduled = nameof(Tick), + ScheduledAt = nameof(ScheduledAt), + Public = true +)] +public partial struct TickTimer +{ + [PrimaryKey] + [AutoInc] + public ulong ScheduledId; + public ScheduleAt ScheduledAt; +} + +[SpacetimeDB.Reducer] +public static void Tick(ReducerContext ctx, TickTimer timer) +{ + // timer row is auto-deleted after this reducer runs +} + +// One-time: fires once at a specific time +var at = new ScheduleAt.Time(DateTimeOffset.UtcNow.AddSeconds(10)); +// Repeating: fires on an interval +var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5)); + +ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); +``` + +## Custom Types + +```csharp +[SpacetimeDB.Type] +public enum Status { Online, Away, Offline } + +[SpacetimeDB.Type] +public partial struct Point { public float X; public float Y; } + +// Tagged enum (discriminated union): +[SpacetimeDB.Type] +public partial record MyUnion : SpacetimeDB.TaggedEnum<(string Text, int Number)>; +``` + +## Optional Fields + +```csharp +[SpacetimeDB.Table(Accessor = "Player")] +public partial struct Player +{ + [PrimaryKey, AutoInc] + public ulong Id; + public string Name; + public string? Nickname; + public uint? HighScore; +} +``` + +## Complete Example + +```csharp +using SpacetimeDB; + +[SpacetimeDB.Table(Accessor = "Entity", Public = true)] +public partial struct Entity +{ + [PrimaryKey] + public Identity Identity; + public string Name; + public bool Active; +} + +[SpacetimeDB.Table(Accessor = "Record", Public = true)] +public partial struct Record +{ + [PrimaryKey] + [AutoInc] + public ulong Id; + public Identity Owner; + public uint Value; + public Timestamp CreatedAt; +} + +public static partial class Module +{ + [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] + public static void OnConnect(ReducerContext ctx) + { + var existing = ctx.Db.Entity.Identity.Find(ctx.Sender); + if (existing is not null) + ctx.Db.Entity.Identity.Update(existing.Value with { Active = true }); + } + + [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] + public static void OnDisconnect(ReducerContext ctx) + { + var existing = ctx.Db.Entity.Identity.Find(ctx.Sender); + if (existing is not null) + ctx.Db.Entity.Identity.Update(existing.Value with { Active = false }); + } + + [SpacetimeDB.Reducer] + public static void CreateEntity(ReducerContext ctx, string name) + { + if (ctx.Db.Entity.Identity.Find(ctx.Sender) is not null) + throw new Exception("already exists"); + ctx.Db.Entity.Insert(new Entity { Identity = ctx.Sender, Name = name, Active = true }); + } + + [SpacetimeDB.Reducer] + public static void AddRecord(ReducerContext ctx, uint value) + { + if (ctx.Db.Entity.Identity.Find(ctx.Sender) is null) + throw new Exception("not found"); + ctx.Db.Record.Insert(new Record { + Id = 0, + Owner = ctx.Sender, + Value = value, + CreatedAt = ctx.Timestamp, + }); + } +} +``` diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md new file mode 100644 index 00000000000..0c97041b1ec --- /dev/null +++ b/skills/rust-server/SKILL.md @@ -0,0 +1,284 @@ +--- +name: rust-server +description: SpacetimeDB Rust server module SDK reference. Use when writing tables, reducers, or module logic in Rust. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: server + language: rust + cursor_globs: "**/*.rs" + cursor_always_apply: true +--- + +# SpacetimeDB Rust SDK Reference + +## Imports + +```rust +use spacetimedb::{ + reducer, table, Identity, ReducerContext, SpacetimeType, Table, + ConnectionId, ScheduleAt, TimeDuration, Timestamp, Uuid, +}; +``` + +**`Table` is required** — without it, `ctx.db.*.insert()`, `.iter()`, `.find()` etc. won't compile (`no method named 'insert' found`). + +## Tables + +`#[spacetimedb::table(...)]` on a `pub struct` — `accessor` must be snake_case: + +```rust +#[spacetimedb::table(accessor = entity, public)] +pub struct Entity { + #[primary_key] + #[auto_inc] + pub id: u64, + pub owner: Identity, + pub name: String, + pub active: bool, +} +``` + +Options: `accessor = snake_case` (required), `public`, `scheduled(reducer_fn)`, `index(...)` + +`ctx.db` accessors use the `accessor` name (snake_case). + +## Column Types + +| Rust type | Notes | +|-----------|-------| +| `u8` / `u16` / `u32` / `u64` / `u128` | unsigned integers | +| `i8` / `i16` / `i32` / `i64` / `i128` | signed integers | +| `f32` / `f64` | floats | +| `bool` | boolean | +| `String` | text | +| `Vec` | list/array | +| `Identity` | user identity | +| `ConnectionId` | connection handle | +| `Timestamp` | server timestamp (microseconds since epoch) | +| `TimeDuration` | duration in microseconds | +| `Uuid` | UUID | +| `Option` | nullable column | + +## Column Attributes + +```rust +#[primary_key] // primary key +#[auto_inc] // auto-increment (use 0 as placeholder on insert) +#[unique] // unique constraint +#[index(btree)] // btree index (enables .filter() on this column) +``` + +## Indexes + +Prefer `#[index(btree)]` inline for single-column. Multi-column uses table-level: + +```rust +// Inline (preferred for single-column): +#[index(btree)] +pub author_id: u64, +// Access: ctx.db.post().author_id().filter(author_id) + +// Multi-column (table-level): +#[spacetimedb::table(accessor = membership, public, + index(accessor = by_group_user, btree(columns = [group_id, user_id])) +)] +pub struct Membership { pub group_id: u64, pub user_id: Identity, ... } +// Access: ctx.db.membership().by_group_user().filter((group_id, &user_id)) +``` + +When you frequently look up rows by multiple columns, prefer a multi-column index over filtering by one column and looping over the results. + +## Reducers + +```rust +#[spacetimedb::reducer] +pub fn create_entity(ctx: &ReducerContext, name: String) { + ctx.db.entity().insert(Entity { id: 0, owner: ctx.sender(), name, active: true }); +} + +// Reducers can return Result<(), String> or Result<(), E> where E: Display +#[spacetimedb::reducer] +pub fn validate_entity(ctx: &ReducerContext, name: String) -> Result<(), String> { + if name.is_empty() { + return Err("Name cannot be empty".to_string()); + } + ctx.db.entity().try_insert(Entity { id: 0, owner: ctx.sender(), name, active: true })?; + Ok(()) +} +``` + +Note: `insert()` panics on constraint violations. Use `try_insert()` with `?` when returning `Result`. + +## DB Operations + +```rust +ctx.db.entity().insert(Entity { id: 0, name: "Sample".into() }); // Insert (0 for autoInc) +ctx.db.entity().id().find(entity_id); // Find by PK → Option +ctx.db.entity().identity().find(ctx.sender()); // Find by unique column → Option +ctx.db.item().author_id().filter(author_id); // Filter by index → iterator +ctx.db.entity().iter(); // All rows → iterator +ctx.db.entity().count(); // Count rows +ctx.db.entity().id().update(Entity { ..existing, name: new_name }); // Update (spread + override) +ctx.db.entity().id().delete(entity_id); // Delete by PK +ctx.db.entity().name().delete("Alice"); // Delete by indexed column +``` + +Note: `iter()` and `filter()` return iterators. Collect to Vec if you need `.sort()`, `.filter()`, `.map()`. + +Range queries on btree indexes: `filter(18..=65)`, `filter(18..)`, `filter(..18)`. + +## Lifecycle Hooks + +```rust +#[spacetimedb::reducer(init)] +pub fn init(ctx: &ReducerContext) { ... } + +#[spacetimedb::reducer(client_connected)] +pub fn on_connect(ctx: &ReducerContext) { ... } + +#[spacetimedb::reducer(client_disconnected)] +pub fn on_disconnect(ctx: &ReducerContext) { ... } +``` + +## Views + +```rust +// Anonymous view (same result for all clients): +use spacetimedb::{view, AnonymousViewContext}; + +#[view(accessor = active_users, public)] +fn active_users(ctx: &AnonymousViewContext) -> Vec { + ctx.db.entity().iter().filter(|e| e.active).collect() +} + +// Per-user view (result varies by sender): +use spacetimedb::{view, ViewContext}; + +#[view(accessor = my_profile, public)] +fn my_profile(ctx: &ViewContext) -> Option { + ctx.db.entity().identity().find(ctx.sender()) +} +``` + +## Authentication & Timestamps + +```rust +// Auth: ctx.sender() is the caller's Identity +if row.owner != ctx.sender() { + panic!("unauthorized"); + // or: return Err(anyhow::anyhow!("unauthorized")); +} + +// Server timestamps +ctx.db.item().insert(Item { id: 0, owner: ctx.sender(), created_at: ctx.timestamp, .. }); + +// Timestamp arithmetic +let expiry = ctx.timestamp + TimeDuration::from_micros(delay_micros); + +// Client: Timestamp → milliseconds since epoch +timestamp.to_micros_since_unix_epoch() / 1000 +``` + +## Scheduled Tables + +```rust +#[spacetimedb::table(accessor = tick_timer, scheduled(tick), public)] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: spacetimedb::ScheduleAt, +} + +#[spacetimedb::reducer] +pub fn tick(ctx: &ReducerContext, timer: TickTimer) { + // timer row is auto-deleted after this reducer runs +} + +// One-time: fires once at a specific time +let at = ScheduleAt::Time(ctx.timestamp + std::time::Duration::from_secs(10)); +// Repeating: fires on an interval +let at = ScheduleAt::Interval(std::time::Duration::from_secs(5).into()); + +ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, scheduled_at: at }); +``` + +## Logging + +```rust +log::info!("Player connected: {:?}", ctx.sender()); +log::warn!("Low health: {}", hp); +log::error!("Failed to find entity"); +``` + +## Custom Types + +```rust +#[derive(SpacetimeType)] +pub enum Status { Online, Away, Offline } + +#[derive(SpacetimeType)] +pub struct Point { x: f32, y: f32 } +``` + +## Complete Example + +```rust +// src/lib.rs +use spacetimedb::{Identity, ReducerContext, SpacetimeType, Table, Timestamp}; + +#[spacetimedb::table(accessor = entity, public)] +pub struct Entity { + #[primary_key] + pub identity: Identity, + pub name: String, + pub active: bool, +} + +#[spacetimedb::table(accessor = record, public)] +pub struct Record { + #[primary_key] + #[auto_inc] + pub id: u64, + pub owner: Identity, + pub value: u32, + pub created_at: Timestamp, +} + +#[spacetimedb::reducer(client_connected)] +pub fn on_connect(ctx: &ReducerContext) { + if let Some(existing) = ctx.db.entity().identity().find(ctx.sender()) { + ctx.db.entity().identity().update(Entity { active: true, ..existing }); + } +} + +#[spacetimedb::reducer(client_disconnected)] +pub fn on_disconnect(ctx: &ReducerContext) { + if let Some(existing) = ctx.db.entity().identity().find(ctx.sender()) { + ctx.db.entity().identity().update(Entity { active: false, ..existing }); + } +} + +#[spacetimedb::reducer] +pub fn create_entity(ctx: &ReducerContext, name: String) { + if ctx.db.entity().identity().find(ctx.sender()).is_some() { + panic!("already exists"); + } + ctx.db.entity().insert(Entity { identity: ctx.sender(), name, active: true }); +} + +#[spacetimedb::reducer] +pub fn add_record(ctx: &ReducerContext, value: u32) { + if ctx.db.entity().identity().find(ctx.sender()).is_none() { + panic!("not found"); + } + ctx.db.record().insert(Record { + id: 0, + owner: ctx.sender(), + value, + created_at: ctx.timestamp, + }); +} +``` diff --git a/skills/spacetimedb-concepts/SKILL.md b/skills/spacetimedb-concepts/SKILL.md deleted file mode 100644 index 42cfcc6d4aa..00000000000 --- a/skills/spacetimedb-concepts/SKILL.md +++ /dev/null @@ -1,345 +0,0 @@ ---- -name: spacetimedb-concepts -description: Understand SpacetimeDB architecture and core concepts. Use when learning SpacetimeDB or making architectural decisions. -license: Apache-2.0 -metadata: - author: clockworklabs - version: "2.0" ---- - -# SpacetimeDB Core Concepts - -SpacetimeDB is a relational database that is also a server. It lets you upload application logic directly into the database via WebAssembly modules, eliminating the traditional web/game server layer entirely. - ---- - -## Critical Rules (Read First) - -These five rules prevent the most common SpacetimeDB mistakes: - -1. **Reducers are transactional** — they do not return data to callers. Use subscriptions to read data. -2. **Reducers must be deterministic** — no filesystem, network, timers, or random. All state must come from tables. -3. **Read data via tables/subscriptions** — not reducer return values. Clients get data through subscribed queries. -4. **Auto-increment IDs are not sequential** — gaps are normal, do not use for ordering. Use timestamps or explicit sequence columns. -5. **`ctx.sender()` is the authenticated principal** — never trust identity passed as arguments. Always use `ctx.sender()` for authorization. - ---- - -## Feature Implementation Checklist - -When implementing a feature that spans backend and client: - -1. **Backend:** Define table(s) to store the data -2. **Backend:** Define reducer(s) to mutate the data -3. **Client:** Subscribe to the table(s) -4. **Client:** Call the reducer(s) from UI — **do not skip this step** -5. **Client:** Render the data from the table(s) - -**Common mistake:** Building backend tables/reducers but forgetting to wire up the client to call them. - ---- - -## Debugging Checklist - -When things are not working: - -1. Is SpacetimeDB server running? (`spacetime start`) -2. Is the module published? (`spacetime publish`) -3. Are client bindings generated? (`spacetime generate`) -4. Check server logs for errors (`spacetime logs `) -5. **Is the reducer actually being called from the client?** - ---- - -## CLI Commands - -```bash -spacetime start -spacetime publish --module-path -spacetime publish --clear-database -y --module-path -spacetime generate --lang --out-dir --module-path -spacetime logs -``` - ---- - -## What SpacetimeDB Is - -SpacetimeDB combines a database and application server into a single deployable unit. Clients connect directly to the database and execute application logic inside it. The system is optimized for real-time applications requiring maximum speed and minimum latency. - -Key characteristics: - -- **In-memory execution**: Application state is served from memory for very low-latency access -- **Persistent storage**: Data is automatically persisted to a write-ahead log (WAL) for durability -- **Real-time synchronization**: Changes are automatically pushed to subscribed clients -- **Single deployment**: No separate servers, containers, or infrastructure to manage - -## The Five Zen Principles - -1. **Everything is a Table**: Your entire application state lives in tables. No separate cache layer, no Redis, no in-memory state to synchronize. -2. **Everything is Persistent**: SpacetimeDB persists state by default (for example via WAL-backed durability). -3. **Everything is Real-Time**: Clients are replicas of server state. Subscribe to data and it flows automatically. -4. **Everything is Transactional**: Every reducer runs atomically. Either all changes succeed or all roll back. -5. **Everything is Programmable**: Modules are real code (Rust, C#, TypeScript) running inside the database. - -## Tables - -Tables store all data in SpacetimeDB. They use the relational model and support SQL queries for subscriptions. - -### Defining Tables - -Tables are defined using language-specific attributes. In 2.0, use `accessor` (not `name`) for the API name: - -**Rust:** -```rust -#[spacetimedb::table(accessor = player, public)] -pub struct Player { - #[primary_key] - #[auto_inc] - id: u32, - #[index(btree)] - name: String, - #[unique] - email: String, -} -``` - -**C#:** -```csharp -[SpacetimeDB.Table(Accessor = "Player", Public = true)] -public partial struct Player -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public uint Id; - [SpacetimeDB.Index.BTree] - public string Name; - [SpacetimeDB.Unique] - public string Email; -} -``` - -**TypeScript:** -```typescript -const players = table( - { name: 'players', public: true }, - { - id: t.u32().primaryKey().autoInc(), - name: t.string().index('btree'), - email: t.string().unique(), - } -); -``` - -### Table Visibility - -- **Private tables** (default): Only accessible by reducers and the database owner -- **Public tables**: Exposed for client read access through subscriptions. Writes still require reducers. - -### Table Design Principles - -Organize data by access pattern, not by entity: - -**Decomposed approach (recommended):** -``` -Player PlayerState PlayerStats -id <-- player_id player_id -name position_x total_kills - position_y total_deaths - velocity_x play_time -``` - -Benefits: reduced bandwidth, cache efficiency, schema evolution, semantic clarity. - -## Reducers - -Reducers are transactional functions that modify database state. They are the primary client-invoked mutation path; procedures can also mutate tables by running explicit transactions. - -### Key Properties - -- **Transactional**: Run in isolated database transactions -- **Atomic**: Either all changes succeed or all roll back -- **Isolated**: Cannot interact with the outside world (no network, no filesystem) -- **Callable**: Clients invoke reducers as remote procedure calls - -### Critical Reducer Rules - -1. **No global state**: Relying on static variables is undefined behavior -2. **No side effects**: Reducers cannot make network requests or access files -3. **Store state in tables**: All persistent state must be in tables -4. **No return data**: Reducers do not return data to callers — use subscriptions -5. **Must be deterministic**: No random, no timers, no external I/O - -### Defining Reducers - -**Rust:** -```rust -#[spacetimedb::reducer] -pub fn create_user(ctx: &ReducerContext, name: String, email: String) -> Result<(), String> { - if name.is_empty() { - return Err("Name cannot be empty".to_string()); - } - ctx.db.user().insert(User { id: 0, name, email }); - Ok(()) -} -``` - -**C#:** -```csharp -[SpacetimeDB.Reducer] -public static void CreateUser(ReducerContext ctx, string name, string email) -{ - if (string.IsNullOrEmpty(name)) - throw new ArgumentException("Name cannot be empty"); - ctx.Db.User.Insert(new User { Id = 0, Name = name, Email = email }); -} -``` - -### ReducerContext - -Every reducer receives a `ReducerContext` providing: -- **Database**: `ctx.db` (Rust field, TS property) / `ctx.Db` (C# property) -- **Sender**: `ctx.sender()` (Rust method) / `ctx.Sender` (C# property) / `ctx.sender` (TS property) -- **Connection ID**: `ctx.connection_id()` (Rust method) / `ctx.ConnectionId` (C# property) / `ctx.connectionId` (TS property) -- **Timestamp**: `ctx.timestamp` (Rust field, TS property) / `ctx.Timestamp` (C# property) - -## Event Tables (2.0) - -Event tables are the preferred way to broadcast reducer-specific data to clients. - -```rust -#[table(accessor = damage_event, public, event)] -pub struct DamageEvent { - pub target: Identity, - pub amount: u32, -} - -#[reducer] -fn deal_damage(ctx: &ReducerContext, target: Identity, amount: u32) { - ctx.db.damage_event().insert(DamageEvent { target, amount }); -} -``` - -Clients subscribe to event tables and use `on_insert` callbacks. Event tables must be subscribed explicitly and are excluded from `subscribe_to_all_tables()`. - -## Subscriptions - -Subscriptions replicate database rows to clients in real-time. - -### How Subscriptions Work - -1. **Subscribe**: Register SQL queries describing needed data -2. **Receive initial data**: All matching rows are sent immediately -3. **Receive updates**: Real-time updates when subscribed rows change -4. **React to changes**: Use callbacks (`onInsert`, `onDelete`, `onUpdate`) - -### Subscription Best Practices - -1. **Group subscriptions by lifetime**: Keep always-needed data separate from temporary subscriptions -2. **Subscribe before unsubscribing**: When updating subscriptions, subscribe to new data first -3. **Avoid overlapping queries**: Distinct queries returning overlapping data cause redundant processing -4. **Use indexes**: Queries on indexed columns are efficient; full table scans are expensive - -## Modules - -Modules are WebAssembly bundles containing application logic that runs inside the database. - -### Module Components - -- **Tables**: Define the data schema -- **Reducers**: Define callable functions that modify state -- **Views**: Define read-only computed queries -- **Event Tables**: Broadcast reducer-specific data to clients (2.0) -- **Procedures**: (Beta) Functions that can have side effects (HTTP requests) - -### Module Languages - -Server-side modules can be written in: Rust, C#, TypeScript (beta) - -### Module Lifecycle - -1. **Write**: Define tables and reducers in your chosen language -2. **Compile**: Build to WebAssembly using the SpacetimeDB CLI -3. **Publish**: Upload to a SpacetimeDB host with `spacetime publish` -4. **Hot-swap**: Republish to update code without disconnecting clients - -## Identity - -Identity is SpacetimeDB's authentication system based on OpenID Connect (OIDC). - -- **Identity**: A long-lived, globally unique identifier for a user. -- **ConnectionId**: Identifies a specific client connection. - -```rust -#[spacetimedb::reducer] -pub fn do_something(ctx: &ReducerContext) { - let caller_identity = ctx.sender(); // Who is calling? - // NEVER trust identity passed as a reducer argument -} -``` - -### Authentication Providers - -SpacetimeDB works with many OIDC providers, including SpacetimeAuth (built-in), Auth0, Clerk, Keycloak, Google, and GitHub. - -## When to Use SpacetimeDB - -### Ideal Use Cases - -- **Real-time games**: MMOs, multiplayer games, turn-based games -- **Collaborative applications**: Document editing, whiteboards, design tools -- **Chat and messaging**: Real-time communication with presence -- **Live dashboards**: Streaming analytics and monitoring - -### Key Decision Factors - -Choose SpacetimeDB when you need: -- Sub-10ms latency for reads and writes -- Automatic real-time synchronization -- Transactional guarantees for all operations -- Simplified architecture (no separate cache, queue, or server) - -### Less Suitable For - -- **Batch analytics**: Optimized for OLTP, not OLAP -- **Large blob storage**: Better suited for structured relational data -- **Stateless APIs**: Traditional REST APIs do not need real-time sync - -## Common Patterns - -**Authentication check in reducer:** -```rust -#[spacetimedb::reducer] -fn admin_action(ctx: &ReducerContext) -> Result<(), String> { - let admin = ctx.db.admin().identity().find(&ctx.sender()) - .ok_or("Not an admin")?; - Ok(()) -} -``` - -**Scheduled reducer:** -```rust -#[spacetimedb::table(accessor = reminder, scheduled(send_reminder))] -pub struct Reminder { - #[primary_key] - #[auto_inc] - id: u64, - scheduled_at: ScheduleAt, - message: String, -} - -#[spacetimedb::reducer] -fn send_reminder(ctx: &ReducerContext, reminder: Reminder) { - log::info!("Reminder: {}", reminder.message); -} -``` - ---- - -## Editing Behavior - -When modifying SpacetimeDB code: - -- Make the smallest change necessary -- Do NOT touch unrelated files, configs, or dependencies -- Do NOT invent new SpacetimeDB APIs — use only what exists in docs or this repo diff --git a/skills/spacetimedb-csharp/SKILL.md b/skills/spacetimedb-csharp/SKILL.md deleted file mode 100644 index 2dc4ce5d9c7..00000000000 --- a/skills/spacetimedb-csharp/SKILL.md +++ /dev/null @@ -1,646 +0,0 @@ ---- -name: spacetimedb-csharp -description: Build C# modules and clients for SpacetimeDB. Covers server-side module development and client SDK integration. -license: Apache-2.0 -metadata: - author: clockworklabs - version: "2.0" - tested_with: "SpacetimeDB 2.0, .NET 8 SDK" ---- - -# SpacetimeDB C# SDK - -This skill provides guidance for building C# server-side modules and C# clients that connect to SpacetimeDB 2.0. - ---- - -## HALLUCINATED APIs — DO NOT USE - -**These APIs DO NOT EXIST. LLMs frequently hallucinate them.** - -```csharp -// WRONG — these table access patterns do not exist -ctx.db.tableName // Wrong casing — use ctx.Db -ctx.Db.tableName // Wrong casing — accessor must match exactly -ctx.Db.TableName.Get(id) // Use Find, not Get -ctx.Db.TableName.FindById(id) // Use index accessor: ctx.Db.TableName.Id.Find(id) -ctx.Db.table.field_name.Find(x) // Wrong! Use PascalCase: ctx.Db.Table.FieldName.Find(x) -Optional field; // Use C# nullable: string? field - -// WRONG — missing partial keyword -public struct MyTable { } // Must be "partial struct" -public class Module { } // Must be "static partial class" - -// WRONG — non-partial types -[SpacetimeDB.Table(Accessor = "Player")] -public struct Player { } // WRONG — missing partial! - -// WRONG — sum type syntax (VERY COMMON MISTAKE) -public partial struct Shape : TaggedEnum<(Circle, Rectangle)> { } // WRONG: struct, missing names -public partial record Shape : TaggedEnum<(Circle, Rectangle)> { } // WRONG: missing variant names -public partial class Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { } // WRONG: class - -// WRONG — Index attribute without full qualification -[Index.BTree(Accessor = "idx", Columns = new[] { "Col" })] // Ambiguous with System.Index! -[SpacetimeDB.Index.BTree(Accessor = "idx", Columns = ["Col"])] // Valid with modern C# collection expressions - -// WRONG — old 1.0 patterns -[SpacetimeDB.Table(Name = "Player")] // Use Accessor, not Name (2.0) - // Use SpacetimeDB.Runtime -.WithModuleName("my-db") // Use .WithDatabaseName() (2.0) -ScheduleAt.Time(futureTime) // Use new ScheduleAt.Time(futureTime) - -// WRONG — lifecycle hooks starting with "On" -[SpacetimeDB.Reducer(ReducerKind.ClientConnected)] -public static void OnClientConnected(ReducerContext ctx) { } // STDB0010 error! - -// WRONG — non-deterministic code in reducers -var random = new Random(); // Use ctx.Rng -var guid = Guid.NewGuid(); // Not allowed -var now = DateTime.Now; // Use ctx.Timestamp - -// WRONG — collection parameters -int[] itemIds = { 1, 2, 3 }; -_conn.Reducers.ProcessItems(itemIds); // Generated code expects List! -``` - -### CORRECT PATTERNS - -```csharp -using SpacetimeDB; - -// CORRECT TABLE — must be partial struct, use Accessor -[SpacetimeDB.Table(Accessor = "Player", Public = true)] -public partial struct Player -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - [SpacetimeDB.Index.BTree] - public Identity OwnerId; - public string Name; -} - -// CORRECT MODULE — must be static partial class -public static partial class Module -{ - [SpacetimeDB.Reducer] - public static void CreatePlayer(ReducerContext ctx, string name) - { - ctx.Db.Player.Insert(new Player { Id = 0, OwnerId = ctx.Sender, Name = name }); - } -} - -// CORRECT DATABASE ACCESS — PascalCase, index-based lookups -var player = ctx.Db.Player.Id.Find(playerId); // Unique/PK: returns nullable -foreach (var p in ctx.Db.Player.OwnerId.Filter(ctx.Sender)) { } // BTree: returns IEnumerable - -// CORRECT SUM TYPE — partial record with named tuple elements -[SpacetimeDB.Type] -public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { } - -// CORRECT — collection parameters use List -_conn.Reducers.ProcessItems(new List { 1, 2, 3 }); -``` - ---- - -## Common Mistakes Table - -| Wrong | Right | Error | -|-------|-------|-------| -| Wrong .csproj name | `StdbModule.csproj` | Publish fails silently | -| .NET 9 SDK | .NET 8 SDK only | WASI compilation fails | -| Missing WASI workload | `dotnet workload install wasi-experimental` | Build fails | -| async/await in reducers | Synchronous only | Not supported | -| `table.Name.Update(...)` | `table.Id.Update(...)` | Update only via primary key (2.0) | -| Not calling `FrameTick()` | `conn.FrameTick()` in Update loop | No callbacks fire | -| Accessing `conn.Db` from background thread | Copy data in callback | Data races | - ---- - -## Hard Requirements - -1. **Tables and Module MUST be `partial`** — required for code generation -2. **Use `Accessor =` in table attributes** — `Name =` is only for SQL compatibility (2.0) -3. **Project file MUST be named `StdbModule.csproj`** — CLI requirement -4. **Requires .NET 8 SDK** — .NET 9 and newer not yet supported -5. **Install WASI workload** — `dotnet workload install wasi-experimental` -6. **Procedures are supported** — use `[SpacetimeDB.Procedure]` with `ProcedureContext` when needed -7. **Reducers must be deterministic** — no filesystem, network, timers, or `Random` -8. **Add `Public = true`** — if clients need to subscribe to a table -9. **Use `T?` for nullable fields** — not `Optional` -10. **Pass `0` for auto-increment** — to trigger ID generation on insert -11. **Sum types must be `partial record`** — not struct or class -12. **Fully qualify Index attribute** — `[SpacetimeDB.Index.BTree]` to avoid System.Index ambiguity -13. **Update only via primary key** — use delete+insert for non-PK changes (2.0) -14. **Use `SpacetimeDB.Runtime` package** — not `ServerSdk` (2.0) -15. **Use `List` for collection parameters** — not arrays -16. **`Identity` is in `SpacetimeDB` namespace** — not `SpacetimeDB.Types` - ---- - -## Server-Side Module Development - -### Table Definition - -```csharp -using SpacetimeDB; - -[SpacetimeDB.Table(Accessor = "Player", Public = true)] -public partial struct Player -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - - [SpacetimeDB.Index.BTree] - public Identity OwnerId; - - public string Name; - public Timestamp CreatedAt; -} - -// Multi-column index (use fully-qualified attribute!) -[SpacetimeDB.Table(Accessor = "Score", Public = true)] -[SpacetimeDB.Index.BTree(Accessor = "by_player_game", Columns = new[] { "PlayerId", "GameId" })] -public partial struct Score -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - public Identity PlayerId; - public string GameId; - public int Points; -} -``` - -### Field Attributes - -```csharp -[SpacetimeDB.PrimaryKey] // Exactly one per table (required) -[SpacetimeDB.AutoInc] // Auto-increment (integer fields only) -[SpacetimeDB.Unique] // Unique constraint -[SpacetimeDB.Index.BTree] // Single-column B-tree index -[SpacetimeDB.Default(value)] // Default value for new columns -``` - -### SpacetimeDB Column Types - -```csharp -Identity // User identity (SpacetimeDB namespace, not SpacetimeDB.Types) -Timestamp // Timestamp (use ctx.Timestamp server-side, never DateTime.Now) -ScheduleAt // For scheduled tables -T? // Nullable (e.g., string?) -List // Collections (use List, not arrays) -``` - -Standard C# primitives (`bool`, `byte`..`ulong`, `float`, `double`, `string`) are all supported. - -### Insert with Auto-Increment - -```csharp -var player = ctx.Db.Player.Insert(new Player -{ - Id = 0, // Pass 0 to trigger auto-increment - OwnerId = ctx.Sender, - Name = name, - CreatedAt = ctx.Timestamp -}); -ulong newId = player.Id; // Insert returns the row with generated ID -``` - -### Module and Reducers - -```csharp -using SpacetimeDB; - -public static partial class Module -{ - [SpacetimeDB.Reducer] - public static void CreateTask(ReducerContext ctx, string title) - { - if (string.IsNullOrEmpty(title)) - throw new Exception("Title cannot be empty"); - - ctx.Db.Task.Insert(new Task - { - Id = 0, - OwnerId = ctx.Sender, - Title = title, - Completed = false - }); - } - - [SpacetimeDB.Reducer] - public static void CompleteTask(ReducerContext ctx, ulong taskId) - { - if (ctx.Db.Task.Id.Find(taskId) is not Task task) - throw new Exception("Task not found"); - if (task.OwnerId != ctx.Sender) - throw new Exception("Not authorized"); - - ctx.Db.Task.Id.Update(task with { Completed = true }); - } - - [SpacetimeDB.Reducer] - public static void DeleteTask(ReducerContext ctx, ulong taskId) - { - ctx.Db.Task.Id.Delete(taskId); - } -} -``` - -### Lifecycle Reducers - -```csharp -public static partial class Module -{ - [SpacetimeDB.Reducer(ReducerKind.Init)] - public static void Init(ReducerContext ctx) - { - Log.Info("Module initialized"); - } - - // CRITICAL: no "On" prefix! - [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] - public static void ClientConnected(ReducerContext ctx) - { - Log.Info($"Client connected: {ctx.Sender}"); - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) - { - ctx.Db.User.Identity.Update(user with { Online = true }); - } - else - { - ctx.Db.User.Insert(new User { Identity = ctx.Sender, Online = true }); - } - } - - [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] - public static void ClientDisconnected(ReducerContext ctx) - { - if (ctx.Db.User.Identity.Find(ctx.Sender) is User user) - { - ctx.Db.User.Identity.Update(user with { Online = false }); - } - } -} -``` - -### Event Tables (2.0) - -Reducer callbacks are removed in 2.0. Use event tables + `OnInsert` instead. - -```csharp -[SpacetimeDB.Table(Accessor = "DamageEvent", Public = true, Event = true)] -public partial struct DamageEvent -{ - public Identity Target; - public uint Amount; -} - -[SpacetimeDB.Reducer] -public static void DealDamage(ReducerContext ctx, Identity target, uint amount) -{ - ctx.Db.DamageEvent.Insert(new DamageEvent { Target = target, Amount = amount }); -} -``` - -Client subscribes and uses `OnInsert`: -```csharp -conn.Db.DamageEvent.OnInsert += (ctx, evt) => { - PlayDamageAnimation(evt.Target, evt.Amount); -}; -``` - -Event tables must be subscribed explicitly — they are excluded from `SubscribeToAllTables()`. - -### Database Access - -```csharp -// Find by primary key — returns nullable, use pattern matching -if (ctx.Db.Task.Id.Find(taskId) is Task task) { /* use task */ } - -// Update by primary key (2.0: only primary key has .Update) -ctx.Db.Task.Id.Update(task with { Title = newTitle }); - -// Delete by primary key -ctx.Db.Task.Id.Delete(taskId); - -// Find by unique index — returns nullable -if (ctx.Db.Player.Username.Find("alice") is Player player) { } - -// Filter by B-tree index — returns iterator -foreach (var task in ctx.Db.Task.OwnerId.Filter(ctx.Sender)) { } - -// Full table scan — avoid for large tables -foreach (var task in ctx.Db.Task.Iter()) { } -var count = ctx.Db.Task.Count; -``` - -### Custom Types and Sum Types - -```csharp -[SpacetimeDB.Type] -public partial struct Position { public int X; public int Y; } - -// Sum types MUST be partial record with named tuple -[SpacetimeDB.Type] -public partial struct Circle { public int Radius; } -[SpacetimeDB.Type] -public partial struct Rectangle { public int Width; public int Height; } -[SpacetimeDB.Type] -public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { } - -// Creating sum type values -var circle = new Shape.Circle(new Circle { Radius = 10 }); -``` - -### Scheduled Tables - -```csharp -[SpacetimeDB.Table(Accessor = "Reminder", Scheduled = nameof(Module.SendReminder))] -public partial struct Reminder -{ - [SpacetimeDB.PrimaryKey] - [SpacetimeDB.AutoInc] - public ulong Id; - public string Message; - public ScheduleAt ScheduledAt; -} - -public static partial class Module -{ - [SpacetimeDB.Reducer] - public static void SendReminder(ReducerContext ctx, Reminder reminder) - { - Log.Info($"Reminder: {reminder.Message}"); - } - - [SpacetimeDB.Reducer] - public static void CreateReminder(ReducerContext ctx, string message, ulong delaySecs) - { - ctx.Db.Reminder.Insert(new Reminder - { - Id = 0, - Message = message, - ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + TimeSpan.FromSeconds(delaySecs)) - }); - } -} -``` - -### Logging - -```csharp -Log.Debug("Debug message"); -Log.Info("Information"); -Log.Warn("Warning"); -Log.Error("Error occurred"); -Log.Exception("Critical failure"); // Logs at error level -``` - -### ReducerContext API - -```csharp -ctx.Sender // Identity of the caller -ctx.Timestamp // Current timestamp -ctx.Db // Database access -ctx.Identity // Module's own identity -ctx.ConnectionId // Connection ID (nullable) -ctx.SenderAuth // Authorization context (JWT claims, internal call detection) -ctx.Rng // Deterministic random number generator -``` - -### Error Handling - -Throwing an exception in a reducer rolls back the entire transaction: - -```csharp -[SpacetimeDB.Reducer] -public static void TransferCredits(ReducerContext ctx, Identity toUser, uint amount) -{ - if (ctx.Db.User.Identity.Find(ctx.Sender) is not User sender) - throw new Exception("Sender not found"); - - if (sender.Credits < amount) - throw new Exception("Insufficient credits"); - - ctx.Db.User.Identity.Update(sender with { Credits = sender.Credits - amount }); - - if (ctx.Db.User.Identity.Find(toUser) is User receiver) - ctx.Db.User.Identity.Update(receiver with { Credits = receiver.Credits + amount }); -} -``` - ---- - -## Project Setup - -### Required .csproj (MUST be named `StdbModule.csproj`) - -```xml - - - net8.0 - wasi-wasm - Exe - enable - enable - - - - - -``` - -### Prerequisites - -```bash -# Install .NET 8 SDK (required, not .NET 9) -# Install WASI workload -dotnet workload install wasi-experimental -``` - ---- - -## Client SDK - -### Installation - -```bash -dotnet add package SpacetimeDB.ClientSDK -``` - -### Generate Module Bindings - -```bash -spacetime generate --lang csharp --out-dir module_bindings --module-path PATH_TO_MODULE -``` - -This creates `SpacetimeDBClient.g.cs`, `Tables/*.g.cs`, `Reducers/*.g.cs`, and `Types/*.g.cs`. - -### Connection Setup - -```csharp -using SpacetimeDB; -using SpacetimeDB.Types; - -var conn = DbConnection.Builder() - .WithUri("http://localhost:3000") - .WithDatabaseName("my-database") - .WithToken(savedToken) - .OnConnect(OnConnected) - .OnConnectError(err => Console.Error.WriteLine($"Failed: {err}")) - .OnDisconnect((conn, err) => { if (err != null) Console.Error.WriteLine(err); }) - .Build(); - -void OnConnected(DbConnection conn, Identity identity, string authToken) -{ - // Save authToken to persistent storage for reconnection - Console.WriteLine($"Connected: {identity}"); - conn.SubscriptionBuilder() - .OnApplied(OnSubscriptionApplied) - .SubscribeToAllTables(); -} -``` - -### Critical: FrameTick - -**The SDK does NOT automatically process messages.** You must call `FrameTick()` regularly. - -```csharp -// Console application -while (running) { conn.FrameTick(); Thread.Sleep(16); } - -// Unity: call conn?.FrameTick() in Update() -``` - -**Warning**: Do NOT call `FrameTick()` from a background thread. It modifies `conn.Db` and can cause data races. - -### Subscribing to Tables - -```csharp -// SQL queries -conn.SubscriptionBuilder() - .OnApplied(OnSubscriptionApplied) - .OnError((ctx, err) => Console.Error.WriteLine($"Subscription failed: {err}")) - .Subscribe(new[] { - "SELECT * FROM player", - "SELECT * FROM message WHERE sender = :sender" - }); - -// Subscribe to all tables (development only) -conn.SubscriptionBuilder() - .OnApplied(OnSubscriptionApplied) - .SubscribeToAllTables(); - -// Subscription handle for later unsubscribe -SubscriptionHandle handle = conn.SubscriptionBuilder() - .OnApplied(ctx => Console.WriteLine("Applied")) - .Subscribe(new[] { "SELECT * FROM player" }); - -handle.UnsubscribeThen(ctx => Console.WriteLine("Unsubscribed")); -``` - -**Warning**: `SubscribeToAllTables()` cannot be mixed with `Subscribe()` on the same connection. - -### Accessing the Client Cache - -```csharp -// Iterate all rows -foreach (var player in ctx.Db.Player.Iter()) { Console.WriteLine(player.Name); } - -// Count rows -int playerCount = ctx.Db.Player.Count; - -// Find by unique/primary key — returns nullable -Player? player = ctx.Db.Player.Identity.Find(someIdentity); -if (player != null) { Console.WriteLine(player.Name); } - -// Filter by BTree index — returns IEnumerable -foreach (var p in ctx.Db.Player.Level.Filter(1)) { } -``` - -### Row Event Callbacks - -```csharp -ctx.Db.Player.OnInsert += (EventContext ctx, Player player) => { - Console.WriteLine($"Player joined: {player.Name}"); -}; - -ctx.Db.Player.OnDelete += (EventContext ctx, Player player) => { - Console.WriteLine($"Player left: {player.Name}"); -}; - -ctx.Db.Player.OnUpdate += (EventContext ctx, Player oldRow, Player newRow) => { - Console.WriteLine($"Player {oldRow.Name} renamed to {newRow.Name}"); -}; - -// Checking event source -ctx.Db.Player.OnInsert += (EventContext ctx, Player player) => { - switch (ctx.Event) - { - case Event.SubscribeApplied: - break; // Initial subscription data - case Event.Reducer(var reducerEvent): - Console.WriteLine($"Reducer: {reducerEvent.Reducer}"); - break; - } -}; -``` - -### Calling Reducers - -```csharp -ctx.Reducers.SendMessage("Hello, world!"); -ctx.Reducers.CreatePlayer("NewPlayer"); - -// Reducer completion callbacks -conn.Reducers.OnSendMessage += (ReducerEventContext ctx, string text) => { - if (ctx.Event.Status is Status.Committed) - Console.WriteLine($"Message sent: {text}"); - else if (ctx.Event.Status is Status.Failed(var reason)) - Console.Error.WriteLine($"Send failed: {reason}"); -}; - -// Unhandled reducer errors -conn.OnUnhandledReducerError += (ReducerEventContext ctx, Exception ex) => { - Console.Error.WriteLine($"Reducer error: {ex.Message}"); -}; -``` - -### Identity and Authentication - -```csharp -// In OnConnect callback — save token for reconnection -void OnConnected(DbConnection conn, Identity identity, string authToken) -{ - // Save authToken to persistent storage (file, config, PlayerPrefs, etc.) - SaveToken(authToken); -} - -// Reconnect with saved token -string savedToken = LoadToken(); -DbConnection.Builder() - .WithUri("http://localhost:3000") - .WithDatabaseName("my-database") - .WithToken(savedToken) - .OnConnect(OnConnected) - .Build(); - -// Pass null or omit WithToken for anonymous connection -``` - ---- - -## Commands - -```bash -spacetime start -spacetime publish --module-path -spacetime publish --clear-database -y --module-path -spacetime generate --lang csharp --out-dir /SpacetimeDB --module-path -spacetime logs -``` diff --git a/skills/spacetimedb-rust/SKILL.md b/skills/spacetimedb-rust/SKILL.md deleted file mode 100644 index 346247b55e7..00000000000 --- a/skills/spacetimedb-rust/SKILL.md +++ /dev/null @@ -1,556 +0,0 @@ ---- -name: spacetimedb-rust -description: Develop SpacetimeDB server modules in Rust. Use when writing reducers, tables, or module logic. -license: Apache-2.0 -metadata: - author: clockworklabs - version: "2.0" ---- - -# SpacetimeDB Rust Module Development - -SpacetimeDB modules are WebAssembly applications that run inside the database. They define tables to store data and reducers to modify data. Clients connect directly to the database and execute application logic inside it. - -> **Tested with:** SpacetimeDB 2.0+ APIs - ---- - -## HALLUCINATED APIs — DO NOT USE - -**These APIs/patterns are incorrect. LLMs frequently hallucinate them.** - -Both macro forms are valid in 2.0: `#[spacetimedb::table(...)]` / `#[table(...)]` and `#[spacetimedb::reducer]` / `#[reducer]`. - -```rust -#[derive(Table)] // Tables use #[table] attribute, not derive -#[derive(Reducer)] // Reducers use #[reducer] attribute - -// WRONG — SpacetimeType on tables -#[derive(SpacetimeType)] // DO NOT use on #[table] structs! -#[table(accessor = my_table)] -pub struct MyTable { ... } - -// WRONG — mutable context -pub fn my_reducer(ctx: &mut ReducerContext, ...) { } // Should be &ReducerContext - -// WRONG — table access without parentheses -ctx.db.player // Should be ctx.db.player() -ctx.db.player.find(id) // Should be ctx.db.player().id().find(&id) - -// WRONG — old 1.0 patterns -ctx.sender // Use ctx.sender() — method, not field (2.0) -.with_module_name("db") // Use .with_database_name() (2.0) -ctx.db.user().name().update(..) // Update only via primary key (2.0) -``` - -### CORRECT PATTERNS: - -```rust -use spacetimedb::{table, reducer, Table, ReducerContext, Identity, Timestamp}; -use spacetimedb::SpacetimeType; // Only for custom types, NOT tables - -// CORRECT TABLE — accessor, not name; no SpacetimeType derive! -#[table(accessor = player, public)] -pub struct Player { - #[primary_key] - pub id: u64, - pub name: String, -} - -// CORRECT REDUCER — immutable context, sender() is a method -#[reducer] -pub fn create_player(ctx: &ReducerContext, name: String) { - ctx.db.player().insert(Player { id: 0, name }); -} - -// CORRECT TABLE ACCESS — methods with parentheses, sender() method -let player = ctx.db.player().id().find(&player_id); -let caller = ctx.sender(); -``` - -### DO NOT: -- **Derive `SpacetimeType` on `#[table]` structs** — the macro handles this -- **Use mutable context** — `&ReducerContext`, not `&mut ReducerContext` -- **Forget `Table` trait import** — required for table operations -- **Use field access for tables** — `ctx.db.player()` not `ctx.db.player` -- **Use `ctx.sender`** — it's `ctx.sender()` (method) in 2.0 - ---- - -## Common Mistakes Table - -| Wrong | Right | Error | -|-------|-------|-------| -| `#[table(accessor = "my_table")]` | `#[table(accessor = my_table)]` | String literals not allowed | -| Missing `public` on table | Add `public` flag | Clients can't subscribe | -| Network/filesystem in reducer | Use procedures instead | Sandbox violation | -| Panic for expected errors | Return `Result<(), String>` | WASM instance destroyed | - ---- - -## Hard Requirements - -1. **DO NOT derive `SpacetimeType` on `#[table]` structs** — the macro handles this -2. **Import `Table` trait** — required for all table operations -3. **Use `&ReducerContext`** — not `&mut ReducerContext` -4. **Tables are methods** — `ctx.db.table()` not `ctx.db.table` -5. **Use `ctx.sender()`** — method call, not field access (2.0) -6. **Use `accessor =` for API handles** — `name = "..."` is optional canonical naming in table/index attributes -7. **Reducers must be deterministic** — no filesystem, network, timers, or external RNG -8. **Use `ctx.rng()`** — not `rand` crate for random numbers -9. **Add `public` flag** — if clients need to subscribe to a table -10. **Update only via primary key** — use delete+insert for non-PK changes (2.0) - ---- - -## Project Setup - -```toml -[package] -name = "my-module" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] - -[dependencies] -spacetimedb = { workspace = true } -log = "0.4" -``` - -### Essential Imports - -```rust -use spacetimedb::{ReducerContext, Table}; -use spacetimedb::{Identity, Timestamp, ConnectionId, ScheduleAt}; -``` - -## Table Definitions - -```rust -#[spacetimedb::table(accessor = player, public)] -pub struct Player { - #[primary_key] - #[auto_inc] - id: u64, - name: String, - score: u32, -} -``` - -### Table Attributes - -| Attribute | Description | -|-----------|-------------| -| `accessor = identifier` | Required. The API name used in `ctx.db.{accessor}()` | -| `public` | Makes table visible to clients via subscriptions | -| `scheduled(function_name)` | Creates a schedule table that triggers the named reducer or procedure | -| `index(accessor = idx, btree(columns = [a, b]))` | Multi-column index | - -### Column Attributes - -| Attribute | Description | -|-----------|-------------| -| `#[primary_key]` | Unique identifier for the row (one per table max) | -| `#[unique]` | Enforces uniqueness, enables `find()` method | -| `#[auto_inc]` | Auto-generates unique integer values when inserting 0 | -| `#[index(btree)]` | Creates a B-tree index for efficient lookups | - -### Supported Column Types - -**Primitives**: `u8`-`u256`, `i8`-`i256`, `f32`, `f64`, `bool`, `String` - -**SpacetimeDB Types**: `Identity`, `ConnectionId`, `Timestamp`, `Uuid`, `ScheduleAt` - -**Collections**: `Vec`, `Option`, `Result` - -**Custom Types**: Any struct/enum with `#[derive(SpacetimeType)]` - ---- - -## Reducers - -```rust -#[spacetimedb::reducer] -pub fn create_player(ctx: &ReducerContext, name: String) -> Result<(), String> { - if name.is_empty() { - return Err("Name cannot be empty".to_string()); - } - ctx.db.player().insert(Player { id: 0, name, score: 0 }); - Ok(()) -} -``` - -### Reducer Rules - -1. First parameter must be `&ReducerContext` -2. Return `()`, `Result<(), String>`, or `Result<(), E>` where `E: Display` -3. All changes roll back on panic or `Err` return -4. Must import `Table` trait: `use spacetimedb::Table;` - -### ReducerContext - -```rust -ctx.db // Database access -ctx.sender() // Identity of the caller (method, not field!) -ctx.connection_id() // Option (None for scheduled/system reducers) -ctx.timestamp // Invocation timestamp -ctx.identity() // Module's own identity -ctx.rng() // Deterministic RNG (method, not field!) -``` - ---- - -## Table Operations - -### Insert - -```rust -// Insert returns the row with auto_inc values populated -let player = ctx.db.player().insert(Player { id: 0, name: "Alice".into(), score: 100 }); -log::info!("Created player with id: {}", player.id); -``` - -### Find and Filter - -```rust -// Find by unique/primary key — returns Option -if let Some(player) = ctx.db.player().id().find(&123) { - log::info!("Found: {}", player.name); -} - -// Optional clarity: typed literals can avoid inference ambiguity -if let Some(player) = ctx.db.player().id().find(&123u64) { - log::info!("Found: {}", player.name); -} - -// Filter by indexed column — returns iterator -for player in ctx.db.player().name().filter(&"Alice".to_string()) { - log::info!("Player: {}", player.name); -} - -// Full table scan -for player in ctx.db.player().iter() { } -let total = ctx.db.player().count(); -``` - -### Update - -```rust -// Update via primary key (2.0: only primary key has update) -if let Some(player) = ctx.db.player().id().find(&123) { - ctx.db.player().id().update(Player { score: player.score + 10, ..player }); -} - -// For non-PK changes: delete + insert -if let Some(old) = ctx.db.player().id().find(&id) { - ctx.db.player().id().delete(&id); - ctx.db.player().insert(Player { name: new_name, ..old }); -} -``` - -### Delete - -```rust -// Delete by primary key -ctx.db.player().id().delete(&123); - -// Delete by indexed column (collect first to avoid iterator invalidation) -let to_remove: Vec = ctx.db.player().name().filter(&"Alice".to_string()) - .map(|p| p.id) - .collect(); -for id in to_remove { - ctx.db.player().id().delete(&id); -} -``` - ---- - -## Indexes - -```rust -// Single-column index -#[spacetimedb::table(accessor = player, public)] -pub struct Player { - #[primary_key] - id: u64, - #[index(btree)] - level: u32, - name: String, -} - -// Multi-column index -#[spacetimedb::table( - accessor = score, public, - index(accessor = by_player_level, btree(columns = [player_id, level])) -)] -pub struct Score { - player_id: u32, - level: u32, - points: i64, -} - -// Multi-column index querying: prefix match (first column only) -for s in ctx.db.score().by_player_level().filter(&(42,)) { - log::info!("Player 42, any level: {} pts", s.points); -} - -// Full match (both columns) -for s in ctx.db.score().by_player_level().filter(&(42, 5)) { - log::info!("Player 42, level 5: {} pts", s.points); -} -``` - ---- - -## Event Tables (2.0) - -Reducer callbacks are removed in 2.0. Use event tables + `on_insert` instead. - -```rust -#[table(accessor = damage_event, public, event)] -pub struct DamageEvent { - pub target: Identity, - pub amount: u32, -} - -#[reducer] -fn deal_damage(ctx: &ReducerContext, target: Identity, amount: u32) { - ctx.db.damage_event().insert(DamageEvent { target, amount }); -} -``` - -Client subscribes and uses `on_insert`: -```rust -conn.db.damage_event().on_insert(|ctx, event| { - play_damage_animation(event.target, event.amount); -}); -``` - -Event tables must be subscribed explicitly — they are excluded from `subscribe_to_all_tables()`. - ---- - -## Lifecycle Reducers - -```rust -#[spacetimedb::reducer(init)] -pub fn init(ctx: &ReducerContext) -> Result<(), String> { - log::info!("Database initializing..."); - ctx.db.config().insert(Config { - id: 0, - max_players: 100, - game_mode: "default".to_string(), - }); - Ok(()) -} - -#[spacetimedb::reducer(client_connected)] -pub fn on_connect(ctx: &ReducerContext) -> Result<(), String> { - let caller = ctx.sender(); - log::info!("Client connected: {}", caller); - - if let Some(user) = ctx.db.user().identity().find(&caller) { - ctx.db.user().identity().update(User { online: true, ..user }); - } else { - ctx.db.user().insert(User { - identity: caller, - name: format!("User-{}", &caller.to_hex()[..8]), - online: true, - }); - } - Ok(()) -} - -#[spacetimedb::reducer(client_disconnected)] -pub fn on_disconnect(ctx: &ReducerContext) -> Result<(), String> { - let caller = ctx.sender(); - if let Some(user) = ctx.db.user().identity().find(&caller) { - ctx.db.user().identity().update(User { online: false, ..user }); - } - Ok(()) -} -``` - ---- - -## Scheduled Reducers - -```rust -use spacetimedb::ScheduleAt; -use std::time::Duration; - -#[spacetimedb::table(accessor = game_tick_schedule, scheduled(game_tick))] -pub struct GameTickSchedule { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, -} - -#[spacetimedb::reducer] -fn game_tick(ctx: &ReducerContext, schedule: GameTickSchedule) { - if !ctx.sender_auth().is_internal() { return; } - log::info!("Game tick at {:?}", ctx.timestamp); -} - -// Schedule at interval (e.g., in init reducer) -ctx.db.game_tick_schedule().insert(GameTickSchedule { - scheduled_id: 0, - scheduled_at: ScheduleAt::Interval(Duration::from_millis(100).into()), -}); - -// Schedule at specific time -let run_at = ctx.timestamp + Duration::from_secs(delay_secs); -ctx.db.reminder_schedule().insert(ReminderSchedule { - scheduled_id: 0, - scheduled_at: ScheduleAt::Time(run_at), -}); -``` - ---- - -## Identity and Authentication - -```rust -#[spacetimedb::table(accessor = user, public)] -pub struct User { - #[primary_key] - identity: Identity, - name: String, - online: bool, -} - -#[spacetimedb::reducer] -pub fn set_name(ctx: &ReducerContext, new_name: String) -> Result<(), String> { - let caller = ctx.sender(); - let user = ctx.db.user().identity().find(&caller) - .ok_or("User not found — connect first")?; - ctx.db.user().identity().update(User { name: new_name, ..user }); - Ok(()) -} -``` - -### Owner-Only Reducer Pattern - -```rust -fn require_owner(ctx: &ReducerContext, entity_owner: &Identity) -> Result<(), String> { - if ctx.sender() != *entity_owner { - Err("Not authorized: you don't own this entity".to_string()) - } else { - Ok(()) - } -} - -#[spacetimedb::reducer] -pub fn rename_character(ctx: &ReducerContext, char_id: u64, new_name: String) -> Result<(), String> { - let character = ctx.db.character().id().find(&char_id) - .ok_or("Character not found")?; - require_owner(ctx, &character.owner)?; - ctx.db.character().id().update(Character { name: new_name, ..character }); - Ok(()) -} -``` - ---- - -## Error Handling - -```rust -// Sender error — return Err (user sees message, transaction rolls back cleanly) -#[spacetimedb::reducer] -pub fn transfer(ctx: &ReducerContext, to: Identity, amount: u64) -> Result<(), String> { - let sender = ctx.db.wallet().identity().find(&ctx.sender()) - .ok_or("Wallet not found")?; - if sender.balance < amount { - return Err("Insufficient balance".to_string()); - } - // ... proceed with transfer - Ok(()) -} - -// Programmer error — panic (destroys the WASM instance, expensive!) -// Only use for truly impossible states -#[spacetimedb::reducer] -pub fn process(ctx: &ReducerContext, id: u64) { - let item = ctx.db.item().id().find(&id) - .expect("BUG: item should exist at this point"); - // ... -} -``` - -Prefer `Result<(), String>` for all expected failure cases. Panics destroy and recreate the WASM instance. - ---- - -## Procedures (Beta) - -> Procedures are behind the `unstable` feature in `spacetimedb`. -> In `Cargo.toml`: `spacetimedb = { version = "...", features = ["unstable"] }` - -```rust -use spacetimedb::{procedure, ProcedureContext}; - -#[procedure] -fn save_external_data(ctx: &mut ProcedureContext, url: String) -> Result<(), String> { - let data = fetch_from_url(&url)?; - ctx.try_with_tx(|tx| { - tx.db.external_data().insert(ExternalData { id: 0, content: data }); - Ok(()) - })?; - Ok(()) -} -``` - -| Reducers | Procedures | -|----------|------------| -| `&ReducerContext` (immutable) | `&mut ProcedureContext` (mutable) | -| Direct `ctx.db` access | Must use `ctx.with_tx()` | -| No HTTP/network | HTTP allowed | -| No return values | Can return data | - ---- - -## Custom Types - -```rust -use spacetimedb::SpacetimeType; - -#[derive(SpacetimeType)] -pub enum PlayerStatus { Active, Idle, Away } - -#[derive(SpacetimeType)] -pub struct Position { x: f32, y: f32, z: f32 } - -// Use in table (DO NOT derive SpacetimeType on the table!) -#[spacetimedb::table(accessor = player, public)] -pub struct Player { - #[primary_key] - id: u64, - status: PlayerStatus, - position: Position, -} -``` - ---- - -## Commands - -```bash -spacetime build -spacetime publish my_database --module-path . -spacetime publish my_database --clear-database --module-path . -spacetime logs my_database -spacetime call my_database create_player "Alice" -spacetime sql my_database "SELECT * FROM player" -spacetime generate --lang rust --out-dir /src/module_bindings --module-path -``` - -## Important Constraints - -1. **No Global State**: Static/global variables are undefined behavior across reducer calls -2. **No Side Effects**: Reducers cannot make network requests or file I/O -3. **Deterministic Execution**: Use `ctx.rng()` and `ctx.new_uuid_*()` for randomness -4. **Transactional**: All reducer changes roll back on failure -5. **Isolated**: Reducers don't see concurrent changes until commit diff --git a/skills/spacetimedb-typescript/SKILL.md b/skills/spacetimedb-typescript/SKILL.md deleted file mode 100644 index 3e2f38f5c05..00000000000 --- a/skills/spacetimedb-typescript/SKILL.md +++ /dev/null @@ -1,489 +0,0 @@ ---- -name: spacetimedb-typescript -description: Build TypeScript clients for SpacetimeDB. Use when connecting to SpacetimeDB from web apps, Node.js, Deno, Bun, or other JavaScript runtimes. -license: Apache-2.0 -metadata: - author: clockworklabs - version: "2.0" ---- - -# SpacetimeDB TypeScript SDK - -Build real-time TypeScript clients that connect directly to SpacetimeDB modules. The SDK provides type-safe database access, automatic synchronization, and reactive updates for web apps, Node.js, Deno, Bun, and other JavaScript runtimes. - ---- - -## HALLUCINATED APIs — DO NOT USE - -**These APIs DO NOT EXIST. LLMs frequently hallucinate them.** - -```typescript -// WRONG PACKAGE — does not exist -import { SpacetimeDBClient } from "@clockworklabs/spacetimedb-sdk"; - -// WRONG — these methods don't exist -SpacetimeDBClient.connect(...); -SpacetimeDBClient.call("reducer_name", [...]); -connection.call("reducer_name", [arg1, arg2]); - -// WRONG — positional reducer arguments -conn.reducers.doSomething("value"); // WRONG! - -// WRONG — old 1.0 patterns -spacetimedb.reducer('reducer_name', params, fn); // Use export const name = spacetimedb.reducer(params, fn) -schema(myTable); // Use schema({ myTable }) -schema(t1, t2, t3); // Use schema({ t1, t2, t3 }) -scheduled: 'run_cleanup' // Use scheduled: () => run_cleanup -.withModuleName('db') // Use .withDatabaseName('db') (2.0) -setReducerFlags.x('NoSuccessNotify') // Removed in 2.0 -``` - -### CORRECT PATTERNS: - -```typescript -// CORRECT IMPORTS -import { DbConnection, tables } from './module_bindings'; // Generated! -import { SpacetimeDBProvider, useTable } from 'spacetimedb/react'; -import { Identity } from 'spacetimedb'; - -// CORRECT REDUCER CALLS — object syntax, not positional! -conn.reducers.doSomething({ value: 'test' }); -conn.reducers.updateItem({ itemId: 1n, newValue: 42 }); - -// CORRECT DATA ACCESS — useTable returns [rows, isReady] -const [items, isReady] = useTable(tables.item); -``` - -### DO NOT: -- **Invent hooks** like `useItems()`, `useData()` — use `useTable(tables.tableName)` -- **Import from fake packages** — only `spacetimedb`, `spacetimedb/react`, `./module_bindings` - ---- - -## Common Mistakes Table - -### Server-side errors - -| Wrong | Right | Error | -|-------|-------|-------| -| Missing `package.json` | Create `package.json` | "could not detect language" | -| Missing `tsconfig.json` | Create `tsconfig.json` | "TsconfigNotFound" | -| Entrypoint not at `src/index.ts` | Use `src/index.ts` | Module won't bundle | -| `indexes` in COLUMNS (2nd arg) | `indexes` in OPTIONS (1st arg) of `table()` | "reading 'tag'" error | -| Index without `algorithm` | `algorithm: 'btree'` | "reading 'tag'" error | -| `filter({ ownerId })` | `filter(ownerId)` | "does not exist in type 'Range'" | -| `.filter()` on unique column | `.find()` on unique column | TypeError | -| `insert({ ...without id })` | `insert({ id: 0n, ... })` | "Property 'id' is missing" | -| `const id = table.insert(...)` | `const row = table.insert(...)` | `.insert()` returns ROW, not ID | -| `.unique()` + explicit index | Just use `.unique()` | "name is used for multiple entities" | -| Import spacetimedb from index.ts | Import from schema.ts | "Cannot access before initialization" | -| Incorrect multi-column `.filter()` range shape | Match index prefix/tuple shape | Empty results or range/type errors | -| `.iter()` in views | Use index lookups only | Views can't scan tables | -| `ctx.db` in procedures | `ctx.withTx(tx => tx.db...)` | Procedures need explicit transactions | - -### Client-side errors - -| Wrong | Right | Error | -|-------|-------|-------| -| Inline `connectionBuilder` | `useMemo(() => ..., [])` | Reconnects every render | -| `const rows = useTable(table)` | `const [rows, isReady] = useTable(table)` | Tuple destructuring | -| Optimistic UI updates | Let subscriptions drive state | Desync issues | -| `` | `connectionBuilder={...}` | Wrong prop name | - ---- - -## Hard Requirements - -1. **`schema({ table })`** — use a single tables object; optional module settings are allowed as a second argument -2. **Reducer/procedure names from exports** — `export const name = spacetimedb.reducer(params, fn)`; never `reducer('name', ...)` -3. **Reducer calls use object syntax** — `{ param: 'value' }` not positional args -4. **Import `DbConnection` from `./module_bindings`** — not from `spacetimedb` -5. **DO NOT edit generated bindings** — regenerate with `spacetime generate` -6. **Indexes go in OPTIONS (1st arg)** — not in COLUMNS (2nd arg) of `table()` -7. **Use BigInt for u64/i64 fields** — `0n`, `1n`, not `0`, `1` -8. **Reducers are transactional** — they do not return data -9. **Reducers must be deterministic** — no filesystem, network, timers, random -10. **Views should use index lookups** — `.iter()` causes severe performance issues -11. **Procedures need `ctx.withTx()`** — `ctx.db` doesn't exist in procedures -12. **Sum type values** — use `{ tag: 'variant', value: payload }` not `{ variant: payload }` -13. **Use `.withDatabaseName()`** — not `.withModuleName()` (2.0) - ---- - -## Installation - -```bash -npm install spacetimedb -``` - -For Node.js environments without native fetch/WebSocket support, install `undici`. - -## Generating Type Bindings - -```bash -spacetime generate --lang typescript --out-dir ./src/module_bindings --module-path ./server -``` - -## Client Connection - -```typescript -import { DbConnection } from './module_bindings'; - -const connection = DbConnection.builder() - .withUri('ws://localhost:3000') - .withDatabaseName('my_database') - .withToken(localStorage.getItem('spacetimedb_token') ?? undefined) - .onConnect((conn, identity, token) => { - // identity: your unique Identity for this database - console.log('Connected as:', identity.toHexString()); - - // Save token for reconnection (preserves identity across sessions) - localStorage.setItem('spacetimedb_token', token); - - conn.subscriptionBuilder() - .onApplied(() => console.log('Cache ready')) - .subscribe('SELECT * FROM player'); - }) - .onDisconnect((ctx) => console.log('Disconnected')) - .onConnectError((ctx, error) => console.error('Connection failed:', error)) - .build(); -``` - -## Subscribing to Tables - -```typescript -// Basic subscription -connection.subscriptionBuilder() - .onApplied((ctx) => console.log('Cache ready')) - .subscribe('SELECT * FROM player'); - -// Multiple queries -connection.subscriptionBuilder() - .subscribe(['SELECT * FROM player', 'SELECT * FROM game_state']); - -// Subscribe to all tables (development only — cannot mix with Subscribe) -connection.subscriptionBuilder().subscribeToAllTables(); - -// Subscription handle for later unsubscribe -const handle = connection.subscriptionBuilder() - .onApplied(() => console.log('Subscribed')) - .subscribe('SELECT * FROM player'); - -handle.unsubscribeThen(() => console.log('Unsubscribed')); -``` - -## Accessing Table Data - -```typescript -for (const player of connection.db.player.iter()) { console.log(player.name); } -const players = Array.from(connection.db.player.iter()); -const count = connection.db.player.count(); -const player = connection.db.player.id.find(42n); -``` - -## Table Event Callbacks - -```typescript -connection.db.player.onInsert((ctx, player) => console.log('New:', player.name)); -connection.db.player.onDelete((ctx, player) => console.log('Left:', player.name)); -connection.db.player.onUpdate((ctx, old, new_) => console.log(`${old.score} -> ${new_.score}`)); -``` - -## Calling Reducers - -**CRITICAL: Use object syntax, not positional arguments.** - -```typescript -connection.reducers.createPlayer({ name: 'Alice', location: { x: 0, y: 0 } }); -``` - -### Snake_case to camelCase conversion -- Server: `export const do_something = spacetimedb.reducer(...)` -- Client: `conn.reducers.doSomething({ ... })` - ---- - -## Identity and Authentication - -- `identity` and `token` are provided in the `onConnect` callback (see Client Connection above) -- `identity.toHexString()` for display or logging -- Omit `.withToken()` for anonymous connection — server assigns a new identity -- Pass a stale/invalid token: server issues a new identity and token in `onConnect` - ---- - -## Error Handling - -Connection-level errors (`.onConnectError`, `.onDisconnect`) are shown in the Client Connection example above. - -```typescript -// Subscription error -connection.subscriptionBuilder() - .onApplied(() => console.log('Subscribed')) - .onError((ctx) => console.error('Subscription error:', ctx.event)) - .subscribe('SELECT * FROM player'); -``` - ---- - -## Server-Side Module Development - -### Table Definition - -```typescript -import { schema, table, t } from 'spacetimedb/server'; - -export const Task = table({ - name: 'task', - public: true, - indexes: [{ name: 'task_owner_id', algorithm: 'btree', columns: ['ownerId'] }] -}, { - id: t.u64().primaryKey().autoInc(), - ownerId: t.identity(), - title: t.string(), - createdAt: t.timestamp(), -}); -``` - -### Column types - -```typescript -t.identity() // User identity -t.u64() // Unsigned 64-bit integer (use for IDs) -t.string() // Text -t.bool() // Boolean -t.timestamp() // Timestamp -t.scheduleAt() // For scheduled tables only -t.object('Name', {}) // Product types (nested objects) -t.enum('Name', {}) // Sum types (tagged unions) -t.string().optional() // Nullable -``` - -> BigInt syntax: All `u64`/`i64` fields use `0n`, `1n`, not `0`, `1`. - -### Schema export - -```typescript -const spacetimedb = schema({ Task, Player }); -export default spacetimedb; -``` - -### Reducer Definition (2.0) - -**Name comes from the export — NOT from a string argument.** - -```typescript -import spacetimedb from './schema'; -import { t, SenderError } from 'spacetimedb/server'; - -export const create_task = spacetimedb.reducer( - { title: t.string() }, - (ctx, { title }) => { - if (!title) throw new SenderError('title required'); - ctx.db.task.insert({ id: 0n, ownerId: ctx.sender, title, createdAt: ctx.timestamp }); - } -); -``` - -### Update Pattern - -```typescript -const existing = ctx.db.task.id.find(taskId); -if (!existing) throw new SenderError('Task not found'); -ctx.db.task.id.update({ ...existing, title: newTitle, updatedAt: ctx.timestamp }); -``` - -### Lifecycle Hooks - -```typescript -spacetimedb.clientConnected((ctx) => { /* ctx.sender is the connecting identity */ }); -spacetimedb.clientDisconnected((ctx) => { /* clean up */ }); -``` - ---- - -## Event Tables (2.0) - -Reducer callbacks are removed in 2.0. Use event tables + `onInsert` instead. - -```typescript -export const DamageEvent = table( - { name: 'damage_event', public: true, event: true }, - { target: t.identity(), amount: t.u32() } -); - -export const deal_damage = spacetimedb.reducer( - { target: t.identity(), amount: t.u32() }, - (ctx, { target, amount }) => { - ctx.db.damageEvent.insert({ target, amount }); - } -); -``` - -Client subscribes and uses `onInsert`: -```typescript -conn.db.damageEvent.onInsert((ctx, evt) => { - playDamageAnimation(evt.target, evt.amount); -}); -``` - -Event tables must be subscribed explicitly — they are excluded from `subscribeToAllTables()`. - ---- - -## Views - -### ViewContext vs AnonymousViewContext - -```typescript -// ViewContext — has ctx.sender, result varies per user -spacetimedb.view({ name: 'my_items', public: true }, t.array(Item.rowType), (ctx) => { - return [...ctx.db.item.by_owner.filter(ctx.sender)]; -}); - -// AnonymousViewContext — no ctx.sender, same result for everyone (better perf) -spacetimedb.anonymousView({ name: 'leaderboard', public: true }, t.array(Player.rowType), (ctx) => { - return ctx.from.player.where(p => p.score.gt(1000)); -}); -``` - -Views can only use index lookups — `.iter()` is NOT allowed. - ---- - -## Scheduled Tables - -```typescript -export const CleanupJob = table({ - name: 'cleanup_job', - scheduled: () => run_cleanup // function returning the exported reducer -}, { - scheduledId: t.u64().primaryKey().autoInc(), - scheduledAt: t.scheduleAt(), - targetId: t.u64(), -}); - -export const run_cleanup = spacetimedb.reducer( - { arg: CleanupJob.rowType }, - (ctx, { arg }) => { /* arg.scheduledId, arg.targetId available */ } -); - -// Schedule a job -import { ScheduleAt } from 'spacetimedb'; -ctx.db.cleanupJob.insert({ - scheduledId: 0n, - scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 60_000_000n), - targetId: someId -}); -``` - -### ScheduleAt on Client - -```typescript -// ScheduleAt is a tagged union on the client -// { tag: 'Time', value: Timestamp } or { tag: 'Interval', value: TimeDuration } -const schedule = row.scheduledAt; -if (schedule.tag === 'Time') { - const date = new Date(Number(schedule.value.microsSinceUnixEpoch / 1000n)); -} -``` - ---- - -## Timestamps - -### Server-side -```typescript -ctx.db.item.insert({ id: 0n, createdAt: ctx.timestamp }); -const future = ctx.timestamp.microsSinceUnixEpoch + 300_000_000n; -``` - -### Client-side -```typescript -// Timestamps are objects with BigInt, not numbers -const date = new Date(Number(row.createdAt.microsSinceUnixEpoch / 1000n)); -``` - ---- - -## Procedures (Beta) - -```typescript -export const fetch_data = spacetimedb.procedure( - { url: t.string() }, t.string(), - (ctx, { url }) => { - const response = ctx.http.fetch(url); - ctx.withTx(tx => { tx.db.myTable.insert({ id: 0n, content: response.text() }); }); - return response.text(); - } -); -``` - -Procedures don't have `ctx.db` — use `ctx.withTx(tx => tx.db...)`. - ---- - -## React Integration - -```tsx -import { useMemo } from 'react'; -import { SpacetimeDBProvider, useTable } from 'spacetimedb/react'; -import { DbConnection, tables } from './module_bindings'; - -function Root() { - const connectionBuilder = useMemo(() => - DbConnection.builder() - .withUri('ws://localhost:3000') - .withDatabaseName('my_game') - .withToken(localStorage.getItem('auth_token') || undefined) - .onConnect((conn, identity, token) => { - localStorage.setItem('auth_token', token); - conn.subscriptionBuilder().subscribe(tables.player); - }), - [] - ); - - return ( - - - - ); -} - -function PlayerList() { - const [players, isReady] = useTable(tables.player); - if (!isReady) return
Loading...
; - return
    {players.map(p =>
  • {p.name}
    • )}
; -} -``` - ---- - -## Project Structure - -### Server (`backend/spacetimedb/`) -``` -src/schema.ts -> Tables, export spacetimedb -src/index.ts -> Reducers, lifecycle, import schema -package.json -> { "type": "module", "dependencies": { "spacetimedb": "^2.0.0" } } -tsconfig.json -> Standard config -``` - -### Client (`client/`) -``` -src/module_bindings/ -> Generated (spacetime generate) -src/main.tsx -> Provider, connection setup -src/App.tsx -> UI components -``` - ---- - -## Commands - -```bash -spacetime start -spacetime publish --module-path -spacetime publish --clear-database -y --module-path -spacetime generate --lang typescript --out-dir /src/module_bindings --module-path -spacetime logs -``` diff --git a/skills/typescript-client/SKILL.md b/skills/typescript-client/SKILL.md new file mode 100644 index 00000000000..e68961a787a --- /dev/null +++ b/skills/typescript-client/SKILL.md @@ -0,0 +1,108 @@ +--- +name: typescript-client +description: SpacetimeDB TypeScript/React client SDK reference. Use when building web clients that connect to SpacetimeDB. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: client + language: typescript + cursor_globs: "**/*.tsx,**/*.ts" + cursor_always_apply: true +--- + +# SpacetimeDB TypeScript Client + +## React — main.tsx + +```typescript +import React, { useEffect, useMemo } from 'react'; +import ReactDOM from 'react-dom/client'; +import { SpacetimeDBProvider } from 'spacetimedb/react'; +import { DbConnection } from './module_bindings'; +import { MODULE_NAME, SPACETIMEDB_URI } from './config'; +import App from './App'; + +function Root() { + const connectionBuilder = useMemo(() => + DbConnection.builder() + .withUri(SPACETIMEDB_URI) + .withDatabaseName(MODULE_NAME) + .withToken(localStorage.getItem('auth_token') || undefined), + [] + ); + return ( + + + + ); +} + +ReactDOM.createRoot(document.getElementById('root')!).render(); +``` + +## React — App.tsx + +```typescript +import { useTable, useSpacetimeDB } from 'spacetimedb/react'; +import { DbConnection, tables } from './module_bindings'; + +function App() { + const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB(); + const conn = getConnection() as DbConnection | null; + + // Save auth token + useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]); + + // Subscribe when connected — prefer typed query builders over raw SQL + useEffect(() => { + if (!conn || !isActive) return; + conn.subscriptionBuilder() + .onApplied(() => setSubscribed(true)) + .subscribe([tables.entity, tables.record]); + // Or with filters: tables.entity.where(r => r.active.eq(true)) + // Or raw SQL: 'SELECT * FROM entity' + }, [conn, isActive]); + + // Reactive data — returns [rows, isReady] + const [entities, entitiesReady] = useTable(tables.entity); + const [records, recordsReady] = useTable(tables.record); + + // useTable with row callbacks + const [onlineUsers] = useTable( + tables.entity.where(r => r.active.eq(true)), + { + onInsert: (user) => console.log('User connected:', user.name), + onDelete: (user) => console.log('User disconnected:', user.name), + onUpdate: (oldUser, newUser) => console.log('Updated:', newUser.name), + } + ); + + // Call reducers with object syntax + conn?.reducers.addRecord({ data }); + + // Compare identities + const isMe = row.owner.toHexString() === myIdentity?.toHexString(); +} +``` + +## Vanilla (non-React) + +```typescript +import { DbConnection, tables } from './module_bindings'; + +const conn = DbConnection.builder() + .withUri('wss://maincloud.spacetimedb.com') + .withDatabaseName('my_module') + .onConnect((ctx) => { + ctx.subscriptionBuilder() + .onApplied(() => console.log('Ready')) + .subscribe([tables.user, tables.message]); + }) + .build(); + +// Row callbacks +conn.db.user.onInsert((ctx, user) => console.log('Joined:', user.name)); +conn.db.user.onDelete((ctx, user) => console.log('Left:', user.name)); +conn.db.user.onUpdate((ctx, oldUser, newUser) => console.log('Updated:', newUser.name)); +``` diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md new file mode 100644 index 00000000000..13d8cb95970 --- /dev/null +++ b/skills/typescript-server/SKILL.md @@ -0,0 +1,250 @@ +--- +name: typescript-server +description: SpacetimeDB TypeScript server module SDK reference. Use when writing tables, reducers, or module logic in TypeScript. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: server + language: typescript + cursor_globs: "**/*.ts" + cursor_always_apply: true +--- + +# SpacetimeDB TypeScript SDK Reference + +## Imports + +```typescript +import { schema, table, t } from 'spacetimedb/server'; +import { SenderError } from 'spacetimedb/server'; +import { ScheduleAt } from 'spacetimedb'; // for scheduled tables only +``` + +## Tables + +`table(OPTIONS, COLUMNS)` — two arguments. The `name` field MUST be snake_case: + +```typescript +const entity = table( + { name: 'entity', public: true }, + { + identity: t.identity().primaryKey(), + name: t.string(), + active: t.bool(), + } +); +``` + +Options: `name` (snake_case, recommended), `public: true`, `event: true`, `scheduled: (): any => reducerRef`, `indexes: [...]` + +`ctx.db` accessors are the camelCase form of the table's `name` field. + +## Column Types + +| Builder | JS type | Notes | +|---------|---------|-------| +| `t.u64()` | bigint | Use `0n` literals | +| `t.i64()` | bigint | Use `0n` literals | +| `t.u32()` / `t.i32()` | number | | +| `t.f64()` / `t.f32()` | number | | +| `t.bool()` | boolean | | +| `t.string()` | string | | +| `t.identity()` | Identity | | +| `t.connectionId()` | ConnectionId | | +| `t.timestamp()` | Timestamp | | +| `t.timeDuration()` | TimeDuration | | +| `t.scheduleAt()` | ScheduleAt | | + +Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')` + +Optional columns: `nickname: t.option(t.string())` + +## Indexes + +Prefer inline `.index('btree')` for single-column. Use named indexes only for multi-column: + +```typescript +// Inline (preferred for single-column): +authorId: t.u64().index('btree'), +// Access: ctx.db.post.authorId.filter(authorId); + +// Multi-column (named): +indexes: [{ accessor: 'by_group_user', algorithm: 'btree', columns: ['groupId', 'userId'] }] +// Access: ctx.db.membership.by_group_user.filter([groupId, userId]); +``` + +When you frequently look up rows by multiple columns, prefer a multi-column index over filtering by one column and looping over the results. Multi-column filter takes an array matching the index column order. You can omit trailing columns to do a prefix scan. + +## Schema Export + +```typescript +const spacetimedb = schema({ entity, record }); // ONE object, not spread args +export default spacetimedb; +``` + +## Reducers + +Export name becomes the reducer name: + +```typescript +export const createEntity = spacetimedb.reducer( + { name: t.string(), age: t.i32() }, + (ctx, { name, age }) => { + ctx.db.entity.insert({ identity: ctx.sender, name, age, active: true }); + } +); + +// No arguments — just the callback: +export const doReset = spacetimedb.reducer((ctx) => { ... }); +``` + +## DB Operations + +```typescript +ctx.db.entity.insert({ id: 0n, name: 'Sample' }); // Insert (0n for autoInc) +ctx.db.entity.id.find(entityId); // Find by PK → row | null +ctx.db.entity.identity.find(ctx.sender); // Find by unique column +[...ctx.db.item.authorId.filter(authorId)]; // Filter → spread to Array +[...ctx.db.entity.iter()]; // All rows → Array +ctx.db.entity.id.update({ ...existing, name: newName }); // Update (spread + override) +ctx.db.entity.id.delete(entityId); // Delete by PK +``` + +Note: `iter()` and `filter()` return iterators. Spread to Array for `.sort()`, `.filter()`, `.map()`. + +## Lifecycle Hooks + +MUST be `export const` — bare calls are silently ignored: + +```typescript +export const init = spacetimedb.init((ctx) => { ... }); +export const onConnect = spacetimedb.clientConnected((ctx) => { ... }); +export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); +``` + +## Authentication & Timestamps + +```typescript +// Auth: ctx.sender is the caller's Identity +if (!row.owner.equals(ctx.sender)) throw new SenderError('unauthorized'); + +// Server timestamps +ctx.db.item.insert({ id: 0n, createdAt: ctx.timestamp }); + +// Client: Timestamp → Date +new Date(Number(row.createdAt.microsSinceUnixEpoch / 1000n)); +``` + +## Scheduled Tables + +```typescript +const tickTimer = table({ + name: 'tick_timer', + scheduled: (): any => tick, // (): any => breaks circular dep +}, { + scheduled_id: t.u64().primaryKey().autoInc(), + scheduled_at: t.scheduleAt(), +}); + +export const tick = spacetimedb.reducer( + { timer: tickTimer.rowType }, + (ctx, { timer }) => { /* timer row auto-deleted after this runs */ } +); + +// One-time: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + delayMicros) +// Repeating: ScheduleAt.interval(60_000_000n) +``` + +## Custom Types + +```typescript +// Product type (struct): +const Position = t.object('Position', { x: t.i32(), y: t.i32() }); +const entity = table({ name: 'entity' }, { + id: t.u64().primaryKey().autoInc(), + pos: Position, +}); + +// Sum type (tagged union): +const Shape = t.enum('Shape', { + circle: t.i32(), + rectangle: t.object('Rect', { w: t.i32(), h: t.i32() }), +}); +// Values: { tag: 'circle', value: 10 } +``` + +## Views + +```typescript +// Anonymous view (same for all clients): +export const activeUsers = spacetimedb.anonymousView( + { name: 'active_users', public: true }, + t.array(entity.rowType), + (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) +); + +// Per-user view (varies by ctx.sender): +export const myProfile = spacetimedb.view( + { name: 'my_profile', public: true }, + t.option(entity.rowType), + (ctx) => ctx.db.entity.identity.find(ctx.sender) ?? undefined +); +``` + +## Complete Example + +```typescript +import { schema, table, t } from 'spacetimedb/server'; + +const entity = table( + { name: 'entity', public: true }, + { + identity: t.identity().primaryKey(), + name: t.string(), + active: t.bool(), + } +); + +const record = table( + { + name: 'record', + public: true, + indexes: [{ accessor: 'by_owner', algorithm: 'btree', columns: ['owner'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + owner: t.identity(), + value: t.u32(), + } +); + +const spacetimedb = schema({ entity, record }); +export default spacetimedb; + +export const onConnect = spacetimedb.clientConnected((ctx) => { + const existing = ctx.db.entity.identity.find(ctx.sender); + if (existing) ctx.db.entity.identity.update({ ...existing, active: true }); +}); + +export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { + const existing = ctx.db.entity.identity.find(ctx.sender); + if (existing) ctx.db.entity.identity.update({ ...existing, active: false }); +}); + +export const createEntity = spacetimedb.reducer( + { name: t.string() }, + (ctx, { name }) => { + if (ctx.db.entity.identity.find(ctx.sender)) throw new Error('already exists'); + ctx.db.entity.insert({ identity: ctx.sender, name, active: true }); + } +); + +export const addRecord = spacetimedb.reducer( + { value: t.u32() }, + (ctx, { value }) => { + if (!ctx.db.entity.identity.find(ctx.sender)) throw new Error('not found'); + ctx.db.record.insert({ id: 0n, owner: ctx.sender, value }); + } +); +``` diff --git a/skills/spacetimedb-unity/SKILL.md b/skills/unity/SKILL.md similarity index 70% rename from skills/spacetimedb-unity/SKILL.md rename to skills/unity/SKILL.md index 56260f9f2f6..bbbde4e5f1d 100644 --- a/skills/spacetimedb-unity/SKILL.md +++ b/skills/unity/SKILL.md @@ -1,44 +1,20 @@ --- -name: spacetimedb-unity +name: unity description: Integrate SpacetimeDB with Unity game projects. Use when building Unity clients with MonoBehaviour lifecycle, FrameTick, and PlayerPrefs token persistence. license: Apache-2.0 metadata: author: clockworklabs version: "2.0" + role: client + language: csharp + cursor_globs: "**/*.cs" + cursor_always_apply: false tested_with: "SpacetimeDB 2.0, Unity 2022.3+" --- # SpacetimeDB Unity Integration -This skill covers Unity-specific patterns for connecting to SpacetimeDB. For server-side module development and general C# SDK usage, see the `spacetimedb-csharp` skill. - ---- - -## HALLUCINATED APIs — DO NOT USE - -```csharp -// WRONG — these do not exist in Unity SDK -SpacetimeDBClient.instance.Connect(...); // Use DbConnection.Builder() -SpacetimeDBClient.instance.Subscribe(...); // Use conn.SubscriptionBuilder() -NetworkManager.RegisterReducer(...); // SpacetimeDB is not a Unity networking plugin - -// WRONG — old 1.0 patterns -.WithModuleName("my-db") // Use .WithDatabaseName() (2.0) -ScheduleAt.Time(futureTime) // Use new ScheduleAt.Time(futureTime) -``` - ---- - -## Common Mistakes - -| Wrong | Right | Error | -|-------|-------|-------| -| Not calling `FrameTick()` | `conn?.FrameTick()` in `Update()` | No callbacks fire | -| Accessing `conn.Db` from background thread | Copy data in callback, use on main thread | Data races / crashes | -| Forgetting `DontDestroyOnLoad` | Add to manager `Awake()` | Connection lost on scene load | -| Connecting in `Update()` | Connect in `Start()` or on user action | Reconnects every frame | -| Not saving auth token | `PlayerPrefs.SetString(...)` in `OnConnect` | New identity every session | -| Missing generated bindings | Run `spacetime generate --lang csharp` | Compile errors | +This skill covers Unity-specific patterns for connecting to SpacetimeDB. For server-side module development, see the `csharp-server` skill. --- @@ -140,47 +116,12 @@ public class SpacetimeManager : MonoBehaviour ## FrameTick — Critical -**`FrameTick()` must be called every frame in `Update()`.** The SDK queues all network messages and only processes them when you call `FrameTick()`. Without it: -- No callbacks fire (OnInsert, OnUpdate, OnDelete, reducer callbacks) -- The client appears frozen - -```csharp -void Update() -{ - Connection?.FrameTick(); -} -``` +**`FrameTick()` must be called every frame in `Update()`.** The SDK queues all network messages and only processes them when you call `FrameTick()`. Without it, no callbacks fire and the client appears frozen. See the `Update()` method in the SpacetimeManager above. **Thread safety**: `FrameTick()` processes messages on the calling thread (the main thread in Unity). Do NOT call it from a background thread. Do NOT access `conn.Db` from background threads. --- -## Subscribing to Tables - -Subscribe in the `OnConnected` callback: - -```csharp -private void OnConnected(DbConnection conn, Identity identity, string authToken) -{ - // ...save token... - - // Development: subscribe to all - conn.SubscriptionBuilder() - .OnApplied(OnSubscriptionApplied) - .SubscribeToAllTables(); - - // Production: subscribe to specific tables - conn.SubscriptionBuilder() - .OnApplied(OnSubscriptionApplied) - .Subscribe(new[] { - "SELECT * FROM player", - "SELECT * FROM game_state" - }); -} -``` - ---- - ## Row Callbacks for Game State Register callbacks to update Unity GameObjects when table data changes. @@ -279,14 +220,3 @@ The SpacetimeDB SDK uses code generation. If you encounter issues with IL2CPP bu ### Token Persistence Token save/load via `PlayerPrefs` is demonstrated in the SpacetimeManager singleton above. If the token is stale or invalid, the server issues a new identity and token in the `OnConnect` callback. ---- - -## Commands - -```bash -spacetime start -spacetime publish --module-path -spacetime publish --clear-database -y --module-path -spacetime generate --lang csharp --out-dir Assets/SpacetimeDB/module_bindings --module-path -spacetime logs -``` diff --git a/skills/unreal/SKILL.md b/skills/unreal/SKILL.md new file mode 100644 index 00000000000..0ba6aac04e3 --- /dev/null +++ b/skills/unreal/SKILL.md @@ -0,0 +1,443 @@ +--- +name: unreal +description: SpacetimeDB Unreal Engine client SDK reference. Use when building Unreal Engine clients that connect to SpacetimeDB. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "2.0" + role: client + language: cpp + cursor_globs: "**/*.cpp,**/*.h" + cursor_always_apply: true +--- + +# SpacetimeDB Unreal Engine Integration + +This skill covers Unreal Engine-specific patterns for connecting to SpacetimeDB. For server-side module development, see the `rust-server` or `csharp-server` skills. + +--- + +## Installation + +Add the SpacetimeDB Unreal SDK as a plugin: + +1. Create a `Plugins` folder in your Unreal project root if it does not exist. +2. Copy the `SpacetimeDbSdk` folder into `Plugins/`. +3. Right-click your `.uproject` file and select **Generate Visual Studio project files**. +4. Add `"SpacetimeDbSdk"` to your module's `Build.cs`: + +```csharp +PublicDependencyModuleNames.AddRange(new string[] { "SpacetimeDbSdk" }); +``` + +--- + +## Generate Module Bindings + +```bash +spacetime generate --lang unrealcpp \ + --uproject-dir \ + --module-path \ + --unreal-module-name +``` + +This generates C++ bindings in `ModuleBindings/` inside your project. Include the generated header: + +```cpp +#include "ModuleBindings/SpacetimeDBClient.g.h" +``` + +Regenerate whenever you change module tables, reducers, or types. + +--- + +## GameManager Actor Pattern + +The recommended pattern is a singleton Actor that owns the connection. Enable ticking so `FrameTick` is called every frame. + +### Header (GameManager.h) + +```cpp +#pragma once +#include "CoreMinimal.h" +#include "GameFramework/Actor.h" +#include "ModuleBindings/SpacetimeDBClient.g.h" +#include "GameManager.generated.h" + +class UDbConnection; + +UCLASS() +class AGameManager : public AActor +{ + GENERATED_BODY() +public: + AGameManager(); + static AGameManager* Instance; + + UPROPERTY(EditAnywhere, Category="SpacetimeDB") + FString ServerUri = TEXT("127.0.0.1:3000"); + + UPROPERTY(EditAnywhere, Category="SpacetimeDB") + FString DatabaseName = TEXT("my-module"); + + UPROPERTY(BlueprintReadOnly, Category="SpacetimeDB") + UDbConnection* Conn = nullptr; + + UPROPERTY(BlueprintReadOnly, Category="SpacetimeDB") + FSpacetimeDBIdentity LocalIdentity; + +protected: + virtual void BeginPlay() override; + virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; +public: + virtual void Tick(float DeltaTime) override; + +private: + UFUNCTION() void HandleConnect(UDbConnection* InConn, FSpacetimeDBIdentity Identity, const FString& Token); + UFUNCTION() void HandleConnectError(const FString& Error); + UFUNCTION() void HandleDisconnect(UDbConnection* InConn, const FString& Error); + UFUNCTION() void HandleSubscriptionApplied(FSubscriptionEventContext& Context); +}; +``` + +### Source (GameManager.cpp) + +```cpp +#include "GameManager.h" +#include "Connection/Credentials.h" + +AGameManager* AGameManager::Instance = nullptr; + +AGameManager::AGameManager() +{ + PrimaryActorTick.bCanEverTick = true; + PrimaryActorTick.bStartWithTickEnabled = true; +} + +void AGameManager::BeginPlay() +{ + Super::BeginPlay(); + Instance = this; + + FOnConnectDelegate ConnectDelegate; + BIND_DELEGATE_SAFE(ConnectDelegate, this, AGameManager, HandleConnect); + FOnDisconnectDelegate DisconnectDelegate; + BIND_DELEGATE_SAFE(DisconnectDelegate, this, AGameManager, HandleDisconnect); + FOnConnectErrorDelegate ConnectErrorDelegate; + BIND_DELEGATE_SAFE(ConnectErrorDelegate, this, AGameManager, HandleConnectError); + + UCredentials::Init(TEXT(".spacetime_token")); + FString Token = UCredentials::LoadToken(); + + UDbConnectionBuilder* Builder = UDbConnection::Builder() + ->WithUri(ServerUri) + ->WithDatabaseName(DatabaseName) + ->OnConnect(ConnectDelegate) + ->OnDisconnect(DisconnectDelegate) + ->OnConnectError(ConnectErrorDelegate); + + if (!Token.IsEmpty()) + { + Builder->WithToken(Token); + } + + Conn = Builder->Build(); +} + +void AGameManager::EndPlay(const EEndPlayReason::Type EndPlayReason) +{ + if (Conn) { Conn->Disconnect(); Conn = nullptr; } + if (Instance == this) { Instance = nullptr; } + Super::EndPlay(EndPlayReason); +} + +void AGameManager::Tick(float DeltaTime) +{ + if (Conn && Conn->IsActive()) + { + Conn->FrameTick(); + } +} + +void AGameManager::HandleConnect(UDbConnection* InConn, FSpacetimeDBIdentity Identity, const FString& Token) +{ + LocalIdentity = Identity; + UCredentials::SaveToken(Token); + + FOnSubscriptionApplied AppliedDelegate; + BIND_DELEGATE_SAFE(AppliedDelegate, this, AGameManager, HandleSubscriptionApplied); + Conn->SubscriptionBuilder() + ->OnApplied(AppliedDelegate) + ->SubscribeToAllTables(); +} + +void AGameManager::HandleConnectError(const FString& Error) +{ + UE_LOG(LogTemp, Error, TEXT("Connection error: %s"), *Error); +} + +void AGameManager::HandleDisconnect(UDbConnection* InConn, const FString& Error) +{ + UE_LOG(LogTemp, Warning, TEXT("Disconnected: %s"), *Error); +} + +void AGameManager::HandleSubscriptionApplied(FSubscriptionEventContext& Context) +{ + UE_LOG(LogTemp, Log, TEXT("Subscription applied - game state loaded")); +} +``` + +--- + +## FrameTick -- Critical + +**You must either call `Conn->FrameTick()` every frame in your Actor's `Tick()`, or call `Conn->SetAutoTicking(true)` once at startup.** The SDK queues all network messages and only processes them on tick. Without one of these, no callbacks fire and the client appears frozen. + +--- + +## Connection Builder + +Build a connection with the builder pattern. All builder methods return pointers for chaining with `->`. + +```cpp +UDbConnection* Conn = UDbConnection::Builder() + ->WithUri(TEXT("127.0.0.1:3000")) + ->WithDatabaseName(TEXT("my-module")) + ->WithToken(SavedToken) // optional + ->WithCompression(ESpacetimeDBCompression::Gzip) // optional + ->OnConnect(ConnectDelegate) + ->OnConnectError(ErrorDelegate) + ->OnDisconnect(DisconnectDelegate) + ->Build(); +``` + +### OnConnect callback signature + +```cpp +UFUNCTION() +void OnConnected(UDbConnection* Connection, FSpacetimeDBIdentity Identity, const FString& Token); +``` + +Save the `Token` for future reconnection. The `Identity` is the user's persistent identifier. + +--- + +## Subscribing to Tables + +After connecting, subscribe to receive table data: + +```cpp +// Subscribe to all public tables +Conn->SubscriptionBuilder() + ->OnApplied(AppliedDelegate) + ->SubscribeToAllTables(); + +// Subscribe to specific queries +TArray Queries = { TEXT("SELECT * FROM player"), TEXT("SELECT * FROM entity") }; +Conn->SubscriptionBuilder() + ->OnApplied(AppliedDelegate) + ->OnError(ErrorDelegate) + ->Subscribe(Queries); +``` + +### Subscription Handle + +`Subscribe` and `SubscribeToAllTables` return a `USubscriptionHandle*`: + +```cpp +USubscriptionHandle* Handle = Conn->SubscriptionBuilder()->...->Subscribe(Queries); +Handle->IsActive(); // true while subscription is live +Handle->Unsubscribe(); // cancel the subscription +Handle->UnsubscribeThen(OnEndDelegate); // cancel with callback +Handle->GetQuerySqls(); // get the SQL queries +``` + +--- + +## Reading the Client Cache + +Access tables through `Conn->Db`: + +```cpp +// Find by unique/primary key (returns by value; default-constructed if not found) +FUserType User = Conn->Db->User->Identity->Find(SomeIdentity); + +// Filter by BTree index +TArray LevelFive = Conn->Db->Player->Level->Filter(5); + +// Iterate all rows +TArray AllEntities = Conn->Db->Entity->Iter(); + +// Count +int32 Total = Conn->Db->Player->Count(); +``` + +--- + +## Row Callbacks + +Register callbacks on table objects. Callbacks use Unreal dynamic multicast delegates. + +```cpp +// OnInsert +Conn->Db->User->OnInsert.AddDynamic(this, &AMyActor::OnUserInsert); + +// OnDelete +Conn->Db->User->OnDelete.AddDynamic(this, &AMyActor::OnUserDelete); + +// OnUpdate (only fires for rows with a primary key) +Conn->Db->User->OnUpdate.AddDynamic(this, &AMyActor::OnUserUpdate); +``` + +### Callback signatures (must be UFUNCTION) + +```cpp +UFUNCTION() +void OnUserInsert(const FEventContext& Context, const FUserType& NewRow); + +UFUNCTION() +void OnUserDelete(const FEventContext& Context, const FUserType& DeletedRow); + +UFUNCTION() +void OnUserUpdate(const FEventContext& Context, const FUserType& OldRow, const FUserType& NewRow); +``` + +Register callbacks before connecting or in `HandleSubscriptionApplied`. + +--- + +## Calling Reducers + +Invoke reducers through `Conn->Reducers`: + +```cpp +Conn->Reducers->SendMessage(TEXT("Hello!")); +Conn->Reducers->SetName(TEXT("Alice")); +Conn->Reducers->MovePlayer(1.0f, 0.0f); +``` + +### Reducer Result Callbacks + +Observe when a reducer you called completes: + +```cpp +Conn->Reducers->OnSendMessage.AddDynamic(this, &AMyActor::OnSendMessageResult); +``` + +```cpp +UFUNCTION() +void OnSendMessageResult(const FReducerEventContext& Context, const FString& Text) +{ + UE_LOG(LogTemp, Log, TEXT("SendMessage result for: %s"), *Text); +} +``` + +These delegates fire only for reducer calls made by this connection, not for other clients' calls. + +--- + +## Delegate Binding with BIND_DELEGATE_SAFE + +Use the `BIND_DELEGATE_SAFE` macro to safely bind delegates to member functions: + +```cpp +FOnConnectDelegate ConnectDelegate; +BIND_DELEGATE_SAFE(ConnectDelegate, this, AMyActor, HandleConnect); +``` + +This is the recommended pattern for all SpacetimeDB delegate bindings in C++. + +--- + +## Identity and ConnectionId + +```cpp +// FSpacetimeDBIdentity -- 256-bit unique user identifier, persists across connections +FSpacetimeDBIdentity Identity; +Identity.ToHex(); + +// FSpacetimeDBConnectionId -- 128-bit per-session connection identifier +FSpacetimeDBConnectionId ConnId = Conn->GetConnectionId(); + +// From any context +FSpacetimeDBIdentity Id; +bool Found = Context.TryGetIdentity(Id); +FSpacetimeDBConnectionId CId = Context.GetConnectionId(); +``` + +--- + +## Token Persistence + +Use the built-in `UCredentials` helper to save and load tokens: + +```cpp +UCredentials::Init(TEXT(".spacetime_token")); +FString Token = UCredentials::LoadToken(); +// ... after connect: +UCredentials::SaveToken(Token); +``` + +--- + +## Context Types + +All callbacks receive a context struct that provides access to `Db` and `Reducers`: + +| Type | Used In | +|------|---------| +| `FEventContext` | Table row callbacks (OnInsert, OnDelete, OnUpdate) | +| `FReducerEventContext` | Reducer result callbacks | +| `FSubscriptionEventContext` | Subscription lifecycle callbacks (OnApplied, OnError) | +| `FErrorContext` | Error callbacks | + +All inherit from `FContextBase` which provides: + +```cpp +Context.Db // URemoteTables* -- client cache +Context.Reducers // URemoteReducers* -- invoke reducers +Context.SubscriptionBuilder() // start a new subscription +``` + +--- + +## Blueprint Integration + +All core classes are Blueprint-accessible via `UFUNCTION(BlueprintCallable)` and `UPROPERTY(BlueprintReadOnly/BlueprintAssignable)`: + +- `UDbConnection::Builder()` and all builder methods are `BlueprintCallable`. +- Table callbacks (`OnInsert`, `OnDelete`, `OnUpdate`) are `BlueprintAssignable` delegates. +- Reducer invoke methods and result delegates are Blueprint-accessible. +- `Conn->Db` and `Conn->Reducers` are `BlueprintReadOnly` properties. +- Generated row types are `BlueprintType` USTRUCTs with `BlueprintReadWrite` properties. + +This means you can build the entire connection and callback flow in Blueprints without writing C++. + +--- + +## Unreal-Specific Considerations + +### Auto Ticking Alternative + +`UDbConnection` inherits from `FTickableGameObject`, but auto ticking is **off by default**. You have two options: + +```cpp +// Option 1: Call FrameTick() manually in your Actor's Tick() (shown in GameManager above) +void Tick(float DeltaTime) { Conn->FrameTick(); } + +// Option 2: Enable auto ticking — the SDK then processes messages every frame automatically +Conn->SetAutoTicking(true); +``` + +Pick one. Without either, no callbacks fire. + +### Compression + +```cpp +Builder->WithCompression(ESpacetimeDBCompression::Gzip); // default +Builder->WithCompression(ESpacetimeDBCompression::None); // no compression +``` + +### Generated Types + +Codegen produces USTRUCTs prefixed with `F` (e.g., `FUserType`, `FEntityType`) and table classes prefixed with `U` (e.g., `UUserTable`). Row types use `GENERATED_BODY()` and `UPROPERTY()` for full reflection support. diff --git a/tools/xtask-llm-benchmark/.gitignore b/tools/xtask-llm-benchmark/.gitignore new file mode 100644 index 00000000000..a1e03960fba --- /dev/null +++ b/tools/xtask-llm-benchmark/.gitignore @@ -0,0 +1 @@ +runs/ diff --git a/tools/xtask-llm-benchmark/Cargo.toml b/tools/xtask-llm-benchmark/Cargo.toml index 947a9d7c76a..d71b7844132 100644 --- a/tools/xtask-llm-benchmark/Cargo.toml +++ b/tools/xtask-llm-benchmark/Cargo.toml @@ -26,8 +26,6 @@ urlencoding = "2.1.3" reqwest = { version = "0.12", features = ["json"] } futures = "0.3.31" regex = "1" -tempfile = "3.23.0" -fs2 = "0.4.3" heck = "0.5.0" thiserror = "2.0.17" diff --git a/tools/xtask-llm-benchmark/src/api/client.rs b/tools/xtask-llm-benchmark/src/api/client.rs new file mode 100644 index 00000000000..c036ef39b13 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/api/client.rs @@ -0,0 +1,353 @@ +use anyhow::{Context, Result}; +use serde_json::json; + +use crate::bench::results_merge::{canonical_mode, normalize_model_names}; +use crate::bench::types::RunOutcome; +use crate::results::schema::Results; + +/// HTTP client for the SpacetimeDB LLM benchmark API (spacetime-web Postgres). +/// +/// Supports two POST endpoints that already exist in spacetime-web: +/// - `POST /api/llm-benchmark-upload` — upload benchmark results +/// - `POST /api/llm-benchmark-tasks` — upload task catalog +#[derive(Clone)] +pub struct ApiClient { + client: reqwest::blocking::Client, + base_url: String, + api_key: String, +} + +impl ApiClient { + pub fn new(base_url: &str, api_key: &str) -> Result { + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(120)) + .build() + .context("failed to build HTTP client")?; + Ok(Self { + client, + base_url: base_url.trim_end_matches('/').to_string(), + api_key: api_key.to_string(), + }) + } + + /// Build from environment variables `LLM_BENCHMARK_UPLOAD_URL` and `LLM_BENCHMARK_API_KEY`. + /// Returns `None` if `LLM_BENCHMARK_UPLOAD_URL` is not set. + pub fn from_env() -> Result> { + let url = match std::env::var("LLM_BENCHMARK_UPLOAD_URL") { + Ok(u) if !u.is_empty() => u, + _ => return Ok(None), + }; + let key = + std::env::var("LLM_BENCHMARK_API_KEY").context("LLM_BENCHMARK_API_KEY required when UPLOAD_URL is set")?; + Self::new(&url, &key).map(Some) + } + + /// Upload a batch of run outcomes for a single (lang, mode) combination. + /// Normalizes model names and sanitizes volatile fields before upload. + /// If `analysis` is provided, it is stored in the `llm_benchmark_analysis` table. + pub fn upload_batch( + &self, + lang: &str, + mode: &str, + hash: &str, + outcomes: &[RunOutcome], + analysis: Option<&str>, + ) -> Result { + if outcomes.is_empty() { + return Ok(0); + } + + let mode = canonical_mode(mode); + + // Build in-memory Results so we can normalize model names + let mut results = Results::default(); + { + use crate::bench::results_merge::{canonical_model_name, ensure_lang, ensure_mode, ensure_model}; + + for r in outcomes { + let lang_v = ensure_lang(&mut results, &r.lang); + let mode_v = ensure_mode(lang_v, mode, Some(r.hash.clone())); + let canonical_name = canonical_model_name(&r.model_name); + let model_v = ensure_model(mode_v, &canonical_name); + model_v.route_api_model = r.route_api_model.clone(); + + let mut sanitized = r.clone(); + sanitized.sanitize_for_commit(); + model_v.tasks.insert(r.task.clone(), sanitized); + } + } + normalize_model_names(&mut results); + + let url = format!("{}/api/llm-benchmark-upload", self.base_url); + let mut total_uploaded = 0usize; + + for lang_entry in &results.languages { + for mode_entry in &lang_entry.modes { + // Serialize models and inject analysis into each model object if provided + let mut models_json = serde_json::to_value(&mode_entry.models)?; + if let Some(text) = analysis + && let Some(arr) = models_json.as_array_mut() + { + for model in arr { + model["analysis"] = json!(text); + } + } + + let payload = json!({ + "lang": lang_entry.lang, + "mode": mode_entry.mode, + "hash": mode_entry.hash, + "models": models_json, + }); + + let resp = self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .header("Content-Type", "application/json") + .json(&payload) + .send() + .with_context(|| format!("upload failed for {}/{}", lang_entry.lang, mode_entry.mode))?; + + if resp.status().is_success() { + let body: serde_json::Value = resp.json().unwrap_or_default(); + let inserted = body["inserted"].as_u64().unwrap_or(0); + total_uploaded += inserted as usize; + println!( + "\u{1f4e4} uploaded {}/{}: {} results", + lang_entry.lang, mode_entry.mode, inserted + ); + } else { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + eprintln!( + "\u{26a0}\u{fe0f} upload failed for {}/{}: {} \u{2014} {}", + lang_entry.lang, mode_entry.mode, status, body + ); + } + } + } + + let _ = lang; + let _ = hash; + Ok(total_uploaded) + } + + /// Upload the task catalog to `POST /api/llm-benchmark-tasks`, derived from + /// the benchmarks directory structure on disk. + pub fn upload_task_catalog(&self, bench_root: &std::path::Path) -> Result { + use std::collections::BTreeMap; + use std::fs; + + let mut categories: BTreeMap> = BTreeMap::new(); + + let cats = fs::read_dir(bench_root).with_context(|| format!("read_dir {}", bench_root.display()))?; + for cat_entry in cats.filter_map(|e| e.ok()) { + if !cat_entry.file_type().map(|t| t.is_dir()).unwrap_or(false) { + continue; + } + let cat_name = cat_entry.file_name().to_string_lossy().to_string(); + let cat_path = cat_entry.path(); + + let tasks = match fs::read_dir(&cat_path) { + Ok(rd) => rd, + Err(_) => continue, + }; + for task_entry in tasks.filter_map(|e| e.ok()) { + if !task_entry.file_type().map(|t| t.is_dir()).unwrap_or(false) { + continue; + } + let task_name = task_entry.file_name().to_string_lossy().to_string(); + + // Read first line of prompt.md as description + let prompt_path = task_entry.path().join("prompt.md"); + let description = fs::read_to_string(&prompt_path) + .ok() + .and_then(|s| s.lines().next().map(|l| l.trim_start_matches('#').trim().to_string())) + .unwrap_or_default(); + + // Humanize task_name for title + let title = task_name + .trim_start_matches(|c: char| c == 't' || c == '_' || c.is_ascii_digit()) + .replace('_', " ") + .trim() + .to_string(); + let title = if title.is_empty() { + task_name.clone() + } else { + title + .split_whitespace() + .map(|w| { + let mut c = w.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().to_string() + c.as_str(), + } + }) + .collect::>() + .join(" ") + }; + + // Read golden answers per language + let answers_dir = task_entry.path().join("answers"); + let mut golden_answers = serde_json::Map::new(); + for (lang, file) in [("rust", "rust.rs"), ("csharp", "csharp.cs"), ("typescript", "typescript.ts")] { + let path = answers_dir.join(file); + if let Ok(content) = fs::read_to_string(&path) { + golden_answers.insert(lang.to_string(), json!(content)); + } + } + + categories.entry(cat_name.clone()).or_default().push(json!({ + "id": task_name, + "title": title, + "description": description, + "golden_answers": golden_answers, + })); + } + } + + let url = format!("{}/api/llm-benchmark-tasks", self.base_url); + let payload = json!({ "categories": categories }); + + let resp = self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .header("Content-Type", "application/json") + .json(&payload) + .send() + .context("upload task catalog failed")?; + + if resp.status().is_success() { + let body: serde_json::Value = resp.json().unwrap_or_default(); + let upserted = body["upserted"].as_u64().unwrap_or(0) as usize; + println!("\u{1f4e4} uploaded task catalog: {} tasks", upserted); + Ok(upserted) + } else { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + anyhow::bail!("task catalog upload failed: {} \u{2014} {}", status, body); + } + } + + /// Fetch available run dates from `GET /api/llm-benchmark-results?dates=true`. + pub fn fetch_run_dates( + &self, + lang: Option<&str>, + mode: Option<&str>, + ) -> Result> { + let mut params = vec!["dates=true".to_string()]; + if let Some(l) = lang { + params.push(format!("lang={}", urlencoding::encode(l))); + } + if let Some(m) = mode { + params.push(format!("mode={}", urlencoding::encode(m))); + } + let url = format!("{}/api/llm-benchmark-results?{}", self.base_url, params.join("&")); + + let resp = self + .client + .get(&url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .send() + .context("fetch run dates failed")?; + + if resp.status().is_success() { + let body: serde_json::Value = resp.json().context("parse dates response")?; + Ok(body["dates"] + .as_array() + .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect()) + .unwrap_or_default()) + } else { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + anyhow::bail!("fetch run dates failed: {} \u{2014} {}", status, body); + } + } + + /// Fetch failure results from `GET /api/llm-benchmark-results?failures=true`. + pub fn fetch_failures( + &self, + lang: Option<&str>, + mode: Option<&str>, + model: Option<&str>, + date: Option<&str>, + ) -> Result<(Vec, Option)> { + let mut params = vec!["failures=true".to_string()]; + if let Some(l) = lang { + params.push(format!("lang={}", urlencoding::encode(l))); + } + if let Some(m) = mode { + params.push(format!("mode={}", urlencoding::encode(m))); + } + if let Some(m) = model { + params.push(format!("model={}", urlencoding::encode(m))); + } + if let Some(d) = date { + params.push(format!("date={}", urlencoding::encode(d))); + } + let url = format!("{}/api/llm-benchmark-results?{}", self.base_url, params.join("&")); + + let resp = self + .client + .get(&url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .send() + .context("fetch failures failed")?; + + if resp.status().is_success() { + let body: serde_json::Value = resp.json().context("parse failures response")?; + let date = body["date"].as_str().map(String::from); + let results = body["results"] + .as_array() + .cloned() + .unwrap_or_default(); + Ok((results, date)) + } else { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + anyhow::bail!("fetch failures failed: {} \u{2014} {}", status, body); + } + } + + /// Upload analysis for a specific (lang, mode, model) via the upload endpoint. + pub fn upload_analysis( + &self, + lang: &str, + mode: &str, + model_name: &str, + analysis: &str, + date: &str, + ) -> Result<()> { + let payload = json!({ + "lang": lang, + "mode": mode, + "hash": null, + "models": [{ + "name": model_name, + "tasks": {}, + "analysis": analysis, + }], + }); + + let url = format!("{}/api/llm-benchmark-upload", self.base_url); + let resp = self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", self.api_key)) + .header("Content-Type", "application/json") + .json(&payload) + .send() + .with_context(|| format!("upload analysis failed for {}/{}/{}", lang, mode, model_name))?; + + if resp.status().is_success() { + println!(" uploaded analysis for {}/{}/{}/{}", lang, mode, model_name, date); + Ok(()) + } else { + let status = resp.status(); + let body = resp.text().unwrap_or_default(); + anyhow::bail!("upload analysis failed: {} \u{2014} {}", status, body); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/api/mod.rs b/tools/xtask-llm-benchmark/src/api/mod.rs new file mode 100644 index 00000000000..e6b4f78a9eb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/api/mod.rs @@ -0,0 +1,3 @@ +mod client; + +pub use client::ApiClient; diff --git a/tools/xtask-llm-benchmark/src/bench/analysis.rs b/tools/xtask-llm-benchmark/src/bench/analysis.rs new file mode 100644 index 00000000000..2d9d4da7d77 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/bench/analysis.rs @@ -0,0 +1,218 @@ +use crate::bench::types::RunOutcome; +use crate::eval::ScoreDetails; +use crate::llm::prompt::BuiltPrompt; +use crate::llm::segmentation::Segment; +use crate::llm::{LlmProvider, ModelRoute}; +use anyhow::Result; +use spacetimedb_data_structures::map::HashMap; +use std::path::Path; + +pub async fn run_analysis( + outcomes: &[RunOutcome], + lang: &str, + mode: &str, + model_name: &str, + bench_root: &Path, + llm: &dyn LlmProvider, +) -> Result> { + let failures: Vec<&RunOutcome> = outcomes + .iter() + .filter(|o| o.passed_tests < o.total_tests && o.llm_output.is_some()) + .collect(); + + if failures.is_empty() { + return Ok(None); + } + + let prompt = build_prompt(lang, mode, model_name, bench_root, &failures); + + let route = ModelRoute::new( + "gpt-4.1-mini", + crate::llm::types::Vendor::OpenAi, + "gpt-4.1-mini", + Some("openai/gpt-4.1-mini"), + ); + + let built = BuiltPrompt { + system: Some(system_prompt()), + static_prefix: None, + segments: vec![Segment::new("user", prompt)], + search_enabled: false, + }; + + let response = llm.generate(&route, &built).await?; + Ok(Some(response.text)) +} + +pub fn system_prompt() -> String { + String::from(SYSTEM_PROMPT) +} + +pub const SYSTEM_PROMPT: &str = "\ +You summarize LLM benchmark failures for SpacetimeDB into structured markdown. \ +Each failure includes the model's generated code, the scorer error, and the golden (correct) answer when available. \ +Write in third person for a public benchmark page. Do not address the reader."; + +fn context_description(mode: &str) -> &'static str { + match mode { + "guidelines" => "the SpacetimeDB AI guidelines (concise cheat-sheets for code generation)", + "docs" => "SpacetimeDB markdown documentation", + "rustdoc_json" => "SpacetimeDB rustdoc JSON (auto-generated API reference)", + "llms.md" => "the SpacetimeDB llms.md file", + "no_context" | "none" | "no_guidelines" => "no documentation (testing base model knowledge only)", + "search" => "web search results (no local docs)", + _ => "unspecified context", + } +} + +fn has_context(mode: &str) -> bool { + !matches!(mode, "no_context" | "none" | "no_guidelines" | "search") +} + +fn context_name(mode: &str) -> &'static str { + match mode { + "guidelines" => "AI guidelines", + "docs" => "documentation", + "rustdoc_json" => "rustdoc", + "llms.md" => "llms.md", + _ => "context", + } +} + +/// Read the golden answer for a task from disk. +/// Scans `bench_root///answers/{rust.rs,csharp.cs,typescript.ts}`. +fn read_golden(bench_root: &Path, task_id: &str, lang: &str) -> Option { + let answer_file = match lang { + "rust" => "rust.rs", + "csharp" => "csharp.cs", + "typescript" => "typescript.ts", + _ => return None, + }; + + // Scan categories to find the task directory + let Ok(cats) = std::fs::read_dir(bench_root) else { + return None; + }; + for cat in cats.filter_map(|e| e.ok()) { + let task_dir = cat.path().join(task_id); + let path = task_dir.join("answers").join(answer_file); + if path.is_file() { + return std::fs::read_to_string(&path).ok(); + } + } + None +} + +pub fn build_prompt( + lang: &str, + mode: &str, + model_name: &str, + bench_root: &Path, + failures: &[&RunOutcome], +) -> String { + let lang_display = match lang { + "rust" => "Rust", + "csharp" => "C#", + "typescript" => "TypeScript", + _ => lang, + }; + + let mut prompt = format!( + "{model_name} was given {ctx} and asked to generate {lang_display} SpacetimeDB modules. \ + It failed {count} tasks.\n\n", + ctx = context_description(mode), + count = failures.len(), + ); + + for f in failures.iter().take(15) { + prompt.push_str(&format!( + "### {} ({}/{})\n", + f.task, f.passed_tests, f.total_tests + )); + + let reasons = f + .scorer_details + .as_ref() + .map(extract_reasons) + .unwrap_or_default(); + if !reasons.is_empty() { + prompt.push_str(&format!("Error: {}\n", reasons.join("; "))); + } + + if let Some(ref out) = f.llm_output { + prompt.push_str(&format!( + "Generated:\n```{}\n{}\n```\n", + lang, + truncate(out, 1500) + )); + } + + if let Some(golden) = read_golden(bench_root, &f.task, lang) { + prompt.push_str(&format!( + "Expected:\n```{}\n{}\n```\n", + lang, + truncate(&golden, 1500) + )); + } + + prompt.push('\n'); + } + + if failures.len() > 15 { + prompt.push_str(&format!( + "({} more failures not shown)\n\n", + failures.len() - 15 + )); + } + + prompt.push_str(&analysis_instructions(mode)); + prompt +} + +pub fn analysis_instructions(mode: &str) -> String { + let fix_line = if has_context(mode) { + let name = context_name(mode); + format!("5. **{name} gap:** What's missing or unclear in the {name} that led to this mistake\n") + } else { + String::new() + }; + + format!( + "\ +--- + +Group failures by root cause pattern. Use this exact structure for each group: + +### [Pattern Name] (N tasks) + +1. **What the model wrote:** Show the relevant incorrect lines from the generated code +2. **What was expected:** Show the relevant lines from the golden answer +3. **What the error says:** Quote the scorer error that identifies the problem +4. **Why this happened:** Why the model likely made this mistake (e.g. confused with another framework, hallucinated API, singular vs plural naming) +5. **Affected tasks:** list of task IDs +{fix_line} +Rules: +- Group tasks that fail for the same reason. Do not repeat the same analysis per task. +- Show only the relevant lines, not entire files. +- Skip provider errors (timeouts, 429s) with a brief note. +" + ) +} + +fn extract_reasons(details: &HashMap) -> Vec { + details + .iter() + .filter_map(|(name, score)| { + score + .failure_reason() + .map(|r| format!("{}: {}", name, truncate(&r, 150))) + }) + .collect() +} + +fn truncate(s: &str, max: usize) -> &str { + match s.char_indices().nth(max) { + Some((i, _)) => &s[..i], + None => s, + } +} diff --git a/tools/xtask-llm-benchmark/src/bench/mod.rs b/tools/xtask-llm-benchmark/src/bench/mod.rs index c7ab52c31c8..61e38e92871 100644 --- a/tools/xtask-llm-benchmark/src/bench/mod.rs +++ b/tools/xtask-llm-benchmark/src/bench/mod.rs @@ -1,5 +1,6 @@ +pub mod analysis; pub mod publishers; -pub(crate) mod results_merge; +pub mod results_merge; pub mod runner; mod templates; pub mod types; diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 92e69b0317c..c2bcdcd90bc 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -294,7 +294,11 @@ impl Publisher for TypeScriptPublisher { Some(p) => Command::new(p), None => Command::new("pnpm"), }; - pnpm_cmd.arg("install").arg("--ignore-workspace").current_dir(source); + pnpm_cmd + .arg("install") + .arg("--ignore-workspace") + .current_dir(source) + .env("CI", "true"); // When using NODEJS_DIR, prepend it to PATH so pnpm.cmd can find node. if let Some(ref dir) = pnpm_exe && let Some(parent) = dir.parent() diff --git a/tools/xtask-llm-benchmark/src/bench/results_merge.rs b/tools/xtask-llm-benchmark/src/bench/results_merge.rs index b6f8df78295..2d64ca75169 100644 --- a/tools/xtask-llm-benchmark/src/bench/results_merge.rs +++ b/tools/xtask-llm-benchmark/src/bench/results_merge.rs @@ -1,33 +1,28 @@ -use anyhow::{Context, Result}; -use fs2::FileExt; -use std::collections::BTreeMap; -use std::fs; -use std::io::{Read, Write}; -use std::path::Path; -use tempfile::NamedTempFile; - -use crate::bench::types::RunOutcome; use crate::results::schema::{LangEntry, ModeEntry, ModelEntry, Results}; -fn load_results(path: &Path) -> Result { - if !path.exists() { - return Ok(Results::default()); +/// Normalize all model names in loaded results and merge duplicates. +pub fn normalize_model_names(root: &mut Results) { + for lang in &mut root.languages { + for mode in &mut lang.modes { + let mut merged: Vec = Vec::new(); + for mut model in mode.models.drain(..) { + let canonical = canonical_model_name(&model.name); + model.name = canonical; + if let Some(existing) = merged.iter_mut().find(|m| m.name == model.name) { + // Merge tasks from duplicate into existing entry + for (task_id, outcome) in model.tasks { + existing.tasks.insert(task_id, outcome); + } + if existing.route_api_model.is_none() { + existing.route_api_model = model.route_api_model; + } + } else { + merged.push(model); + } + } + mode.models = merged; + } } - let mut f = fs::File::open(path)?; - let mut s = String::new(); - f.read_to_string(&mut s)?; - let root: Results = serde_json::from_str(&s).with_context(|| format!("failed parsing {}", path.display()))?; - Ok(root) -} - -fn save_atomic(path: &Path, root: &Results) -> Result<()> { - let parent = path.parent().context("no parent dir for results path")?; - fs::create_dir_all(parent)?; - let mut tmp = NamedTempFile::new_in(parent)?; - serde_json::to_writer_pretty(&mut tmp, root)?; - tmp.flush()?; - tmp.persist(path)?; - Ok(()) } pub fn ensure_lang<'a>(root: &'a mut Results, lang: &str) -> &'a mut LangEntry { @@ -37,12 +32,11 @@ pub fn ensure_lang<'a>(root: &'a mut Results, lang: &str) -> &'a mut LangEntry { root.languages.push(LangEntry { lang: lang.to_string(), modes: Vec::new(), - golden_answers: BTreeMap::new(), }); root.languages.last_mut().unwrap() } -fn ensure_mode<'a>(lang_v: &'a mut LangEntry, mode: &str, hash: Option) -> &'a mut ModeEntry { +pub fn ensure_mode<'a>(lang_v: &'a mut LangEntry, mode: &str, hash: Option) -> &'a mut ModeEntry { if let Some(i) = lang_v.modes.iter().position(|m| m.mode == mode) { if let Some(h) = hash { lang_v.modes[i].hash = Some(h); @@ -57,57 +51,46 @@ fn ensure_mode<'a>(lang_v: &'a mut LangEntry, mode: &str, hash: Option) lang_v.modes.last_mut().unwrap() } -fn ensure_model<'a>(mode_v: &'a mut ModeEntry, name: &str) -> &'a mut ModelEntry { +pub fn ensure_model<'a>(mode_v: &'a mut ModeEntry, name: &str) -> &'a mut ModelEntry { if let Some(i) = mode_v.models.iter().position(|m| m.name == name) { return &mut mode_v.models[i]; } mode_v.models.push(ModelEntry { name: name.to_string(), route_api_model: None, - tasks: Default::default(), // HashMap + tasks: Default::default(), }); mode_v.models.last_mut().unwrap() } -pub fn merge_task_runs(path: &Path, mode: &str, runs: &[RunOutcome]) -> Result<()> { - if runs.is_empty() { - return Ok(()); +/// Normalize mode aliases to their canonical names before saving. +pub fn canonical_mode(mode: &str) -> &str { + match mode { + "none" | "no_guidelines" => "no_context", + other => other, } +} - let lock_path = path.with_extension("lock"); - let lock = fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(&lock_path)?; - lock.lock_exclusive()?; - - let res = (|| -> Result<()> { - let mut root = load_results(path)?; - - for r in runs { - let lang_v = ensure_lang(&mut root, &r.lang); - - // Always bump the mode hash to the latest run's hash - let mode_v = ensure_mode(lang_v, mode, Some(r.hash.clone())); - - let model_v = ensure_model(mode_v, &r.model_name); - - // Always replace with the latest value (even if None) - model_v.route_api_model = r.route_api_model.clone(); - - // Sanitize volatile fields before saving to reduce git diff noise - let mut sanitized = r.clone(); - sanitized.sanitize_for_commit(); - - // Always overwrite the task result - model_v.tasks.insert(r.task.clone(), sanitized); +/// Normalize model names so that OpenRouter-style IDs and case variants +/// resolve to the canonical display name from model_routes. +pub fn canonical_model_name(name: &str) -> String { + use crate::llm::model_routes::default_model_routes; + let lower = name.to_ascii_lowercase(); + for route in default_model_routes() { + // Match by openrouter model id (e.g. "anthropic/claude-sonnet-4.6") + if let Some(ref or) = route.openrouter_model + && lower == or.to_ascii_lowercase() + { + return route.display_name.to_string(); } - - save_atomic(path, &root) - })(); - - let _ = lock.unlock(); - res + // Match by api model id (e.g. "claude-sonnet-4-6") + if lower == route.api_model.to_ascii_lowercase() { + return route.display_name.to_string(); + } + // Match by case-insensitive display name (e.g. "claude sonnet 4.6") + if lower == route.display_name.to_ascii_lowercase() { + return route.display_name.to_string(); + } + } + name.to_string() } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 632aa64fac0..896d6af1ab7 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -11,13 +11,12 @@ use tokio::sync::Mutex; use tokio::task; use crate::bench::publishers::{DotnetPublisher, SpacetimeRustPublisher, TypeScriptPublisher}; -use crate::bench::results_merge::merge_task_runs; use crate::bench::templates::materialize_project; use crate::bench::types::{BenchRunContext, PublishParams, RunContext, RunOneError}; pub(crate) use crate::bench::types::{RunOutcome, TaskPaths}; use crate::bench::utils::{ - bench_concurrency, bench_csharp_concurrency, category_slug, debug_llm, fmt_dur, print_llm_output, sanitize_db_name, - task_slug, work_server_dir_scoped, + bench_concurrency, bench_csharp_concurrency, bench_rust_concurrency, category_slug, debug_llm, fmt_dur, + print_llm_output, sanitize_db_name, task_slug, work_server_dir_scoped, }; use crate::bench::Publisher; use crate::eval::{Lang, ScoreDetails}; @@ -175,7 +174,7 @@ impl TaskRunner { let category = category_slug(&task.root); let task_id = task_slug(&task.root); - let route_tag = sanitize_db_name(cfg.route.display_name); + let route_tag = sanitize_db_name(&cfg.route.display_name); let golden_db = sanitize_db_name(&format!("{}-{}-golden", category, task_id)); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", category, task_id, route_tag)); @@ -187,16 +186,80 @@ impl TaskRunner { let prompt_builder = (spec.make_prompt)(cfg.lang); println!("→ [{}] {}: building prompt", cfg.lang_name, cfg.route.display_name); - let prompt = prompt_builder.build_segmented(cfg.context); + let prompt = prompt_builder.build_segmented(cfg.mode, cfg.context); println!("→ [{}] {}: calling provider", cfg.lang_name, cfg.route.display_name); - let llm_output = tokio::time::timeout(std::time::Duration::from_secs(90), cfg.llm.generate(cfg.route, &prompt)) - .await - .map_err(|_| RunOneError::Other(anyhow!("LLM call timed out")))? - .map_err(RunOneError::Other)?; + let mut gen_start = Instant::now(); + let llm_result = { + const MAX_ATTEMPTS: u32 = 3; + // Slow models (Gemini 3.1 Pro, DeepSeek Reasoner) can take 8+ minutes on large contexts. + let timeout_secs = match cfg.route.display_name.to_ascii_lowercase() { + n if n.contains("gemini") || n.contains("deepseek") => 600, + _ => 300, + }; + let mut last_err: anyhow::Error = anyhow!("no attempts made"); + let mut result = None; + for attempt in 1..=MAX_ATTEMPTS { + gen_start = Instant::now(); + let r = tokio::time::timeout( + std::time::Duration::from_secs(timeout_secs), + cfg.llm.generate(cfg.route, &prompt), + ) + .await; + match r { + Ok(Ok(output)) => { + result = Some(output); + break; + } + Ok(Err(e)) => { + let msg = format!("{e:#}"); + let retryable = msg.contains("timed out") + || msg.contains("429") + || msg.contains("502") + || msg.contains("503") + || msg.contains("504") + || msg.contains("rate limit"); + if retryable && attempt < MAX_ATTEMPTS { + let delay = if msg.contains("429") || msg.contains("rate limit") { + 60 + } else { + 30 + }; + eprintln!( + "⚠️ [{}/{}] provider error (attempt {}/{}), retrying in {delay}s: {}", + cfg.lang_name, cfg.route.display_name, attempt, MAX_ATTEMPTS, msg + ); + tokio::time::sleep(std::time::Duration::from_secs(delay)).await; + last_err = e; + } else { + return Err(RunOneError::Other(e)); + } + } + Err(_) => { + if attempt < MAX_ATTEMPTS { + eprintln!( + "⚠️ [{}/{}] LLM call timed out after {timeout_secs}s (attempt {}/{}), retrying in 30s", + cfg.lang_name, cfg.route.display_name, attempt, MAX_ATTEMPTS + ); + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + last_err = anyhow!("LLM call timed out after {timeout_secs}s"); + } else { + return Err(RunOneError::Other(anyhow!( + "LLM call timed out after {timeout_secs}s ({MAX_ATTEMPTS} attempts)" + ))); + } + } + } + } + result.ok_or_else(|| RunOneError::Other(last_err))? + }; + let generation_duration_ms = Some(gen_start.elapsed().as_millis() as u64); + let input_tokens = llm_result.input_tokens; + let output_tokens = llm_result.output_tokens; + let llm_output = llm_result.text; if debug_llm() { - print_llm_output(cfg.route.display_name, &task_id, &llm_output); + print_llm_output(&cfg.route.display_name, &task_id, &llm_output); } let publish_error: Option = self @@ -296,12 +359,60 @@ impl TaskRunner { .into_owned(), ), scorer_details: Some(scorer_details), + input_tokens, + output_tokens, + generation_duration_ms, started_at: Some(started), finished_at: Some(finished), }) } } +/// Partition task results into (good outcomes with LLM output, tasks to retry). +/// Tasks where the LLM never responded (provider error) are returned for retry. +/// Tasks where the LLM responded but the code failed to compile/score are kept as outcomes. +fn partition_results( + results: Vec<(TaskPaths, Result)>, + lang_name: &str, + route: &ModelRoute, + hash: &str, +) -> (Vec, Vec) { + let mut good = Vec::new(); + let mut retry = Vec::new(); + + for (task, r) in results { + match r { + Ok(v) => { + if v.llm_output.is_some() { + good.push(v); + } else { + // Outcome with no LLM output = provider error recorded as result + retry.push(task); + } + } + Err(RunOneError::WithOutput { msg, llm_output }) => { + // LLM responded but something else failed — keep as a result + eprintln!("\u{26a0}\u{fe0f} task failed but has output: {msg}"); + good.push(build_fail_outcome( + &task, + lang_name, + route, + hash, + anyhow::anyhow!(msg), + Some(llm_output), + )); + } + Err(RunOneError::Other(e)) => { + // No output at all — provider error, retry + eprintln!("\u{26a0}\u{fe0f} provider error, will retry: {e:?}"); + retry.push(task); + } + } + } + + (good, retry) +} + pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Result> { let total_wall = Instant::now(); @@ -316,6 +427,7 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu let lang_name = cfg.lang.as_str(); let buf = match cfg.lang { Lang::CSharp => bench_csharp_concurrency(), + Lang::Rust => bench_rust_concurrency(), _ => bench_concurrency(), }; @@ -335,6 +447,7 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu let run_cfg = RunContext { lang_name: &lang_name, lang, + mode: cfg.mode, route, context, hash, @@ -356,48 +469,117 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu .collect() .await; - let mut outcomes = Vec::new(); - let mut errs = 0usize; + let (mut good, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); - for (task, r) in results { - match r { - Ok(v) => outcomes.push(v), - Err(RunOneError::WithOutput { msg, llm_output }) => { - errs += 1; - eprintln!("⚠️ task failed but continuing: {msg}"); - outcomes.push(build_fail_outcome( - &task, - lang_name, - cfg.route, - cfg.hash, - anyhow::anyhow!(msg), - Some(llm_output), - )); - } - Err(RunOneError::Other(e)) => { - errs += 1; - eprintln!("⚠️ task failed but continuing: {e:?}"); - outcomes.push(build_fail_outcome(&task, lang_name, cfg.route, cfg.hash, e, None)); - } + // Retry provider-error tasks until all pass or none make progress + const MAX_RETRY_ROUNDS: usize = 3; + for round in 1..=MAX_RETRY_ROUNDS { + if retry_tasks.is_empty() { + break; + } + eprintln!( + "[runner] retry round {}/{}: {} tasks with provider errors", + round, MAX_RETRY_ROUNDS, retry_tasks.len() + ); + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + + let retry_results: Vec<(TaskPaths, Result)> = + futures::stream::iter(retry_tasks.drain(..).map(|task| { + let runner = &runner; + let route = cfg.route; + let lang = cfg.lang; + let lang_name = lang_name.to_string(); + let context = cfg.context; + let hash = cfg.hash; + let llm = cfg.llm; + let host = cfg.host.clone(); + + async move { + let started = Utc::now(); + let run_cfg = RunContext { + lang_name: &lang_name, + lang, + mode: cfg.mode, + route, + context, + hash, + llm, + host, + }; + let res = runner.run_one(&task, &run_cfg).await; + (task, res.map(|mut o| { o.started_at.get_or_insert(started); o })) + } + })) + .buffer_unordered(buf) + .collect() + .await; + + let (new_good, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + + if new_good.is_empty() && !still_failing.is_empty() { + // No progress — provider is likely down. Give up on these tasks. + eprintln!( + "[runner] no tasks recovered in retry round {} — provider may be down, dropping {} tasks", + round, + still_failing.len() + ); + break; } + + good.extend(new_good); + if still_failing.is_empty() { + retry_tasks = still_failing; + break; + } + retry_tasks = still_failing; + } + + let dropped = retry_tasks.len(); + if dropped > 0 { + eprintln!( + "[runner] {} tasks still failing after retries — excluded from upload", + dropped + ); } - println!("[runner] completed batch: ok={} err={}", outcomes.len(), errs); + println!( + "[runner] completed batch: {} uploadable results", + good.len() + ); + + if cfg.dry_run { + eprintln!("[dry-run] skipping upload ({} outcomes)", good.len()); + } else if !good.is_empty() { + let analysis = + match crate::bench::analysis::run_analysis(&good, cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, cfg.bench_root, cfg.llm).await { + Ok(Some(text)) => { + eprintln!("[runner] generated analysis for {}/{}", cfg.lang.as_str(), cfg.mode); + Some(text) + } + Ok(None) => None, + Err(e) => { + eprintln!("[runner] analysis failed (non-fatal): {e}"); + None + } + }; - if !outcomes.is_empty() { - merge_task_runs(&cfg.details_path, cfg.mode, &outcomes)?; + if let Some(ref api) = cfg.api_client { + api.upload_batch(cfg.lang.as_str(), cfg.mode, cfg.hash, &good, analysis.as_deref())?; + } else { + eprintln!("[runner] no API client configured; skipping upload"); + } } else { - eprintln!("[runner] no successful runs; not calling merge_task_runs"); + eprintln!("[runner] no uploadable results; skipping upload"); } println!( - "✓ [{}] {}: total {}", + "\u{2713} [{}] {}: total {}", lang_name, cfg.route.display_name, fmt_dur(total_wall.elapsed()) ); - Ok(outcomes) + Ok(good) } // run only selected tasks by selectors like 1/01/001 or t_001 @@ -415,7 +597,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> let selected: Vec = tasks .into_iter() .filter(|t| { - let name = t.root.file_name().and_then(|x| x.to_str()).unwrap_or(""); + let name = t.root.file_name().and_then(|x: &std::ffi::OsStr| x.to_str()).unwrap_or(""); wanted.iter().any(|w| name.starts_with(w)) }) .collect(); @@ -433,6 +615,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> let lang_name = cfg.lang.as_str(); let buf = match cfg.lang { Lang::CSharp => bench_csharp_concurrency(), + Lang::Rust => bench_rust_concurrency(), _ => bench_concurrency(), }; @@ -451,6 +634,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> let run_cfg = RunContext { lang_name: &lang_name, lang, + mode: cfg.mode, route, context, hash, @@ -472,44 +656,109 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> .collect() .await; - let mut outcomes = Vec::with_capacity(results.len()); - let mut errs = 0usize; + let (mut good, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); - for (task, r) in results { - match r { - Ok(v) => outcomes.push(v), - Err(RunOneError::WithOutput { msg, llm_output }) => { - errs += 1; - eprintln!("⚠️ task failed but continuing: {msg}"); - outcomes.push(build_fail_outcome( - &task, - lang_name, - cfg.route, - cfg.hash, - anyhow::anyhow!(msg), - Some(llm_output), - )); + // Retry provider-error tasks until all pass or none make progress + let mut dropped = 0usize; + { + const MAX_RETRY_ROUNDS: usize = 3; + let mut pending = retry_tasks; + for round in 1..=MAX_RETRY_ROUNDS { + if pending.is_empty() { + break; } - Err(RunOneError::Other(e)) => { - errs += 1; - eprintln!("⚠️ task failed but continuing: {e:?}"); - outcomes.push(build_fail_outcome(&task, lang_name, cfg.route, cfg.hash, e, None)); + eprintln!( + "[runner] retry round {}/{}: {} tasks with provider errors", + round, MAX_RETRY_ROUNDS, pending.len() + ); + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + + let retry_results: Vec<(TaskPaths, Result)> = + futures::stream::iter(pending.drain(..).map(|task| { + let runner = &runner; + let route = cfg.route; + let lang = cfg.lang; + let lang_name = lang_name.to_string(); + let context = cfg.context; + let hash = cfg.hash; + let llm = cfg.llm; + let host = cfg.host.clone(); + + async move { + let started = Utc::now(); + let run_cfg = RunContext { + lang_name: &lang_name, + lang, + mode: cfg.mode, + route, + context, + hash, + llm, + host, + }; + let res = runner.run_one(&task, &run_cfg).await; + (task, res.map(|mut o| { o.started_at.get_or_insert(started); o })) + } + })) + .buffer_unordered(buf) + .collect() + .await; + + let (new_good, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + + if new_good.is_empty() && !still_failing.is_empty() { + eprintln!( + "[runner] no tasks recovered in retry round {} — provider may be down, dropping {} tasks", + round, + still_failing.len() + ); + dropped = still_failing.len(); + break; } + + good.extend(new_good); + pending = still_failing; } + dropped += pending.len(); } - if !outcomes.is_empty() { - merge_task_runs(&cfg.details_path, cfg.mode, &outcomes)?; + if dropped > 0 { + eprintln!( + "[runner] {} tasks still failing after retries — excluded from upload", + dropped + ); + } + + if cfg.dry_run { + eprintln!("[dry-run] skipping upload ({} outcomes)", good.len()); + } else if !good.is_empty() { + let analysis = + match crate::bench::analysis::run_analysis(&good, cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, cfg.bench_root, cfg.llm).await { + Ok(Some(text)) => { + eprintln!("[runner] generated analysis for {}/{}", cfg.lang.as_str(), cfg.mode); + Some(text) + } + Ok(None) => None, + Err(e) => { + eprintln!("[runner] analysis failed (non-fatal): {e}"); + None + } + }; + + if let Some(ref api) = cfg.api_client { + api.upload_batch(cfg.lang.as_str(), cfg.mode, cfg.hash, &good, analysis.as_deref())?; + } else { + eprintln!("[runner] no API client configured; skipping upload"); + } } println!( - "✓ [{}] {}: total {} (err={})", + "\u{2713} [{}] {}: total {}", lang_name, cfg.route.display_name, fmt_dur(total_wall.elapsed()), - errs ); - Ok(outcomes) + Ok(good) } pub async fn run_selected_or_all_for_model_async_for_lang(ctx: &BenchRunContext<'_>) -> Result> { @@ -526,7 +775,8 @@ pub async fn run_selected_or_all_for_model_async_for_lang(ctx: &BenchRunContext< lang: ctx.lang, selectors: Option::from(sels), host: ctx.host.clone(), - details_path: ctx.details_path.clone(), + api_client: ctx.api_client.clone(), + dry_run: ctx.dry_run, }; return run_selected_for_model_async_for_lang(&sel_cfg).await; } @@ -546,7 +796,7 @@ pub async fn build_goldens_only_for_lang( let filtered: Vec = all .into_iter() .filter(|t| { - let name = t.root.file_name().and_then(|x| x.to_str()).unwrap_or(""); + let name = t.root.file_name().and_then(|x: &std::ffi::OsStr| x.to_str()).unwrap_or(""); wanted.iter().any(|w| name.starts_with(w)) }) .collect(); @@ -567,6 +817,7 @@ pub async fn build_goldens_only_for_lang( let lang_name = lang.as_str(); let buf = match lang { Lang::CSharp => bench_csharp_concurrency(), + Lang::Rust => bench_rust_concurrency(), _ => bench_concurrency(), }; @@ -654,6 +905,9 @@ fn build_fail_outcome( scorer_details: Some(sd), vendor: route.vendor.slug().to_string(), + input_tokens: None, + output_tokens: None, + generation_duration_ms: None, started_at: Some(now), finished_at: Some(now), } diff --git a/tools/xtask-llm-benchmark/src/bench/templates.rs b/tools/xtask-llm-benchmark/src/bench/templates.rs index 879c5efa1dc..b5fa5f6add3 100644 --- a/tools/xtask-llm-benchmark/src/bench/templates.rs +++ b/tools/xtask-llm-benchmark/src/bench/templates.rs @@ -76,7 +76,8 @@ fn copy_tree_with_templates(src: &Path, dst: &Path) -> Result<()> { let p = entry.path(); let rel = p.strip_prefix(from)?; let out_path = to.join(rel); - if entry.file_type()?.is_dir() { + // Use p.metadata() (follows symlinks) so directory symlinks recurse correctly. + if p.metadata().map(|m| m.is_dir()).unwrap_or(false) { recurse(&p, &out_path)?; } else if out_path.extension().and_then(|e| e.to_str()) == Some("tmpl") { let rendered_path = out_path.with_extension(""); @@ -130,7 +131,7 @@ fn inject_rust(root: &Path, llm_code: &str) -> anyhow::Result<()> { if !sdk_path.is_dir() { bail!("local Rust SDK not found at {}", sdk_path.display()); } - let replacement = format!(r#"{{ path = "{}" }}"#, relative); + let replacement = format!(r#"spacetimedb = {{ path = "{}" }}"#, relative); let cargo_toml = root.join("Cargo.toml"); let mut toml = fs::read_to_string(&cargo_toml).with_context(|| format!("read {}", cargo_toml.display()))?; toml = toml.replace( diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index 1b0754a56bf..4ff39354553 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -1,3 +1,4 @@ +use crate::api::ApiClient; use crate::eval::{Lang, ScoreDetails}; use crate::llm::types::Vendor; use crate::llm::{LlmProvider, ModelRoute}; @@ -45,6 +46,13 @@ pub struct RunOutcome { #[serde(default)] pub vendor: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub generation_duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub started_at: Option>, #[serde(skip_serializing_if = "Option::is_none")] @@ -52,11 +60,10 @@ pub struct RunOutcome { } impl RunOutcome { - /// Strip volatile fields that change every run (timestamps, ports, paths) + /// Strip volatile fields that change every run (ports, paths) /// to reduce git diff noise when committing results. + /// Timestamps (started_at, finished_at) are preserved for score history tracking. pub fn sanitize_for_commit(&mut self) { - self.started_at = None; - self.finished_at = None; self.work_dir_golden = None; self.work_dir_llm = None; self.golden_db = None; @@ -104,6 +111,7 @@ pub enum RunOneError { pub struct RunContext<'a> { pub lang_name: &'a str, pub lang: Lang, + pub mode: &'a str, pub route: &'a ModelRoute, pub context: &'a str, pub hash: &'a str, @@ -112,9 +120,11 @@ pub struct RunContext<'a> { } impl<'a> RunContext<'a> { + #[allow(clippy::too_many_arguments)] pub fn new( lang_name: &'a str, lang: Lang, + mode: &'a str, route: &'a ModelRoute, context: &'a str, hash: &'a str, @@ -124,6 +134,7 @@ impl<'a> RunContext<'a> { Self { lang_name, lang, + mode, route, context, hash, @@ -143,7 +154,8 @@ pub struct BenchRunContext<'a> { pub lang: Lang, pub selectors: Option<&'a [String]>, pub host: Option, - pub details_path: PathBuf, + pub api_client: Option, + pub dry_run: bool, } pub struct RunConfig { @@ -157,6 +169,8 @@ pub struct RunConfig { pub categories: Option>, pub model_filter: Option>>, pub host: Option, - /// Path to the details.json file where results will be merged - pub details_path: PathBuf, + /// API client for uploading results to the remote database + pub api_client: Option, + /// When true, run benchmarks but don't upload results + pub dry_run: bool, } diff --git a/tools/xtask-llm-benchmark/src/bench/utils.rs b/tools/xtask-llm-benchmark/src/bench/utils.rs index df643d356e7..a8ccddc23e5 100644 --- a/tools/xtask-llm-benchmark/src/bench/utils.rs +++ b/tools/xtask-llm-benchmark/src/bench/utils.rs @@ -100,6 +100,15 @@ pub fn bench_concurrency() -> usize { .unwrap_or(8) } +/// Concurrency for Rust/WASM builds. Lower default to avoid cargo registry +/// lock contention that causes STATUS_STACK_BUFFER_OVERRUN on Windows. +pub fn bench_rust_concurrency() -> usize { + env::var("LLM_BENCH_RUST_CONCURRENCY") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(2) +} + /// Concurrency for C# builds. Lower default than Rust due to dotnet/WASI SDK /// instability under high parallelism (causes SIGSEGV and "Pipe is broken" errors). pub fn bench_csharp_concurrency() -> usize { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/csharp.cs new file mode 100644 index 00000000000..39ef97c93b2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/csharp.cs @@ -0,0 +1,41 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Message", Public = true)] + public partial struct Message + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [SpacetimeDB.Index.BTree] + public Identity Owner; + public string Text; + } + + [Reducer] + public static void CreateMessage(ReducerContext ctx, string text) + { + ctx.Db.Message.Insert(new Message + { + Id = 0, + Owner = ctx.Sender, + Text = text, + }); + } + + [Reducer] + public static void DeleteMessage(ReducerContext ctx, ulong id) + { + var msg = ctx.Db.Message.Id.Find(id); + if (msg is not Message m) + { + throw new Exception("not found"); + } + if (m.Owner != ctx.Sender) + { + throw new Exception("unauthorized"); + } + ctx.Db.Message.Id.Delete(id); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/rust.rs new file mode 100644 index 00000000000..2f85ee68cff --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/rust.rs @@ -0,0 +1,30 @@ +use spacetimedb::Identity; +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = message, public)] +pub struct Message { + #[primary_key] + #[auto_inc] + pub id: u64, + #[index(btree)] + pub owner: Identity, + pub text: String, +} + +#[reducer] +pub fn create_message(ctx: &ReducerContext, text: String) { + ctx.db.message().insert(Message { + id: 0, + owner: ctx.sender(), + text, + }); +} + +#[reducer] +pub fn delete_message(ctx: &ReducerContext, id: u64) { + let msg = ctx.db.message().id().find(id).expect("not found"); + if msg.owner != ctx.sender() { + panic!("unauthorized"); + } + ctx.db.message().id().delete(id); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/typescript.ts new file mode 100644 index 00000000000..9a26bc7cf68 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/answers/typescript.ts @@ -0,0 +1,38 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const message = table({ + name: 'message', + public: true, +}, { + id: t.u64().primaryKey().autoInc(), + owner: t.identity().index('btree'), + text: t.string(), +}); + +const spacetimedb = schema({ message }); +export default spacetimedb; + +export const create_message = spacetimedb.reducer( + { text: t.string() }, + (ctx, { text }) => { + ctx.db.message.insert({ + id: 0n, + owner: ctx.sender, + text, + }); + } +); + +export const delete_message = spacetimedb.reducer( + { id: t.u64() }, + (ctx, { id }) => { + const msg = ctx.db.message.id.find(id); + if (!msg) { + throw new Error("not found"); + } + if (!msg.owner.equals(ctx.sender)) { + throw new Error("unauthorized"); + } + ctx.db.message.id.delete(id); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt new file mode 100644 index 00000000000..d1bbef5075b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in C# that defines a table of messages owned by users, and reducers that enforce ownership using ctx.Sender. + +TABLES +- Message + - Struct: Message + - Fields: + - Id: ulong (primary key, AutoInc) + - Owner: Identity (index BTree) + - Text: string + +REDUCERS +- CreateMessage(ctx, string text) + - Insert a new Message with Owner set to ctx.Sender + - Use Id: 0 for AutoInc + +- DeleteMessage(ctx, ulong id) + - Find the message by Id + - If not found, throw exception with "not found" + - If the message's Owner does not equal ctx.Sender, throw exception with "unauthorized" + - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt new file mode 100644 index 00000000000..58689ccd927 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in Rust that defines a table of messages owned by users, and reducers that enforce ownership using ctx.sender. + +TABLES +- message + - Struct: Message + - public table + - Fields: + - id: u64 (primary key, auto_inc) + - owner: Identity (index btree) + - text: String + +REDUCERS +- create_message(ctx, text: String) + - Insert a new Message with owner set to ctx.sender + - Use id: 0 for auto_inc + +- delete_message(ctx, id: u64) + - Find the message by id + - If not found, panic with "not found" + - If the message's owner does not equal ctx.sender, panic with "unauthorized" + - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt new file mode 100644 index 00000000000..3eb1729d5af --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript that defines a table of messages owned by users, and reducers that enforce ownership using ctx.sender. + +TABLES +- message + - public table + - Fields: + - id: number (u64, primary key, autoInc) + - owner: Identity (index btree) + - text: string + +REDUCERS +- create_message(ctx, { text: string }) + - Insert a new message with owner set to ctx.sender() + - Use id: 0n for autoInc + +- delete_message(ctx, { id: number (u64) }) + - Find the message by id + - If not found, throw "not found" + - If the message's owner does not equal ctx.sender(), throw "unauthorized" + - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/csharp.cs new file mode 100644 index 00000000000..674605878dc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/csharp.cs @@ -0,0 +1,40 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "UserInternal")] + public partial struct UserInternal + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Name; + public string Email; + public string PasswordHash; + } + + [Table(Accessor = "UserPublic", Public = true)] + public partial struct UserPublic + { + [PrimaryKey] + public ulong Id; + public string Name; + } + + [Reducer] + public static void RegisterUser(ReducerContext ctx, string name, string email, string passwordHash) + { + var intern = ctx.Db.UserInternal.Insert(new UserInternal + { + Id = 0, + Name = name, + Email = email, + PasswordHash = passwordHash, + }); + ctx.Db.UserPublic.Insert(new UserPublic + { + Id = intern.Id, + Name = name, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/rust.rs new file mode 100644 index 00000000000..d41588c9fe3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/rust.rs @@ -0,0 +1,29 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = user_internal)] +pub struct UserInternal { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, + pub email: String, + pub password_hash: String, +} + +#[table(accessor = user_public, public)] +pub struct UserPublic { + #[primary_key] + pub id: u64, + pub name: String, +} + +#[reducer] +pub fn register_user(ctx: &ReducerContext, name: String, email: String, password_hash: String) { + let internal = ctx.db.user_internal().insert(UserInternal { + id: 0, + name: name.clone(), + email, + password_hash, + }); + ctx.db.user_public().insert(UserPublic { id: internal.id, name }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/typescript.ts new file mode 100644 index 00000000000..720fd37ec17 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/answers/typescript.ts @@ -0,0 +1,37 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const userInternal = table({ + name: 'user_internal', +}, { + id: t.u64().primaryKey().autoInc(), + name: t.string(), + email: t.string(), + passwordHash: t.string(), +}); + +const userPublic = table({ + name: 'user_public', + public: true, +}, { + id: t.u64().primaryKey(), + name: t.string(), +}); + +const spacetimedb = schema({ userInternal, userPublic }); +export default spacetimedb; + +export const register_user = spacetimedb.reducer( + { name: t.string(), email: t.string(), passwordHash: t.string() }, + (ctx, { name, email, passwordHash }) => { + const internal = ctx.db.userInternal.insert({ + id: 0n, + name, + email, + passwordHash, + }); + ctx.db.userPublic.insert({ + id: internal.id, + name, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/csharp.txt new file mode 100644 index 00000000000..69e2026892d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/csharp.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in C# that defines a private table for sensitive data and a public table for client-visible data, with a reducer that copies safe fields from private to public. + +TABLES +- UserInternal + - Struct: UserInternal + - private table (do not set Public = true) + - Fields: + - Id: ulong (primary key, AutoInc) + - Name: string + - Email: string + - PasswordHash: string + +- UserPublic + - Struct: UserPublic + - public table (Public = true) + - Fields: + - Id: ulong (primary key) + - Name: string + +REDUCERS +- RegisterUser(ctx, string name, string email, string passwordHash) + - Insert into UserInternal (Id: 0 for AutoInc) + - Also insert into UserPublic with just the Id and Name (safe fields only) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/rust.txt new file mode 100644 index 00000000000..3b3ae7cf0e1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/rust.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in Rust that defines a private table for sensitive data and a public table for client-visible data, with a reducer that copies safe fields from private to public. + +TABLES +- user_internal + - Struct: UserInternal + - private table (default, no public attribute) + - Fields: + - id: u64 (primary key, auto_inc) + - name: String + - email: String + - password_hash: String + +- user_public + - Struct: UserPublic + - public table + - Fields: + - id: u64 (primary key) + - name: String + +REDUCERS +- register_user(ctx, name: String, email: String, password_hash: String) + - Insert into user_internal (id: 0 for auto_inc) + - Also insert into user_public with just the id and name (safe fields only) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt new file mode 100644 index 00000000000..b4386b12f9c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in TypeScript that defines a private table for sensitive data and a public table for client-visible data, with a reducer that copies safe fields from private to public. + +TABLES +- userInternal + - private table (do not set public: true) + - Fields: + - id: number (u64, primary key, autoInc) + - name: string + - email: string + - passwordHash: string + +- userPublic + - public table (set public: true) + - Fields: + - id: number (u64, primary key) + - name: string + +REDUCERS +- register_user(ctx, { name: string, email: string, passwordHash: string }) + - Insert into userInternal (id: 0n for autoInc) + - Also insert into userPublic with just the id and name (safe fields only) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/csharp.cs new file mode 100644 index 00000000000..498eb336e7e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/csharp.cs @@ -0,0 +1,43 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "User")] + public partial struct User + { + [PrimaryKey] + public Identity Identity; + public string Name; + } + + [Table(Accessor = "Message", Public = true)] + public partial struct Message + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [SpacetimeDB.Index.BTree] + public Identity Sender; + public string Text; + } + + [Reducer] + public static void Register(ReducerContext ctx, string name) + { + if (ctx.Db.User.Identity.Find(ctx.Sender) is not null) + { + throw new Exception("already registered"); + } + ctx.Db.User.Insert(new User { Identity = ctx.Sender, Name = name }); + } + + [Reducer] + public static void PostMessage(ReducerContext ctx, string text) + { + if (ctx.Db.User.Identity.Find(ctx.Sender) is null) + { + throw new Exception("not registered"); + } + ctx.Db.Message.Insert(new Message { Id = 0, Sender = ctx.Sender, Text = text }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/rust.rs new file mode 100644 index 00000000000..df55ea4f4cb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/rust.rs @@ -0,0 +1,41 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table(accessor = user)] +pub struct User { + #[primary_key] + pub identity: Identity, + pub name: String, +} + +#[table(accessor = message, public)] +pub struct Message { + #[primary_key] + #[auto_inc] + pub id: u64, + #[index(btree)] + pub sender: Identity, + pub text: String, +} + +#[reducer] +pub fn register(ctx: &ReducerContext, name: String) { + if ctx.db.user().identity().find(ctx.sender()).is_some() { + panic!("already registered"); + } + ctx.db.user().insert(User { + identity: ctx.sender(), + name, + }); +} + +#[reducer] +pub fn post_message(ctx: &ReducerContext, text: String) { + if ctx.db.user().identity().find(ctx.sender()).is_none() { + panic!("not registered"); + } + ctx.db.message().insert(Message { + id: 0, + sender: ctx.sender(), + text, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/typescript.ts new file mode 100644 index 00000000000..bffb86dc9a4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/typescript.ts @@ -0,0 +1,40 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const user = table({ + name: 'user', +}, { + identity: t.identity().primaryKey(), + name: t.string(), +}); + +const message = table({ + name: 'message', + public: true, +}, { + id: t.u64().primaryKey().autoInc(), + sender: t.identity().index('btree'), + text: t.string(), +}); + +const spacetimedb = schema({ user, message }); +export default spacetimedb; + +export const register = spacetimedb.reducer( + { name: t.string() }, + (ctx, { name }) => { + if (ctx.db.user.identity.find(ctx.sender)) { + throw new Error('already registered'); + } + ctx.db.user.insert({ identity: ctx.sender, name }); + } +); + +export const post_message = spacetimedb.reducer( + { text: t.string() }, + (ctx, { text }) => { + if (!ctx.db.user.identity.find(ctx.sender)) { + throw new Error('not registered'); + } + ctx.db.message.insert({ id: 0n, sender: ctx.sender, text }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt new file mode 100644 index 00000000000..142af581a0f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt @@ -0,0 +1,26 @@ +Write a SpacetimeDB backend module in C# that gates actions behind a registration check. + +TABLES +- User + - Struct: User + - private table + - Fields: + - Identity: Identity (primary key) + - Name: string + +- Message + - Struct: Message + - public table + - Fields: + - Id: ulong (primary key, AutoInc) + - Sender: Identity (index BTree) + - Text: string + +REDUCERS +- Register(ctx, string name) + - If a User row already exists for ctx.Sender, throw exception with "already registered" + - Otherwise insert a new User with Identity = ctx.Sender and the given Name + +- PostMessage(ctx, string text) + - If no User row exists for ctx.Sender, throw exception with "not registered" + - Otherwise insert a Message with Sender = ctx.Sender and the given Text, Id = 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt new file mode 100644 index 00000000000..16e386b3009 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt @@ -0,0 +1,26 @@ +Write a SpacetimeDB backend module in Rust that gates actions behind a registration check. + +TABLES +- user + - Struct: User + - private table + - Fields: + - identity: Identity (primary key) + - name: String + +- message + - Struct: Message + - public table + - Fields: + - id: u64 (primary key, auto_inc) + - sender: Identity (index btree) + - text: String + +REDUCERS +- register(ctx, name: String) + - If a User row already exists for ctx.sender, panic with "already registered" + - Otherwise insert a new User with identity = ctx.sender and the given name + +- post_message(ctx, text: String) + - If no User row exists for ctx.sender, panic with "not registered" + - Otherwise insert a Message with sender = ctx.sender and the given text, id = 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt new file mode 100644 index 00000000000..a4e2d6e8340 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt @@ -0,0 +1,24 @@ +Write a SpacetimeDB backend module in TypeScript that gates actions behind a registration check. + +TABLES +- user + - private table + - Fields: + - identity: Identity (primary key) + - name: string + +- message + - public table + - Fields: + - id: number (u64, primary key, autoInc) + - sender: Identity (index btree) + - text: string + +REDUCERS +- register(ctx, { name: string }) + - If a user row already exists for ctx.sender, throw "already registered" + - Otherwise insert a new user with identity = ctx.sender and the given name + +- post_message(ctx, { text: string }) + - If no user row exists for ctx.sender, throw "not registered" + - Otherwise insert a message with sender = ctx.sender and the given text, id = 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/csharp.cs new file mode 100644 index 00000000000..0f05ae487cf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/csharp.cs @@ -0,0 +1,42 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Config")] + public partial struct Config + { + [PrimaryKey] + public uint Id; + public Identity Admin; + } + + [Table(Accessor = "AdminLog", Public = true)] + public partial struct AdminLog + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Action; + } + + [Reducer] + public static void BootstrapAdmin(ReducerContext ctx) + { + if (ctx.Db.Config.Id.Find(0u) is not null) + { + throw new Exception("already bootstrapped"); + } + ctx.Db.Config.Insert(new Config { Id = 0, Admin = ctx.Sender }); + } + + [Reducer] + public static void AdminAction(ReducerContext ctx, string action) + { + var config = ctx.Db.Config.Id.Find(0u) ?? throw new Exception("not bootstrapped"); + if (config.Admin != ctx.Sender) + { + throw new Exception("not admin"); + } + ctx.Db.AdminLog.Insert(new AdminLog { Id = 0, Action = action }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/rust.rs new file mode 100644 index 00000000000..a69a973584b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/rust.rs @@ -0,0 +1,36 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table(accessor = config)] +pub struct Config { + #[primary_key] + pub id: u32, + pub admin: Identity, +} + +#[table(accessor = admin_log, public)] +pub struct AdminLog { + #[primary_key] + #[auto_inc] + pub id: u64, + pub action: String, +} + +#[reducer] +pub fn bootstrap_admin(ctx: &ReducerContext) { + if ctx.db.config().id().find(0).is_some() { + panic!("already bootstrapped"); + } + ctx.db.config().insert(Config { + id: 0, + admin: ctx.sender(), + }); +} + +#[reducer] +pub fn admin_action(ctx: &ReducerContext, action: String) { + let config = ctx.db.config().id().find(0).expect("not bootstrapped"); + if ctx.sender() != config.admin { + panic!("not admin"); + } + ctx.db.admin_log().insert(AdminLog { id: 0, action }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/typescript.ts new file mode 100644 index 00000000000..73af87a66ee --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/answers/typescript.ts @@ -0,0 +1,39 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const config = table({ + name: 'config', +}, { + id: t.u32().primaryKey(), + admin: t.identity(), +}); + +const admin_log = table({ + name: 'admin_log', + public: true, +}, { + id: t.u64().primaryKey().autoInc(), + action: t.string(), +}); + +const spacetimedb = schema({ config, admin_log }); +export default spacetimedb; + +export const bootstrap_admin = spacetimedb.reducer( + {}, + (ctx) => { + if (ctx.db.config.id.find(0)) { + throw new Error('already bootstrapped'); + } + ctx.db.config.insert({ id: 0, admin: ctx.sender }); + } +); + +export const admin_action = spacetimedb.reducer( + { action: t.string() }, + (ctx, { action }) => { + const config = ctx.db.config.id.find(0); + if (!config) throw new Error('not bootstrapped'); + if (!config.admin.equals(ctx.sender)) throw new Error('not admin'); + ctx.db.admin_log.insert({ id: 0n, action }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt new file mode 100644 index 00000000000..b8443e4e736 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt @@ -0,0 +1,26 @@ +Write a SpacetimeDB backend module in C# that implements a one-time admin bootstrap pattern. + +TABLES +- Config + - Struct: Config + - private table + - Fields: + - Id: uint (primary key) + - Admin: Identity + +- AdminLog + - Struct: AdminLog + - public table + - Fields: + - Id: ulong (primary key, AutoInc) + - Action: string + +REDUCERS +- BootstrapAdmin(ctx) + - If a Config row already exists (Id = 0), throw exception with "already bootstrapped" + - Otherwise insert Config { Id = 0, Admin = ctx.Sender } + +- AdminAction(ctx, string action) + - Load the Config row with Id = 0; if missing, throw exception with "not bootstrapped" + - If ctx.Sender does not equal config.Admin, throw exception with "not admin" + - Insert an AdminLog row with the given Action, Id = 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt new file mode 100644 index 00000000000..61ab5cf6b29 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt @@ -0,0 +1,26 @@ +Write a SpacetimeDB backend module in Rust that implements a one-time admin bootstrap pattern. + +TABLES +- config + - Struct: Config + - private table + - Fields: + - id: u32 (primary key) + - admin: Identity + +- admin_log + - Struct: AdminLog + - public table + - Fields: + - id: u64 (primary key, auto_inc) + - action: String + +REDUCERS +- bootstrap_admin(ctx) + - If a Config row already exists (id = 0), panic with "already bootstrapped" + - Otherwise insert Config { id: 0, admin: ctx.sender } + +- admin_action(ctx, action: String) + - Load the Config row with id = 0; if missing, panic with "not bootstrapped" + - If ctx.sender does not equal config.admin, panic with "not admin" + - Insert an AdminLog row with the given action, id = 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt new file mode 100644 index 00000000000..4bfd7a4bb89 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt @@ -0,0 +1,24 @@ +Write a SpacetimeDB backend module in TypeScript that implements a one-time admin bootstrap pattern. + +TABLES +- config + - private table + - Fields: + - id: number (u32, primary key) + - admin: Identity + +- admin_log + - public table + - Fields: + - id: number (u64, primary key, autoInc) + - action: string + +REDUCERS +- bootstrap_admin(ctx) + - If a config row already exists (id = 0), throw "already bootstrapped" + - Otherwise insert config { id: 0, admin: ctx.sender } + +- admin_action(ctx, { action: string }) + - Load the config row with id = 0; if missing, throw "not bootstrapped" + - If ctx.sender does not equal config.admin, throw "not admin" + - Insert an admin_log row with the given action, id = 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/csharp.cs new file mode 100644 index 00000000000..346dc0a241a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/csharp.cs @@ -0,0 +1,50 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "User")] + public partial struct User + { + [PrimaryKey] + public Identity Identity; + public string Role; + } + + [Reducer] + public static void Register(ReducerContext ctx) + { + if (ctx.Db.User.Identity.Find(ctx.Sender) is not null) + { + throw new Exception("already registered"); + } + ctx.Db.User.Insert(new User { Identity = ctx.Sender, Role = "member" }); + } + + [Reducer] + public static void Promote(ReducerContext ctx, Identity target) + { + var caller = ctx.Db.User.Identity.Find(ctx.Sender) ?? throw new Exception("not registered"); + if (caller.Role != "admin") + { + throw new Exception("not admin"); + } + var targetUser = ctx.Db.User.Identity.Find(target) ?? throw new Exception("target not registered"); + ctx.Db.User.Identity.Update(targetUser with { Role = "admin" }); + } + + [Reducer] + public static void MemberAction(ReducerContext ctx) + { + _ = ctx.Db.User.Identity.Find(ctx.Sender) ?? throw new Exception("not registered"); + } + + [Reducer] + public static void AdminAction(ReducerContext ctx) + { + var user = ctx.Db.User.Identity.Find(ctx.Sender) ?? throw new Exception("not registered"); + if (user.Role != "admin") + { + throw new Exception("not admin"); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/rust.rs new file mode 100644 index 00000000000..35ae48c1bb6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/rust.rs @@ -0,0 +1,43 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table(accessor = user)] +pub struct User { + #[primary_key] + pub identity: Identity, + pub role: String, +} + +#[reducer] +pub fn register(ctx: &ReducerContext) { + if ctx.db.user().identity().find(ctx.sender()).is_some() { + panic!("already registered"); + } + ctx.db.user().insert(User { + identity: ctx.sender(), + role: "member".to_string(), + }); +} + +#[reducer] +pub fn promote(ctx: &ReducerContext, target: Identity) { + let caller = ctx.db.user().identity().find(ctx.sender()).expect("not registered"); + if caller.role != "admin" { + panic!("not admin"); + } + let mut target_user = ctx.db.user().identity().find(target).expect("target not registered"); + target_user.role = "admin".to_string(); + ctx.db.user().identity().update(target_user); +} + +#[reducer] +pub fn member_action(ctx: &ReducerContext) { + ctx.db.user().identity().find(ctx.sender()).expect("not registered"); +} + +#[reducer] +pub fn admin_action(ctx: &ReducerContext) { + let user = ctx.db.user().identity().find(ctx.sender()).expect("not registered"); + if user.role != "admin" { + panic!("not admin"); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/typescript.ts new file mode 100644 index 00000000000..2936d7e214c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/typescript.ts @@ -0,0 +1,49 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const user = table({ + name: 'user', +}, { + identity: t.identity().primaryKey(), + role: t.string(), +}); + +const spacetimedb = schema({ user }); +export default spacetimedb; + +export const register = spacetimedb.reducer( + {}, + (ctx) => { + if (ctx.db.user.identity.find(ctx.sender)) { + throw new Error('already registered'); + } + ctx.db.user.insert({ identity: ctx.sender, role: 'member' }); + } +); + +export const promote = spacetimedb.reducer( + { target: t.identity() }, + (ctx, { target }) => { + const caller = ctx.db.user.identity.find(ctx.sender); + if (!caller) throw new Error('not registered'); + if (caller.role !== 'admin') throw new Error('not admin'); + const targetUser = ctx.db.user.identity.find(target); + if (!targetUser) throw new Error('target not registered'); + ctx.db.user.identity.update({ ...targetUser, role: 'admin' }); + } +); + +export const member_action = spacetimedb.reducer( + {}, + (ctx) => { + if (!ctx.db.user.identity.find(ctx.sender)) throw new Error('not registered'); + } +); + +export const admin_action = spacetimedb.reducer( + {}, + (ctx) => { + const user = ctx.db.user.identity.find(ctx.sender); + if (!user) throw new Error('not registered'); + if (user.role !== 'admin') throw new Error('not admin'); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt new file mode 100644 index 00000000000..d0efc9dc564 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt @@ -0,0 +1,27 @@ +Write a SpacetimeDB backend module in C# that implements role-based access control. + +TABLES +- User + - Struct: User + - private table + - Fields: + - Identity: Identity (primary key) + - Role: string ("admin" or "member") + +REDUCERS +- Register(ctx) + - If a User row already exists for ctx.Sender, throw exception with "already registered" + - Insert User { Identity = ctx.Sender, Role = "member" } + +- Promote(ctx, Identity target) + - Load the User row for ctx.Sender; if missing, throw exception with "not registered" + - If the caller's Role is not "admin", throw exception with "not admin" + - Load the User row for target; if missing, throw exception with "target not registered" + - Update that row's Role to "admin" + +- MemberAction(ctx) + - Load the User row for ctx.Sender; if missing, throw exception with "not registered" + +- AdminAction(ctx) + - Load the User row for ctx.Sender; if missing, throw exception with "not registered" + - If the Role is not "admin", throw exception with "not admin" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt new file mode 100644 index 00000000000..73b4a876bfb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt @@ -0,0 +1,29 @@ +Write a SpacetimeDB backend module in Rust that implements role-based access control. + +TABLES +- user + - Struct: User + - private table + - Fields: + - identity: Identity (primary key) + - role: String ("admin" or "member") + +REDUCERS +- register(ctx) + - If a User row already exists for ctx.sender, panic with "already registered" + - Insert User { identity: ctx.sender, role: "member" } + +- promote(ctx, target: Identity) + - Load the User row for ctx.sender; if missing, panic with "not registered" + - If the caller's role is not "admin", panic with "not admin" + - Load the User row for target; if missing, panic with "target not registered" + - Update that row's role to "admin" + +- member_action(ctx) + - Load the User row for ctx.sender; if missing, panic with "not registered" + - (No further logic needed; presence of the row is sufficient) + +- admin_action(ctx) + - Load the User row for ctx.sender; if missing, panic with "not registered" + - If the role is not "admin", panic with "not admin" + - (No further logic needed beyond the guard) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt new file mode 100644 index 00000000000..60cb6e06bbf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt @@ -0,0 +1,26 @@ +Write a SpacetimeDB backend module in TypeScript that implements role-based access control. + +TABLES +- user + - private table + - Fields: + - identity: Identity (primary key) + - role: string ("admin" or "member") + +REDUCERS +- register(ctx) + - If a user row already exists for ctx.sender, throw "already registered" + - Insert user { identity: ctx.sender, role: "member" } + +- promote(ctx, { target: Identity }) + - Load the user row for ctx.sender; if missing, throw "not registered" + - If the caller's role is not "admin", throw "not admin" + - Load the user row for target; if missing, throw "target not registered" + - Update that row's role to "admin" + +- member_action(ctx) + - Load the user row for ctx.sender; if missing, throw "not registered" + +- admin_action(ctx) + - Load the user row for ctx.sender; if missing, throw "not registered" + - If the role is not "admin", throw "not admin" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/csharp.cs new file mode 100644 index 00000000000..6ac27ad542e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/csharp.cs @@ -0,0 +1,64 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Admin")] + public partial struct Admin + { + [PrimaryKey] + public Identity Identity; + } + + [Table(Accessor = "Banned")] + public partial struct Banned + { + [PrimaryKey] + public Identity Identity; + } + + [Table(Accessor = "Player", Public = true)] + public partial struct Player + { + [PrimaryKey] + public Identity Identity; + public string Name; + } + + [Reducer] + public static void AddAdmin(ReducerContext ctx, Identity target) + { + if (ctx.Db.Admin.Identity.Find(ctx.Sender) is null) + { + throw new Exception("not admin"); + } + try { ctx.Db.Admin.Insert(new Admin { Identity = target }); } catch { } + } + + [Reducer] + public static void BanPlayer(ReducerContext ctx, Identity target) + { + if (ctx.Db.Admin.Identity.Find(ctx.Sender) is null) + { + throw new Exception("not admin"); + } + ctx.Db.Banned.Insert(new Banned { Identity = target }); + if (ctx.Db.Player.Identity.Find(target) is not null) + { + ctx.Db.Player.Identity.Delete(target); + } + } + + [Reducer] + public static void JoinGame(ReducerContext ctx, string name) + { + if (ctx.Db.Banned.Identity.Find(ctx.Sender) is not null) + { + throw new Exception("banned"); + } + if (ctx.Db.Player.Identity.Find(ctx.Sender) is not null) + { + throw new Exception("already in game"); + } + ctx.Db.Player.Insert(new Player { Identity = ctx.Sender, Name = name }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/rust.rs new file mode 100644 index 00000000000..69b7ed62165 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/rust.rs @@ -0,0 +1,53 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table(accessor = admin)] +pub struct Admin { + #[primary_key] + pub identity: Identity, +} + +#[table(accessor = banned)] +pub struct Banned { + #[primary_key] + pub identity: Identity, +} + +#[table(accessor = player, public)] +pub struct Player { + #[primary_key] + pub identity: Identity, + pub name: String, +} + +#[reducer] +pub fn add_admin(ctx: &ReducerContext, target: Identity) { + if ctx.db.admin().identity().find(ctx.sender()).is_none() { + panic!("not admin"); + } + let _ = ctx.db.admin().try_insert(Admin { identity: target }); +} + +#[reducer] +pub fn ban_player(ctx: &ReducerContext, target: Identity) { + if ctx.db.admin().identity().find(ctx.sender()).is_none() { + panic!("not admin"); + } + ctx.db.banned().insert(Banned { identity: target }); + if ctx.db.player().identity().find(target).is_some() { + ctx.db.player().identity().delete(target); + } +} + +#[reducer] +pub fn join_game(ctx: &ReducerContext, name: String) { + if ctx.db.banned().identity().find(ctx.sender()).is_some() { + panic!("banned"); + } + if ctx.db.player().identity().find(ctx.sender()).is_some() { + panic!("already in game"); + } + ctx.db.player().insert(Player { + identity: ctx.sender(), + name, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/typescript.ts new file mode 100644 index 00000000000..b5f622dfab1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/answers/typescript.ts @@ -0,0 +1,52 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const admin = table({ + name: 'admin', +}, { + identity: t.identity().primaryKey(), +}); + +const banned = table({ + name: 'banned', +}, { + identity: t.identity().primaryKey(), +}); + +const player = table({ + name: 'player', + public: true, +}, { + identity: t.identity().primaryKey(), + name: t.string(), +}); + +const spacetimedb = schema({ admin, banned, player }); +export default spacetimedb; + +export const add_admin = spacetimedb.reducer( + { target: t.identity() }, + (ctx, { target }) => { + if (!ctx.db.admin.identity.find(ctx.sender)) throw new Error('not admin'); + try { ctx.db.admin.insert({ identity: target }); } catch {} + } +); + +export const ban_player = spacetimedb.reducer( + { target: t.identity() }, + (ctx, { target }) => { + if (!ctx.db.admin.identity.find(ctx.sender)) throw new Error('not admin'); + ctx.db.banned.insert({ identity: target }); + if (ctx.db.player.identity.find(target)) { + ctx.db.player.identity.delete(target); + } + } +); + +export const join_game = spacetimedb.reducer( + { name: t.string() }, + (ctx, { name }) => { + if (ctx.db.banned.identity.find(ctx.sender)) throw new Error('banned'); + if (ctx.db.player.identity.find(ctx.sender)) throw new Error('already in game'); + ctx.db.player.insert({ identity: ctx.sender, name }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt new file mode 100644 index 00000000000..5337d4c112e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt @@ -0,0 +1,35 @@ +Write a SpacetimeDB backend module in C# that implements a ban list enforced on game entry. + +TABLES +- Admin + - Struct: Admin + - private table + - Fields: + - Identity: Identity (primary key) + +- Banned + - Struct: Banned + - private table + - Fields: + - Identity: Identity (primary key) + +- Player + - Struct: Player + - public table + - Fields: + - Identity: Identity (primary key) + - Name: string + +REDUCERS +- AddAdmin(ctx, Identity target) + - If ctx.Sender is not in the Admin table, throw exception with "not admin" + - Insert Admin { Identity = target } (ignore if already present) + +- BanPlayer(ctx, Identity target) + - If ctx.Sender is not in the Admin table, throw exception with "not admin" + - Insert Banned { Identity = target } + - Remove the Player row for target if one exists + +- JoinGame(ctx, string name) + - If ctx.Sender is in the Banned table, throw exception with "banned" + - Insert Player { Identity = ctx.Sender, Name = name }; if already present, throw exception with "already in game" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt new file mode 100644 index 00000000000..21717530997 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt @@ -0,0 +1,35 @@ +Write a SpacetimeDB backend module in Rust that implements a ban list enforced on game entry. + +TABLES +- admin + - Struct: Admin + - private table + - Fields: + - identity: Identity (primary key) + +- banned + - Struct: Banned + - private table + - Fields: + - identity: Identity (primary key) + +- player + - Struct: Player + - public table + - Fields: + - identity: Identity (primary key) + - name: String + +REDUCERS +- add_admin(ctx, target: Identity) + - If ctx.sender is not in the admin table, panic with "not admin" + - Insert Admin { identity: target } (ignore if already present) + +- ban_player(ctx, target: Identity) + - If ctx.sender is not in the admin table, panic with "not admin" + - Insert Banned { identity: target } + - Remove the Player row for target if one exists + +- join_game(ctx, name: String) + - If ctx.sender is in the banned table, panic with "banned" + - Insert Player { identity: ctx.sender, name }; if already present, panic with "already in game" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt new file mode 100644 index 00000000000..7400f7f0374 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt @@ -0,0 +1,33 @@ +Write a SpacetimeDB backend module in TypeScript that implements a ban list enforced on game entry. + +TABLES +- admin + - private table + - Fields: + - identity: Identity (primary key) + +- banned + - private table + - Fields: + - identity: Identity (primary key) + +- player + - public table + - Fields: + - identity: Identity (primary key) + - name: string + +REDUCERS +- add_admin(ctx, { target: Identity }) + - If ctx.sender is not in the admin table, throw "not admin" + - Insert { identity: target } into admin (ignore if already present) + +- ban_player(ctx, { target: Identity }) + - If ctx.sender is not in the admin table, throw "not admin" + - Insert { identity: target } into banned + - Remove the player row for target if one exists + +- join_game(ctx, { name: string }) + - If ctx.sender is in the banned table, throw "banned" + - If a player row already exists for ctx.sender, throw "already in game" + - Insert player { identity: ctx.sender, name } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/csharp.cs new file mode 100644 index 00000000000..3ada95fc066 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/csharp.cs @@ -0,0 +1,42 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "RateLimit")] + public partial struct RateLimit + { + [PrimaryKey] + public Identity Identity; + public ulong LastCallUs; + } + + [Table(Accessor = "ActionLog", Public = true)] + public partial struct ActionLog + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public Identity Identity; + public string Payload; + } + + [Reducer] + public static void LimitedAction(ReducerContext ctx, string payload) + { + ulong now = (ulong)ctx.Timestamp.MicrosecondsSinceUnixEpoch; + var entry = ctx.Db.RateLimit.Identity.Find(ctx.Sender); + if (entry is RateLimit r) + { + if (now - r.LastCallUs < 1_000_000UL) + { + throw new Exception("rate limited"); + } + ctx.Db.RateLimit.Identity.Update(r with { LastCallUs = now }); + } + else + { + ctx.Db.RateLimit.Insert(new RateLimit { Identity = ctx.Sender, LastCallUs = now }); + } + ctx.Db.ActionLog.Insert(new ActionLog { Id = 0, Identity = ctx.Sender, Payload = payload }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/rust.rs new file mode 100644 index 00000000000..f6748729bb3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/rust.rs @@ -0,0 +1,41 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table(accessor = rate_limit)] +pub struct RateLimit { + #[primary_key] + pub identity: Identity, + pub last_call_us: i64, +} + +#[table(accessor = action_log, public)] +pub struct ActionLog { + #[primary_key] + #[auto_inc] + pub id: u64, + pub identity: Identity, + pub payload: String, +} + +#[reducer] +pub fn limited_action(ctx: &ReducerContext, payload: String) { + let now = ctx.timestamp.to_micros_since_unix_epoch(); + if let Some(entry) = ctx.db.rate_limit().identity().find(ctx.sender()) { + if now.saturating_sub(entry.last_call_us) < 1_000_000 { + panic!("rate limited"); + } + ctx.db.rate_limit().identity().update(RateLimit { + identity: ctx.sender(), + last_call_us: now, + }); + } else { + ctx.db.rate_limit().insert(RateLimit { + identity: ctx.sender(), + last_call_us: now, + }); + } + ctx.db.action_log().insert(ActionLog { + id: 0, + identity: ctx.sender(), + payload, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/typescript.ts new file mode 100644 index 00000000000..85dbb3cae26 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/answers/typescript.ts @@ -0,0 +1,35 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const rate_limit = table({ + name: 'rate_limit', +}, { + identity: t.identity().primaryKey(), + last_call_us: t.u64(), +}); + +const action_log = table({ + name: 'action_log', + public: true, +}, { + id: t.u64().primaryKey().autoInc(), + identity: t.identity(), + payload: t.string(), +}); + +const spacetimedb = schema({ rate_limit, action_log }); +export default spacetimedb; + +export const limited_action = spacetimedb.reducer( + { payload: t.string() }, + (ctx, { payload }) => { + const now = BigInt(ctx.timestamp.microsSinceUnixEpoch); + const entry = ctx.db.rate_limit.identity.find(ctx.sender); + if (entry) { + if (now - entry.last_call_us < 1_000_000n) throw new Error('rate limited'); + ctx.db.rate_limit.identity.update({ ...entry, last_call_us: now }); + } else { + ctx.db.rate_limit.insert({ identity: ctx.sender, last_call_us: now }); + } + ctx.db.action_log.insert({ id: 0n, identity: ctx.sender, payload }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt new file mode 100644 index 00000000000..8fa3e429326 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in C# that enforces per-identity rate limiting. + +TABLES +- RateLimit + - Struct: RateLimit + - private table + - Fields: + - Identity: Identity (primary key) + - LastCallUs: ulong (microsecond timestamp of the last allowed call) + +- ActionLog + - Struct: ActionLog + - public table + - Fields: + - Id: ulong (primary key, AutoInc) + - Identity: Identity + - Payload: string + +REDUCERS +- LimitedAction(ctx, string payload) + - Get the current time in microseconds: ctx.Timestamp.ToUnixTimeMilliseconds() * 1000 + - Look up the RateLimit row for ctx.Sender + - If a row exists and (now - LastCallUs) < 1_000_000, throw exception with "rate limited" + - Insert or update the RateLimit row with LastCallUs = now + - Insert an ActionLog row with Identity = ctx.Sender and the given Payload, Id = 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt new file mode 100644 index 00000000000..db178fe9cfc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in Rust that enforces per-identity rate limiting. + +TABLES +- rate_limit + - Struct: RateLimit + - private table + - Fields: + - identity: Identity (primary key) + - last_call_us: i64 (microsecond timestamp of the last allowed call) + +- action_log + - Struct: ActionLog + - public table + - Fields: + - id: u64 (primary key, auto_inc) + - identity: Identity + - payload: String + +REDUCERS +- limited_action(ctx, payload: String) + - Get the current time in microseconds as i64: ctx.timestamp.to_micros_since_unix_epoch() + - Look up the RateLimit row for ctx.sender + - If a row exists and (now - last_call_us) < 1_000_000, panic with "rate limited" + - Insert or update the RateLimit row with last_call_us = now + - Insert an ActionLog row with identity = ctx.sender and the given payload, id = 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt new file mode 100644 index 00000000000..2af01a1dc16 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in TypeScript that enforces per-identity rate limiting. + +TABLES +- rate_limit + - private table + - Fields: + - identity: Identity (primary key) + - last_call_us: number (u64, microsecond timestamp of the last allowed call) + +- action_log + - public table + - Fields: + - id: number (u64, primary key, autoInc) + - identity: Identity + - payload: string + +REDUCERS +- limited_action(ctx, { payload: string }) + - Get the current time in microseconds: ctx.timestamp.microsSinceUnixEpoch (a BigInt) + - Look up the rate_limit row for ctx.sender + - If a row exists and (now - last_call_us) < 1_000_000n, throw "rate limited" + - Insert or update the rate_limit row with last_call_us = now + - Insert an action_log row with identity = ctx.sender and the given payload, id = 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/csharp.cs new file mode 100644 index 00000000000..6005314be50 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/csharp.cs @@ -0,0 +1,55 @@ +using SpacetimeDB; +using System.Linq; + +public static partial class Module +{ + [Table(Accessor = "Document", Public = true)] + public partial struct Document + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [SpacetimeDB.Index.BTree] + public Identity Owner; + public string Title; + } + + [Table(Accessor = "DocumentShare", Public = true)] + public partial struct DocumentShare + { + [SpacetimeDB.Index.BTree] + public ulong DocumentId; + [SpacetimeDB.Index.BTree] + public Identity SharedWith; + } + + [Reducer] + public static void CreateDocument(ReducerContext ctx, string title) + { + ctx.Db.Document.Insert(new Document { Id = 0, Owner = ctx.Sender, Title = title }); + } + + [Reducer] + public static void ShareDocument(ReducerContext ctx, ulong documentId, Identity target) + { + var doc = ctx.Db.Document.Id.Find(documentId) ?? throw new Exception("not found"); + if (doc.Owner != ctx.Sender) + { + throw new Exception("not owner"); + } + ctx.Db.DocumentShare.Insert(new DocumentShare { DocumentId = documentId, SharedWith = target }); + } + + [Reducer] + public static void EditDocument(ReducerContext ctx, ulong documentId, string newTitle) + { + var doc = ctx.Db.Document.Id.Find(documentId) ?? throw new Exception("not found"); + bool isOwner = doc.Owner == ctx.Sender; + bool isShared = ctx.Db.DocumentShare.DocumentId.Filter(documentId).Any(s => s.SharedWith == ctx.Sender); + if (!isOwner && !isShared) + { + throw new Exception("unauthorized"); + } + ctx.Db.Document.Id.Update(doc with { Title = newTitle }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/rust.rs new file mode 100644 index 00000000000..59e51fac9dc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/rust.rs @@ -0,0 +1,57 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table(accessor = document, public)] +pub struct Document { + #[primary_key] + #[auto_inc] + pub id: u64, + #[index(btree)] + pub owner: Identity, + pub title: String, +} + +#[table(accessor = document_share, public)] +pub struct DocumentShare { + #[index(btree)] + pub document_id: u64, + #[index(btree)] + pub shared_with: Identity, +} + +#[reducer] +pub fn create_document(ctx: &ReducerContext, title: String) { + ctx.db.document().insert(Document { + id: 0, + owner: ctx.sender(), + title, + }); +} + +#[reducer] +pub fn share_document(ctx: &ReducerContext, document_id: u64, target: Identity) { + let doc = ctx.db.document().id().find(document_id).expect("not found"); + if doc.owner != ctx.sender() { + panic!("not owner"); + } + ctx.db.document_share().insert(DocumentShare { + document_id, + shared_with: target, + }); +} + +#[reducer] +pub fn edit_document(ctx: &ReducerContext, document_id: u64, new_title: String) { + let mut doc = ctx.db.document().id().find(document_id).expect("not found"); + let is_owner = doc.owner == ctx.sender(); + let is_shared = ctx + .db + .document_share() + .document_id() + .filter(document_id) + .any(|s| s.shared_with == ctx.sender()); + if !is_owner && !is_shared { + panic!("unauthorized"); + } + doc.title = new_title; + ctx.db.document().id().update(doc); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/typescript.ts new file mode 100644 index 00000000000..76c8db3b677 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/answers/typescript.ts @@ -0,0 +1,51 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const document = table({ + name: 'document', + public: true, +}, { + id: t.u64().primaryKey().autoInc(), + owner: t.identity().index('btree'), + title: t.string(), +}); + +const document_share = table({ + name: 'document_share', + public: true, +}, { + document_id: t.u64().index('btree'), + shared_with: t.identity().index('btree'), +}); + +const spacetimedb = schema({ document, document_share }); +export default spacetimedb; + +export const create_document = spacetimedb.reducer( + { title: t.string() }, + (ctx, { title }) => { + ctx.db.document.insert({ id: 0n, owner: ctx.sender, title }); + } +); + +export const share_document = spacetimedb.reducer( + { document_id: t.u64(), target: t.identity() }, + (ctx, { document_id, target }) => { + const doc = ctx.db.document.id.find(document_id); + if (!doc) throw new Error('not found'); + if (!doc.owner.equals(ctx.sender)) throw new Error('not owner'); + ctx.db.document_share.insert({ document_id, shared_with: target }); + } +); + +export const edit_document = spacetimedb.reducer( + { document_id: t.u64(), new_title: t.string() }, + (ctx, { document_id, new_title }) => { + const doc = ctx.db.document.id.find(document_id); + if (!doc) throw new Error('not found'); + const isOwner = doc.owner.equals(ctx.sender); + const isShared = [...ctx.db.document_share.document_id.filter(document_id)] + .some(s => s.shared_with.equals(ctx.sender)); + if (!isOwner && !isShared) throw new Error('unauthorized'); + ctx.db.document.id.update({ ...doc, title: new_title }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt new file mode 100644 index 00000000000..cf07439c658 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt @@ -0,0 +1,33 @@ +Write a SpacetimeDB backend module in C# that implements document ownership and explicit sharing. + +TABLES +- Document + - Struct: Document + - public table + - Fields: + - Id: ulong (primary key, AutoInc) + - Owner: Identity (index BTree) + - Title: string + +- DocumentShare + - Struct: DocumentShare + - public table + - Fields: + - DocumentId: ulong (index BTree) + - SharedWith: Identity (index BTree) + +REDUCERS +- CreateDocument(ctx, string title) + - Insert a Document with Owner = ctx.Sender and the given Title, Id = 0 for AutoInc + +- ShareDocument(ctx, ulong documentId, Identity target) + - Load the Document by Id; if missing, throw exception with "not found" + - If Document.Owner != ctx.Sender, throw exception with "not owner" + - Insert a DocumentShare { DocumentId = documentId, SharedWith = target } + +- EditDocument(ctx, ulong documentId, string newTitle) + - Load the Document by Id; if missing, throw exception with "not found" + - The caller is authorized if they are the Owner OR if a DocumentShare row exists + with matching DocumentId and SharedWith == ctx.Sender + - If neither condition is true, throw exception with "unauthorized" + - Update the Document's Title to newTitle diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt new file mode 100644 index 00000000000..791afb14297 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt @@ -0,0 +1,33 @@ +Write a SpacetimeDB backend module in Rust that implements document ownership and explicit sharing. + +TABLES +- document + - Struct: Document + - public table + - Fields: + - id: u64 (primary key, auto_inc) + - owner: Identity (index btree) + - title: String + +- document_share + - Struct: DocumentShare + - public table + - Fields: + - document_id: u64 (index btree) + - shared_with: Identity (index btree) + +REDUCERS +- create_document(ctx, title: String) + - Insert a Document with owner = ctx.sender and the given title, id = 0 for auto_inc + +- share_document(ctx, document_id: u64, target: Identity) + - Load the Document by id; if missing, panic with "not found" + - If document.owner != ctx.sender, panic with "not owner" + - Insert a DocumentShare { document_id, shared_with: target } + +- edit_document(ctx, document_id: u64, new_title: String) + - Load the Document by id; if missing, panic with "not found" + - The caller is authorized if they are the owner OR if a DocumentShare row exists + with matching document_id and shared_with == ctx.sender + - If neither condition is true, panic with "unauthorized" + - Update the Document's title to new_title diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt new file mode 100644 index 00000000000..2d7c257b147 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt @@ -0,0 +1,31 @@ +Write a SpacetimeDB backend module in TypeScript that implements document ownership and explicit sharing. + +TABLES +- document + - public table + - Fields: + - id: number (u64, primary key, autoInc) + - owner: Identity (index btree) + - title: string + +- document_share + - public table + - Fields: + - document_id: number (u64, index btree) + - shared_with: Identity (index btree) + +REDUCERS +- create_document(ctx, { title: string }) + - Insert a document with owner = ctx.sender and the given title, id = 0n for autoInc + +- share_document(ctx, { document_id: number (u64), target: Identity }) + - Load the document by id; if missing, throw "not found" + - If document.owner != ctx.sender, throw "not owner" + - Insert a document_share { document_id, shared_with: target } + +- edit_document(ctx, { document_id: number (u64), new_title: string }) + - Load the document by id; if missing, throw "not found" + - The caller is authorized if they are the owner OR if a document_share row exists + with matching document_id and shared_with == ctx.sender + - If neither condition is true, throw "unauthorized" + - Update the document's title to new_title diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/answers/typescript.ts index 3ef100dd454..87d000255bc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/answers/typescript.ts @@ -2,7 +2,7 @@ import { ScheduleAt } from 'spacetimedb'; import { table, schema, t } from 'spacetimedb/server'; const tickTimer = table({ - name: 'tickTimer', + name: 'tick_timer', scheduled: (): any => tick, }, { scheduledId: t.u64().primaryKey().autoInc(), diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/csharp.cs index b0d4d6fdf32..8772175b472 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/csharp.cs @@ -5,14 +5,20 @@ public static partial class Module [Table(Accessor = "User")] public partial struct User { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public string Name; public int Age; public bool Active; } [Reducer] - public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active) + public static void InsertUser(ReducerContext ctx, string name, int age, bool active) + { + ctx.Db.User.Insert(new User { Id = 0, Name = name, Age = age, Active = active }); + } + + [Reducer] + public static void UpdateUser(ReducerContext ctx, ulong id, string name, int age, bool active) { ctx.Db.User.Id.Update(new User { Id = id, Name = name, Age = age, Active = active }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/rust.rs index edc2974a4bb..b9a2d62ac36 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/rust.rs @@ -1,15 +1,26 @@ -use spacetimedb::{reducer, table, ReducerContext}; +use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = user)] pub struct User { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub name: String, pub age: i32, pub active: bool, } #[reducer] -pub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { +pub fn insert_user(ctx: &ReducerContext, name: String, age: i32, active: bool) { + ctx.db.user().insert(User { + id: 0, + name, + age, + active, + }); +} + +#[reducer] +pub fn update_user(ctx: &ReducerContext, id: u64, name: String, age: i32, active: bool) { ctx.db.user().id().update(User { id, name, age, active }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/typescript.ts index c5cb5dd9bfd..ec1e150fee2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/answers/typescript.ts @@ -5,7 +5,7 @@ const user = table( name: 'user', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), name: t.string(), age: t.i32(), active: t.bool(), @@ -15,8 +15,15 @@ const user = table( const spacetimedb = schema({ user }); export default spacetimedb; +export const insertUser = spacetimedb.reducer( + { name: t.string(), age: t.i32(), active: t.bool() }, + (ctx, { name, age, active }) => { + ctx.db.user.insert({ id: 0n, name, age, active }); + } +); + export const updateUser = spacetimedb.reducer( - { id: t.i32(), name: t.string(), age: t.i32(), active: t.bool() }, + { id: t.u64(), name: t.string(), age: t.i32(), active: t.bool() }, (ctx, { id, name, age, active }) => { ctx.db.user.id.update({ id, name, age, active }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/spec.rs index 528b1ac0aa5..146cb2fba1d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/spec.rs @@ -1,5 +1,5 @@ use crate::eval::defaults::{ - default_schema_parity_scorers, make_reducer_data_parity_scorer, make_sql_exec_both_scorer, + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, }; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; use serde_json::Value; @@ -12,21 +12,18 @@ pub fn spec() -> BenchmarkSpec { let casing = casing_for_lang(lang); let sb = SqlBuilder::new(casing); let user_table = table_name("user", lang); - let seed = sb.insert_values( - &user_table, - &["id", "name", "age", "active"], - &["1", "'Alice'", "30", "true"], - ); let select = sb.select_by_id(&user_table, &["id", "name", "age", "active"], "id", 1); - let reducer_name = ident("UpdateUser", crate::eval::Casing::Snake); + let insert_reducer = ident("InsertUser", crate::eval::Casing::Snake); + let update_reducer = ident("UpdateUser", crate::eval::Casing::Snake); - v.push(make_sql_exec_both_scorer( + // Seed a user row via reducer on both DBs (auto-inc assigns id=1) + v.push(make_reducer_call_both_scorer( host_url, file!(), route_tag, - &seed, - "seed_users_row", - time::Duration::from_secs(10), + &insert_reducer, + vec![Value::from("Alice"), Value::from(30), Value::from(true)], + "seed_user_via_reducer", )); v.push(make_reducer_data_parity_scorer( @@ -34,7 +31,7 @@ pub fn spec() -> BenchmarkSpec { ReducerDataParityConfig { src_file: file!(), route_tag, - reducer: reducer_name.into(), + reducer: update_reducer.into(), args: vec![ Value::from(1), Value::from("Alice2"), diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/csharp.txt index 34673332260..b096f21ea6d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/csharp.txt @@ -1,14 +1,15 @@ -Write a SpacetimeDB backend module in C# that defines one table and a reducer that updates a row. +Write a SpacetimeDB backend module in C# that defines one table and two reducers: one that inserts a row and one that updates a row. TABLE - User - Struct: User - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Name: string - Age: int - Active: bool REDUCERS -- UpdateUser: given id:int, name:string, age:int, active:bool, update the row in User with Id=id to exactly these values +- InsertUser: given name:string, age:int, active:bool, insert a new row (Id is auto-assigned) +- UpdateUser: given id:ulong, name:string, age:int, active:bool, update the row in User with Id=id to exactly these values - (Id=id, Name=name, Age=age, Active=active) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/rust.txt index c582352d51d..aee86db7d22 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/rust.txt @@ -1,14 +1,15 @@ -Write a SpacetimeDB backend module in Rust that defines one table and a reducer that updates a row. +Write a SpacetimeDB backend module in Rust that defines one table and two reducers: one that inserts a row and one that updates a row. TABLE - user - Struct: User - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - name: String - age: i32 - active: bool REDUCERS -- update_user: given id:i32, name:String, age:i32, active:bool, update the row in user with id=id to exactly these values +- insert_user: given name:String, age:i32, active:bool, insert a new row (id is auto-assigned) +- update_user: given id:u64, name:String, age:i32, active:bool, update the row in user with id=id to exactly these values - (id=id, name=name, age=age, active=active) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt index 96d6d1db415..32e7ab2ef5f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt @@ -1,13 +1,14 @@ -Write a SpacetimeDB backend module in TypeScript that defines one table and a reducer that updates a row. +Write a SpacetimeDB backend module in TypeScript that defines one table and two reducers: one that inserts a row and one that updates a row. TABLE - user - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean REDUCERS +- insertUser: given name:string, age:number, active:boolean, insert a new row (id is auto-assigned) - updateUser: given id:number, name:string, age:number, active:boolean, update the row with matching id - (id=id, name=name, age=age, active=active) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/csharp.cs index 569c9a8921a..ebe0269634d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/csharp.cs @@ -5,14 +5,20 @@ public static partial class Module [Table(Accessor = "User")] public partial struct User { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public string Name; public int Age; public bool Active; } [Reducer] - public static void DeleteUser(ReducerContext ctx, int id) + public static void InsertUser(ReducerContext ctx, string name, int age, bool active) + { + ctx.Db.User.Insert(new User { Id = 0, Name = name, Age = age, Active = active }); + } + + [Reducer] + public static void DeleteUser(ReducerContext ctx, ulong id) { ctx.Db.User.Id.Delete(id); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/rust.rs index dcf95fdba73..95160381602 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/rust.rs @@ -1,15 +1,26 @@ -use spacetimedb::{reducer, table, ReducerContext}; +use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = user)] pub struct User { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub name: String, pub age: i32, pub active: bool, } #[reducer] -pub fn delete_user(ctx: &ReducerContext, id: i32) { +pub fn insert_user(ctx: &ReducerContext, name: String, age: i32, active: bool) { + ctx.db.user().insert(User { + id: 0, + name, + age, + active, + }); +} + +#[reducer] +pub fn delete_user(ctx: &ReducerContext, id: u64) { ctx.db.user().id().delete(id); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/typescript.ts index 60a04c8fc93..64ab5f17ec3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/answers/typescript.ts @@ -5,7 +5,7 @@ const user = table( name: 'user', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), name: t.string(), age: t.i32(), active: t.bool(), @@ -15,8 +15,15 @@ const user = table( const spacetimedb = schema({ user }); export default spacetimedb; +export const insertUser = spacetimedb.reducer( + { name: t.string(), age: t.i32(), active: t.bool() }, + (ctx, { name, age, active }) => { + ctx.db.user.insert({ id: 0n, name, age, active }); + } +); + export const deleteUser = spacetimedb.reducer( - { id: t.i32() }, + { id: t.u64() }, (ctx, { id }) => { ctx.db.user.id.delete(id); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs index 665062fe405..c2e9ead6f8d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs @@ -1,4 +1,6 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer, make_sql_exec_both_scorer}; +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_sql_count_scorer, +}; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig, SqlBuilder}; use serde_json::Value; use std::time; @@ -10,21 +12,18 @@ pub fn spec() -> BenchmarkSpec { let casing = casing_for_lang(lang); let sb = SqlBuilder::new(casing); let user_table = table_name("user", lang); - let seed = sb.insert_values( - &user_table, - &["id", "name", "age", "active"], - &["1", "'Alice'", "30", "true"], - ); let count = sb.count_by_id(&user_table, "id", 1); - let reducer_name = ident("DeleteUser", crate::eval::Casing::Snake); + let insert_reducer = ident("InsertUser", crate::eval::Casing::Snake); + let delete_reducer = ident("DeleteUser", crate::eval::Casing::Snake); - v.push(make_sql_exec_both_scorer( + // Seed a user row via reducer on both DBs (auto-inc assigns id=1) + v.push(make_reducer_call_both_scorer( host_url, file!(), route_tag, - &seed, - "seed_users_row", - time::Duration::from_secs(10), + &insert_reducer, + vec![Value::from("Alice"), Value::from(30), Value::from(true)], + "seed_user_via_reducer", )); v.push(make_reducer_sql_count_scorer( @@ -32,7 +31,7 @@ pub fn spec() -> BenchmarkSpec { ReducerSqlCountConfig { src_file: file!(), route_tag, - reducer: reducer_name.into(), + reducer: delete_reducer.into(), args: vec![Value::from(1)], sql_count_query: count.clone(), expected_count: 0, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/csharp.txt index 84e11759781..c2c6d3d550f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/csharp.txt @@ -1,13 +1,14 @@ -Write a SpacetimeDB backend module in C# that defines one table and a reducer that deletes a row. +Write a SpacetimeDB backend module in C# that defines one table and two reducers: one that inserts a row and one that deletes a row. TABLE - User - Struct: User - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Name: string - Age: int - Active: bool REDUCERS -- DeleteUser: given id:int, delete the row in User with that Id +- InsertUser: given name:string, age:int, active:bool, insert a new row (Id is auto-assigned) +- DeleteUser: given id:ulong, delete the row in User with that Id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/rust.txt index 6548b7ff2b5..470f424df10 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/rust.txt @@ -1,13 +1,14 @@ -Write a SpacetimeDB backend module in Rust that defines one table and a reducer that deletes a row. +Write a SpacetimeDB backend module in Rust that defines one table and two reducers: one that inserts a row and one that deletes a row. TABLE - user - Struct: User - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - name: String - age: i32 - active: bool REDUCERS -- delete_user: given id:i32, delete the row in user with that id +- insert_user: given name:String, age:i32, active:bool, insert a new row (id is auto-assigned) +- delete_user: given id:u64, delete the row in user with that id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt index 7f970963be1..c8df7425f4a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt @@ -1,12 +1,13 @@ -Write a SpacetimeDB backend module in TypeScript that defines one table and a reducer that deletes a row. +Write a SpacetimeDB backend module in TypeScript that defines one table and two reducers: one that inserts a row and one that deletes a row. TABLE - user - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean REDUCERS +- insertUser: given name:string, age:number, active:boolean, insert a new row (id is auto-assigned) - deleteUser: given id:number, delete the row with matching id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/csharp.cs index f84ed24b33c..26ca3aadae3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/csharp.cs @@ -5,7 +5,7 @@ public static partial class Module [Table(Accessor = "User")] public partial struct User { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public string Name; public int Age; public bool Active; @@ -14,8 +14,8 @@ public partial struct User [Reducer] public static void Crud(ReducerContext ctx) { - ctx.Db.User.Insert(new User { Id = 1, Name = "Alice", Age = 30, Active = true }); - ctx.Db.User.Insert(new User { Id = 2, Name = "Bob", Age = 22, Active = false }); + ctx.Db.User.Insert(new User { Id = 0, Name = "Alice", Age = 30, Active = true }); + ctx.Db.User.Insert(new User { Id = 0, Name = "Bob", Age = 22, Active = false }); ctx.Db.User.Id.Update(new User { Id = 1, Name = "Alice2", Age = 31, Active = false }); ctx.Db.User.Id.Delete(2); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/rust.rs index 34709b65610..33846652e28 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/rust.rs @@ -3,7 +3,8 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = user)] pub struct User { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub name: String, pub age: i32, pub active: bool, @@ -12,13 +13,13 @@ pub struct User { #[reducer] pub fn crud(ctx: &ReducerContext) { ctx.db.user().insert(User { - id: 1, + id: 0, name: "Alice".into(), age: 30, active: true, }); ctx.db.user().insert(User { - id: 2, + id: 0, name: "Bob".into(), age: 22, active: false, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/typescript.ts index 6a83d4bf4d0..5fa379e30a2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/answers/typescript.ts @@ -5,7 +5,7 @@ const user = table( name: 'user', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), name: t.string(), age: t.i32(), active: t.bool(), @@ -16,8 +16,8 @@ const spacetimedb = schema({ user }); export default spacetimedb; export const crud = spacetimedb.reducer(ctx => { - ctx.db.user.insert({ id: 1, name: 'Alice', age: 30, active: true }); - ctx.db.user.insert({ id: 2, name: 'Bob', age: 22, active: false }); - ctx.db.user.id.update({ id: 1, name: 'Alice2', age: 31, active: false }); - ctx.db.user.id.delete(2); + ctx.db.user.insert({ id: 0n, name: 'Alice', age: 30, active: true }); + ctx.db.user.insert({ id: 0n, name: 'Bob', age: 22, active: false }); + ctx.db.user.id.update({ id: 1n, name: 'Alice2', age: 31, active: false }); + ctx.db.user.id.delete(2n); }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/csharp.txt index ef84f8d7ba8..7ef756f4f31 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/csharp.txt @@ -4,14 +4,14 @@ TABLE - User - Struct: User - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Name: string - Age: int - Active: bool REDUCERS - Crud: perform these steps in order - - insert (Id=1, Name="Alice", Age=30, Active=true) - - insert (Id=2, Name="Bob", Age=22, Active=false) + - insert (Name="Alice", Age=30, Active=true) — Id is auto-assigned + - insert (Name="Bob", Age=22, Active=false) — Id is auto-assigned - update (Id=1, Name="Alice2", Age=31, Active=false) - delete Id=2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/rust.txt index 1a16a95fa80..3e1e7a8de7d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/rust.txt @@ -1,17 +1,17 @@ -Write a SpacetimeDB backend module in Rust that defines one table and a reducer that performs insert, update, and delete in one call. +Write a SpacetimeDB backend module in Rust that defines one table and a reducer that performs insert, update, and delete in one call. TABLE - user - Struct: User - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - name: String - age: i32 - active: bool REDUCERS - crud: perform these steps in order - - insert (id=1, name="Alice", age=30, active=true) - - insert (id=2, name="Bob", age=22, active=false) + - insert (name="Alice", age=30, active=true) — id is auto-assigned + - insert (name="Bob", age=22, active=false) — id is auto-assigned - update (id=1, name="Alice2", age=31, active=false) - delete id=2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt index 5c905b6e7ee..f49502a3877 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt @@ -3,14 +3,14 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and a re TABLE - user - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean REDUCERS - crud: no arguments, performs the following operations in order: - 1. Insert (id=1, name="Alice", age=30, active=true) - 2. Insert (id=2, name="Bob", age=22, active=false) - 3. Update row with id=1 to (id=1, name="Alice2", age=31, active=false) + 1. Insert (name="Alice", age=30, active=true) — id is auto-assigned + 2. Insert (name="Bob", age=22, active=false) — id is auto-assigned + 3. Update row with id=1 to (name="Alice2", age=31, active=false) 4. Delete row with id=2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/csharp.cs index bf5ba6d2c1c..86667423779 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/csharp.cs @@ -5,7 +5,7 @@ public static partial class Module [Table(Accessor = "User")] public partial struct User { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public string Name; public int Age; public bool Active; @@ -14,12 +14,18 @@ public partial struct User [Table(Accessor = "Result")] public partial struct Result { - [PrimaryKey] public int Id; + [PrimaryKey] public ulong Id; public string Name; } [Reducer] - public static void LookupUserName(ReducerContext ctx, int id) + public static void InsertUser(ReducerContext ctx, string name, int age, bool active) + { + ctx.Db.User.Insert(new User { Id = 0, Name = name, Age = age, Active = active }); + } + + [Reducer] + public static void LookupUserName(ReducerContext ctx, ulong id) { var u = ctx.Db.User.Id.Find(id); if (u.HasValue) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/rust.rs index fef6e5703d7..cc196aae267 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/rust.rs @@ -3,7 +3,8 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = user)] pub struct User { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub name: String, pub age: i32, pub active: bool, @@ -12,12 +13,22 @@ pub struct User { #[table(accessor = result)] pub struct ResultRow { #[primary_key] - pub id: i32, + pub id: u64, pub name: String, } #[reducer] -pub fn lookup_user_name(ctx: &ReducerContext, id: i32) { +pub fn insert_user(ctx: &ReducerContext, name: String, age: i32, active: bool) { + ctx.db.user().insert(User { + id: 0, + name, + age, + active, + }); +} + +#[reducer] +pub fn lookup_user_name(ctx: &ReducerContext, id: u64) { if let Some(u) = ctx.db.user().id().find(id) { ctx.db.result().insert(ResultRow { id: u.id, name: u.name }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/typescript.ts index 6b1d533f870..e854bafd189 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/answers/typescript.ts @@ -5,7 +5,7 @@ const user = table( name: 'user', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), name: t.string(), age: t.i32(), active: t.bool(), @@ -17,7 +17,7 @@ const result = table( name: 'result', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey(), name: t.string(), } ); @@ -25,8 +25,15 @@ const result = table( const spacetimedb = schema({ user, result }); export default spacetimedb; +export const insertUser = spacetimedb.reducer( + { name: t.string(), age: t.i32(), active: t.bool() }, + (ctx, { name, age, active }) => { + ctx.db.user.insert({ id: 0n, name, age, active }); + } +); + export const lookupUserName = spacetimedb.reducer( - { id: t.i32() }, + { id: t.u64() }, (ctx, { id }) => { const u = ctx.db.user.id.find(id); if (u) { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/spec.rs index 690b637d333..29474a68de4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/spec.rs @@ -1,5 +1,5 @@ use crate::eval::defaults::{ - default_schema_parity_scorers, make_reducer_data_parity_scorer, make_sql_exec_both_scorer, + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, }; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; use serde_json::Value; @@ -11,27 +11,21 @@ pub fn spec() -> BenchmarkSpec { let casing = casing_for_lang(lang); let sb = SqlBuilder::new(casing); - let reducer_name = ident("LookupUserName", crate::eval::Casing::Snake); - let user_table = table_name("user", lang); + let insert_reducer = ident("InsertUser", crate::eval::Casing::Snake); + let lookup_reducer = ident("LookupUserName", crate::eval::Casing::Snake); let result_table = table_name("result", lang); - // Seed a user row in both DBs so the lookup has something to find - let seed_users = sb.insert_values( - &user_table, - &["id", "name", "age", "active"], - &["1", "'Alice'", "30", "true"], - ); - - v.push(make_sql_exec_both_scorer( + // Seed a user row via reducer on both DBs (auto-inc assigns id=1) + v.push(make_reducer_call_both_scorer( host_url, file!(), route_tag, - &seed_users, - "seed_user_row", - Duration::from_secs(10), + &insert_reducer, + vec![Value::from("Alice"), Value::from(30), Value::from(true)], + "seed_user_via_reducer", )); - // After calling the reducer, the projection should be present in results + // After calling the lookup reducer, the projection should be present in results let select_result = sb.select_by_id(&result_table, &["id", "name"], "id", 1); v.push(make_reducer_data_parity_scorer( @@ -39,7 +33,7 @@ pub fn spec() -> BenchmarkSpec { ReducerDataParityConfig { src_file: file!(), route_tag, - reducer: reducer_name.into(), + reducer: lookup_reducer.into(), args: vec![Value::from(1)], select_query: select_result.clone(), id_str: "index_lookup_projection_parity", diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt index e823320fafb..7de9b538bee 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt @@ -1,10 +1,10 @@ -Write a SpacetimeDB backend module in C# that defines two tables and a reducer that looks up a row by primary-key index and writes a projection to another table. +Write a SpacetimeDB backend module in C# that defines two tables and two reducers: one that inserts a user row, and one that looks up a row by primary-key index and writes a projection to another table. TABLES - User - Struct: User - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Name: string - Age: int - Active: bool @@ -12,8 +12,9 @@ TABLES - Result - Struct: Result - Fields: - - Id: int (primary key) + - Id: ulong (primary key) - Name: string REDUCERS -- LookupUserName: given id:int, find the User row with that Id using the index and insert (Id, Name) into Result. +- InsertUser: given name:string, age:int, active:bool, insert a new row into User (Id is auto-assigned) +- LookupUserName: given id:ulong, find the User row with that Id using the index and insert (Id, Name) into Result. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt index 15c8a641362..1e5546dfe01 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt @@ -1,10 +1,10 @@ -Write a SpacetimeDB backend module in Rust that defines two tables and a reducer that looks up a row by primary-key index and writes a projection to another table. +Write a SpacetimeDB backend module in Rust that defines two tables and two reducers: one that inserts a user row, and one that looks up a row by primary-key index and writes a projection to another table. TABLES - user - Struct: User - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - name: String - age: i32 - active: bool @@ -12,8 +12,9 @@ TABLES - result - Struct: ResultRow - Fields: - - id: i32 (primary key) + - id: u64 (primary key) - name: String REDUCERS -- lookup_user_name: given id:i32, find the user row with that id using the index and insert (id, name) into result. +- insert_user: given name:String, age:i32, active:bool, insert a new row into user (id is auto-assigned) +- lookup_user_name: given id:u64, find the user row with that id using the index and insert (id, name) into result. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt index f332df37f28..20a8c2bec56 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt @@ -3,15 +3,16 @@ Write a SpacetimeDB backend module in TypeScript that demonstrates looking up ro TABLES - user - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean - result - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key) - name: string REDUCERS +- insertUser: given name:string, age:number, active:boolean, insert a new row into user (id is auto-assigned) - lookupUserName: given id:number, find user by id, insert projection (id, name) into result table diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/csharp.cs index 73567b8e1a8..5bf7f28be81 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/csharp.cs @@ -5,7 +5,7 @@ public static partial class Module [Table(Accessor = "User")] public partial struct User { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public string Name; public int Age; public bool Active; @@ -14,7 +14,7 @@ public partial struct User [Reducer(ReducerKind.Init)] public static void Init(ReducerContext ctx) { - ctx.Db.User.Insert(new User { Id = 1, Name = "Alice", Age = 30, Active = true }); - ctx.Db.User.Insert(new User { Id = 2, Name = "Bob", Age = 22, Active = false }); + ctx.Db.User.Insert(new User { Id = 0, Name = "Alice", Age = 30, Active = true }); + ctx.Db.User.Insert(new User { Id = 0, Name = "Bob", Age = 22, Active = false }); } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/rust.rs index 793ba56e285..d2332b63d16 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/rust.rs @@ -3,7 +3,8 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = user)] pub struct User { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub name: String, pub age: i32, pub active: bool, @@ -12,13 +13,13 @@ pub struct User { #[reducer(init)] pub fn init(ctx: &ReducerContext) { ctx.db.user().insert(User { - id: 1, + id: 0, name: "Alice".into(), age: 30, active: true, }); ctx.db.user().insert(User { - id: 2, + id: 0, name: "Bob".into(), age: 22, active: false, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/typescript.ts index 53a3cd0f900..5682789187f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/answers/typescript.ts @@ -5,7 +5,7 @@ const user = table( name: 'user', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), name: t.string(), age: t.i32(), active: t.bool(), @@ -16,6 +16,6 @@ const spacetimedb = schema({ user }); export default spacetimedb; export const init = spacetimedb.init(ctx => { - ctx.db.user.insert({ id: 1, name: 'Alice', age: 30, active: true }); - ctx.db.user.insert({ id: 2, name: 'Bob', age: 22, active: false }); + ctx.db.user.insert({ id: 0n, name: 'Alice', age: 30, active: true }); + ctx.db.user.insert({ id: 0n, name: 'Bob', age: 22, active: false }); }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/csharp.txt index 8234736e20d..e97362c55d4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/csharp.txt @@ -4,12 +4,12 @@ TABLE - User - Struct: User - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Name: string - Age: int - Active: bool REDUCERS -- Init: insert exactly these rows on initialization - - (Id=1, Name="Alice", Age=30, Active=true) - - (Id=2, Name="Bob", Age=22, Active=false) +- Init: insert exactly these rows on initialization (IDs are auto-assigned) + - (Name="Alice", Age=30, Active=true) + - (Name="Bob", Age=22, Active=false) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/rust.txt index e555ef1337e..c93cff4ac49 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/rust.txt @@ -1,15 +1,15 @@ -Write a SpacetimeDB backend module in Rust that defines one table and an init reducer that seeds rows on database initialization. +Write a SpacetimeDB backend module in Rust that defines one table and an init reducer that seeds rows on database initialization. TABLE - user - Struct: User - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - name: String - age: i32 - active: bool REDUCERS -- init: insert exactly these rows on initialization - - (id=1, name="Alice", age=30, active=true) - - (id=2, name="Bob", age=22, active=false) +- init: insert exactly these rows on initialization (IDs are auto-assigned) + - (name="Alice", age=30, active=true) + - (name="Bob", age=22, active=false) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt index dae7a2062f8..53107203c63 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt @@ -3,12 +3,12 @@ Write a SpacetimeDB backend module in TypeScript that demonstrates the init redu TABLE - user - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean REDUCERS -- init: insert exactly two rows on database initialization: - 1. (id=1, name="Alice", age=30, active=true) - 2. (id=2, name="Bob", age=22, active=false) +- init: insert exactly two rows on database initialization (IDs are auto-assigned): + 1. (name="Alice", age=30, active=true) + 2. (name="Bob", age=22, active=false) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt index 31757356e29..0aa6dc0401d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt @@ -6,6 +6,6 @@ TABLE - id: bigint (u64, primary key, auto-increment) - kind: string -REDUCERS -- clientConnected: insert event with kind="connected" -- clientDisconnected: insert event with kind="disconnected" +LIFECYCLE HOOKS (use spacetimedb.clientConnected / spacetimedb.clientDisconnected) +- onConnect: insert event with kind="connected" +- onDisconnect: insert event with kind="disconnected" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/csharp.cs new file mode 100644 index 00000000000..412858e8827 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/csharp.cs @@ -0,0 +1,26 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Reminder", Scheduled = nameof(SendReminder), ScheduledAt = nameof(Reminder.ScheduledAt))] + public partial struct Reminder + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public string Message; + } + + [Reducer] + public static void SendReminder(ReducerContext ctx, Reminder row) { } + + [Reducer(ReducerKind.Init)] + public static void Init(ReducerContext ctx) + { + var delay = new TimeDuration { Microseconds = 60_000_000 }; + ctx.Db.Reminder.Insert(new Reminder + { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + delay), + Message = "Hello!", + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/rust.rs new file mode 100644 index 00000000000..d33b199b057 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/rust.rs @@ -0,0 +1,24 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = reminder, scheduled(send_reminder))] +pub struct Reminder { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, + message: String, +} + +#[reducer] +pub fn send_reminder(_ctx: &ReducerContext, _row: Reminder) {} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + let fire_at = ctx.timestamp + Duration::from_secs(60); + ctx.db.reminder().insert(Reminder { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(fire_at), + message: "Hello!".to_string(), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/typescript.ts new file mode 100644 index 00000000000..bf684147237 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/answers/typescript.ts @@ -0,0 +1,28 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const reminder = table({ + name: 'reminder', + scheduled: (): any => sendReminder, +}, { + scheduledId: t.u64().primaryKey().autoInc(), + scheduledAt: t.scheduleAt(), + message: t.string(), +}); + +const spacetimedb = schema({ reminder }); +export default spacetimedb; + +export const sendReminder = spacetimedb.reducer( + { timer: reminder.rowType }, + (_ctx, _args) => {} +); + +export const init = spacetimedb.init(ctx => { + const fireAt = ctx.timestamp.microsSinceUnixEpoch + 60_000_000n; + ctx.db.reminder.insert({ + scheduledId: 0n, + scheduledAt: ScheduleAt.time(fireAt), + message: 'Hello!', + }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt new file mode 100644 index 00000000000..d4fec2a3d61 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# that schedules a one-shot reducer to fire at a specific future time. + +TABLE +- Reminder + - Fields: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt (the schedule column) + - Message: string + - Scheduling: + - Scheduled = nameof(SendReminder) + - ScheduledAt = nameof(Reminder.ScheduledAt) + +REDUCERS +- SendReminder: scheduled reducer triggered by Reminder (receives a Reminder row, does nothing) +- Init (ReducerKind.Init): inserts one row into Reminder using ScheduleAt.Time with a TimeDuration of 60 seconds (60_000_000 microseconds) added to ctx.Timestamp diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt new file mode 100644 index 00000000000..8e601abb547 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt @@ -0,0 +1,16 @@ +Write a SpacetimeDB backend module in Rust that schedules a one-shot reducer to fire at a specific future time. + +TABLE +- reminder + - Struct: Reminder + - Fields: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - message: String + - Scheduling: + - reducer: send_reminder + +REDUCERS +- send_reminder: scheduled reducer triggered by reminder (receives a Reminder row, does nothing) +- init: inserts one row into reminder, scheduling send_reminder to fire 60 seconds in the future using ScheduleAt::Time + - Use ctx.timestamp + Duration::from_secs(60) to compute the fire time diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt new file mode 100644 index 00000000000..7396a8784d9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in TypeScript that schedules a one-shot reducer to fire at a specific future time. + +TABLE +- reminder + - Fields: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: scheduleAt + - message: string + - Scheduling: scheduled to call the send_reminder reducer + +REDUCERS +- send_reminder: scheduled reducer triggered by reminder (receives a reminder row argument, does nothing) +- init: inserts one row into reminder scheduling send_reminder to fire 60 seconds in the future using ScheduleAt.time + - Use ctx.timestamp + 60_000_000n microseconds to compute the fire time as a BigInt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/csharp.cs new file mode 100644 index 00000000000..659168a48cc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/csharp.cs @@ -0,0 +1,29 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "CleanupJob", Scheduled = nameof(RunCleanup), ScheduledAt = nameof(CleanupJob.ScheduledAt))] + public partial struct CleanupJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + } + + [Reducer] + public static void RunCleanup(ReducerContext ctx, CleanupJob row) { } + + [Reducer(ReducerKind.Init)] + public static void Init(ReducerContext ctx) + { + ctx.Db.CleanupJob.Insert(new CleanupJob + { + ScheduledAt = new ScheduleAt.Interval(new TimeDuration { Microseconds = 60_000_000 }), + }); + } + + [Reducer] + public static void CancelCleanup(ReducerContext ctx, ulong scheduledId) + { + ctx.Db.CleanupJob.ScheduledId.Delete(scheduledId); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/rust.rs new file mode 100644 index 00000000000..1d41fd1bfe8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/rust.rs @@ -0,0 +1,26 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = cleanup_job, scheduled(run_cleanup))] +pub struct CleanupJob { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn run_cleanup(_ctx: &ReducerContext, _row: CleanupJob) {} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + ctx.db.cleanup_job().insert(CleanupJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Interval(Duration::from_secs(60).into()), + }); +} + +#[reducer] +pub fn cancel_cleanup(ctx: &ReducerContext, scheduled_id: u64) { + ctx.db.cleanup_job().scheduled_id().delete(&scheduled_id); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/typescript.ts new file mode 100644 index 00000000000..e0edbeef00d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/answers/typescript.ts @@ -0,0 +1,32 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const cleanupJob = table({ + name: 'cleanup_job', + scheduled: (): any => runCleanup, +}, { + scheduledId: t.u64().primaryKey().autoInc(), + scheduledAt: t.scheduleAt(), +}); + +const spacetimedb = schema({ cleanupJob }); +export default spacetimedb; + +export const runCleanup = spacetimedb.reducer( + { timer: cleanupJob.rowType }, + (_ctx, _args) => {} +); + +export const init = spacetimedb.init(ctx => { + ctx.db.cleanupJob.insert({ + scheduledId: 0n, + scheduledAt: ScheduleAt.interval(60_000_000n), + }); +}); + +export const cancelCleanup = spacetimedb.reducer( + { scheduledId: t.u64() }, + (ctx, { scheduledId }) => { + ctx.db.cleanupJob.scheduledId.delete(scheduledId); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt new file mode 100644 index 00000000000..d775e92fe0f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# that schedules a reducer and allows canceling it before it fires. + +TABLE +- CleanupJob + - Fields: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt (the schedule column) + - Scheduling: + - Scheduled = nameof(RunCleanup) + - ScheduledAt = nameof(CleanupJob.ScheduledAt) + +REDUCERS +- RunCleanup: scheduled reducer triggered by CleanupJob (receives a CleanupJob row, does nothing) +- Init (ReducerKind.Init): inserts one row into CleanupJob with a 60-second interval (60_000_000 microseconds) +- CancelCleanup(ctx, scheduledId: ulong): deletes the CleanupJob row with the given scheduledId, canceling the pending scheduled call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt new file mode 100644 index 00000000000..949b98b99f7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in Rust that schedules a reducer and allows canceling it before it fires. + +TABLE +- cleanup_job + - Struct: CleanupJob + - Fields: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - Scheduling: + - reducer: run_cleanup + +REDUCERS +- run_cleanup: scheduled reducer triggered by cleanup_job (receives a CleanupJob row, does nothing) +- init: inserts one row into cleanup_job with a 60-second interval +- cancel_cleanup(ctx, scheduled_id: u64): deletes the CleanupJob row with the given scheduled_id, canceling the pending scheduled call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt new file mode 100644 index 00000000000..5c28363f7a3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in TypeScript that schedules a reducer and allows canceling it before it fires. + +TABLE +- cleanupJob + - Fields: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: scheduleAt + - Scheduling: scheduled to call the run_cleanup reducer + +REDUCERS +- run_cleanup: scheduled reducer triggered by cleanupJob (receives a cleanupJob row argument, does nothing) +- init: inserts one row into cleanupJob with a 60-second interval (60_000_000n microseconds) +- cancel_cleanup(ctx, { scheduledId: u64 }): deletes the cleanupJob row with the given scheduledId, canceling the pending scheduled call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/csharp.cs new file mode 100644 index 00000000000..92b236e14c2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/csharp.cs @@ -0,0 +1,28 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "OnlinePlayer", Public = true)] + public partial struct OnlinePlayer + { + [PrimaryKey] + public Identity Identity; + public Timestamp ConnectedAt; + } + + [Reducer(ReducerKind.ClientConnected)] + public static void ClientConnected(ReducerContext ctx) + { + ctx.Db.OnlinePlayer.Insert(new OnlinePlayer + { + Identity = ctx.Sender, + ConnectedAt = ctx.Timestamp, + }); + } + + [Reducer(ReducerKind.ClientDisconnected)] + public static void ClientDisconnected(ReducerContext ctx) + { + ctx.Db.OnlinePlayer.Identity.Delete(ctx.Sender); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/rust.rs new file mode 100644 index 00000000000..a030105663e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/rust.rs @@ -0,0 +1,21 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table, Timestamp}; + +#[table(accessor = online_player, public)] +pub struct OnlinePlayer { + #[primary_key] + pub identity: Identity, + pub connected_at: Timestamp, +} + +#[reducer(client_connected)] +pub fn client_connected(ctx: &ReducerContext) { + ctx.db.online_player().insert(OnlinePlayer { + identity: ctx.sender(), + connected_at: ctx.timestamp, + }); +} + +#[reducer(client_disconnected)] +pub fn client_disconnected(ctx: &ReducerContext) { + ctx.db.online_player().identity().delete(&ctx.sender()); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/typescript.ts new file mode 100644 index 00000000000..6387227f6d1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/answers/typescript.ts @@ -0,0 +1,23 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const onlinePlayer = table({ + name: 'online_player', + public: true, +}, { + identity: t.identity().primaryKey(), + connectedAt: t.timestamp(), +}); + +const spacetimedb = schema({ onlinePlayer }); +export default spacetimedb; + +export const clientConnected = spacetimedb.clientConnected(ctx => { + ctx.db.onlinePlayer.insert({ + identity: ctx.sender, + connectedAt: ctx.timestamp, + }); +}); + +export const clientDisconnected = spacetimedb.clientDisconnected(ctx => { + ctx.db.onlinePlayer.identity.delete(ctx.sender); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/csharp.txt new file mode 100644 index 00000000000..bf28e233a69 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/csharp.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in C# that maintains a table of currently connected players using lifecycle reducers. + +TABLE +- OnlinePlayer (public) + - Fields: + - Identity: Identity (primary key) + - ConnectedAt: Timestamp + +REDUCERS +- ClientConnected (ReducerKind.ClientConnected): inserts an OnlinePlayer row using ctx.Sender as the identity and ctx.Timestamp as ConnectedAt +- ClientDisconnected (ReducerKind.ClientDisconnected): deletes the OnlinePlayer row for ctx.Sender diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/rust.txt new file mode 100644 index 00000000000..6489899584b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/rust.txt @@ -0,0 +1,12 @@ +Write a SpacetimeDB backend module in Rust that maintains a table of currently connected players using lifecycle reducers. + +TABLE +- online_player (public) + - Struct: OnlinePlayer + - Fields: + - identity: Identity (primary key) + - connected_at: Timestamp + +REDUCERS +- client_connected: lifecycle reducer that inserts an OnlinePlayer row using ctx.sender() as the identity and ctx.timestamp as connected_at +- client_disconnected: lifecycle reducer that deletes the OnlinePlayer row for ctx.sender() diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/typescript.txt new file mode 100644 index 00000000000..8efba7e8231 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/typescript.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in TypeScript that maintains a table of currently connected players using lifecycle reducers. + +TABLE +- onlinePlayer (public) + - Fields: + - identity: identity (primary key) + - connectedAt: timestamp + +REDUCERS +- clientConnected: lifecycle reducer (spacetimedb.clientConnected) that inserts an onlinePlayer row using ctx.sender as the identity and ctx.timestamp as connectedAt +- clientDisconnected: lifecycle reducer (spacetimedb.clientDisconnected) that deletes the onlinePlayer row for ctx.sender diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/csharp.cs new file mode 100644 index 00000000000..9b9d99b4683 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/csharp.cs @@ -0,0 +1,23 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "DamageEvent", Public = true, Event = true)] + public partial struct DamageEvent + { + public ulong EntityId; + public uint Damage; + public string Source; + } + + [Reducer] + public static void DealDamage(ReducerContext ctx, ulong entityId, uint damage, string source) + { + ctx.Db.DamageEvent.Insert(new DamageEvent + { + EntityId = entityId, + Damage = damage, + Source = source, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/rust.rs new file mode 100644 index 00000000000..4d235291d12 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/rust.rs @@ -0,0 +1,17 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = damage_event, public, event)] +pub struct DamageEvent { + pub entity_id: u64, + pub damage: u32, + pub source: String, +} + +#[reducer] +fn deal_damage(ctx: &ReducerContext, entity_id: u64, damage: u32, source: String) { + ctx.db.damage_event().insert(DamageEvent { + entity_id, + damage, + source, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/typescript.ts new file mode 100644 index 00000000000..c535f30746b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/answers/typescript.ts @@ -0,0 +1,21 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const damageEvent = table({ + name: 'damage_event', + public: true, + event: true, +}, { + entityId: t.u64(), + damage: t.u32(), + source: t.string(), +}); + +const spacetimedb = schema({ damageEvent }); +export default spacetimedb; + +export const deal_damage = spacetimedb.reducer( + { entityId: t.u64(), damage: t.u32(), source: t.string() }, + (ctx, { entityId, damage, source }) => { + ctx.db.damageEvent.insert({ entityId, damage, source }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/csharp.txt new file mode 100644 index 00000000000..076f2f7d0c5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/csharp.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in C# that defines an event table for damage notifications and a reducer that publishes events to it. + +TABLES +- DamageEvent + - Struct: DamageEvent + - public event table (rows exist only for the duration of the transaction, set Event = true) + - Fields: + - EntityId: ulong + - Damage: uint + - Source: string + +REDUCERS +- DealDamage(ctx, ulong entityId, uint damage, string source) + - Insert a DamageEvent row into the event table diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/rust.txt new file mode 100644 index 00000000000..97e3e23684a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/rust.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in Rust that defines an event table for damage notifications and a reducer that publishes events to it. + +TABLES +- damage_event + - Struct: DamageEvent + - public event table (rows exist only for the duration of the transaction) + - Fields: + - entity_id: u64 + - damage: u32 + - source: String + +REDUCERS +- deal_damage(ctx, entity_id: u64, damage: u32, source: String) + - Insert a DamageEvent row into the event table diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/typescript.txt new file mode 100644 index 00000000000..df4bac40f02 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/typescript.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in TypeScript that defines an event table for damage notifications and a reducer that publishes events to it. + +TABLES +- damageEvent + - public event table (rows exist only for the duration of the transaction, set event: true) + - Fields: + - entityId: number (u64) + - damage: number (u32) + - source: string + +REDUCERS +- deal_damage(ctx, { entityId: number (u64), damage: number (u32), source: string }) + - Insert a row into the damageEvent event table diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/csharp.cs new file mode 100644 index 00000000000..63c611abfaf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/csharp.cs @@ -0,0 +1,27 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Player")] + public partial struct Player + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Name; + public string? Nickname; + public uint? HighScore; + } + + [Reducer] + public static void CreatePlayer(ReducerContext ctx, string name, string? nickname, uint? highScore) + { + ctx.Db.Player.Insert(new Player + { + Id = 0, + Name = name, + Nickname = nickname, + HighScore = highScore, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/rust.rs new file mode 100644 index 00000000000..ff440debcfd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/rust.rs @@ -0,0 +1,21 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = player)] +pub struct Player { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, + pub nickname: Option, + pub high_score: Option, +} + +#[reducer] +pub fn create_player(ctx: &ReducerContext, name: String, nickname: Option, high_score: Option) { + ctx.db.player().insert(Player { + id: 0, + name, + nickname, + high_score, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/typescript.ts new file mode 100644 index 00000000000..d500dd8c3e1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/answers/typescript.ts @@ -0,0 +1,25 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const player = table({ + name: 'player', +}, { + id: t.u64().primaryKey().autoInc(), + name: t.string(), + nickname: t.option(t.string()), + highScore: t.option(t.u32()), +}); + +const spacetimedb = schema({ player }); +export default spacetimedb; + +export const create_player = spacetimedb.reducer( + { name: t.string(), nickname: t.option(t.string()), highScore: t.option(t.u32()) }, + (ctx, { name, nickname, highScore }) => { + ctx.db.player.insert({ + id: 0n, + name, + nickname, + highScore, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt new file mode 100644 index 00000000000..04c9a2a10b9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. + +TABLES +- Player + - Struct: Player + - Fields: + - Id: ulong (primary key, AutoInc) + - Name: string + - Nickname: string? (nullable) + - HighScore: uint? (nullable) + +REDUCERS +- CreatePlayer(ctx, string name, string? nickname, uint? highScore) + - Insert a Player row, passing through the nullable fields as-is + - Use Id: 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt new file mode 100644 index 00000000000..ac9b1d00a50 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in Rust that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. + +TABLES +- player + - Struct: Player + - Fields: + - id: u64 (primary key, auto_inc) + - name: String + - nickname: Option + - high_score: Option + +REDUCERS +- create_player(ctx, name: String, nickname: Option, high_score: Option) + - Insert a Player row, passing through the optional fields as-is + - Use id: 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt new file mode 100644 index 00000000000..48712f8abea --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in TypeScript that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. + +TABLES +- player + - Fields: + - id: number (u64, primary key, autoInc) + - name: string + - nickname: optional string (use t.option(t.string())) + - highScore: optional number (use t.option(t.u32())) + +REDUCERS +- create_player(ctx, { name: string, nickname: optional string, highScore: optional number (u32) }) + - Insert a player row, passing through the optional fields as-is + - Use id: 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/csharp.cs new file mode 100644 index 00000000000..6a8a0172dbc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/csharp.cs @@ -0,0 +1,36 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Author")] + public partial struct Author + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Name; + } + + [Table(Accessor = "Post")] + public partial struct Post + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [SpacetimeDB.Index.BTree] + public ulong AuthorId; + public string Title; + } + + [Reducer] + public static void DeleteAuthor(ReducerContext ctx, ulong authorId) + { + // Delete all posts by this author + foreach (var p in ctx.Db.Post.AuthorId.Filter(authorId)) + { + ctx.Db.Post.Id.Delete(p.Id); + } + // Delete the author + ctx.Db.Author.Id.Delete(authorId); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/rust.rs new file mode 100644 index 00000000000..a6d20b5395c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/rust.rs @@ -0,0 +1,29 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = author)] +pub struct Author { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table(accessor = post)] +pub struct Post { + #[primary_key] + #[auto_inc] + pub id: u64, + #[index(btree)] + pub author_id: u64, + pub title: String, +} + +#[reducer] +pub fn delete_author(ctx: &ReducerContext, author_id: u64) { + // Delete all posts by this author + for p in ctx.db.post().author_id().filter(&author_id) { + ctx.db.post().id().delete(p.id); + } + // Delete the author + ctx.db.author().id().delete(author_id); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/typescript.ts new file mode 100644 index 00000000000..e7104755233 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/answers/typescript.ts @@ -0,0 +1,31 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const author = table({ + name: 'author', +}, { + id: t.u64().primaryKey().autoInc(), + name: t.string(), +}); + +const post = table({ + name: 'post', +}, { + id: t.u64().primaryKey().autoInc(), + authorId: t.u64().index('btree'), + title: t.string(), +}); + +const spacetimedb = schema({ author, post }); +export default spacetimedb; + +export const delete_author = spacetimedb.reducer( + { authorId: t.u64() }, + (ctx, { authorId }) => { + // Delete all posts by this author + for (const p of ctx.db.post.authorId.filter(authorId)) { + ctx.db.post.id.delete(p.id); + } + // Delete the author + ctx.db.author.id.delete(authorId); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt new file mode 100644 index 00000000000..c3d32fd1d12 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in C# that defines an Author table and a Post table with a foreign key, and a reducer that deletes an author along with all their posts. + +TABLES +- Author + - Struct: Author + - Fields: + - Id: ulong (primary key, AutoInc) + - Name: string + +- Post + - Struct: Post + - Fields: + - Id: ulong (primary key, AutoInc) + - AuthorId: ulong (index BTree) + - Title: string + +REDUCERS +- DeleteAuthor(ctx, ulong authorId) + - First, iterate all posts with matching AuthorId and delete each one + - Then delete the author by Id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt new file mode 100644 index 00000000000..8a3c25ba08a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in Rust that defines an author table and a post table with a foreign key, and a reducer that deletes an author along with all their posts. + +TABLES +- author + - Struct: Author + - Fields: + - id: u64 (primary key, auto_inc) + - name: String + +- post + - Struct: Post + - Fields: + - id: u64 (primary key, auto_inc) + - author_id: u64 (index btree) + - title: String + +REDUCERS +- delete_author(ctx, author_id: u64) + - First, iterate all posts with matching author_id and delete each one + - Then delete the author by id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt new file mode 100644 index 00000000000..5cb6d02e7b6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in TypeScript that defines an author table and a post table with a foreign key, and a reducer that deletes an author along with all their posts. + +TABLES +- author + - Fields: + - id: number (u64, primary key, autoInc) + - name: string + +- post + - Fields: + - id: number (u64, primary key, autoInc) + - authorId: number (u64, index btree) + - title: string + +REDUCERS +- delete_author(ctx, { authorId: number (u64) }) + - First, iterate all posts with matching authorId and delete each one + - Then delete the author by id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/csharp.cs new file mode 100644 index 00000000000..43e39178e51 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/csharp.cs @@ -0,0 +1,47 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Order")] + public partial struct Order + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [SpacetimeDB.Index.BTree] + public string Category; + public ulong Amount; + public bool Fulfilled; + } + + [Table(Accessor = "CategoryStats")] + public partial struct CategoryStats + { + [PrimaryKey] + public string Category; + public ulong TotalAmount; + public uint OrderCount; + } + + [Reducer] + public static void ComputeStats(ReducerContext ctx, string category) + { + ulong totalAmount = 0; + uint orderCount = 0; + + foreach (var o in ctx.Db.Order.Category.Filter(category)) + { + totalAmount += o.Amount; + orderCount += 1; + } + + // Upsert: delete existing then insert + ctx.Db.CategoryStats.Category.Delete(category); + ctx.Db.CategoryStats.Insert(new CategoryStats + { + Category = category, + TotalAmount = totalAmount, + OrderCount = orderCount, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/rust.rs new file mode 100644 index 00000000000..03d4478e841 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/rust.rs @@ -0,0 +1,39 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = order)] +pub struct Order { + #[primary_key] + #[auto_inc] + pub id: u64, + #[index(btree)] + pub category: String, + pub amount: u64, + pub fulfilled: bool, +} + +#[table(accessor = category_stats)] +pub struct CategoryStats { + #[primary_key] + pub category: String, + pub total_amount: u64, + pub order_count: u32, +} + +#[reducer] +pub fn compute_stats(ctx: &ReducerContext, category: String) { + let mut total_amount: u64 = 0; + let mut order_count: u32 = 0; + + for o in ctx.db.order().category().filter(&category) { + total_amount += o.amount; + order_count += 1; + } + + // Upsert: delete existing then insert + let _ = ctx.db.category_stats().category().delete(category.clone()); + ctx.db.category_stats().insert(CategoryStats { + category, + total_amount, + order_count, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/typescript.ts new file mode 100644 index 00000000000..54627ee47bb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/answers/typescript.ts @@ -0,0 +1,42 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const order = table({ + name: 'order', +}, { + id: t.u64().primaryKey().autoInc(), + category: t.string().index('btree'), + amount: t.u64(), + fulfilled: t.bool(), +}); + +const categoryStats = table({ + name: 'category_stats', +}, { + category: t.string().primaryKey(), + totalAmount: t.u64(), + orderCount: t.u32(), +}); + +const spacetimedb = schema({ order, categoryStats }); +export default spacetimedb; + +export const compute_stats = spacetimedb.reducer( + { category: t.string() }, + (ctx, { category }) => { + let totalAmount = 0n; + let orderCount = 0; + + for (const o of ctx.db.order.category.filter(category)) { + totalAmount += o.amount; + orderCount += 1; + } + + // Upsert: delete existing then insert + ctx.db.categoryStats.category.delete(category); + ctx.db.categoryStats.insert({ + category, + totalAmount, + orderCount, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt new file mode 100644 index 00000000000..2f761d52a22 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in C# that defines an Order table and a CategoryStats table, with a reducer that computes aggregate statistics from filtered orders. + +TABLES +- Order + - Struct: Order + - Fields: + - Id: ulong (primary key, AutoInc) + - Category: string (index BTree) + - Amount: ulong + - Fulfilled: bool + +- CategoryStats + - Struct: CategoryStats + - Fields: + - Category: string (primary key) + - TotalAmount: ulong + - OrderCount: uint + +REDUCERS +- ComputeStats(ctx, string category) + - Filter all orders matching the given category + - Count the number of matching orders and sum their amounts + - Insert (or update if exists) a CategoryStats row with the results diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt new file mode 100644 index 00000000000..ebe29cf6a76 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in Rust that defines an order table and a stats table, with a reducer that computes aggregate statistics from filtered orders. + +TABLES +- order + - Struct: Order + - Fields: + - id: u64 (primary key, auto_inc) + - category: String (index btree) + - amount: u64 + - fulfilled: bool + +- category_stats + - Struct: CategoryStats + - Fields: + - category: String (primary key) + - total_amount: u64 + - order_count: u32 + +REDUCERS +- compute_stats(ctx, category: String) + - Filter all orders matching the given category + - Count the number of matching orders and sum their amounts + - Insert (or update if exists) a CategoryStats row with the results diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt new file mode 100644 index 00000000000..092c5eeb4ce --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in TypeScript that defines an order table and a stats table, with a reducer that computes aggregate statistics from filtered orders. + +TABLES +- order + - Fields: + - id: number (u64, primary key, autoInc) + - category: string (index btree) + - amount: number (u64) + - fulfilled: boolean + +- categoryStats + - Fields: + - category: string (primary key) + - totalAmount: number (u64) + - orderCount: number (u32) + +REDUCERS +- compute_stats(ctx, { category: string }) + - Filter all orders matching the given category + - Count the number of matching orders and sum their amounts + - Insert (or update if exists) a categoryStats row with the results diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/csharp.cs new file mode 100644 index 00000000000..2529d1e8bc0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/csharp.cs @@ -0,0 +1,53 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Customer")] + public partial struct Customer + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Name; + } + + [Table(Accessor = "Order")] + public partial struct Order + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [SpacetimeDB.Index.BTree] + public ulong CustomerId; + public string Product; + public uint Amount; + } + + [Table(Accessor = "OrderDetail")] + public partial struct OrderDetail + { + [PrimaryKey] + public ulong OrderId; + public string CustomerName; + public string Product; + public uint Amount; + } + + [Reducer] + public static void BuildOrderDetails(ReducerContext ctx) + { + foreach (var o in ctx.Db.Order.Iter()) + { + if (ctx.Db.Customer.Id.Find(o.CustomerId) is Customer c) + { + ctx.Db.OrderDetail.Insert(new OrderDetail + { + OrderId = o.Id, + CustomerName = c.Name, + Product = o.Product, + Amount = o.Amount, + }); + } + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/rust.rs new file mode 100644 index 00000000000..42a91bb252f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/rust.rs @@ -0,0 +1,43 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = customer)] +pub struct Customer { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table(accessor = order)] +pub struct Order { + #[primary_key] + #[auto_inc] + pub id: u64, + #[index(btree)] + pub customer_id: u64, + pub product: String, + pub amount: u32, +} + +#[table(accessor = order_detail)] +pub struct OrderDetail { + #[primary_key] + pub order_id: u64, + pub customer_name: String, + pub product: String, + pub amount: u32, +} + +#[reducer] +pub fn build_order_details(ctx: &ReducerContext) { + for o in ctx.db.order().iter() { + if let Some(c) = ctx.db.customer().id().find(o.customer_id) { + ctx.db.order_detail().insert(OrderDetail { + order_id: o.id, + customer_name: c.name, + product: o.product, + amount: o.amount, + }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/typescript.ts new file mode 100644 index 00000000000..7cf358e949b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/answers/typescript.ts @@ -0,0 +1,45 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const customer = table({ + name: 'customer', +}, { + id: t.u64().primaryKey().autoInc(), + name: t.string(), +}); + +const order = table({ + name: 'order', +}, { + id: t.u64().primaryKey().autoInc(), + customerId: t.u64().index('btree'), + product: t.string(), + amount: t.u32(), +}); + +const orderDetail = table({ + name: 'order_detail', +}, { + orderId: t.u64().primaryKey(), + customerName: t.string(), + product: t.string(), + amount: t.u32(), +}); + +const spacetimedb = schema({ customer, order, orderDetail }); +export default spacetimedb; + +export const build_order_details = spacetimedb.reducer( + (ctx) => { + for (const o of ctx.db.order.iter()) { + const c = ctx.db.customer.id.find(o.customerId); + if (c) { + ctx.db.orderDetail.insert({ + orderId: o.id, + customerName: c.name, + product: o.product, + amount: o.amount, + }); + } + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt new file mode 100644 index 00000000000..6dfd47c6580 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt @@ -0,0 +1,30 @@ +Write a SpacetimeDB backend module in C# that defines a Customer table, an Order table with a foreign key to Customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. + +TABLES +- Customer + - Struct: Customer + - Fields: + - Id: ulong (primary key, AutoInc) + - Name: string + +- Order + - Struct: Order + - Fields: + - Id: ulong (primary key, AutoInc) + - CustomerId: ulong (index BTree) + - Product: string + - Amount: uint + +- OrderDetail + - Struct: OrderDetail + - Fields: + - OrderId: ulong (primary key) + - CustomerName: string + - Product: string + - Amount: uint + +REDUCERS +- BuildOrderDetails(ctx) + - Iterate all orders + - For each order, look up the customer by CustomerId + - If the customer is found, insert an OrderDetail row with the joined data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt new file mode 100644 index 00000000000..21170c6e3c1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt @@ -0,0 +1,30 @@ +Write a SpacetimeDB backend module in Rust that defines a customer table, an order table with a foreign key to customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. + +TABLES +- customer + - Struct: Customer + - Fields: + - id: u64 (primary key, auto_inc) + - name: String + +- order + - Struct: Order + - Fields: + - id: u64 (primary key, auto_inc) + - customer_id: u64 (index btree) + - product: String + - amount: u32 + +- order_detail + - Struct: OrderDetail + - Fields: + - order_id: u64 (primary key) + - customer_name: String + - product: String + - amount: u32 + +REDUCERS +- build_order_details(ctx) + - Iterate all orders + - For each order, look up the customer by customer_id + - If the customer is found, insert an OrderDetail row with the joined data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt new file mode 100644 index 00000000000..1dae0185b3d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt @@ -0,0 +1,27 @@ +Write a SpacetimeDB backend module in TypeScript that defines a customer table, an order table with a foreign key to customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. + +TABLES +- customer + - Fields: + - id: number (u64, primary key, autoInc) + - name: string + +- order + - Fields: + - id: number (u64, primary key, autoInc) + - customerId: number (u64, index btree) + - product: string + - amount: number (u32) + +- orderDetail + - Fields: + - orderId: number (u64, primary key) + - customerName: string + - product: string + - amount: number (u32) + +REDUCERS +- build_order_details(ctx) + - Iterate all orders + - For each order, look up the customer by customerId + - If the customer is found, insert an orderDetail row with the joined data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/csharp.cs new file mode 100644 index 00000000000..994e7baf153 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/csharp.cs @@ -0,0 +1,26 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Account")] + public partial struct Account + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [Unique] + public string Email; + public string DisplayName; + } + + [Reducer] + public static void CreateAccount(ReducerContext ctx, string email, string displayName) + { + ctx.Db.Account.Insert(new Account + { + Id = 0, + Email = email, + DisplayName = displayName, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/rust.rs new file mode 100644 index 00000000000..016daeb4e3d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/rust.rs @@ -0,0 +1,20 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = account)] +pub struct Account { + #[primary_key] + #[auto_inc] + pub id: u64, + #[unique] + pub email: String, + pub display_name: String, +} + +#[reducer] +pub fn create_account(ctx: &ReducerContext, email: String, display_name: String) { + ctx.db.account().insert(Account { + id: 0, + email, + display_name, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/typescript.ts new file mode 100644 index 00000000000..5faccc6b9bb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/answers/typescript.ts @@ -0,0 +1,23 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const account = table({ + name: 'account', +}, { + id: t.u64().primaryKey().autoInc(), + email: t.string().unique(), + displayName: t.string(), +}); + +const spacetimedb = schema({ account }); +export default spacetimedb; + +export const create_account = spacetimedb.reducer( + { email: t.string(), displayName: t.string() }, + (ctx, { email, displayName }) => { + ctx.db.account.insert({ + id: 0n, + email, + displayName, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt new file mode 100644 index 00000000000..433a4df53aa --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in C# that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. + +TABLES +- Account + - Struct: Account + - Fields: + - Id: ulong (primary key, AutoInc) + - Email: string (Unique) + - DisplayName: string + +REDUCERS +- CreateAccount(ctx, string email, string displayName) + - Insert an Account row with the given email and displayName + - Use Id: 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt new file mode 100644 index 00000000000..2482f76d578 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in Rust that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. + +TABLES +- account + - Struct: Account + - Fields: + - id: u64 (primary key, auto_inc) + - email: String (unique) + - display_name: String + +REDUCERS +- create_account(ctx, email: String, display_name: String) + - Insert an Account row with the given email and display_name + - Use id: 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt new file mode 100644 index 00000000000..b1e90089c63 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in TypeScript that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. + +TABLES +- account + - Fields: + - id: number (u64, primary key, autoInc) + - email: string (unique) + - displayName: string + +REDUCERS +- create_account(ctx, { email: string, displayName: string }) + - Insert an account row with the given email and displayName + - Use id: 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/csharp.cs new file mode 100644 index 00000000000..bbdf8653a82 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/csharp.cs @@ -0,0 +1,23 @@ +using SpacetimeDB; +using System.Collections.Generic; +using System.Linq; + +public static partial class Module +{ + [Table(Accessor = "Announcement", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "Active", Columns = new[] { nameof(Announcement.Active) })] + public partial struct Announcement + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Message; + public bool Active; + } + + [SpacetimeDB.View(Accessor = "ActiveAnnouncements", Public = true)] + public static List ActiveAnnouncements(AnonymousViewContext ctx) + { + return ctx.Db.Announcement.Active.Filter(true).ToList(); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/rust.rs new file mode 100644 index 00000000000..aa44bfd481c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/rust.rs @@ -0,0 +1,16 @@ +use spacetimedb::{table, view, AnonymousViewContext}; + +#[table(accessor = announcement, public)] +pub struct Announcement { + #[primary_key] + #[auto_inc] + pub id: u64, + pub message: String, + #[index(btree)] + pub active: bool, +} + +#[view(accessor = active_announcements, public)] +fn active_announcements(ctx: &AnonymousViewContext) -> Vec { + ctx.db.announcement().active().filter(true).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/typescript.ts new file mode 100644 index 00000000000..296ae12e239 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/answers/typescript.ts @@ -0,0 +1,21 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const announcement = table({ + name: 'announcement', + public: true, +}, { + id: t.u64().primaryKey().autoInc(), + message: t.string(), + active: t.bool().index('btree'), +}); + +const spacetimedb = schema({ announcement }); +export default spacetimedb; + +export const activeAnnouncements = spacetimedb.anonymousView( + { name: 'active_announcements', public: true }, + t.array(announcement.rowType), + (ctx) => { + return Array.from(ctx.db.announcement.active.filter(true)); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/csharp.txt new file mode 100644 index 00000000000..aafc1c283c6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/csharp.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in C# that defines a public anonymous view over a table. + +TABLE +- Announcement (public) + - Fields: + - Id: ulong (primary key, auto-increment) + - Message: string + - Active: bool (btree index) + +VIEW +- ActiveAnnouncements (AnonymousViewContext, public) + - Returns: IEnumerable + - Body: return all rows where Active == true using the btree index filter diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/rust.txt new file mode 100644 index 00000000000..c149a7514c0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/rust.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in Rust that defines a public anonymous view over a table. + +TABLE +- announcement (public) + - Struct: Announcement + - Fields: + - id: u64 (primary key, auto-increment) + - message: String + - active: bool (btree index) + +VIEW +- active_announcements (AnonymousViewContext, public) + - Returns: Vec + - Body: return all rows where active == true using the btree index diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/typescript.txt new file mode 100644 index 00000000000..968e7cce71b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/tasks/typescript.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in TypeScript that defines a public anonymous view over a table. + +TABLE +- announcement (public) + - Fields: + - id: u64 (primary key, auto-increment) + - message: string + - active: bool (btree index) + +VIEW +- activeAnnouncements (anonymousView, public) + - Returns: array of announcement rows + - Body: return all rows where active == true using the btree index filter diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/csharp.cs new file mode 100644 index 00000000000..936f5cb8f87 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/csharp.cs @@ -0,0 +1,22 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Profile", Public = true)] + public partial struct Profile + { + [PrimaryKey] + [AutoInc] + public ulong Id; + [Unique] + public Identity Identity; + public string Name; + public string Bio; + } + + [SpacetimeDB.View(Accessor = "MyProfile", Public = true)] + public static Profile? MyProfile(ViewContext ctx) + { + return ctx.Db.Profile.Identity.Find(ctx.Sender) as Profile?; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/rust.rs new file mode 100644 index 00000000000..a336ab0e51a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/rust.rs @@ -0,0 +1,17 @@ +use spacetimedb::{table, view, Identity, ViewContext}; + +#[table(accessor = profile, public)] +pub struct Profile { + #[primary_key] + #[auto_inc] + pub id: u64, + #[unique] + pub identity: Identity, + pub name: String, + pub bio: String, +} + +#[view(accessor = my_profile, public)] +fn my_profile(ctx: &ViewContext) -> Option { + ctx.db.profile().identity().find(ctx.sender()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/typescript.ts new file mode 100644 index 00000000000..5c36f9ef000 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/answers/typescript.ts @@ -0,0 +1,22 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const profile = table({ + name: 'profile', + public: true, +}, { + id: t.u64().primaryKey().autoInc(), + identity: t.identity().unique(), + name: t.string(), + bio: t.string(), +}); + +const spacetimedb = schema({ profile }); +export default spacetimedb; + +export const my_profile = spacetimedb.view( + { name: 'my_profile', public: true }, + t.option(profile.rowType), + (ctx) => { + return ctx.db.profile.identity.find(ctx.sender) ?? undefined; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt new file mode 100644 index 00000000000..7cc46d493b7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt @@ -0,0 +1,17 @@ +Write a SpacetimeDB backend module in C# that defines a table of user profiles and a per-user view that returns only the calling user's profile. + +TABLES +- Profile + - Struct: Profile + - Fields: + - Id: ulong (primary key, AutoInc) + - Identity: Identity (unique) + - Name: string + - Bio: string + +VIEWS +- MyProfile + - per-user view (uses ViewContext, not AnonymousViewContext) + - public + - Returns: Profile? (the single profile matching ctx.Sender, or null) + - Logic: find the profile by the Identity column matching ctx.Sender diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt new file mode 100644 index 00000000000..7016433ee2d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in Rust that defines a table of user profiles and a per-user view that returns only the calling user's profile. + +TABLES +- profile + - Struct: Profile + - public table + - Fields: + - id: u64 (primary key, auto_inc) + - identity: Identity (unique) + - name: String + - bio: String + +VIEWS +- my_profile + - per-user view (uses ViewContext, not AnonymousViewContext) + - public + - Returns: Option (the single profile row matching ctx.sender(), or None) + - Logic: look up the profile by the identity column matching ctx.sender() diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt new file mode 100644 index 00000000000..d2088770afd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt @@ -0,0 +1,17 @@ +Write a SpacetimeDB backend module in TypeScript that defines a table of user profiles and a per-user view that returns only the calling user's profile. + +TABLES +- profile + - public table + - Fields: + - id: number (u64, primary key, autoInc) + - identity: Identity (unique) + - name: string + - bio: string + +VIEWS +- my_profile + - per-user view (uses spacetimedb.view, not spacetimedb.anonymousView) + - public + - Returns: option of profile row (the single profile matching ctx.sender(), or undefined) + - Logic: find the profile by the identity column matching ctx.sender() diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/csharp.cs new file mode 100644 index 00000000000..27912411a0a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/csharp.cs @@ -0,0 +1,41 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Product")] + public partial struct Product + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Name; + [SpacetimeDB.Index.BTree] + public uint Price; + } + + [Table(Accessor = "PriceRangeResult")] + public partial struct PriceRangeResult + { + [PrimaryKey] + public ulong ProductId; + public string Name; + public uint Price; + } + + [Reducer] + public static void FindInPriceRange(ReducerContext ctx, uint minPrice, uint maxPrice) + { + foreach (var p in ctx.Db.Product.Iter()) + { + if (p.Price >= minPrice && p.Price <= maxPrice) + { + ctx.Db.PriceRangeResult.Insert(new PriceRangeResult + { + ProductId = p.Id, + Name = p.Name, + Price = p.Price, + }); + } + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/rust.rs new file mode 100644 index 00000000000..0eb1b055fbc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/rust.rs @@ -0,0 +1,32 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = product)] +pub struct Product { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, + #[index(btree)] + pub price: u32, +} + +#[table(accessor = price_range_result)] +pub struct PriceRangeResult { + #[primary_key] + pub product_id: u64, + pub name: String, + pub price: u32, +} + +#[reducer] +pub fn find_in_price_range(ctx: &ReducerContext, min_price: u32, max_price: u32) { + for p in ctx.db.product().iter() { + if p.price >= min_price && p.price <= max_price { + ctx.db.price_range_result().insert(PriceRangeResult { + product_id: p.id, + name: p.name, + price: p.price, + }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/typescript.ts new file mode 100644 index 00000000000..cba00bf23cc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/answers/typescript.ts @@ -0,0 +1,35 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const product = table({ + name: 'product', +}, { + id: t.u64().primaryKey().autoInc(), + name: t.string(), + price: t.u32().index('btree'), +}); + +const priceRangeResult = table({ + name: 'price_range_result', +}, { + productId: t.u64().primaryKey(), + name: t.string(), + price: t.u32(), +}); + +const spacetimedb = schema({ product, priceRangeResult }); +export default spacetimedb; + +export const find_in_price_range = spacetimedb.reducer( + { minPrice: t.u32(), maxPrice: t.u32() }, + (ctx, { minPrice, maxPrice }) => { + for (const p of ctx.db.product.iter()) { + if (p.price >= minPrice && p.price <= maxPrice) { + ctx.db.priceRangeResult.insert({ + productId: p.id, + name: p.name, + price: p.price, + }); + } + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt new file mode 100644 index 00000000000..03b091f6fe2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in C# that defines a Product table with a BTree-indexed Price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. + +TABLES +- Product + - Struct: Product + - Fields: + - Id: ulong (primary key, AutoInc) + - Name: string + - Price: uint (index BTree) + +- PriceRangeResult + - Struct: PriceRangeResult + - Fields: + - ProductId: ulong (primary key) + - Name: string + - Price: uint + +REDUCERS +- FindInPriceRange(ctx, uint minPrice, uint maxPrice) + - Iterate all products + - For each product where Price >= minPrice AND Price <= maxPrice, insert a PriceRangeResult row diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt new file mode 100644 index 00000000000..7396da179b4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in Rust that defines a product table with a BTree-indexed price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. + +TABLES +- product + - Struct: Product + - Fields: + - id: u64 (primary key, auto_inc) + - name: String + - price: u32 (index btree) + +- price_range_result + - Struct: PriceRangeResult + - Fields: + - product_id: u64 (primary key) + - name: String + - price: u32 + +REDUCERS +- find_in_price_range(ctx, min_price: u32, max_price: u32) + - Iterate all products + - For each product where price >= min_price AND price <= max_price, insert a PriceRangeResult row diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt new file mode 100644 index 00000000000..87e20bd524d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt @@ -0,0 +1,19 @@ +Write a SpacetimeDB backend module in TypeScript that defines a product table with a BTree-indexed price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. + +TABLES +- product + - Fields: + - id: number (u64, primary key, autoInc) + - name: string + - price: number (u32, index btree) + +- priceRangeResult + - Fields: + - productId: number (u64, primary key) + - name: string + - price: number (u32) + +REDUCERS +- find_in_price_range(ctx, { minPrice: number (u32), maxPrice: number (u32) }) + - Iterate all products + - For each product where price >= minPrice AND price <= maxPrice, insert a priceRangeResult row diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/csharp.cs new file mode 100644 index 00000000000..194c5494a90 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/csharp.cs @@ -0,0 +1,43 @@ +using SpacetimeDB; +using System.Linq; + +public static partial class Module +{ + [Table(Accessor = "Player")] + public partial struct Player + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Name; + public ulong Score; + } + + [Table(Accessor = "Leaderboard")] + public partial struct LeaderboardEntry + { + [PrimaryKey] + public uint Rank; + public string PlayerName; + public ulong Score; + } + + [Reducer] + public static void BuildLeaderboard(ReducerContext ctx, uint limit) + { + var players = ctx.Db.Player.Iter() + .OrderByDescending(p => p.Score) + .Take((int)limit) + .ToList(); + + for (int i = 0; i < players.Count; i++) + { + ctx.Db.Leaderboard.Insert(new LeaderboardEntry + { + Rank = (uint)(i + 1), + PlayerName = players[i].Name, + Score = players[i].Score, + }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/rust.rs new file mode 100644 index 00000000000..79c83caee3b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/rust.rs @@ -0,0 +1,32 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = player)] +pub struct Player { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, + pub score: u64, +} + +#[table(accessor = leaderboard)] +pub struct LeaderboardEntry { + #[primary_key] + pub rank: u32, + pub player_name: String, + pub score: u64, +} + +#[reducer] +pub fn build_leaderboard(ctx: &ReducerContext, limit: u32) { + let mut players: Vec = ctx.db.player().iter().collect(); + players.sort_by(|a, b| b.score.cmp(&a.score)); + + for (i, p) in players.into_iter().take(limit as usize).enumerate() { + ctx.db.leaderboard().insert(LeaderboardEntry { + rank: (i as u32) + 1, + player_name: p.name, + score: p.score, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/typescript.ts new file mode 100644 index 00000000000..64897b29310 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/answers/typescript.ts @@ -0,0 +1,37 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const player = table({ + name: 'player', +}, { + id: t.u64().primaryKey().autoInc(), + name: t.string(), + score: t.u64(), +}); + +const leaderboard = table({ + name: 'leaderboard', +}, { + rank: t.u32().primaryKey(), + playerName: t.string(), + score: t.u64(), +}); + +const spacetimedb = schema({ player, leaderboard }); +export default spacetimedb; + +export const build_leaderboard = spacetimedb.reducer( + { limit: t.u32() }, + (ctx, { limit }) => { + const players = Array.from(ctx.db.player.iter()); + players.sort((a, b) => (b.score > a.score ? 1 : b.score < a.score ? -1 : 0)); + + for (let i = 0; i < Math.min(limit, players.length); i++) { + const p = players[i]; + ctx.db.leaderboard.insert({ + rank: i + 1, + playerName: p.name, + score: p.score, + }); + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt new file mode 100644 index 00000000000..f59eca1f12c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in C# that defines a Player table with scores, a Leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. + +TABLES +- Player + - Struct: Player + - Fields: + - Id: ulong (primary key, AutoInc) + - Name: string + - Score: ulong + +- Leaderboard + - Struct: LeaderboardEntry + - Fields: + - Rank: uint (primary key) + - PlayerName: string + - Score: ulong + +REDUCERS +- BuildLeaderboard(ctx, uint limit) + - Collect all players into a list, sort by Score descending + - Take the first `limit` entries + - Insert each into Leaderboard with Rank starting at 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt new file mode 100644 index 00000000000..cfc38860762 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in Rust that defines a player table with scores, a leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. + +TABLES +- player + - Struct: Player + - Fields: + - id: u64 (primary key, auto_inc) + - name: String + - score: u64 + +- leaderboard + - Struct: LeaderboardEntry + - Fields: + - rank: u32 (primary key) + - player_name: String + - score: u64 + +REDUCERS +- build_leaderboard(ctx, limit: u32) + - Collect all players into a Vec, sort by score descending + - Take the first `limit` entries + - Insert each into leaderboard with rank starting at 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt new file mode 100644 index 00000000000..d888f39528e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript that defines a player table with scores, a leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. + +TABLES +- player + - Fields: + - id: number (u64, primary key, autoInc) + - name: string + - score: number (u64) + +- leaderboard + - Fields: + - rank: number (u32, primary key) + - playerName: string + - score: number (u64) + +REDUCERS +- build_leaderboard(ctx, { limit: number (u32) }) + - Collect all players into an array, sort by score descending + - Take the first `limit` entries + - Insert each into leaderboard with rank starting at 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/csharp.cs new file mode 100644 index 00000000000..ae60e2f299f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/csharp.cs @@ -0,0 +1,39 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Task")] + public partial struct Task + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Title; + public bool Completed; + } + + [Table(Accessor = "FirstIncomplete")] + public partial struct FirstIncomplete + { + [PrimaryKey] + public ulong TaskId; + public string Title; + } + + [Reducer] + public static void FindFirstIncomplete(ReducerContext ctx) + { + foreach (var t in ctx.Db.Task.Iter()) + { + if (!t.Completed) + { + ctx.Db.FirstIncomplete.Insert(new FirstIncomplete + { + TaskId = t.Id, + Title = t.Title, + }); + return; + } + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/rust.rs new file mode 100644 index 00000000000..4c96d76298d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/rust.rs @@ -0,0 +1,27 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = task)] +pub struct Task { + #[primary_key] + #[auto_inc] + pub id: u64, + pub title: String, + pub completed: bool, +} + +#[table(accessor = first_incomplete)] +pub struct FirstIncomplete { + #[primary_key] + pub task_id: u64, + pub title: String, +} + +#[reducer] +pub fn find_first_incomplete(ctx: &ReducerContext) { + if let Some(t) = ctx.db.task().iter().find(|t| !t.completed) { + ctx.db.first_incomplete().insert(FirstIncomplete { + task_id: t.id, + title: t.title, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/typescript.ts new file mode 100644 index 00000000000..76c785c19b7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/answers/typescript.ts @@ -0,0 +1,33 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const task = table({ + name: 'task', +}, { + id: t.u64().primaryKey().autoInc(), + title: t.string(), + completed: t.bool(), +}); + +const firstIncomplete = table({ + name: 'first_incomplete', +}, { + taskId: t.u64().primaryKey(), + title: t.string(), +}); + +const spacetimedb = schema({ task, firstIncomplete }); +export default spacetimedb; + +export const find_first_incomplete = spacetimedb.reducer( + (ctx) => { + for (const t of ctx.db.task.iter()) { + if (!t.completed) { + ctx.db.firstIncomplete.insert({ + taskId: t.id, + title: t.title, + }); + return; + } + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt new file mode 100644 index 00000000000..c89d54ed9a6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in C# that defines a Task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. + +TABLES +- Task + - Struct: Task + - Fields: + - Id: ulong (primary key, AutoInc) + - Title: string + - Completed: bool + +- FirstIncomplete + - Struct: FirstIncomplete + - Fields: + - TaskId: ulong (primary key) + - Title: string + +REDUCERS +- FindFirstIncomplete(ctx) + - Iterate through the Task table + - Find the first task where Completed is false + - If found, insert it into FirstIncomplete diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt new file mode 100644 index 00000000000..dec8df0b43e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in Rust that defines a task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. + +TABLES +- task + - Struct: Task + - Fields: + - id: u64 (primary key, auto_inc) + - title: String + - completed: bool + +- first_incomplete + - Struct: FirstIncomplete + - Fields: + - task_id: u64 (primary key) + - title: String + +REDUCERS +- find_first_incomplete(ctx) + - Iterate through the task table + - Find the first task where completed is false + - If found, insert it into first_incomplete diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt new file mode 100644 index 00000000000..1a660b24679 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt @@ -0,0 +1,19 @@ +Write a SpacetimeDB backend module in TypeScript that defines a task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. + +TABLES +- task + - Fields: + - id: number (u64, primary key, autoInc) + - title: string + - completed: boolean + +- firstIncomplete + - Fields: + - taskId: number (u64, primary key) + - title: string + +REDUCERS +- find_first_incomplete(ctx) + - Iterate through the task table + - Find the first task where completed is false + - If found, insert it into firstIncomplete diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/csharp.cs new file mode 100644 index 00000000000..d8e0ff3bbf0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/csharp.cs @@ -0,0 +1,36 @@ +using SpacetimeDB; +using System.Collections.Generic; + +public static partial class Module +{ + [Table(Accessor = "Order")] + public partial struct Order + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Category; + public uint Amount; + } + + [Table(Accessor = "DistinctCategory")] + public partial struct DistinctCategory + { + [PrimaryKey] + public string Category; + } + + [Reducer] + public static void CollectDistinctCategories(ReducerContext ctx) + { + var categories = new HashSet(); + foreach (var o in ctx.Db.Order.Iter()) + { + categories.Add(o.Category); + } + foreach (var category in categories) + { + ctx.Db.DistinctCategory.Insert(new DistinctCategory { Category = category }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/rust.rs new file mode 100644 index 00000000000..8b6be727eed --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/rust.rs @@ -0,0 +1,25 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; +use std::collections::BTreeSet; + +#[table(accessor = order)] +pub struct Order { + #[primary_key] + #[auto_inc] + pub id: u64, + pub category: String, + pub amount: u32, +} + +#[table(accessor = distinct_category)] +pub struct DistinctCategory { + #[primary_key] + pub category: String, +} + +#[reducer] +pub fn collect_distinct_categories(ctx: &ReducerContext) { + let categories: BTreeSet = ctx.db.order().iter().map(|o| o.category).collect(); + for category in categories { + ctx.db.distinct_category().insert(DistinctCategory { category }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/typescript.ts new file mode 100644 index 00000000000..83d96e1dc6c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/answers/typescript.ts @@ -0,0 +1,30 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const order = table({ + name: 'order', +}, { + id: t.u64().primaryKey().autoInc(), + category: t.string(), + amount: t.u32(), +}); + +const distinctCategory = table({ + name: 'distinct_category', +}, { + category: t.string().primaryKey(), +}); + +const spacetimedb = schema({ order, distinctCategory }); +export default spacetimedb; + +export const collect_distinct_categories = spacetimedb.reducer( + (ctx) => { + const categories = new Set(); + for (const o of ctx.db.order.iter()) { + categories.add(o.category); + } + for (const category of categories) { + ctx.db.distinctCategory.insert({ category }); + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt new file mode 100644 index 00000000000..5ac96c35a88 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in C# that defines an Order table and a result table, with a reducer that collects all distinct Category values from orders into the result table. + +TABLES +- Order + - Struct: Order + - Fields: + - Id: ulong (primary key, AutoInc) + - Category: string + - Amount: uint + +- DistinctCategory + - Struct: DistinctCategory + - Fields: + - Category: string (primary key) + +REDUCERS +- CollectDistinctCategories(ctx) + - Iterate all orders + - Collect unique Category values (use a HashSet) + - Insert each unique category into DistinctCategory diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt new file mode 100644 index 00000000000..bc7d9e915cd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in Rust that defines an order table and a result table, with a reducer that collects all distinct category values from orders into the result table. + +TABLES +- order + - Struct: Order + - Fields: + - id: u64 (primary key, auto_inc) + - category: String + - amount: u32 + +- distinct_category + - Struct: DistinctCategory + - Fields: + - category: String (primary key) + +REDUCERS +- collect_distinct_categories(ctx) + - Iterate all orders + - Collect unique category values (use a HashSet or BTreeSet) + - Insert each unique category into distinct_category diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt new file mode 100644 index 00000000000..40a7385feb4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in TypeScript that defines an order table and a result table, with a reducer that collects all distinct category values from orders into the result table. + +TABLES +- order + - Fields: + - id: number (u64, primary key, autoInc) + - category: string + - amount: number (u32) + +- distinctCategory + - Fields: + - category: string (primary key) + +REDUCERS +- collect_distinct_categories(ctx) + - Iterate all orders + - Collect unique category values (use a Set) + - Insert each unique category into distinctCategory diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/csharp.cs new file mode 100644 index 00000000000..695d1cfb98a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/csharp.cs @@ -0,0 +1,40 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "User")] + public partial struct User + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Name; + public bool Active; + } + + [Table(Accessor = "UserStats")] + public partial struct UserStats + { + [PrimaryKey] + public string Key; + public ulong Count; + } + + [Reducer] + public static void ComputeUserCounts(ReducerContext ctx) + { + ulong total = 0; + ulong active = 0; + foreach (var u in ctx.Db.User.Iter()) + { + total++; + if (u.Active) + { + active++; + } + } + + ctx.Db.UserStats.Insert(new UserStats { Key = "total", Count = total }); + ctx.Db.UserStats.Insert(new UserStats { Key = "active", Count = active }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/rust.rs new file mode 100644 index 00000000000..3fd62b817d3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/rust.rs @@ -0,0 +1,32 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = user)] +pub struct User { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, + pub active: bool, +} + +#[table(accessor = user_stats)] +pub struct UserStats { + #[primary_key] + pub key: String, + pub count: u64, +} + +#[reducer] +pub fn compute_user_counts(ctx: &ReducerContext) { + let total = ctx.db.user().iter().count() as u64; + let active = ctx.db.user().iter().filter(|u| u.active).count() as u64; + + ctx.db.user_stats().insert(UserStats { + key: "total".to_string(), + count: total, + }); + ctx.db.user_stats().insert(UserStats { + key: "active".to_string(), + count: active, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/typescript.ts new file mode 100644 index 00000000000..83f95707198 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/answers/typescript.ts @@ -0,0 +1,35 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const user = table({ + name: 'user', +}, { + id: t.u64().primaryKey().autoInc(), + name: t.string(), + active: t.bool(), +}); + +const userStats = table({ + name: 'user_stats', +}, { + key: t.string().primaryKey(), + count: t.u64(), +}); + +const spacetimedb = schema({ user, userStats }); +export default spacetimedb; + +export const compute_user_counts = spacetimedb.reducer( + (ctx) => { + let total = 0n; + let active = 0n; + for (const u of ctx.db.user.iter()) { + total += 1n; + if (u.active) { + active += 1n; + } + } + + ctx.db.userStats.insert({ key: 'total', count: total }); + ctx.db.userStats.insert({ key: 'active', count: active }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt new file mode 100644 index 00000000000..1508c532fc8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in C# that defines a User table and a stats table, with a reducer that counts total users and active users without collecting into a list. + +TABLES +- User + - Struct: User + - Fields: + - Id: ulong (primary key, AutoInc) + - Name: string + - Active: bool + +- UserStats + - Struct: UserStats + - Fields: + - Key: string (primary key) + - Count: ulong + +REDUCERS +- ComputeUserCounts(ctx) + - Count total users by iterating (do not collect into a List first) + - Count active users by iterating and checking Active == true + - Insert a UserStats row with Key="total" and the total count + - Insert a UserStats row with Key="active" and the active count diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt new file mode 100644 index 00000000000..24c987810d7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in Rust that defines a user table and a stats table, with a reducer that counts total users and active users without collecting into a Vec. + +TABLES +- user + - Struct: User + - Fields: + - id: u64 (primary key, auto_inc) + - name: String + - active: bool + +- user_stats + - Struct: UserStats + - Fields: + - key: String (primary key) + - count: u64 + +REDUCERS +- compute_user_counts(ctx) + - Count total users by iterating with .count() (do not collect into a Vec) + - Count active users by iterating with .filter(|u| u.active).count() + - Insert a UserStats row with key="total" and the total count + - Insert a UserStats row with key="active" and the active count diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt new file mode 100644 index 00000000000..ec13e16b72b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript that defines a user table and a stats table, with a reducer that counts total users and active users without collecting into an array. + +TABLES +- user + - Fields: + - id: number (u64, primary key, autoInc) + - name: string + - active: boolean + +- userStats + - Fields: + - key: string (primary key) + - count: number (u64) + +REDUCERS +- compute_user_counts(ctx) + - Count total users by iterating (do not collect into an array first) + - Count active users by iterating and checking active === true + - Insert a userStats row with key="total" and the total count + - Insert a userStats row with key="active" and the active count diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/csharp.cs new file mode 100644 index 00000000000..06861035482 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/csharp.cs @@ -0,0 +1,40 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "EventLog")] + [SpacetimeDB.Index.BTree(Accessor = "by_category_severity", Columns = new[] { nameof(EventLog.Category), nameof(EventLog.Severity) })] + public partial struct EventLog + { + [PrimaryKey] + [AutoInc] + public ulong Id; + public string Category; + public uint Severity; + public string Message; + } + + [Table(Accessor = "FilteredEvent")] + public partial struct FilteredEvent + { + [PrimaryKey] + public ulong EventId; + public string Message; + } + + [Reducer] + public static void FilterEvents(ReducerContext ctx, string category, uint severity) + { + foreach (var e in ctx.Db.EventLog.Iter()) + { + if (e.Category == category && e.Severity == severity) + { + ctx.Db.FilteredEvent.Insert(new FilteredEvent + { + EventId = e.Id, + Message = e.Message, + }); + } + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/rust.rs new file mode 100644 index 00000000000..ee45a854216 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/rust.rs @@ -0,0 +1,33 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table( + accessor = event_log, + index(accessor = by_category_severity, btree(columns = [category, severity])) +)] +pub struct EventLog { + #[primary_key] + #[auto_inc] + pub id: u64, + pub category: String, + pub severity: u32, + pub message: String, +} + +#[table(accessor = filtered_event)] +pub struct FilteredEvent { + #[primary_key] + pub event_id: u64, + pub message: String, +} + +#[reducer] +pub fn filter_events(ctx: &ReducerContext, category: String, severity: u32) { + for e in ctx.db.event_log().iter() { + if e.category == category && e.severity == severity { + ctx.db.filtered_event().insert(FilteredEvent { + event_id: e.id, + message: e.message, + }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/typescript.ts new file mode 100644 index 00000000000..26c7dc9b230 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/answers/typescript.ts @@ -0,0 +1,35 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const eventLog = table({ + name: 'event_log', + indexes: [{ name: 'byCategorySeverity', algorithm: 'btree', columns: ['category', 'severity'] }], +}, { + id: t.u64().primaryKey().autoInc(), + category: t.string(), + severity: t.u32(), + message: t.string(), +}); + +const filteredEvent = table({ + name: 'filtered_event', +}, { + eventId: t.u64().primaryKey(), + message: t.string(), +}); + +const spacetimedb = schema({ eventLog, filteredEvent }); +export default spacetimedb; + +export const filter_events = spacetimedb.reducer( + { category: t.string(), severity: t.u32() }, + (ctx, { category, severity }) => { + for (const e of ctx.db.eventLog.iter()) { + if (e.category === category && e.severity === severity) { + ctx.db.filteredEvent.insert({ + eventId: e.id, + message: e.message, + }); + } + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt new file mode 100644 index 00000000000..c1f49d42fac --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in C# that defines an EventLog table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. + +TABLES +- EventLog + - Struct: EventLog + - Fields: + - Id: ulong (primary key, AutoInc) + - Category: string + - Severity: uint + - Message: string + - Index: multi-column btree index on (Category, Severity) + +- FilteredEvent + - Struct: FilteredEvent + - Fields: + - EventId: ulong (primary key) + - Message: string + +REDUCERS +- FilterEvents(ctx, string category, uint severity) + - Iterate all EventLog rows + - For each row where Category matches AND Severity matches, insert into FilteredEvent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt new file mode 100644 index 00000000000..88516a7ab47 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in Rust that defines an event log table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. + +TABLES +- event_log + - Struct: EventLog + - Fields: + - id: u64 (primary key, auto_inc) + - category: String + - severity: u32 + - message: String + - Index: multi-column btree index on (category, severity) + +- filtered_event + - Struct: FilteredEvent + - Fields: + - event_id: u64 (primary key) + - message: String + +REDUCERS +- filter_events(ctx, category: String, severity: u32) + - Iterate all event_log rows + - For each row where category matches AND severity matches, insert into filtered_event diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt new file mode 100644 index 00000000000..ceecf39352a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript that defines an event log table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. + +TABLES +- eventLog + - Fields: + - id: number (u64, primary key, autoInc) + - category: string + - severity: number (u32) + - message: string + - Index: multi-column btree index on (category, severity) + +- filteredEvent + - Fields: + - eventId: number (u64, primary key) + - message: string + +REDUCERS +- filter_events(ctx, { category: string, severity: number (u32) }) + - Iterate all eventLog rows + - For each row where category matches AND severity matches, insert into filteredEvent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/answers/typescript.ts index 3db3a3fe393..8058f96f18f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_013_spacetime_sum_type/answers/typescript.ts @@ -26,6 +26,6 @@ export default spacetimedb; export const setCircle = spacetimedb.reducer( { id: t.i32(), radius: t.i32() }, (ctx, { id, radius }) => { - ctx.db.result.insert({ id, value: { circle: radius } }); + ctx.db.result.insert({ id, value: { tag: 'circle', value: radius } }); } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/csharp.cs index ac99835ffd2..ed5418d332a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/csharp.cs @@ -5,7 +5,7 @@ public static partial class Module [Table(Accessor = "Primitive")] public partial struct Primitive { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public int Count; public long Total; public float Price; @@ -18,7 +18,7 @@ public partial struct Primitive public static void Seed(ReducerContext ctx) { ctx.Db.Primitive.Insert(new Primitive { - Id = 1, + Id = 0, Count = 2, Total = 3000000000, Price = 1.5f, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/rust.rs index 56275bf3a65..b4da70fcfbe 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/rust.rs @@ -3,7 +3,8 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = primitive)] pub struct Primitive { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub count: i32, pub total: i64, pub price: f32, @@ -15,7 +16,7 @@ pub struct Primitive { #[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.primitive().insert(Primitive { - id: 1, + id: 0, count: 2, total: 3_000_000_000, price: 1.5, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/typescript.ts index 86af2a062a2..a67bb3a0bc3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/answers/typescript.ts @@ -5,7 +5,7 @@ const primitive = table( name: 'primitive', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), count: t.i32(), total: t.i64(), price: t.f32(), @@ -20,7 +20,7 @@ export default spacetimedb; export const seed = spacetimedb.reducer(ctx => { ctx.db.primitive.insert({ - id: 1, + id: 0n, count: 2, total: 3000000000n, price: 1.5, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/csharp.txt index 39b796edb42..3206723ba9f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/csharp.txt @@ -4,7 +4,7 @@ TABLE - Primitive - Struct: Primitive - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Count: int - Total: long - Price: float @@ -13,5 +13,5 @@ TABLE - Name: string REDUCERS -- Seed: insert exactly this row into Primitive - - (Id=1, Count=2, Total=3000000000, Price=1.5, Ratio=2.25, Active=true, Name="Alice") +- Seed: insert exactly this row into Primitive (Id is auto-assigned) + - (Count=2, Total=3000000000, Price=1.5, Ratio=2.25, Active=true, Name="Alice") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/rust.txt index 1777bf8c651..db922b17f3b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/rust.txt @@ -1,10 +1,10 @@ -Write a SpacetimeDB backend module in Rust that defines one table and seeds one row. +Write a SpacetimeDB backend module in Rust that defines one table and seeds one row. TABLE - primitive - Struct: Primitive - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - count: i32 - total: i64 - price: f32 @@ -13,5 +13,5 @@ TABLE - name: String REDUCERS -- seed: insert exactly this row into primitive - - (id=1, count=2, total=3000000000, price=1.5, ratio=2.25, active=true, name="Alice") +- seed: insert exactly this row into primitive (id is auto-assigned) + - (count=2, total=3000000000, price=1.5, ratio=2.25, active=true, name="Alice") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt index 269e2664680..ca3775db537 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that tests all elementary/primi TABLE - primitive - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - count: number (i32) - total: bigint (i64) - price: number (f32) @@ -12,5 +12,5 @@ TABLE - name: string REDUCERS -- seed: insert one row with all field types: - - (id=1, count=2, total=3000000000, price=1.5, ratio=2.25, active=true, name="Alice") +- seed: insert one row with all field types (id is auto-assigned): + - (count=2, total=3000000000, price=1.5, ratio=2.25, active=true, name="Alice") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/csharp.cs index f48a6aa1ed6..e3b699c5c19 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/csharp.cs @@ -19,7 +19,7 @@ public partial struct Position [Table(Accessor = "Profile", Public = true)] public partial struct Profile { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public Address Home; public Address Work; public Position Pos; @@ -29,7 +29,7 @@ public partial struct Profile public static void Seed(ReducerContext ctx) { ctx.Db.Profile.Insert(new Profile { - Id = 1, + Id = 0, Home = new Address { Street = "1 Main", Zip = 11111 }, Work = new Address { Street = "2 Broad", Zip = 22222 }, Pos = new Position { X = 7, Y = 9 } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/rust.rs index 60adb24f0db..6e3693de2a4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/rust.rs @@ -15,7 +15,8 @@ pub struct Position { #[table(accessor = profile)] pub struct Profile { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub home: Address, pub work: Address, pub pos: Position, @@ -24,7 +25,7 @@ pub struct Profile { #[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.profile().insert(Profile { - id: 1, + id: 0, home: Address { street: "1 Main".into(), zip: 11111, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/typescript.ts index 1bbb00d949d..3d8ee4a83b9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/answers/typescript.ts @@ -15,7 +15,7 @@ const profile = table( name: 'profile', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), home: Address, work: Address, pos: Position, @@ -27,7 +27,7 @@ export default spacetimedb; export const seed = spacetimedb.reducer(ctx => { ctx.db.profile.insert({ - id: 1, + id: 0n, home: { street: '1 Main', zip: 11111 }, work: { street: '2 Broad', zip: 22222 }, pos: { x: 7, y: 9 }, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/csharp.txt index 12a6b23fe19..e6f022047b9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/csharp.txt @@ -14,11 +14,11 @@ TABLE - Profile (public) - Struct: Profile - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Home: Address - Work: Address - Pos: Position REDUCERS -- Seed: insert exactly one row into Profile - - (Id=1, Home=Address{Street="1 Main", Zip=11111}, Work=Address{Street="2 Broad", Zip=22222}, Pos=Position{X=7, Y=9}) +- Seed: insert exactly one row into Profile (Id is auto-assigned) + - (Home=Address{Street="1 Main", Zip=11111}, Work=Address{Street="2 Broad", Zip=22222}, Pos=Position{X=7, Y=9}) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/rust.txt index e08ce27ac42..857c6bbc1e0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/rust.txt @@ -1,4 +1,4 @@ -Write a SpacetimeDB backend module in Rust that defines product types and uses them as columns in a table. +Write a SpacetimeDB backend module in Rust that defines product types and uses them as columns in a table. TYPES - Struct: Address @@ -14,11 +14,11 @@ TABLE - profile - Struct: Profile - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - home: Address - work: Address - pos: Position REDUCERS -- seed: insert exactly one row into profile - - (id=1, home=Address{street:"1 Main", zip:11111}, work=Address{street:"2 Broad", zip:22222}, pos=Position{x:7, y:9}) +- seed: insert exactly one row into profile (id is auto-assigned) + - (home=Address{street:"1 Main", zip:11111}, work=Address{street:"2 Broad", zip:22222}, pos=Position{x:7, y:9}) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt index 2a01fe158cb..78facfbf087 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt @@ -11,13 +11,13 @@ TYPES TABLE - profile - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - home: Address - work: Address - pos: Position REDUCERS -- seed: insert one row with all nested types: +- seed: insert one row with all nested types (id is auto-assigned): - home: { street: "1 Main", zip: 11111 } - work: { street: "2 Broad", zip: 22222 } - pos: { x: 7, y: 9 } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/csharp.cs index f6d10046228..e5cd8063c29 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/csharp.cs @@ -14,7 +14,7 @@ public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { [Table(Accessor = "Drawing")] public partial struct Drawing { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public Shape A; public Shape B; } @@ -23,7 +23,7 @@ public partial struct Drawing public static void Seed(ReducerContext ctx) { ctx.Db.Drawing.Insert(new Drawing { - Id = 1, + Id = 0, A = new Shape.Circle(new Circle { Radius = 10 }), B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 }) }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/rust.rs index f3e224b819e..4589fc19b2e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/rust.rs @@ -15,7 +15,8 @@ pub enum Shape { #[table(accessor = drawing)] pub struct Drawing { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub a: Shape, pub b: Shape, } @@ -23,7 +24,7 @@ pub struct Drawing { #[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.drawing().insert(Drawing { - id: 1, + id: 0, a: Shape::Circle(10), b: Shape::Rectangle(Rect { width: 4, height: 6 }), }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/typescript.ts index 7f350397c1c..9a6265de194 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/answers/typescript.ts @@ -13,7 +13,7 @@ const Shape = t.enum('Shape', { const drawing = table({ name: 'drawing', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), a: Shape, b: Shape, }); @@ -24,9 +24,9 @@ export default spacetimedb; export const seed = spacetimedb.reducer( ctx => { ctx.db.drawing.insert({ - id: 1, - a: { circle: 10 }, - b: { rectangle: { width: 4, height: 6 } }, + id: 0n, + a: { tag: 'circle', value: 10 }, + b: { tag: 'rectangle', value: { width: 4, height: 6 } }, }); } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/csharp.txt index 3acfc8c2c5e..eb872e09e84 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/csharp.txt @@ -14,10 +14,10 @@ TABLE - Drawing - Struct: Drawing - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - A: Shape - B: Shape REDUCERS -- Seed: insert exactly one row into Drawing - - (Id=1, A=Circle{Radius=10}, B=Rectangle{Width=4, Height=6}) +- Seed: insert exactly one row into Drawing (Id is auto-assigned) + - (A=Circle{Radius=10}, B=Rectangle{Width=4, Height=6}) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/rust.txt index a5a5cef4dac..f462516822b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/rust.txt @@ -1,4 +1,4 @@ -Write a SpacetimeDB backend module in Rust that defines a sum type and uses it in multiple table columns. +Write a SpacetimeDB backend module in Rust that defines a sum type and uses it in multiple table columns. TYPES - Struct: Rect @@ -14,10 +14,10 @@ TABLE - drawing - Struct: Drawing - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - a: Shape - b: Shape REDUCERS -- seed: insert exactly one row into drawing - - (id=1, a=Shape::Circle(10), b=Shape::Rectangle(Rect { width:4, height:6 })) +- seed: insert exactly one row into drawing (id is auto-assigned) + - (a=Shape::Circle(10), b=Shape::Rectangle(Rect { width:4, height:6 })) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt index 70392fea638..07eede1f9eb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt @@ -11,11 +11,11 @@ TYPES TABLE - drawing - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - a: Shape - b: Shape REDUCERS -- seed: insert one row with two Shape values: +- seed: insert one row with two Shape values (id is auto-assigned): - a: Shape.circle(10) - b: Shape.rectangle({ width: 4, height: 6 }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/answers/typescript.ts index afac1e747c5..1a06c3d4db0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/answers/typescript.ts @@ -3,7 +3,7 @@ import { table, schema, t } from 'spacetimedb/server'; const tickTimer = table( { - name: 'tickTimer', + name: 'tick_timer', scheduled: (): any => tick, }, { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/csharp.cs index 4bdb78de75a..751a11b6ee4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/csharp.cs @@ -6,7 +6,7 @@ public static partial class Module [SpacetimeDB.Index.BTree(Accessor = "by_name", Columns = [nameof(Name)])] public partial struct Account { - [SpacetimeDB.PrimaryKey] public int Id; + [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc] public ulong Id; [SpacetimeDB.Unique] public string Email; public string Name; } @@ -14,7 +14,7 @@ public partial struct Account [SpacetimeDB.Reducer] public static void Seed(ReducerContext ctx) { - ctx.Db.Account.Insert(new Account { Id = 1, Email = "a@example.com", Name = "Alice" }); - ctx.Db.Account.Insert(new Account { Id = 2, Email = "b@example.com", Name = "Bob" }); + ctx.Db.Account.Insert(new Account { Id = 0, Email = "a@example.com", Name = "Alice" }); + ctx.Db.Account.Insert(new Account { Id = 0, Email = "b@example.com", Name = "Bob" }); } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/rust.rs index 3e2da6e82f4..94855e2f6ee 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/rust.rs @@ -6,7 +6,8 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; )] pub struct Account { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, #[unique] pub email: String, pub name: String, @@ -15,12 +16,12 @@ pub struct Account { #[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.account().insert(Account { - id: 1, + id: 0, email: "a@example.com".into(), name: "Alice".into(), }); ctx.db.account().insert(Account { - id: 2, + id: 0, email: "b@example.com".into(), name: "Bob".into(), }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/typescript.ts index 85fa8d0ea46..50d9f9c1dae 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/answers/typescript.ts @@ -4,7 +4,7 @@ const account = table({ name: 'account', indexes: [{ name: 'byName', algorithm: 'btree', columns: ['name'] }], }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), email: t.string().unique(), name: t.string(), }); @@ -14,7 +14,7 @@ export default spacetimedb; export const seed = spacetimedb.reducer( ctx => { - ctx.db.account.insert({ id: 1, email: "a@example.com", name: "Alice" }); - ctx.db.account.insert({ id: 2, email: "b@example.com", name: "Bob" }); + ctx.db.account.insert({ id: 0n, email: "a@example.com", name: "Alice" }); + ctx.db.account.insert({ id: 0n, email: "b@example.com", name: "Bob" }); } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/csharp.txt index 39b7f4cfbcd..c9c689d85ae 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/csharp.txt @@ -4,13 +4,13 @@ TABLE - Account (public) - Struct: Account - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Email: string (unique) - Name: string - Indexes: - by_name: btree(Name) REDUCERS -- Seed: insert exactly these rows into Account - - (Id=1, Email="a@example.com", Name="Alice") - - (Id=2, Email="b@example.com", Name="Bob") +- Seed: insert exactly these rows into Account (IDs are auto-assigned) + - (Email="a@example.com", Name="Alice") + - (Email="b@example.com", Name="Bob") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/rust.txt index 34873235c47..f6a689c3213 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/rust.txt @@ -1,16 +1,16 @@ -Write a SpacetimeDB backend module in Rust that defines one table and seeds two rows. +Write a SpacetimeDB backend module in Rust that defines one table and seeds two rows. TABLE - account - Struct: Account - Columns: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - email: String (unique) - name: String - Indexes: - by_name: btree(name) REDUCERS -- seed: insert exactly these rows into account - - (1, "a@example.com", "Alice") - - (2, "b@example.com", "Bob") \ No newline at end of file +- seed: insert exactly these rows into account (IDs are auto-assigned) + - (email="a@example.com", name="Alice") + - (email="b@example.com", name="Bob") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt index 3a4475bd232..685165ab9e9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt @@ -3,12 +3,12 @@ Write a SpacetimeDB backend module in TypeScript that demonstrates unique constr TABLE - account - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - email: string (unique) - name: string - Index: byName (btree on name) REDUCERS -- seed: insert two rows: - - (id=1, email="a@example.com", name="Alice") - - (id=2, email="b@example.com", name="Bob") +- seed: insert two rows (IDs are auto-assigned): + - (email="a@example.com", name="Alice") + - (email="b@example.com", name="Bob") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/csharp.cs index 13012c1eb2b..cf0044c52a9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/csharp.cs @@ -5,14 +5,14 @@ public static partial class Module [Table(Accessor = "User")] public partial struct User { - [PrimaryKey] public int UserId; + [PrimaryKey, AutoInc] public ulong UserId; public string Name; } [Table(Accessor = "Group")] public partial struct Group { - [PrimaryKey] public int GroupId; + [PrimaryKey, AutoInc] public ulong GroupId; public string Title; } @@ -21,22 +21,22 @@ public partial struct Group [SpacetimeDB.Index.BTree(Accessor = "by_group", Columns = new[] { nameof(GroupId) })] public partial struct Membership { - [PrimaryKey] public int Id; - public int UserId; - public int GroupId; + [PrimaryKey, AutoInc] public ulong Id; + public ulong UserId; + public ulong GroupId; } [Reducer] public static void Seed(ReducerContext ctx) { - ctx.Db.User.Insert(new User { UserId = 1, Name = "Alice" }); - ctx.Db.User.Insert(new User { UserId = 2, Name = "Bob" }); + ctx.Db.User.Insert(new User { UserId = 0, Name = "Alice" }); + ctx.Db.User.Insert(new User { UserId = 0, Name = "Bob" }); - ctx.Db.Group.Insert(new Group { GroupId = 10, Title = "Admin" }); - ctx.Db.Group.Insert(new Group { GroupId = 20, Title = "Dev" }); + ctx.Db.Group.Insert(new Group { GroupId = 0, Title = "Admin" }); + ctx.Db.Group.Insert(new Group { GroupId = 0, Title = "Dev" }); - ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 }); - ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 }); - ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 }); + ctx.Db.Membership.Insert(new Membership { Id = 0, UserId = 1, GroupId = 1 }); + ctx.Db.Membership.Insert(new Membership { Id = 0, UserId = 1, GroupId = 2 }); + ctx.Db.Membership.Insert(new Membership { Id = 0, UserId = 2, GroupId = 2 }); } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/rust.rs index a3645cda64c..5feb4da052a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/rust.rs @@ -3,14 +3,16 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = user)] pub struct User { #[primary_key] - pub user_id: i32, + #[auto_inc] + pub user_id: u64, pub name: String, } #[table(accessor = group)] pub struct Group { #[primary_key] - pub group_id: i32, + #[auto_inc] + pub group_id: u64, pub title: String, } @@ -21,44 +23,45 @@ pub struct Group { )] pub struct Membership { #[primary_key] - pub id: i32, - pub user_id: i32, - pub group_id: i32, + #[auto_inc] + pub id: u64, + pub user_id: u64, + pub group_id: u64, } #[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.user().insert(User { - user_id: 1, + user_id: 0, name: "Alice".into(), }); ctx.db.user().insert(User { - user_id: 2, + user_id: 0, name: "Bob".into(), }); ctx.db.group().insert(Group { - group_id: 10, + group_id: 0, title: "Admin".into(), }); ctx.db.group().insert(Group { - group_id: 20, + group_id: 0, title: "Dev".into(), }); ctx.db.membership().insert(Membership { - id: 1, + id: 0, user_id: 1, - group_id: 10, + group_id: 1, }); ctx.db.membership().insert(Membership { - id: 2, + id: 0, user_id: 1, - group_id: 20, + group_id: 2, }); ctx.db.membership().insert(Membership { - id: 3, + id: 0, user_id: 2, - group_id: 20, + group_id: 2, }); } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/typescript.ts index 737e5cfb3c7..d7629137dcc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/answers/typescript.ts @@ -5,7 +5,7 @@ const user = table( name: 'user', }, { - userId: t.i32().primaryKey(), + userId: t.u64().primaryKey().autoInc(), name: t.string(), } ); @@ -15,7 +15,7 @@ const group = table( name: 'group', }, { - groupId: t.i32().primaryKey(), + groupId: t.u64().primaryKey().autoInc(), title: t.string(), } ); @@ -29,9 +29,9 @@ const membership = table( ], }, { - id: t.i32().primaryKey(), - userId: t.i32(), - groupId: t.i32(), + id: t.u64().primaryKey().autoInc(), + userId: t.u64(), + groupId: t.u64(), } ); @@ -39,13 +39,13 @@ const spacetimedb = schema({ user, group, membership }); export default spacetimedb; export const seed = spacetimedb.reducer(ctx => { - ctx.db.user.insert({ userId: 1, name: 'Alice' }); - ctx.db.user.insert({ userId: 2, name: 'Bob' }); + ctx.db.user.insert({ userId: 0n, name: 'Alice' }); + ctx.db.user.insert({ userId: 0n, name: 'Bob' }); - ctx.db.group.insert({ groupId: 10, title: 'Admin' }); - ctx.db.group.insert({ groupId: 20, title: 'Dev' }); + ctx.db.group.insert({ groupId: 0n, title: 'Admin' }); + ctx.db.group.insert({ groupId: 0n, title: 'Dev' }); - ctx.db.membership.insert({ id: 1, userId: 1, groupId: 10 }); - ctx.db.membership.insert({ id: 2, userId: 1, groupId: 20 }); - ctx.db.membership.insert({ id: 3, userId: 2, groupId: 20 }); + ctx.db.membership.insert({ id: 0n, userId: 1n, groupId: 1n }); + ctx.db.membership.insert({ id: 0n, userId: 1n, groupId: 2n }); + ctx.db.membership.insert({ id: 0n, userId: 2n, groupId: 2n }); }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/spec.rs index b19116e2e7c..c37c12a3315 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/spec.rs @@ -22,10 +22,10 @@ pub fn spec() -> BenchmarkSpec { reducer: reducer_name.into(), args: vec![], sql_count_query: format!( - "SELECT COUNT(*) AS n FROM {membership_table} WHERE {user_id}=1 AND {group_id}=10" + "SELECT COUNT(*) AS n FROM {membership_table} WHERE {user_id}=1 AND {group_id}=1" ), expected_count: 1, - id_str: "m2m_has_1_10", + id_str: "m2m_has_1_1", timeout: Duration::from_secs(10), }, )); @@ -34,9 +34,9 @@ pub fn spec() -> BenchmarkSpec { host_url, file!(), route_tag, - &format!("SELECT COUNT(*) AS n FROM {membership_table} WHERE {user_id}=1 AND {group_id}=20"), + &format!("SELECT COUNT(*) AS n FROM {membership_table} WHERE {user_id}=1 AND {group_id}=2"), 1, - "m2m_has_1_20", + "m2m_has_1_2", Duration::from_secs(10), )); @@ -44,9 +44,9 @@ pub fn spec() -> BenchmarkSpec { host_url, file!(), route_tag, - &format!("SELECT COUNT(*) AS n FROM {membership_table} WHERE {user_id}=2 AND {group_id}=20"), + &format!("SELECT COUNT(*) AS n FROM {membership_table} WHERE {user_id}=2 AND {group_id}=2"), 1, - "m2m_has_2_20", + "m2m_has_2_2", Duration::from_secs(10), )); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt index 3ae80432393..6b71011debf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt @@ -4,30 +4,30 @@ TABLES - User - Struct: User - Fields: - - UserId: int (primary key) + - UserId: ulong (primary key, auto-increment) - Name: string - Group - Struct: Group - Fields: - - GroupId: int (primary key) + - GroupId: ulong (primary key, auto-increment) - Title: string - Membership - Struct: Membership - Fields: - - Id: int (primary key) - - UserId: int - - GroupId: int + - Id: ulong (primary key, auto-increment) + - UserId: ulong + - GroupId: ulong - Indexes: - by_user: btree(UserId) - by_group: btree(GroupId) REDUCERS -- Seed: insert exactly these rows - - User: (UserId=1, Name="Alice"), (UserId=2, Name="Bob") - - Group: (GroupId=10, Title="Admin"), (GroupId=20, Title="Dev") +- Seed: insert exactly these rows (IDs are auto-assigned) + - User: (Name="Alice"), (Name="Bob") + - Group: (Title="Admin"), (Title="Dev") - Membership: - - (Id=1, UserId=1, GroupId=10) - - (Id=2, UserId=1, GroupId=20) - - (Id=3, UserId=2, GroupId=20) + - (UserId=1, GroupId=1) + - (UserId=1, GroupId=2) + - (UserId=2, GroupId=2) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt index 1080d4f9acf..a8e727bccf8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt @@ -1,33 +1,33 @@ -Write a SpacetimeDB backend module in Rust that defines three tables modeling a many-to-many relationship and seeds rows. +Write a SpacetimeDB backend module in Rust that defines three tables modeling a many-to-many relationship and seeds rows. TABLES - user - Struct: User - Fields: - - user_id: i32 (primary key) + - user_id: u64 (primary key, auto_inc) - name: String - group - Struct: Group - Fields: - - group_id: i32 (primary key) + - group_id: u64 (primary key, auto_inc) - title: String - membership - Struct: Membership - Fields: - - id: i32 (primary key) - - user_id: i32 - - group_id: i32 + - id: u64 (primary key, auto_inc) + - user_id: u64 + - group_id: u64 - Indexes: - by_user: btree(user_id) - by_group: btree(group_id) REDUCERS -- seed: insert exactly these rows - - user: (user_id=1, name="Alice"), (user_id=2, name="Bob") - - group: (group_id=10, title="Admin"), (group_id=20, title="Dev") +- seed: insert exactly these rows (IDs are auto-assigned) + - user: (name="Alice"), (name="Bob") + - group: (title="Admin"), (title="Dev") - membership: - - (id=1, user_id=1, group_id=10) - - (id=2, user_id=1, group_id=20) - - (id=3, user_id=2, group_id=20) + - (user_id=1, group_id=1) + - (user_id=1, group_id=2) + - (user_id=2, group_id=2) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt index a93353da982..1025911c166 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt @@ -3,25 +3,25 @@ Write a SpacetimeDB backend module in TypeScript that models many-to-many relati TABLES - user - Fields: - - userId: number (i32, primary key) + - userId: number (u64, primary key, autoInc) - name: string - group - Fields: - - groupId: number (i32, primary key) + - groupId: number (u64, primary key, autoInc) - title: string - membership - Fields: - - id: number (i32, primary key) - - userId: number (i32) - - groupId: number (i32) + - id: number (u64, primary key, autoInc) + - userId: number (u64) + - groupId: number (u64) - Indexes: - byUser (btree on userId) - byGroup (btree on groupId) REDUCERS -- seed: insert users, groups, and memberships: - - user: (1, "Alice"), (2, "Bob") - - group: (10, "Admin"), (20, "Dev") - - membership: (1, userId=1, groupId=10), (2, userId=1, groupId=20), (3, userId=2, groupId=20) +- seed: insert users, groups, and memberships (IDs are auto-assigned): + - user: ("Alice"), ("Bob") + - group: ("Admin"), ("Dev") + - membership: (userId=1, groupId=1), (userId=1, groupId=2), (userId=2, groupId=2) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/csharp.cs index cc64eccf5d0..493d0f12ce5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/csharp.cs @@ -3,7 +3,7 @@ public static partial class Module { [Table(Accessor = "Entity")] - public partial struct Entity { [PrimaryKey] public int Id; } + public partial struct Entity { [PrimaryKey, AutoInc] public ulong Id; } [Table(Accessor = "Position")] public partial struct Position @@ -32,8 +32,8 @@ public partial struct NextPosition [Reducer] public static void Seed(ReducerContext ctx) { - ctx.Db.Entity.Insert(new Entity { Id = 1 }); - ctx.Db.Entity.Insert(new Entity { Id = 2 }); + ctx.Db.Entity.Insert(new Entity { Id = 0 }); + ctx.Db.Entity.Insert(new Entity { Id = 0 }); ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 }); ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/rust.rs index 79377c93056..bbd91df2fa2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/rust.rs @@ -3,7 +3,8 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; #[table(accessor = entity)] pub struct Entity { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, } #[table(accessor = position)] @@ -32,12 +33,12 @@ pub struct NextPosition { #[reducer] pub fn seed(ctx: &ReducerContext) { - ctx.db.entity().insert(Entity { id: 1 }); - ctx.db.entity().insert(Entity { id: 2 }); + ctx.db.entity().insert(Entity { id: 0 }); + ctx.db.entity().insert(Entity { id: 0 }); ctx.db.position().insert(Position { entity_id: 1, - x: 1, + x: 0, y: 0, }); ctx.db.position().insert(Position { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/typescript.ts index 5e44657f6a1..ddb439d67e6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/answers/typescript.ts @@ -5,7 +5,7 @@ const entity = table( name: 'entity', }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), } ); @@ -33,7 +33,7 @@ const velocity = table( const nextPosition = table( { - name: 'nextPosition', + name: 'next_position', }, { entityId: t.i32().primaryKey(), @@ -46,8 +46,8 @@ const spacetimedb = schema({ entity, position, velocity, nextPosition }); export default spacetimedb; export const seed = spacetimedb.reducer(ctx => { - ctx.db.entity.insert({ id: 1 }); - ctx.db.entity.insert({ id: 2 }); + ctx.db.entity.insert({ id: 0n }); + ctx.db.entity.insert({ id: 0n }); ctx.db.position.insert({ entityId: 1, x: 0, y: 0 }); ctx.db.position.insert({ entityId: 2, x: 10, y: 0 }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/spec.rs index 1b397152518..d303b8f86c2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/spec.rs @@ -1,5 +1,5 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig, SqlBuilder}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, SqlBuilder}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { @@ -18,59 +18,65 @@ pub fn spec() -> BenchmarkSpec { let position_table = table_name("position", lang); let next_position_table = table_name("next_position", lang); - let base = |reducer: &str| ReducerSqlCountConfig { - src_file: file!(), + // Seed once + v.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + &seed, + vec![], + "ecs_seed", + )); + + v.push(make_sql_count_only_scorer( + host_url, + file!(), route_tag, - reducer: reducer.to_string(), - args: vec![], - sql_count_query: String::new(), - expected_count: 0, - id_str: "", - timeout: Duration::from_secs(10), - }; + format!("SELECT COUNT(*) AS n FROM {position_table}"), + 2, + "ecs_seed_position_count", + Duration::from_secs(10), + )); - v.push(make_reducer_sql_count_scorer( + // Step once + v.push(make_reducer_call_both_scorer( host_url, - ReducerSqlCountConfig { - sql_count_query: format!("SELECT COUNT(*) AS n FROM {position_table}"), - expected_count: 2, - id_str: "ecs_seed_position_count", - ..base(&seed) // or base("seed") if it's a &str - }, + file!(), + route_tag, + &step, + vec![], + "ecs_step", )); - v.push(make_reducer_sql_count_scorer( + // Then just query + v.push(make_sql_count_only_scorer( host_url, - ReducerSqlCountConfig { - sql_count_query: format!("SELECT COUNT(*) AS n FROM {next_position_table}"), - expected_count: 2, - id_str: "ecs_step_next_position_count", - ..base(&step) // or base("step") - }, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {next_position_table}"), + 2, + "ecs_step_next_position_count", + Duration::from_secs(10), )); - v.push(make_reducer_sql_count_scorer( + v.push(make_sql_count_only_scorer( host_url, - ReducerSqlCountConfig { - sql_count_query: format!( - "SELECT COUNT(*) AS n FROM {next_position_table} WHERE {entity_id}=1 AND {x}=1 AND {y}=0", - ), - expected_count: 1, - id_str: "ecs_next_pos_entity1", - ..base(&step) - }, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {next_position_table} WHERE {entity_id}=1 AND {x}=1 AND {y}=0",), + 1, + "ecs_next_pos_entity1", + Duration::from_secs(10), )); - v.push(make_reducer_sql_count_scorer( + v.push(make_sql_count_only_scorer( host_url, - ReducerSqlCountConfig { - sql_count_query: format!( - "SELECT COUNT(*) AS n FROM {next_position_table} WHERE {entity_id}=2 AND {x}=8 AND {y}=3", - ), - expected_count: 1, - id_str: "ecs_next_pos_entity2", - ..base(&step) - }, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {next_position_table} WHERE {entity_id}=2 AND {x}=8 AND {y}=3",), + 1, + "ecs_next_pos_entity2", + Duration::from_secs(10), )); v diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt index eba64c88a55..9e192d2500a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt @@ -4,31 +4,31 @@ TABLES - Entity - Struct: Entity - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - Position - Struct: Position - Fields: - - EntityId: int (primary key) + - EntityId: int (primary key) — NOT auto-increment, this is a foreign key to Entity - X: int - Y: int - Velocity - Struct: Velocity - Fields: - - EntityId: int (primary key) + - EntityId: int (primary key) — NOT auto-increment, this is a foreign key to Entity - VX: int - VY: int - NextPosition - Struct: NextPosition - Fields: - - EntityId: int (primary key) + - EntityId: int (primary key) — NOT auto-increment, this is a foreign key to Entity - X: int - Y: int REDUCERS -- Seed: insert two entities with positions and velocities: - - (Id=1, Pos=(0,0), Vel=(1,0)) - - (Id=2, Pos=(10,0), Vel=(-2,3)) +- Seed: insert two entities with positions and velocities (entity IDs are auto-assigned): + - (Pos=(0,0), Vel=(1,0)) + - (Pos=(10,0), Vel=(-2,3)) - Step: for each Position, find Velocity by EntityId; compute (X+VX, Y+VY) and upsert into NextPosition by EntityId diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt index 47bc5b41a46..a5c801fae43 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt @@ -1,34 +1,34 @@ -Write a SpacetimeDB backend module in Rust that models a minimal ECS and computes next positions. +Write a SpacetimeDB backend module in Rust that models a minimal ECS and computes next positions. TABLES - entity - Struct: Entity - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - position - Struct: Position - Fields: - - entity_id: i32 (primary key) + - entity_id: i32 (primary key) — NOT auto_inc, this is a foreign key to entity - x: i32 - y: i32 - velocity - Struct: Velocity - Fields: - - entity_id: i32 (primary key) + - entity_id: i32 (primary key) — NOT auto_inc, this is a foreign key to entity - vx: i32 - vy: i32 - next_position - Struct: NextPosition - Fields: - - entity_id: i32 (primary key) + - entity_id: i32 (primary key) — NOT auto_inc, this is a foreign key to entity - x: i32 - y: i32 REDUCERS -- seed: insert the two entities with positions/velocities: - - (id=1, pos=(0,0), vel=(1,0)) - - (id=2, pos=(10,0), vel=(-2,3)) +- seed: insert the two entities with positions/velocities (entity IDs are auto-assigned): + - Entity 1: pos=(0,0), vel=(1,0) + - Entity 2: pos=(10,0), vel=(-2,3) - step: for each position, find velocity by entity_id; compute (x+vx, y+vy) and upsert into next_position by entity_id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt index 3b3f6a9014b..443c43aad10 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt @@ -3,28 +3,28 @@ Write a SpacetimeDB backend module in TypeScript that models a minimal Entity Co TABLES - entity - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - position - Fields: - - entityId: number (i32, primary key) + - entityId: number (i32, primary key) — NOT autoInc, this is a foreign key to entity - x: number (i32) - y: number (i32) - velocity - Fields: - - entityId: number (i32, primary key) + - entityId: number (i32, primary key) — NOT autoInc, this is a foreign key to entity - vx: number (i32) - vy: number (i32) - nextPosition - Fields: - - entityId: number (i32, primary key) + - entityId: number (i32, primary key) — NOT autoInc, this is a foreign key to entity - x: number (i32) - y: number (i32) REDUCERS -- seed: insert 2 entities with positions and velocities: +- seed: insert 2 entities with positions and velocities (entity IDs are auto-assigned): - Entity 1: position(0,0), velocity(1,0) - Entity 2: position(10,0), velocity(-2,3) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/csharp.cs index 53cb9f6c42e..e3566cf9a4f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/csharp.cs @@ -6,7 +6,7 @@ public static partial class Module [SpacetimeDB.Index.BTree(Accessor = "by_user_day", Columns = new[] { nameof(UserId), nameof(Day) })] public partial struct Log { - [PrimaryKey] public int Id; + [PrimaryKey, AutoInc] public ulong Id; public int UserId; public int Day; public string Message; @@ -15,8 +15,8 @@ public partial struct Log [Reducer] public static void Seed(ReducerContext ctx) { - ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = "a" }); - ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = "b" }); - ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = "c" }); + ctx.Db.Log.Insert(new Log { Id = 0, UserId = 7, Day = 1, Message = "a" }); + ctx.Db.Log.Insert(new Log { Id = 0, UserId = 7, Day = 2, Message = "b" }); + ctx.Db.Log.Insert(new Log { Id = 0, UserId = 9, Day = 1, Message = "c" }); } } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/rust.rs index 6b6c7a621b1..d5bead5e1e6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/rust.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/rust.rs @@ -6,7 +6,8 @@ use spacetimedb::{reducer, table, ReducerContext, Table}; )] pub struct Log { #[primary_key] - pub id: i32, + #[auto_inc] + pub id: u64, pub user_id: i32, pub day: i32, pub message: String, @@ -15,19 +16,19 @@ pub struct Log { #[reducer] pub fn seed(ctx: &ReducerContext) { ctx.db.log().insert(Log { - id: 1, + id: 0, user_id: 7, day: 1, message: "a".into(), }); ctx.db.log().insert(Log { - id: 2, + id: 0, user_id: 7, day: 2, message: "b".into(), }); ctx.db.log().insert(Log { - id: 3, + id: 0, user_id: 9, day: 1, message: "c".into(), diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/typescript.ts index 66d5bf7a724..5d5fb568d7b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/typescript.ts +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/answers/typescript.ts @@ -4,7 +4,7 @@ const log = table({ name: 'log', indexes: [{ name: 'byUserDay', algorithm: 'btree', columns: ['userId', 'day'] }], }, { - id: t.i32().primaryKey(), + id: t.u64().primaryKey().autoInc(), userId: t.i32(), day: t.i32(), message: t.string(), @@ -15,8 +15,8 @@ export default spacetimedb; export const seed = spacetimedb.reducer( ctx => { - ctx.db.log.insert({ id: 1, userId: 7, day: 1, message: "a" }); - ctx.db.log.insert({ id: 2, userId: 7, day: 2, message: "b" }); - ctx.db.log.insert({ id: 3, userId: 9, day: 1, message: "c" }); + ctx.db.log.insert({ id: 0n, userId: 7, day: 1, message: "a" }); + ctx.db.log.insert({ id: 0n, userId: 7, day: 2, message: "b" }); + ctx.db.log.insert({ id: 0n, userId: 9, day: 1, message: "c" }); } ); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/spec.rs index bf3e6715b60..bbbc46be9d7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/spec.rs @@ -1,5 +1,5 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; -use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig, SqlBuilder}; +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, SqlBuilder}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { @@ -15,53 +15,53 @@ pub fn spec() -> BenchmarkSpec { let user_id = ident("user_id", sb.case); let day = ident("day", sb.case); - let base = |reducer: &str| ReducerSqlCountConfig { - src_file: file!(), + // Seed once via reducer on both DBs + v.push(make_reducer_call_both_scorer( + host_url, + file!(), route_tag, - reducer: reducer.to_string(), - args: vec![], - sql_count_query: String::new(), // override per case - expected_count: 0, // override per case - id_str: "", - timeout: Duration::from_secs(10), - }; + &seed, + vec![], + "mcindex_seed", + )); - v.push(make_reducer_sql_count_scorer( + // Then just query — don't call seed again + v.push(make_sql_count_only_scorer( host_url, - ReducerSqlCountConfig { - sql_count_query: format!("SELECT COUNT(*) AS n FROM {log_table}"), - expected_count: 3, - id_str: "mcindex_seed_count", - ..base(&seed) - }, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {log_table}"), + 3, + "mcindex_seed_count", + Duration::from_secs(10), )); - v.push(make_reducer_sql_count_scorer( + v.push(make_sql_count_only_scorer( host_url, - ReducerSqlCountConfig { - sql_count_query: format!( - "SELECT COUNT(*) AS n FROM {log_table} WHERE {u}=7 AND {d}=1", - u = user_id, - d = day - ), - expected_count: 1, - id_str: "mcindex_lookup_u7_d1", - ..base(&seed) - }, + file!(), + route_tag, + format!( + "SELECT COUNT(*) AS n FROM {log_table} WHERE {u}=7 AND {d}=1", + u = user_id, + d = day + ), + 1, + "mcindex_lookup_u7_d1", + Duration::from_secs(10), )); - v.push(make_reducer_sql_count_scorer( + v.push(make_sql_count_only_scorer( host_url, - ReducerSqlCountConfig { - sql_count_query: format!( - "SELECT COUNT(*) AS n FROM {log_table} WHERE {u}=7 AND {d}=2", - u = user_id, - d = day - ), - expected_count: 1, - id_str: "mcindex_lookup_u7_d2", - ..base(&seed) - }, + file!(), + route_tag, + format!( + "SELECT COUNT(*) AS n FROM {log_table} WHERE {u}=7 AND {d}=2", + u = user_id, + d = day + ), + 1, + "mcindex_lookup_u7_d2", + Duration::from_secs(10), )); v diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/csharp.txt index d8ab6ed41a2..c35f379e54b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/csharp.txt @@ -4,7 +4,7 @@ TABLE - Log - Struct: Log - Fields: - - Id: int (primary key) + - Id: ulong (primary key, auto-increment) - UserId: int - Day: int - Message: string @@ -12,7 +12,7 @@ TABLE - by_user_day: btree(UserId, Day) REDUCERS -- Seed: insert exactly these rows into Log - - (Id=1, UserId=7, Day=1, Message="a") - - (Id=2, UserId=7, Day=2, Message="b") - - (Id=3, UserId=9, Day=1, Message="c") +- Seed: insert exactly these rows into Log (IDs are auto-assigned) + - (UserId=7, Day=1, Message="a") + - (UserId=7, Day=2, Message="b") + - (UserId=9, Day=1, Message="c") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/rust.txt index 111ee2483ff..d70a0539eba 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/rust.txt @@ -1,10 +1,10 @@ -Write a SpacetimeDB backend module in Rust that defines one table with a multi-column B-Tree index and seeds rows. +Write a SpacetimeDB backend module in Rust that defines one table with a multi-column B-Tree index and seeds rows. TABLE - log - Struct: Log - Fields: - - id: i32 (primary key) + - id: u64 (primary key, auto_inc) - user_id: i32 - day: i32 - message: String @@ -12,7 +12,7 @@ TABLE - by_user_day: btree(user_id, day) REDUCERS -- seed: insert exactly these rows into log - - (id=1, user_id=7, day=1, message="a") - - (id=2, user_id=7, day=2, message="b") - - (id=3, user_id=9, day=1, message="c") +- seed: insert exactly these rows into log (IDs are auto-assigned) + - (user_id=7, day=1, message="a") + - (user_id=7, day=2, message="b") + - (user_id=9, day=1, message="c") diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt index 7449f38b60d..9615de680f3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt @@ -3,14 +3,14 @@ Write a SpacetimeDB backend module in TypeScript that demonstrates multi-column TABLE - log - Fields: - - id: number (i32, primary key) + - id: number (u64, primary key, autoInc) - userId: number (i32) - day: number (i32) - message: string - Index: byUserDay (btree on userId, day) REDUCERS -- seed: insert three rows: - - (id=1, userId=7, day=1, message="a") - - (id=2, userId=7, day=2, message="b") - - (id=3, userId=9, day=1, message="c") +- seed: insert three rows (IDs are auto-assigned): + - (userId=7, day=1, message="a") + - (userId=7, day=2, message="b") + - (userId=9, day=1, message="c") diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 12568e725b1..b9ccb0f71ec 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -1,6 +1,6 @@ #![allow(clippy::disallowed_macros, clippy::type_complexity, clippy::enum_variant_names)] -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use clap::{Args, Parser, Subcommand}; use futures::{StreamExt, TryStreamExt}; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet}; @@ -14,17 +14,13 @@ use xtask_llm_benchmark::bench::bench_route_concurrency; use xtask_llm_benchmark::bench::runner::{ build_goldens_only_for_lang, ensure_goldens_built_once, run_selected_or_all_for_model_async_for_lang, }; +use xtask_llm_benchmark::api::ApiClient; use xtask_llm_benchmark::bench::types::{BenchRunContext, RouteRun, RunConfig, RunOutcome}; -use xtask_llm_benchmark::context::constants::{ - docs_benchmark_comment, docs_benchmark_details, docs_benchmark_summary, llm_comparison_details, - llm_comparison_summary, ALL_MODES, -}; -use xtask_llm_benchmark::context::{build_context, compute_processed_context_hash, docs_dir}; +use xtask_llm_benchmark::context::constants::ALL_MODES; +use xtask_llm_benchmark::context::{build_context, compute_processed_context_hash}; use xtask_llm_benchmark::eval::Lang; use xtask_llm_benchmark::llm::types::Vendor; use xtask_llm_benchmark::llm::{default_model_routes, make_provider_from_env, LlmProvider, ModelRoute}; -use xtask_llm_benchmark::results::io::{update_golden_answers_on_disk, write_summary_from_details_file}; -use xtask_llm_benchmark::results::{load_summary, Summary}; #[derive(Clone, Debug)] struct ModelGroup { @@ -80,26 +76,8 @@ enum Commands { /// Run benchmarks / build goldens / compute hashes. Run(RunArgs), - /// Check-only: ensure required mode exists per language and hashes match saved run. - CiCheck(CiCheckArgs), - - /// Quickfix CI by running a minimal OpenAI model set. - CiQuickfix, - - /// Generate markdown comment for GitHub PR (compares against master baseline). - CiComment(CiCommentArgs), - - /// Regenerate summary.json from details.json (optionally custom paths). - Summary(SummaryArgs), - - /// Analyze benchmark failures and generate a human-readable markdown report. + /// Run AI analysis on existing benchmark failures from the database. Analyze(AnalyzeArgs), - - /// Count failures due to HTTP/API errors (429, 503, timeouts, etc.) and list task IDs to rerun. - CountHttpFailures(CountHttpFailuresArgs), - - /// Scan details file for LLM API failures and print rerun commands per (lang, model). Does not run anything. - ScanRerunCommands(ScanRerunCommandsArgs), } #[derive(Args, Debug, Clone)] @@ -145,62 +123,33 @@ struct RunArgs { /// --models "anthropic:Claude 4.5 Sonnet" --models openai:gpt-5 #[arg(long, num_args = 1..)] models: Option>, -} -#[derive(Args, Debug, Clone)] -struct CiCheckArgs { - /// Optional: one or more languages (default: rust,csharp) - #[arg(long, num_args = 1.., value_delimiter = ',')] - lang: Option>, -} - -#[derive(Args, Debug, Clone)] -struct SummaryArgs { - /// Optional input details.json (default: results_path_details()) - details: Option, - - /// Optional output summary.json (default: results_path_summary()) - summary: Option, -} - -#[derive(Args, Debug, Clone)] -struct CountHttpFailuresArgs { - /// Input details.json (default: same as run output, llm-comparison-details.json) + /// Run benchmarks without uploading results #[arg(long)] - details: Option, + dry_run: bool, } #[derive(Args, Debug, Clone)] -struct ScanRerunCommandsArgs { - /// Input details.json (default: llm-comparison-details.json) +struct AnalyzeArgs { + /// Filter by language (e.g. rust, csharp, typescript) #[arg(long)] - details: Option, -} + lang: Option, -#[derive(Args, Debug, Clone)] -struct AnalyzeArgs { - /// Input details.json file (default: docs-benchmark-details.json) + /// Filter by mode (e.g. guidelines, no_context, docs) #[arg(long)] - details: Option, + mode: Option, - /// Output markdown file (default: docs-benchmark-analysis.md) - #[arg(long, short)] - output: Option, + /// Filter by model name (e.g. "Claude Sonnet 4.6") + #[arg(long)] + model: Option, - /// Only analyze failures for a specific language (rust, csharp) + /// Run date (YYYY-MM-DD). If omitted, lists available dates. #[arg(long)] - lang: Option, -} + date: Option, -#[derive(Args, Debug, Clone)] -struct CiCommentArgs { - /// Output markdown file (default: docs-benchmark-comment.md) - #[arg(long, short)] - output: Option, - - /// Git ref to compare against for baseline (default: origin/master) - #[arg(long, default_value = "origin/master")] - baseline_ref: String, + /// Print analysis to stdout instead of uploading + #[arg(long)] + dry_run: bool, } /// Local wrapper so we can parse Vendor without orphan-rule issues. @@ -249,28 +198,29 @@ fn main() -> Result<()> { match cli.command { Commands::Run(args) => cmd_run(args), - Commands::CiCheck(args) => cmd_ci_check(args), - Commands::CiQuickfix => cmd_ci_quickfix(), - Commands::CiComment(args) => cmd_ci_comment(args), - Commands::Summary(args) => cmd_summary(args), Commands::Analyze(args) => cmd_analyze(args), - Commands::CountHttpFailures(args) => cmd_count_http_failures(args), - Commands::ScanRerunCommands(args) => cmd_scan_rerun_commands(args), } } /* ------------------------------ run ------------------------------ */ fn cmd_run(args: RunArgs) -> Result<()> { - // Run command writes to llm-comparison files (for comparing LLM performance) - let details_path = llm_comparison_details(); - let summary_path = llm_comparison_summary(); - - run_benchmarks(args, &details_path, &summary_path) + run_benchmarks(args)?; + Ok(()) } /// Core benchmark runner used by both `run` and `ci-quickfix` -fn run_benchmarks(args: RunArgs, details_path: &Path, summary_path: &Path) -> Result<()> { +fn run_benchmarks(args: RunArgs) -> Result<()> { + let api_client = if args.dry_run { + None + } else { + ApiClient::from_env().context("failed to initialize API client")? + }; + + if api_client.is_none() && !args.dry_run { + eprintln!("[warn] LLM_BENCHMARK_UPLOAD_URL not set; results will not be uploaded"); + } + let mut config = RunConfig { modes: args.modes, hash_only: args.hash_only, @@ -282,11 +232,19 @@ fn run_benchmarks(args: RunArgs, details_path: &Path, summary_path: &Path) -> Re categories: categories_to_set(args.categories), model_filter: model_filter_from_groups(args.models), host: None, - details_path: details_path.to_path_buf(), + api_client: api_client.clone(), + dry_run: args.dry_run, }; let bench_root = find_bench_root(); + // Upload task catalog before running benchmarks + if let Some(ref api) = api_client + && let Err(e) = api.upload_task_catalog(&bench_root) + { + eprintln!("[warn] failed to upload task catalog: {e}"); + } + let modes = config .modes .clone() @@ -315,8 +273,10 @@ fn run_benchmarks(args: RunArgs, details_path: &Path, summary_path: &Path) -> Re ))?; } + let mut all_outcomes: Vec = Vec::new(); + for mode in modes { - run_mode_benchmarks( + let outcomes = run_mode_benchmarks( &mode, config.lang, &config, @@ -324,520 +284,215 @@ fn run_benchmarks(args: RunArgs, details_path: &Path, summary_path: &Path) -> Re runtime.as_ref(), llm_provider.as_ref(), )?; - } - - if !config.goldens_only && !config.hash_only { - fs::create_dir_all(docs_dir().join("llms"))?; - - update_golden_answers_on_disk(details_path, &bench_root, /*all=*/ true, /*overwrite=*/ true)?; - - write_summary_from_details_file(details_path, summary_path)?; - println!("Results written to:"); - println!(" Details: {}", details_path.display()); - println!(" Summary: {}", summary_path.display()); - - // Show HTTP/timeout failures for this run's scope (providers/models we ran) - if let Some((total, http_list)) = load_details_and_http_failures( - details_path, - config.providers_filter.as_ref(), - config.model_filter.as_ref(), - )? && !http_list.is_empty() - { - print_http_failures_summary(total, &http_list); + all_outcomes.extend(outcomes); + } + + // Write local run log on --dry-run so results aren't lost + if args.dry_run && !all_outcomes.is_empty() { + let runs_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("runs"); + let _ = fs::create_dir_all(&runs_dir); + let timestamp = chrono::Utc::now().format("%Y-%m-%d_%H%M%S"); + let log_path = runs_dir.join(format!("run-{timestamp}.json")); + match serde_json::to_string_pretty(&all_outcomes) { + Ok(json) => { + if let Err(e) = fs::write(&log_path, json) { + eprintln!("[warn] failed to write run log: {e}"); + } else { + println!("Run log: {}", log_path.display()); + } + } + Err(e) => eprintln!("[warn] failed to serialize run log: {e}"), } } Ok(()) } -/* --------------------------- ci-check --------------------------- */ - -fn cmd_ci_check(args: CiCheckArgs) -> Result<()> { - // Check-only: - // - Verifies the required modes exist for each language - // - Computes the current context hash and compares against the saved summary hash - // - Does NOT run any providers/models or build goldens - // - // Required mode/lang combinations: - // Rust → "rustdoc_json" - // Rust → "docs" - // CSharp → "docs" - - let langs = args.lang.unwrap_or_else(|| vec![Lang::Rust, Lang::CSharp]); - - // Build the list of (lang, mode) combinations to check - let mut checks: Vec<(Lang, &'static str)> = Vec::new(); - for lang in &langs { - match lang { - Lang::Rust => { - checks.push((Lang::Rust, "rustdoc_json")); - checks.push((Lang::Rust, "docs")); - } - Lang::CSharp => { - checks.push((Lang::CSharp, "docs")); - } - Lang::TypeScript => { - checks.push((Lang::TypeScript, "docs")); +/* ------------------------------ analyze ------------------------------ */ + +fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { + let api = ApiClient::from_env() + .context("failed to initialize API client")? + .context("LLM_BENCHMARK_UPLOAD_URL required for analyze")?; + + // If no date specified, list available dates and exit + if args.date.is_none() { + let dates = api.fetch_run_dates(args.lang.as_deref(), args.mode.as_deref())?; + if dates.is_empty() { + println!("No run dates found."); + } else { + println!("Available run dates:"); + for d in &dates { + println!(" {}", d); } + println!("\nUse --date YYYY-MM-DD to analyze a specific run."); } + return Ok(()); } - // De-dupe, preserve order - let mut seen = HashSet::new(); - checks.retain(|(lang, mode)| seen.insert((lang.as_str().to_string(), mode.to_string()))); + let date = args.date.as_deref().unwrap(); - // Debug hint for how to (re)generate entries - let hint_for = |_lang: Lang| -> &'static str { "Check DEVELOP.md for instructions on how to proceed." }; + // Fetch failures from the API + let (failures, run_date) = api.fetch_failures( + args.lang.as_deref(), + args.mode.as_deref(), + args.model.as_deref(), + Some(date), + )?; - // Load docs-benchmark summary to compare hashes against - let summary_path = docs_benchmark_summary(); - let summary: Summary = - load_summary(&summary_path).with_context(|| format!("load summary file at {:?}", summary_path))?; + let run_date = run_date.unwrap_or_else(|| date.to_string()); - for (lang, mode) in checks { - let lang_str = lang.as_str(); + if failures.is_empty() { + println!("No failures found for date {}.", run_date); + return Ok(()); + } - // Ensure mode exists (non-empty paths) - match xtask_llm_benchmark::context::resolve_mode_paths(mode) { - Ok(paths) if !paths.is_empty() => {} - Ok(_) => bail!( - "CI check FAILED: {}/{} resolved to 0 paths.\n→ {}", - mode, - lang_str, - hint_for(lang) - ), - Err(e) => bail!( - "CI check FAILED: {}/{} not available: {}.\n→ {}", - mode, - lang_str, - e, - hint_for(lang) - ), - } + // Group failures by (lang, mode, model) + let mut groups: std::collections::BTreeMap<(String, String, String), Vec<&serde_json::Value>> = + std::collections::BTreeMap::new(); + for f in &failures { + let lang = f["lang"].as_str().unwrap_or("unknown").to_string(); + let mode = f["mode"].as_str().unwrap_or("unknown").to_string(); + let model = f["modelName"].as_str().unwrap_or("unknown").to_string(); + groups.entry((lang, mode, model)).or_default().push(f); + } - // Compute current context hash (using processed context for lang-specific hash) - let current_hash = compute_processed_context_hash(mode, lang) - .with_context(|| format!("compute processed context hash for `{mode}`/{lang_str}"))?; - - // Find saved hash in summary - let saved_hash = summary - .by_language - .get(lang_str) - .and_then(|lang_sum| lang_sum.modes.get(mode)) - .map(|mode_sum| &mode_sum.hash); - - let saved_hash = match saved_hash { - Some(h) => h, - None => bail!( - "CI check FAILED: no saved entry for {}/{}.\n→ Generate it with: {}", - mode, - lang_str, - hint_for(lang) - ), - }; + println!( + "Found {} failures across {} (lang, mode, model) groups for date {}", + failures.len(), + groups.len(), + run_date + ); - if *saved_hash != current_hash { - bail!( - "CI check FAILED: hash mismatch for {}/{}: saved={} current={}.\n→ Re-run to refresh: {}", - mode, - lang_str, - short_hash(saved_hash.as_str()), - short_hash(¤t_hash), - hint_for(lang) - ); - } + // Initialize LLM provider for analysis + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let provider = make_provider_from_env()?; - println!("CI check OK: {}/{} hash {}", mode, lang_str, short_hash(¤t_hash)); - } + let analysis_route = ModelRoute::new( + "gpt-4.1-mini", + xtask_llm_benchmark::llm::types::Vendor::OpenAi, + "gpt-4.1-mini", + Some("openai/gpt-4.1-mini"), + ); - Ok(()) -} + for ((lang, mode, model), group_failures) in &groups { + println!("\nAnalyzing {}/{}/{} ({} failures)...", lang, mode, model, group_failures.len()); -fn model_filter_from_groups(groups: Option>) -> Option>> { - let groups = groups?; - let mut out: HashMap> = HashMap::new(); + // Build prompt from the JSON failure data + let prompt = build_analysis_prompt_from_json(lang, mode, model, group_failures); - for g in groups { - out.entry(g.vendor).or_default().extend(g.models.into_iter()); - } - Some(out) -} + let built = xtask_llm_benchmark::llm::prompt::BuiltPrompt { + system: Some(xtask_llm_benchmark::bench::analysis::system_prompt()), + static_prefix: None, + segments: vec![xtask_llm_benchmark::llm::segmentation::Segment::new("user", prompt)], + search_enabled: false, + }; -fn cmd_ci_quickfix() -> Result<()> { - // CI quickfix writes to docs-benchmark files (for testing documentation quality) - let details_path = docs_benchmark_details(); - let summary_path = docs_benchmark_summary(); - - println!("Running CI quickfix (GPT-5 only) for docs-benchmark..."); - - // Run Rust benchmarks with rustdoc_json mode - let rust_rustdoc_args = RunArgs { - modes: Some(vec!["rustdoc_json".to_string()]), - lang: Lang::Rust, - hash_only: false, - goldens_only: false, - force: true, - categories: None, - tasks: None, - providers: Some(vec![VendorArg(Vendor::OpenAi)]), - models: Some(vec![ModelGroup { - vendor: Vendor::OpenAi, - models: vec!["gpt-5".to_string()], - }]), - }; - run_benchmarks(rust_rustdoc_args, &details_path, &summary_path)?; - - // Run Rust benchmarks with docs mode (markdown documentation) - let rust_docs_args = RunArgs { - modes: Some(vec!["docs".to_string()]), - lang: Lang::Rust, - hash_only: false, - goldens_only: false, - force: true, - categories: None, - tasks: None, - providers: Some(vec![VendorArg(Vendor::OpenAi)]), - models: Some(vec![ModelGroup { - vendor: Vendor::OpenAi, - models: vec!["gpt-5".to_string()], - }]), - }; - run_benchmarks(rust_docs_args, &details_path, &summary_path)?; - - // Run C# benchmarks with docs mode - let csharp_args = RunArgs { - modes: Some(vec!["docs".to_string()]), - lang: Lang::CSharp, - hash_only: false, - goldens_only: false, - force: true, - categories: None, - tasks: None, - providers: Some(vec![VendorArg(Vendor::OpenAi)]), - models: Some(vec![ModelGroup { - vendor: Vendor::OpenAi, - models: vec!["gpt-5".to_string()], - }]), - }; - run_benchmarks(csharp_args, &details_path, &summary_path)?; - - // Run TypeScript benchmarks with docs mode - let typescript_args = RunArgs { - modes: Some(vec!["docs".to_string()]), - lang: Lang::TypeScript, - hash_only: false, - goldens_only: false, - force: true, - categories: None, - tasks: None, - providers: Some(vec![VendorArg(Vendor::OpenAi)]), - models: Some(vec![ModelGroup { - vendor: Vendor::OpenAi, - models: vec!["gpt-5".to_string()], - }]), - }; - run_benchmarks(typescript_args, &details_path, &summary_path)?; + let analysis = runtime.block_on(provider.generate(&analysis_route, &built))?.text; - println!("CI quickfix complete. Results written to:"); - println!(" Details: {}", details_path.display()); - println!(" Summary: {}", summary_path.display()); + if args.dry_run { + // Save locally + let runs_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("runs"); + let _ = fs::create_dir_all(&runs_dir); + let safe_model = model.replace([' ', '/'], "_"); + let path = runs_dir.join(format!("analysis-{lang}-{mode}-{safe_model}-{run_date}.md")); + if let Err(e) = fs::write(&path, &analysis) { + eprintln!("[warn] failed to write analysis: {e}"); + } else { + println!("Analysis written to: {}", path.display()); + } + } else { + api.upload_analysis(lang, mode, model, &analysis, &run_date)?; + } + } + println!("\nDone."); Ok(()) } -/* --------------------------- ci-comment --------------------------- */ - -fn cmd_ci_comment(args: CiCommentArgs) -> Result<()> { - use std::process::Command; +fn build_analysis_prompt_from_json(lang: &str, mode: &str, model: &str, failures: &[&serde_json::Value]) -> String { + // Reuse the shared prompt builder for the intro + instructions, + // but we need to build the failure list from JSON values instead of RunOutcome. + use xtask_llm_benchmark::bench::analysis::analysis_instructions; - let summary_path = docs_benchmark_summary(); - let output_path = args.output.unwrap_or_else(docs_benchmark_comment); - - // Load current summary - let summary: Summary = - load_summary(&summary_path).with_context(|| format!("load summary file at {:?}", summary_path))?; - - // Try to load baseline from git ref - let baseline: Option = { - let relative_path = "docs/llms/docs-benchmark-summary.json"; - let output = Command::new("git") - .args(["show", &format!("{}:{}", args.baseline_ref, relative_path)]) - .output(); + // Reuse the same context description logic as bench::analysis + let lang_display = match lang { + "rust" => "Rust", + "csharp" => "C#", + "typescript" => "TypeScript", + _ => lang, + }; - match output { - Ok(out) if out.status.success() => { - let json = String::from_utf8_lossy(&out.stdout); - match serde_json::from_str(&json) { - Ok(s) => { - println!("Loaded baseline from {}", args.baseline_ref); - Some(s) - } - Err(e) => { - println!("Warning: Could not parse baseline JSON: {}", e); - None - } - } - } - Ok(out) => { - let stderr = String::from_utf8_lossy(&out.stderr); - println!( - "Note: Could not load baseline from {} (file may not exist yet): {}", - args.baseline_ref, - stderr.trim() - ); - None - } - Err(e) => { - println!("Warning: git command failed: {}", e); - None - } - } + let ctx_desc = match mode { + "guidelines" => "the SpacetimeDB AI guidelines (concise cheat-sheets for code generation)", + "cursor_rules" => "SpacetimeDB Cursor/IDE rules (anti-hallucination guardrails)", + "docs" => "SpacetimeDB markdown documentation", + "rustdoc_json" => "SpacetimeDB rustdoc JSON (auto-generated API reference)", + "llms.md" => "the SpacetimeDB llms.md file", + "no_context" | "none" | "no_guidelines" => "no documentation (testing base model knowledge only)", + "search" => "web search results (no local docs)", + _ => "unspecified context", }; - // Generate markdown - let markdown = generate_comment_markdown(&summary, baseline.as_ref()); + let mut prompt = format!( + "{model} was given {ctx_desc} and asked to generate {lang_display} SpacetimeDB modules. \ + It failed {count} tasks.\n\n", + count = failures.len(), + ); - // Write to file - if let Some(parent) = output_path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(&output_path, &markdown)?; - println!("Comment markdown written to: {}", output_path.display()); + for f in failures.iter().take(15) { + let task_id = f["taskId"].as_str().unwrap_or("?"); + let passed = f["passedTests"].as_u64().unwrap_or(0); + let total = f["totalTests"].as_u64().unwrap_or(0); - Ok(()) -} - -/// Generate the markdown comment for GitHub PR. -fn generate_comment_markdown(summary: &Summary, baseline: Option<&Summary>) -> String { - // Rust with rustdoc_json mode - let rust_rustdoc_results = summary - .by_language - .get("rust") - .and_then(|l| l.modes.get("rustdoc_json")) - .and_then(|m| m.models.get("GPT-5")); - // Rust with docs mode (markdown documentation) - let rust_docs_results = summary - .by_language - .get("rust") - .and_then(|l| l.modes.get("docs")) - .and_then(|m| m.models.get("GPT-5")); - // C# with docs mode - let csharp_results = summary - .by_language - .get("csharp") - .and_then(|l| l.modes.get("docs")) - .and_then(|m| m.models.get("GPT-5")); - // TypeScript with docs mode - let typescript_results = summary - .by_language - .get("typescript") - .and_then(|l| l.modes.get("docs")) - .and_then(|m| m.models.get("GPT-5")); - - let rust_rustdoc_baseline = baseline - .and_then(|b| b.by_language.get("rust")) - .and_then(|l| l.modes.get("rustdoc_json")) - .and_then(|m| m.models.get("GPT-5")); - let rust_docs_baseline = baseline - .and_then(|b| b.by_language.get("rust")) - .and_then(|l| l.modes.get("docs")) - .and_then(|m| m.models.get("GPT-5")); - let csharp_baseline = baseline - .and_then(|b| b.by_language.get("csharp")) - .and_then(|l| l.modes.get("docs")) - .and_then(|m| m.models.get("GPT-5")); - let typescript_baseline = baseline - .and_then(|b| b.by_language.get("typescript")) - .and_then(|l| l.modes.get("docs")) - .and_then(|m| m.models.get("GPT-5")); - - fn format_pct(val: f32) -> String { - format!("{:.1}%", val) - } + prompt.push_str(&format!("### {} ({}/{})\n", task_id, passed, total)); - fn format_diff(current: f32, baseline: Option) -> String { - match baseline { - Some(b) => { - let diff = current - b; - if diff.abs() < 0.1 { - String::new() - } else { - let sign = if diff > 0.0 { "+" } else { "" }; - let arrow = if diff > 0.0 { "⬆️" } else { "⬇️" }; - format!(" {} {}{:.1}%", arrow, sign, diff) - } + if let Some(details) = f["scorerDetails"].as_object() { + let reasons: Vec = details + .iter() + .filter_map(|(name, score)| { + if score["pass"].as_bool() == Some(true) { + return None; + } + let notes = &score["notes"]; + let error = notes["error"] + .as_str() + .or_else(|| notes["stderr"].as_str()) + .or_else(|| notes["diff"].as_str()) + .unwrap_or("failed"); + Some(format!("{}: {}", name, &error[..error.len().min(150)])) + }) + .collect(); + if !reasons.is_empty() { + prompt.push_str(&format!("Error: {}\n", reasons.join("; "))); } - None => String::new(), } - } - let mut md = String::new(); - md.push_str("## LLM Benchmark Results (ci-quickfix)\n\n"); - md.push_str("| Language | Mode | Category | Tests Passed | Task Pass % |\n"); - md.push_str("|----------|------|----------|--------------|-------------|\n"); - - // Rust with rustdoc_json mode - if let Some(results) = rust_rustdoc_results { - let base_cats = rust_rustdoc_baseline.map(|b| &b.categories); - - if let Some(c) = results.categories.get("basics") { - let b = base_cats.and_then(|cats| cats.get("basics")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| Rust | rustdoc_json | basics | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); - } - if let Some(c) = results.categories.get("schema") { - let b = base_cats.and_then(|cats| cats.get("schema")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| Rust | rustdoc_json | schema | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); + if let Some(output) = f["llmOutput"].as_str() { + let truncated = if output.len() > 1500 { &output[..1500] } else { output }; + prompt.push_str(&format!("```{}\n{}\n```\n", lang, truncated)); } - let diff = format_diff( - results.totals.task_pass_pct, - rust_rustdoc_baseline.map(|b| b.totals.task_pass_pct), - ); - md.push_str(&format!( - "| Rust | rustdoc_json | **total** | {}/{} | **{}**{} |\n", - results.totals.passed_tests, - results.totals.total_tests, - format_pct(results.totals.task_pass_pct), - diff - )); + prompt.push('\n'); } - // Rust with docs mode - if let Some(results) = rust_docs_results { - let base_cats = rust_docs_baseline.map(|b| &b.categories); - - if let Some(c) = results.categories.get("basics") { - let b = base_cats.and_then(|cats| cats.get("basics")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| Rust | docs | basics | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); - } - if let Some(c) = results.categories.get("schema") { - let b = base_cats.and_then(|cats| cats.get("schema")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| Rust | docs | schema | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); - } - let diff = format_diff( - results.totals.task_pass_pct, - rust_docs_baseline.map(|b| b.totals.task_pass_pct), - ); - md.push_str(&format!( - "| Rust | docs | **total** | {}/{} | **{}**{} |\n", - results.totals.passed_tests, - results.totals.total_tests, - format_pct(results.totals.task_pass_pct), - diff - )); + if failures.len() > 15 { + prompt.push_str(&format!("({} more failures not shown)\n\n", failures.len() - 15)); } - // C# with docs mode - if let Some(results) = csharp_results { - let base_cats = csharp_baseline.map(|b| &b.categories); - - if let Some(c) = results.categories.get("basics") { - let b = base_cats.and_then(|cats| cats.get("basics")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| C# | docs | basics | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); - } - if let Some(c) = results.categories.get("schema") { - let b = base_cats.and_then(|cats| cats.get("schema")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| C# | docs | schema | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); - } - let diff = format_diff( - results.totals.task_pass_pct, - csharp_baseline.map(|b| b.totals.task_pass_pct), - ); - md.push_str(&format!( - "| C# | docs | **total** | {}/{} | **{}**{} |\n", - results.totals.passed_tests, - results.totals.total_tests, - format_pct(results.totals.task_pass_pct), - diff - )); - } + prompt.push_str(&analysis_instructions(mode)); + prompt +} - // TypeScript with docs mode - if let Some(results) = typescript_results { - let base_cats = typescript_baseline.map(|b| &b.categories); - - if let Some(c) = results.categories.get("basics") { - let b = base_cats.and_then(|cats| cats.get("basics")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| TypeScript | docs | basics | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); - } - if let Some(c) = results.categories.get("schema") { - let b = base_cats.and_then(|cats| cats.get("schema")); - let diff = format_diff(c.task_pass_pct, b.map(|x| x.task_pass_pct)); - md.push_str(&format!( - "| TypeScript | docs | schema | {}/{} | {}{} |\n", - c.passed_tests, - c.total_tests, - format_pct(c.task_pass_pct), - diff - )); - } - let diff = format_diff( - results.totals.task_pass_pct, - typescript_baseline.map(|b| b.totals.task_pass_pct), - ); - md.push_str(&format!( - "| TypeScript | docs | **total** | {}/{} | **{}**{} |\n", - results.totals.passed_tests, - results.totals.total_tests, - format_pct(results.totals.task_pass_pct), - diff - )); - } +fn model_filter_from_groups(groups: Option>) -> Option>> { + let groups = groups?; + let mut out: HashMap> = HashMap::new(); - if baseline.is_some() { - md.push_str("\n_Compared against master branch baseline_\n"); + for g in groups { + out.entry(g.vendor).or_default().extend(g.models.into_iter()); } - md.push_str(&format!("\nGenerated at: {}\n", summary.generated_at)); - - md + Some(out) } /* --------------------------- helpers --------------------------- */ @@ -846,7 +501,7 @@ fn short_hash(s: &str) -> &str { &s[..s.len().min(12)] } -/// Run benchmarks for a single mode. Results are merged into the details file. +/// Run benchmarks for a single mode. fn run_mode_benchmarks( mode: &str, lang: Lang, @@ -854,7 +509,7 @@ fn run_mode_benchmarks( bench_root: &Path, runtime: Option<&Runtime>, llm_provider: Option<&Arc>, -) -> Result<()> { +) -> Result> { let lang_str = lang.as_str(); let context = build_context(mode, Some(lang))?; // Use processed context hash so each lang/mode combination has its own unique hash @@ -864,7 +519,7 @@ fn run_mode_benchmarks( println!("{:<12} [{:<10}] hash: {}", mode, lang_str, short_hash(&hash)); if config.hash_only { - return Ok(()); + return Ok(Vec::new()); } if config.goldens_only { @@ -873,7 +528,7 @@ fn run_mode_benchmarks( rt.block_on(build_goldens_only_for_lang(config.host.clone(), bench_root, lang, sels))?; println!("{:<12} [{:<10}] goldens-only build complete", mode, lang_str); - return Ok(()); + return Ok(Vec::new()); } // Run benchmarks for all matching routes @@ -881,7 +536,7 @@ fn run_mode_benchmarks( if routes.is_empty() { println!("{:<12} [{:<10}] no matching models to run", mode, lang_str); - return Ok(()); + return Ok(Vec::new()); } let runtime = runtime.expect("runtime required for normal runs"); @@ -898,25 +553,36 @@ fn run_mode_benchmarks( &routes, ))?; - // Print summary - for rr in &route_runs { - let total: u32 = rr.outcomes.iter().map(|o| o.total_tests).sum(); - let passed: u32 = rr.outcomes.iter().map(|o| o.passed_tests).sum(); - let pct = if total == 0 { - 0.0 - } else { - (passed as f32 / total as f32) * 100.0 - }; - println!(" ↳ {}: {}/{} passed ({:.1}%)", rr.route_name, passed, total, pct); + // Print summary sorted by pass rate descending + let mut summary: Vec<(&str, u32, u32, f32)> = route_runs + .iter() + .map(|rr| { + let total: u32 = rr.outcomes.iter().map(|o| o.total_tests).sum(); + let passed: u32 = rr.outcomes.iter().map(|o| o.passed_tests).sum(); + let pct = if total == 0 { + 0.0 + } else { + (passed as f32 / total as f32) * 100.0 + }; + (rr.route_name.as_str(), passed, total, pct) + }) + .collect(); + summary.sort_by(|a, b| b.3.partial_cmp(&a.3).unwrap_or(std::cmp::Ordering::Equal)); + for (name, passed, total, pct) in &summary { + println!(" ↳ {}: {}/{} passed ({:.1}%)", name, passed, total, pct); } - Ok(()) + let all_outcomes: Vec = route_runs.into_iter().flat_map(|rr| rr.outcomes).collect(); + Ok(all_outcomes) } /// Routes to run: when `model_filter` is set (from --models), only routes whose vendor and /// model are in that filter are included; vendors not in the filter are excluded. +/// +/// When explicit `openrouter:vendor/model` entries are passed they won't appear in +/// `default_model_routes`, so we synthesize ad-hoc routes for them here. fn filter_routes(config: &RunConfig) -> Vec { - default_model_routes() + let mut routes: Vec = default_model_routes() .iter() .filter(|r| config.providers_filter.as_ref().is_none_or(|f| f.contains(&r.vendor))) .filter(|r| match &config.model_filter { @@ -926,12 +592,35 @@ fn filter_routes(config: &RunConfig) -> Vec { Some(allowed) => { let api = r.api_model.to_ascii_lowercase(); let dn = r.display_name.to_ascii_lowercase(); - allowed.contains(&api) || allowed.contains(&dn) + let or_id = r.openrouter_model.as_ref().map(|m| m.to_ascii_lowercase()); + allowed.contains(&api) + || allowed.contains(&dn) + || or_id.as_ref().map(|m| allowed.contains(m)).unwrap_or(false) } }, }) .cloned() - .collect() + .collect(); + + // Synthesize ad-hoc routes for any vendor:model that isn't in the static list. + // This lets callers pass arbitrary model IDs (e.g. new models, openrouter paths) + // without having to add them to default_model_routes() first. + if let Some(mf) = &config.model_filter { + for (vendor, model_ids) in mf { + for model_id in model_ids { + let already_matched = routes.iter().any(|r| { + r.vendor == *vendor + && (r.api_model == model_id.as_str() + || r.openrouter_model.as_deref() == Some(model_id.as_str())) + }); + if !already_matched { + routes.push(ModelRoute::new(model_id, *vendor, model_id, None)); + } + } + } + } + + routes } #[allow(clippy::too_many_arguments)] @@ -948,14 +637,15 @@ async fn run_many_routes_for_mode( let rbuf = bench_route_concurrency(); let selectors = config.selectors.as_deref(); let host = config.host.clone(); - let details_path = config.details_path.clone(); + let api_client = config.api_client.clone(); + let dry_run = config.dry_run; futures::stream::iter(routes.iter().map(|route| { let host = host.clone(); - let details_path = details_path.clone(); + let api_client = api_client.clone(); async move { - println!("→ running {}", route.display_name); + println!("\u{2192} running {}", route.display_name); let per = BenchRunContext { bench_root, @@ -967,7 +657,8 @@ async fn run_many_routes_for_mode( lang, selectors, host, - details_path, + api_client, + dry_run, }; let outcomes = run_selected_or_all_for_model_async_for_lang(&per).await?; @@ -1104,717 +795,3 @@ fn apply_category_filter( } } } - -fn cmd_summary(args: SummaryArgs) -> Result<()> { - // Default to llm-comparison files (the full benchmark suite) - let in_path = args.details.unwrap_or_else(llm_comparison_details); - let out_path = args.summary.unwrap_or_else(llm_comparison_summary); - - if let Some(parent) = out_path.parent() { - fs::create_dir_all(parent).with_context(|| format!("create dir {}", parent.display()))?; - } - - write_summary_from_details_file(&in_path, &out_path)?; - println!("Summary written to: {}", out_path.display()); - Ok(()) -} - -fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { - use xtask_llm_benchmark::results::schema::Results; - - let details_path = args.details.unwrap_or_else(docs_benchmark_details); - let output_path = args.output.unwrap_or_else(|| { - details_path - .parent() - .unwrap_or(Path::new(".")) - .join("docs-benchmark-analysis.md") - }); - - println!("Analyzing benchmark results from: {}", details_path.display()); - - // Load the details file - let content = - fs::read_to_string(&details_path).with_context(|| format!("Failed to read {}", details_path.display()))?; - let results: Results = serde_json::from_str(&content).with_context(|| "Failed to parse details.json")?; - - // Collect failures - let mut failures: Vec = Vec::new(); - - for lang_entry in &results.languages { - // Skip if filtering by language - if let Some(filter_lang) = &args.lang - && lang_entry.lang != filter_lang.as_str() - { - continue; - } - - let golden_answers = &lang_entry.golden_answers; - - for mode_entry in &lang_entry.modes { - for model_entry in &mode_entry.models { - for (task_id, outcome) in &model_entry.tasks { - if outcome.passed_tests < outcome.total_tests { - // This task has failures - let golden = golden_answers - .get(task_id) - .or_else(|| { - // Try with category prefix stripped - task_id.split('/').next_back().and_then(|t| golden_answers.get(t)) - }) - .map(|g| g.answer.clone()); - - failures.push(FailureInfo { - lang: lang_entry.lang.clone(), - mode: mode_entry.mode.clone(), - model: model_entry.name.clone(), - task: task_id.clone(), - passed: outcome.passed_tests, - total: outcome.total_tests, - llm_output: outcome.llm_output.clone(), - golden_answer: golden, - scorer_details: outcome.scorer_details.clone(), - }); - } - } - } - } - } - - if failures.is_empty() { - println!("No failures found!"); - fs::write( - &output_path, - "# Benchmark Analysis\n\nNo failures found. All tests passed!", - )?; - println!("Analysis written to: {}", output_path.display()); - return Ok(()); - } - - println!("Found {} failing test(s). Generating analysis...", failures.len()); - - // Build prompt for LLM - let prompt = build_analysis_prompt(&failures); - - // Initialize runtime and LLM provider - let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; - let provider = make_provider_from_env()?; - - // Use a fast model for analysis - let route = ModelRoute { - display_name: "gpt-4o-mini", - api_model: "gpt-4o-mini", - vendor: Vendor::OpenAi, - }; - - use xtask_llm_benchmark::llm::prompt::BuiltPrompt; - - let built_prompt = BuiltPrompt { - system: Some( - "You are an expert at analyzing SpacetimeDB benchmark failures. \ - Analyze the test failures and provide actionable insights in markdown format." - .to_string(), - ), - static_prefix: None, - segments: vec![xtask_llm_benchmark::llm::segmentation::Segment::new("user", prompt)], - }; - - let analysis = runtime.block_on(provider.generate(&route, &built_prompt))?; - - // Write markdown output - let markdown = format!( - "# Benchmark Failure Analysis\n\n\ - Generated from: `{}`\n\n\ - ## Summary\n\n\ - - **Total failures analyzed**: {}\n\n\ - ---\n\n\ - {}\n", - details_path.display(), - failures.len(), - analysis - ); - - fs::write(&output_path, markdown)?; - println!("Analysis written to: {}", output_path.display()); - - Ok(()) -} - -fn cmd_count_http_failures(args: CountHttpFailuresArgs) -> Result<()> { - let details_path = args.details.unwrap_or_else(llm_comparison_details); - let Some((total, http_list)) = load_details_and_http_failures(&details_path, None, None)? else { - println!("No details file or no failures at {}", details_path.display()); - return Ok(()); - }; - println!("Total failures: {}", total); - println!("HTTP/API failures (e.g. 429, 503, timeout): {}", http_list.len()); - if !http_list.is_empty() { - print_http_failures_summary(total, &http_list); - } - Ok(()) -} - -fn cmd_scan_rerun_commands(args: ScanRerunCommandsArgs) -> Result<()> { - let details_path = args.details.unwrap_or_else(llm_comparison_details); - let failures = collect_http_failures_full(&details_path)?; - if failures.is_empty() { - println!("No LLM provider API failures found."); - return Ok(()); - } - - let count = failures.len(); - println!("Scanning {} — {} LLM API failures\n", details_path.display(), count); - - // Group by (lang, mode, vendor, api_model) -> sorted task IDs - let mut groups: HashMap<(String, String, String, String), (String, Vec)> = HashMap::new(); - for (lang, mode, vendor, api_model, model_name, task_id) in failures { - let key = (lang.clone(), mode.clone(), vendor.clone(), api_model.clone()); - let entry = groups.entry(key).or_insert_with(|| (model_name, Vec::new())); - if !entry.1.contains(&task_id) { - entry.1.push(task_id); - } - } - for (_, (_, tasks)) in groups.iter_mut() { - tasks.sort(); - } - - let mut keys: Vec<_> = groups.keys().collect(); - keys.sort(); - - for (lang, mode, vendor, api_model) in keys { - let (model_name, tasks) = groups - .get(&(lang.clone(), mode.clone(), vendor.clone(), api_model.clone())) - .unwrap(); - let tasks_arg = tasks.join(","); - println!("# {} + {} ({})", model_name, lang, mode); - println!( - "cargo run --package xtask-llm-benchmark -- run --lang {} --modes {} --models {}:{} --tasks {}", - lang, mode, vendor, api_model, tasks_arg - ); - println!(); - } - Ok(()) -} - -/// True if (vendor, api_model) matches a route in default_model_routes (model_routes.rs). -fn is_model_in_routes(vendor_str: &str, api_model: &str) -> bool { - let Some(vendor) = Vendor::parse(vendor_str) else { - return false; - }; - let api_lower = api_model.to_ascii_lowercase(); - default_model_routes() - .iter() - .any(|r| r.vendor == vendor && r.api_model.to_ascii_lowercase() == api_lower) -} - -/// Collect LLM API failures with full (lang, mode, vendor, api_model, model_name, task_id) for building rerun commands. -/// Only includes models that are in default_model_routes (model_routes.rs). -fn collect_http_failures_full(details_path: &Path) -> Result> { - use xtask_llm_benchmark::results::schema::Results; - - let content = match fs::read_to_string(details_path) { - Ok(c) => c, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(e) => return Err(e).with_context(|| format!("read {}", details_path.display())), - }; - let results: Results = serde_json::from_str(&content).with_context(|| "parse details.json")?; - - let mut out: Vec<(String, String, String, String, String, String)> = Vec::new(); - for lang_entry in &results.languages { - for mode_entry in &lang_entry.modes { - for model_entry in &mode_entry.models { - for (task_id, outcome) in &model_entry.tasks { - if outcome.passed_tests >= outcome.total_tests { - continue; - } - if outcome.llm_output.is_some() { - continue; - } - let Some(err) = get_publish_error_from_outcome(outcome) else { - continue; - }; - if !is_http_like_error(&err) { - continue; - } - let vendor = outcome.vendor.clone(); - let api_model = outcome - .route_api_model - .clone() - .or_else(|| model_entry.route_api_model.clone()) - .or_else(|| { - default_model_routes() - .iter() - .find(|r| r.display_name == model_entry.name) - .map(|r| r.api_model.to_string()) - }) - .unwrap_or_else(|| model_entry.name.to_ascii_lowercase().replace(' ', "-")); - if !is_model_in_routes(&vendor, &api_model) { - continue; - } - out.push(( - lang_entry.lang.clone(), - mode_entry.mode.clone(), - vendor, - api_model, - model_entry.name.clone(), - task_id.clone(), - )); - } - } - } - } - Ok(out) -} - -/// Load details JSON and return (total_failures, list of (lang, mode, model, task_id, vendor, api_model) for HTTP-like failures). -/// When providers_filter/model_filter are set (from --providers/--models), only count outcomes -/// from that run's scope so the summary reflects what we just ran, not the entire file. -/// Returns Ok(None) if file missing or no failures. -fn load_details_and_http_failures( - details_path: &Path, - providers_filter: Option<&HashSet>, - model_filter: Option<&HashMap>>, -) -> Result)>> { - use xtask_llm_benchmark::results::schema::Results; - - let content = match fs::read_to_string(details_path) { - Ok(c) => c, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(e) => return Err(e).with_context(|| format!("read {}", details_path.display())), - }; - let results: Results = serde_json::from_str(&content).with_context(|| "parse details.json")?; - - let mut total_failures = 0u32; - let mut http_failures: Vec<(String, String, String, String, String, String)> = Vec::new(); - - for lang_entry in &results.languages { - for mode_entry in &lang_entry.modes { - for model_entry in &mode_entry.models { - for (task_id, outcome) in &model_entry.tasks { - if outcome.passed_tests < outcome.total_tests { - if !outcome_matches_run_scope( - &outcome.vendor, - &model_entry.name, - outcome.route_api_model.as_deref(), - providers_filter, - model_filter, - ) { - continue; - } - let vendor = outcome.vendor.clone(); - let api_model = outcome - .route_api_model - .clone() - .or_else(|| model_entry.route_api_model.clone()) - .or_else(|| { - default_model_routes() - .iter() - .find(|r| r.display_name == model_entry.name) - .map(|r| r.api_model.to_string()) - }) - .unwrap_or_else(|| model_entry.name.to_ascii_lowercase().replace(' ', "-")); - // Only count/show failures for models in default_model_routes(). - if !is_model_in_routes(&vendor, &api_model) { - continue; - } - total_failures += 1; - // Only count LLM API failures (timeout, 429, etc.): we never got a response. - // If llm_output is present, we got a response and the failure was publish/compile. - if outcome.llm_output.is_some() { - continue; - } - if let Some(err) = get_publish_error_from_outcome(outcome) - && is_http_like_error(&err) - { - http_failures.push(( - lang_entry.lang.clone(), - mode_entry.mode.clone(), - model_entry.name.clone(), - task_id.clone(), - vendor, - api_model, - )); - } - } - } - } - } - } - - if total_failures == 0 { - return Ok(None); - } - Ok(Some((total_failures, http_failures))) -} - -fn outcome_matches_run_scope( - vendor_slug: &str, - model_name: &str, - route_api_model: Option<&str>, - providers_filter: Option<&HashSet>, - model_filter: Option<&HashMap>>, -) -> bool { - let Some(vendor) = Vendor::parse(vendor_slug) else { - return true; // unknown vendor, include - }; - if let Some(pf) = providers_filter - && !pf.contains(&vendor) - { - return false; - } - if let Some(mf) = model_filter - && let Some(allowed) = mf.get(&vendor) - { - let model_lower = model_name.to_ascii_lowercase(); - let api_lower = route_api_model.map(|s| s.to_ascii_lowercase()); - let model_norm = model_lower.replace(' ', "-"); - let matches = allowed.iter().any(|a| { - let al = a.to_ascii_lowercase(); - let a_norm = al.replace(' ', "-"); - model_norm == a_norm - || model_lower == al - || api_lower - .as_ref() - .is_some_and(|api| api == &al || api.contains(al.as_str()) || al.contains(api.as_str())) - }); - if !matches { - return false; - } - } - true -} - -/// Print a summary of HTTP/timeout failures: grouped by (lang, mode, vendor, api_model) with one rerun command per group. -fn print_http_failures_summary( - total_failures: u32, - http_failures: &[(String, String, String, String, String, String)], -) { - // (lang, mode, model, task_id, vendor, api_model) - println!(); - println!("---"); - println!( - "LLM provider API failures (timeout, 429, etc.): {} of {} total failures — rerun these", - http_failures.len(), - total_failures - ); - - let mut groups: HashMap<(String, String, String, String), (String, Vec)> = HashMap::new(); - for (lang, mode, model_name, task_id, vendor, api_model) in http_failures { - let key = (lang.clone(), mode.clone(), vendor.clone(), api_model.clone()); - let entry = groups.entry(key).or_insert_with(|| (model_name.clone(), Vec::new())); - if !entry.1.contains(task_id) { - entry.1.push(task_id.clone()); - } - } - for (_, (_, tasks)) in groups.iter_mut() { - tasks.sort(); - } - - let mut keys: Vec<_> = groups.keys().collect(); - keys.sort(); - - for (lang, mode, vendor, api_model) in keys { - let (model_name, tasks) = groups - .get(&(lang.clone(), mode.clone(), vendor.clone(), api_model.clone())) - .unwrap(); - let tasks_arg = tasks.join(","); - println!("# {} + {} ({})", model_name, lang, mode); - println!( - "cargo run --package xtask-llm-benchmark -- run --lang {} --modes {} --models {}:{} --tasks {}", - lang, mode, vendor, api_model, tasks_arg - ); - println!(); - } - println!("---"); -} - -/// True if the error message looks like an HTTP/API failure (rate limit, server error, timeout, etc.). -fn is_http_like_error(err: &str) -> bool { - let lower = err.to_ascii_lowercase(); - lower.contains("status") - || lower.contains("429") - || lower.contains("503") - || lower.contains("502") - || lower.contains("504") - || lower.contains("500") - || lower.contains("post ") - || lower.contains(" get ") - || lower.contains("timed out") - || lower.contains("timeout") - || lower.contains("too many requests") - || lower.contains("service unavailable") - || lower.contains("bad gateway") - || lower.contains("gateway timeout") - || lower.contains("connection") - || lower.contains("request failed") - || lower.contains("reqwest") -} - -/// Get the error from the publish_error scorer only. LLM API failures (429, timeout, etc.) -/// are recorded there. Scorer-phase errors (schema_parity, call_reducer, sql) are from -/// SpacetimeDB CLI and can also say "timeout" — we must not count those as HTTP failures. -fn get_publish_error_from_outcome(outcome: &RunOutcome) -> Option { - let details = outcome.scorer_details.as_ref()?; - let score = details.get("publish_error")?; - score.notes.get("error").and_then(|v| v.as_str()).map(String::from) -} - -#[allow(dead_code)] -struct FailureInfo { - lang: String, - mode: String, - model: String, - task: String, - passed: u32, - total: u32, - llm_output: Option, - golden_answer: Option, - scorer_details: Option>, -} - -/// Extract concise failure reasons from scorer_details using typed extraction. -fn extract_failure_reasons(details: &HashMap) -> Vec { - details - .iter() - .filter_map(|(scorer_name, score)| { - score - .failure_reason() - .map(|reason| format!("{}: {}", scorer_name, reason)) - }) - .collect() -} - -/// Categorize a failure by its type based on scorer details. -fn categorize_failure(f: &FailureInfo) -> &'static str { - let reasons = f - .scorer_details - .as_ref() - .map(extract_failure_reasons) - .unwrap_or_default(); - - let reasons_str = reasons.join(" "); - if reasons_str.contains("tables differ") { - "table_naming" - } else if reasons_str.contains("timed out") { - "timeout" - } else if reasons_str.contains("publish failed") || reasons_str.contains("compile") { - "compile" - } else { - "other" - } -} - -/// Build the analysis section for failures of a specific language/mode combination. -fn build_mode_section(lang: &str, mode: &str, failures: &[&FailureInfo], prompt: &mut String) { - let lang_display = match lang { - "rust" => "Rust", - "csharp" => "C#", - "typescript" => "TypeScript", - _ => lang, - }; - - prompt.push_str(&format!( - "# {} / {} Failures ({} total)\n\n", - lang_display, - mode, - failures.len() - )); - - // Group by failure type - let table_naming: Vec<_> = failures - .iter() - .filter(|f| categorize_failure(f) == "table_naming") - .collect(); - let compile: Vec<_> = failures.iter().filter(|f| categorize_failure(f) == "compile").collect(); - let timeout: Vec<_> = failures.iter().filter(|f| categorize_failure(f) == "timeout").collect(); - let other: Vec<_> = failures.iter().filter(|f| categorize_failure(f) == "other").collect(); - - // Table naming issues - show detailed examples - if !table_naming.is_empty() { - prompt.push_str(&format!("## Table Naming Issues ({} failures)\n\n", table_naming.len())); - prompt.push_str("The LLM is using incorrect table names. Examples:\n\n"); - - // Show up to 3 detailed examples - for f in table_naming.iter().take(3) { - let reasons = f - .scorer_details - .as_ref() - .map(extract_failure_reasons) - .unwrap_or_default(); - prompt.push_str(&format!("### {}\n", f.task)); - prompt.push_str(&format!("**Failure**: {}\n\n", reasons.join(", "))); - - if let Some(llm_out) = &f.llm_output { - let truncated = truncate_str(llm_out, 1200); - prompt.push_str(&format!("**LLM Output**:\n```\n{}\n```\n\n", truncated)); - } - - if let Some(golden) = &f.golden_answer { - let truncated = truncate_str(golden, 1200); - prompt.push_str(&format!("**Expected**:\n```\n{}\n```\n\n", truncated)); - } - } - if table_naming.len() > 3 { - prompt.push_str(&format!( - "**Additional similar failures**: {}\n\n", - table_naming - .iter() - .skip(3) - .map(|f| f.task.as_str()) - .collect::>() - .join(", ") - )); - } - } - - // Compile/publish errors - show detailed examples with full error messages - if !compile.is_empty() { - prompt.push_str(&format!("## Compile/Publish Errors ({} failures)\n\n", compile.len())); - - for f in compile.iter().take(3) { - let reasons = f - .scorer_details - .as_ref() - .map(extract_failure_reasons) - .unwrap_or_default(); - prompt.push_str(&format!("### {}\n", f.task)); - prompt.push_str(&format!("**Error**: {}\n\n", reasons.join(", "))); - - if let Some(llm_out) = &f.llm_output { - let truncated = truncate_str(llm_out, 1500); - prompt.push_str(&format!("**LLM Output**:\n```\n{}\n```\n\n", truncated)); - } - - if let Some(golden) = &f.golden_answer { - let truncated = truncate_str(golden, 1500); - prompt.push_str(&format!("**Expected (golden)**:\n```\n{}\n```\n\n", truncated)); - } - } - if compile.len() > 3 { - prompt.push_str(&format!( - "**Additional compile failures**: {}\n\n", - compile - .iter() - .skip(3) - .map(|f| f.task.as_str()) - .collect::>() - .join(", ") - )); - } - } - - // Timeout issues - if !timeout.is_empty() { - prompt.push_str(&format!("## Timeout Issues ({} failures)\n\n", timeout.len())); - prompt.push_str("These tasks timed out during execution:\n"); - for f in &timeout { - prompt.push_str(&format!("- {}\n", f.task)); - } - prompt.push('\n'); - } - - // Other failures - show detailed examples - if !other.is_empty() { - prompt.push_str(&format!("## Other Failures ({} failures)\n\n", other.len())); - - // Show up to 5 detailed examples for "other" since they're varied - for f in other.iter().take(5) { - let reasons = f - .scorer_details - .as_ref() - .map(extract_failure_reasons) - .unwrap_or_default(); - prompt.push_str(&format!("### {} - {}/{} tests passed\n", f.task, f.passed, f.total)); - prompt.push_str(&format!("**Failure reason**: {}\n\n", reasons.join(", "))); - - if let Some(llm_out) = &f.llm_output { - let truncated = truncate_str(llm_out, 1200); - prompt.push_str(&format!("**LLM Output**:\n```\n{}\n```\n\n", truncated)); - } - - if let Some(golden) = &f.golden_answer { - let truncated = truncate_str(golden, 1200); - prompt.push_str(&format!("**Expected (golden)**:\n```\n{}\n```\n\n", truncated)); - } - } - if other.len() > 5 { - prompt.push_str(&format!( - "**Additional failures**: {}\n\n", - other - .iter() - .skip(5) - .map(|f| f.task.as_str()) - .collect::>() - .join(", ") - )); - } - } -} - -/// Truncate a string to max_len chars, adding "..." if truncated. -fn truncate_str(s: &str, max_len: usize) -> String { - if s.len() > max_len { - format!("{}...", &s[..max_len]) - } else { - s.to_string() - } -} - -fn build_analysis_prompt(failures: &[FailureInfo]) -> String { - let mut prompt = String::from( - "Analyze the following SpacetimeDB benchmark test failures, organized by language and mode.\n\n\ - IMPORTANT: For each failure you analyze, you MUST include the actual code examples inline to illustrate the problem.\n\ - Show what the LLM generated vs what was expected, highlighting the specific differences.\n\n\ - Focus on SPECIFIC, ACTIONABLE documentation changes.\n\n", - ); - - // Group failures by language AND mode - let rust_rustdoc_failures: Vec<_> = failures - .iter() - .filter(|f| f.lang == "rust" && f.mode == "rustdoc_json") - .collect(); - let rust_docs_failures: Vec<_> = failures - .iter() - .filter(|f| f.lang == "rust" && f.mode == "docs") - .collect(); - let csharp_failures: Vec<_> = failures - .iter() - .filter(|f| f.lang == "csharp" && f.mode == "docs") - .collect(); - let typescript_failures: Vec<_> = failures - .iter() - .filter(|f| f.lang == "typescript" && f.mode == "docs") - .collect(); - - // Build sections for each language/mode combination - // Note: Rust rustdoc_json and Rust docs are separate sections because they use different context sources - if !rust_rustdoc_failures.is_empty() { - build_mode_section("rust", "rustdoc_json", &rust_rustdoc_failures, &mut prompt); - } - - if !rust_docs_failures.is_empty() { - build_mode_section("rust", "docs", &rust_docs_failures, &mut prompt); - } - - if !csharp_failures.is_empty() { - build_mode_section("csharp", "docs", &csharp_failures, &mut prompt); - } - - if !typescript_failures.is_empty() { - build_mode_section("typescript", "docs", &typescript_failures, &mut prompt); - } - - prompt.push_str( - "\n---\n\n## Instructions for your analysis:\n\n\ - For EACH failure or group of similar failures:\n\n\ - 1. **The generated code**: The actual LLM-generated code\n\ - 2. **The golden example**: The expected golden answer\n\ - 3. **The error**: The error message or failure reason (if provided above)\n\ - 4. **Explain the difference**: What specific API/syntax was wrong and caused the failure?\n\ - 5. **Root cause**: What's missing or unclear in the documentation?\n\ - 6. **Recommendation**: Specific fix\n\n\ - Group similar failures together (e.g., if multiple tests fail due to the same issue).\n\ - Use code blocks with syntax highlighting (```rust, ```csharp, or ```typescript).\n", - ); - - prompt -} diff --git a/tools/xtask-llm-benchmark/src/context/combine.rs b/tools/xtask-llm-benchmark/src/context/combine.rs index e889329096a..f9daed5867c 100644 --- a/tools/xtask-llm-benchmark/src/context/combine.rs +++ b/tools/xtask-llm-benchmark/src/context/combine.rs @@ -1,5 +1,5 @@ -use crate::context::constants::docs_dir; -use crate::context::paths::{gather_cursor_rules_files, resolve_mode_paths}; +use crate::context::constants::{docs_dir, is_empty_context_mode}; +use crate::context::paths::{gather_guidelines_files, resolve_mode_paths}; use crate::eval::lang::Lang; use anyhow::{anyhow, bail, Context, Result}; use regex::Regex; @@ -11,9 +11,9 @@ use std::sync::LazyLock; /// Get the base directory for a given mode (used for stripping prefixes to get relative paths). fn base_for_mode(mode: &str) -> Result { Ok(match mode { - "docs" | "llms.md" | "cursor_rules" | "none" => docs_dir(), // rustdoc_json is handled separately in build_context_from_rustdoc_json - _ => bail!("unknown mode `{mode}` for base_for_mode"), + "rustdoc_json" => bail!("rustdoc_json uses a different base; handled separately"), + _ => docs_dir(), }) } @@ -31,7 +31,7 @@ fn stable_rel_path(base: &Path, p: &Path) -> String { /// Build context for the given mode, optionally filtering tabs for a specific language. pub fn build_context(mode: &str, lang: Option) -> Result { - if mode == "none" { + if is_empty_context_mode(mode) { return Ok(String::new()); } if mode == "rustdoc_json" { @@ -39,8 +39,8 @@ pub fn build_context(mode: &str, lang: Option) -> Result { } let base = base_for_mode(mode)?; - let files = if mode == "cursor_rules" { - gather_cursor_rules_files(base.join("static/ai-rules"), lang)? + let files = if mode == "guidelines" { + gather_guidelines_files(base.join("static/ai-guidelines"), lang)? } else { resolve_mode_paths(mode)? }; @@ -49,8 +49,9 @@ pub fn build_context(mode: &str, lang: Option) -> Result { let rel = stable_rel_path(&base, &p); let contents = fs::read_to_string(&p).with_context(|| format!("read {}", rel))?; - // For cursor_rules we don't filter tabs; for other modes filter by lang if specified - let contents = if mode == "cursor_rules" { + // For guidelines we don't filter tabs; for other modes filter by lang if specified + let skip_tab_filter = mode == "guidelines"; + let contents = if skip_tab_filter { contents } else if let Some(lang) = lang { filter_tabs_for_lang(&contents, lang) @@ -59,8 +60,8 @@ pub fn build_context(mode: &str, lang: Option) -> Result { }; // When building for a specific language, skip files that have no content for it - // (e.g. Rust-only pages with no TypeScript tab). Not used for cursor_rules. - if mode != "cursor_rules" && lang.is_some() && contents.trim().is_empty() { + // (e.g. Rust-only pages with no TypeScript tab). Not used for guidelines. + if !skip_tab_filter && lang.is_some() && contents.trim().is_empty() { continue; } diff --git a/tools/xtask-llm-benchmark/src/context/constants.rs b/tools/xtask-llm-benchmark/src/context/constants.rs index 084f34e0401..c652908ecea 100644 --- a/tools/xtask-llm-benchmark/src/context/constants.rs +++ b/tools/xtask-llm-benchmark/src/context/constants.rs @@ -1,46 +1,29 @@ use std::path::PathBuf; pub const DOCS_DIR_DEFAULT: &str = "../../docs"; +pub const SKILLS_DIR_DEFAULT: &str = "../../skills"; pub const RUSTDOC_CRATE_ROOT_DEFAULT: &str = "../../crates/bindings"; -// Docs benchmark files (CI - single "best" model to test documentation quality) -pub const DOCS_BENCHMARK_DETAILS_DEFAULT: &str = "../../docs/llms/docs-benchmark-details.json"; -pub const DOCS_BENCHMARK_SUMMARY_DEFAULT: &str = "../../docs/llms/docs-benchmark-summary.json"; -pub const DOCS_BENCHMARK_COMMENT_DEFAULT: &str = "../../docs/llms/docs-benchmark-comment.md"; - -// LLM comparison files (manual runs - all models to compare LLM performance) -pub const LLM_COMPARISON_DETAILS_DEFAULT: &str = "../../docs/llms/llm-comparison-details.json"; -pub const LLM_COMPARISON_SUMMARY_DEFAULT: &str = "../../docs/llms/llm-comparison-summary.json"; - -pub const ALL_MODES: &[&str] = &["docs", "llms.md", "cursor_rules", "rustdoc_json", "none"]; +pub const ALL_MODES: &[&str] = &[ + "docs", + "llms.md", + "guidelines", + "rustdoc_json", + "no_context", + "none", // alias for no_context (backward compat) + "no_guidelines", // alias for no_context (backward compat) + "search", // no docs context but web search enabled via OpenRouter :online +]; +/// Modes that produce an empty context string (no documentation is injected). #[inline] -pub fn docs_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(DOCS_DIR_DEFAULT) +pub fn is_empty_context_mode(mode: &str) -> bool { + matches!(mode, "no_context" | "none" | "no_guidelines" | "search") } -// Docs benchmark paths (CI) -#[inline] -pub fn docs_benchmark_details() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(DOCS_BENCHMARK_DETAILS_DEFAULT) -} -#[inline] -pub fn docs_benchmark_summary() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(DOCS_BENCHMARK_SUMMARY_DEFAULT) -} #[inline] -pub fn docs_benchmark_comment() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(DOCS_BENCHMARK_COMMENT_DEFAULT) -} - -// LLM comparison paths (manual) -#[inline] -pub fn llm_comparison_details() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(LLM_COMPARISON_DETAILS_DEFAULT) -} -#[inline] -pub fn llm_comparison_summary() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(LLM_COMPARISON_SUMMARY_DEFAULT) +pub fn docs_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(DOCS_DIR_DEFAULT) } #[inline] diff --git a/tools/xtask-llm-benchmark/src/context/hashing.rs b/tools/xtask-llm-benchmark/src/context/hashing.rs index c6395efb379..d13dec28004 100644 --- a/tools/xtask-llm-benchmark/src/context/hashing.rs +++ b/tools/xtask-llm-benchmark/src/context/hashing.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use blake3::Hasher; use std::borrow::Cow; use std::fs; @@ -49,9 +49,8 @@ pub fn compute_processed_context_hash(mode: &str, lang: Lang) -> Result // --- stable base for stripping prefixes --- fn base_for_mode_hashing(mode: &str) -> Result { Ok(match mode { - "docs" | "llms.md" | "cursor_rules" | "none" => docs_dir(), "rustdoc_json" => rustdoc_crate_root(), - _ => bail!("unknown mode `{mode}`"), + _ => docs_dir(), }) } diff --git a/tools/xtask-llm-benchmark/src/context/paths.rs b/tools/xtask-llm-benchmark/src/context/paths.rs index d81ddf9af39..53ab3222485 100644 --- a/tools/xtask-llm-benchmark/src/context/paths.rs +++ b/tools/xtask-llm-benchmark/src/context/paths.rs @@ -1,4 +1,4 @@ -use crate::context::constants::docs_dir; +use crate::context::constants::{docs_dir, is_empty_context_mode}; use crate::context::hashing::gather_docs_files; use crate::context::{rustdoc_crate_root, rustdoc_readme_path}; use crate::eval::lang::Lang; @@ -17,22 +17,23 @@ pub fn resolve_mode_paths(mode: &str) -> Result> { match mode { "docs" => gather_docs_files(), "llms.md" => Ok(vec![docs_dir().join("static/llms.md")]), - "cursor_rules" => gather_cursor_rules_files(docs_dir().join("static/ai-rules"), None), + "guidelines" => gather_guidelines_files(docs_dir().join("static/ai-guidelines"), None), "rustdoc_json" => resolve_rustdoc_json_paths_always(), - "none" => Ok(Vec::new()), - other => bail!("unknown mode `{other}` (expected: docs | llms.md | cursor_rules | rustdoc_json | none)"), + m if is_empty_context_mode(m) => Ok(Vec::new()), + other => bail!( + "unknown mode `{other}` (expected: docs | llms.md | guidelines | rustdoc_json | no_context | search)" + ), } } -/// Cursor rules under docs: include general rules + rules for the given language. -/// General = filename (lowercase) does not contain "typescript", "rust", or "csharp". -/// Lang-specific = filename contains lang (e.g. "typescript" for TypeScript). -pub fn gather_cursor_rules_files(rules_dir: PathBuf, lang: Option) -> Result> { - if !rules_dir.is_dir() { +/// New constructive-only guidelines under docs/static/ai-guidelines/. +/// Files are named spacetimedb-{lang}.md and selected by language tag. +pub fn gather_guidelines_files(guidelines_dir: PathBuf, lang: Option) -> Result> { + if !guidelines_dir.is_dir() { return Ok(Vec::new()); } - let mut out: Vec = fs::read_dir(&rules_dir) - .with_context(|| format!("read cursor rules dir {}", rules_dir.display()))? + let mut out: Vec = fs::read_dir(&guidelines_dir) + .with_context(|| format!("read guidelines dir {}", guidelines_dir.display()))? .filter_map(|e| e.ok()) .map(|e| e.path()) .filter(|p| { @@ -55,13 +56,12 @@ pub fn gather_cursor_rules_files(rules_dir: PathBuf, lang: Option) -> Resu Ok(out) } -// --- hashing resolver stays as you wrote it --- pub fn resolve_mode_paths_hashing(mode: &str) -> Result> { match mode { "docs" => gather_docs_files(), "llms.md" => Ok(vec![docs_dir().join("static/llms.md")]), - "cursor_rules" => gather_cursor_rules_files(docs_dir().join("static/ai-rules"), None), - "none" => Ok(Vec::new()), + "guidelines" => gather_guidelines_files(docs_dir().join("static/ai-guidelines"), None), + m if is_empty_context_mode(m) => Ok(Vec::new()), "rustdoc_json" => { if let Some(p) = rustdoc_readme_path() { Ok(vec![p]) @@ -69,7 +69,9 @@ pub fn resolve_mode_paths_hashing(mode: &str) -> Result> { bail!("README not found under {}", rustdoc_crate_root().display()) } } - other => bail!("unknown mode `{other}` (expected: docs | llms.md | cursor_rules | rustdoc_json | none)"), + other => bail!( + "unknown mode `{other}` (expected: docs | llms.md | guidelines | rustdoc_json | no_context | search)" + ), } } diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index 157e15a2c74..ea40f7a8b93 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -1,6 +1,7 @@ use crate::bench::utils::sanitize_db_name; use crate::eval::scorers::{ - ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, SqlCountOnlyScorer, SqlExecBothScorer, + ReducerCallBothScorer, ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, + SqlCountOnlyScorer, SqlExecBothScorer, }; use crate::eval::{derive_cat_task_from_file, ReducerDataParityConfig, ReducerSqlCountConfig}; use std::time::Duration; @@ -96,3 +97,25 @@ pub fn make_sql_exec_both_scorer( id_str, }) as Box } + +pub fn make_reducer_call_both_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + reducer: &str, + args: Vec, + id_str: &'static str, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + + Box::new(ReducerCallBothScorer { + server: host_url.to_string(), + golden_db, + llm_db, + reducer: reducer.to_string(), + args, + id_str, + }) as Box +} diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 3be2fcc3cc9..2b1c850a562 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -600,3 +600,49 @@ impl Scorer for SqlExecBothScorer { } } } + +pub struct ReducerCallBothScorer { + pub server: String, + pub golden_db: String, + pub llm_db: String, + pub reducer: String, + pub args: Vec, + pub id_str: &'static str, +} + +impl Scorer for ReducerCallBothScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + if debug_llm_verbose() { + eprintln!( + "[dbg] ReducerCallBothScorer: reducer={} args={:?} golden_db={} llm_db={} server={}", + self.reducer, self.args, self.golden_db, self.llm_db, self.server + ); + } + if let Err(e) = call_reducer_json_out(&self.golden_db, &self.reducer, &self.args, Some(&self.server)) { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase":"call_reducer_golden", "error": e, "reducer": self.reducer }), + }; + } + if let Err(e) = call_reducer_json_out(&self.llm_db, &self.reducer, &self.args, Some(&self.server)) { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase":"call_reducer_llm", "error": e, "reducer": self.reducer }), + }; + } + if debug_llm_verbose() { + eprintln!("[dbg] ReducerCallBothScorer: success"); + } + ScoreDetails { + pass: true, + partial: 1.0, + notes: json!({ "reducer": self.reducer, "args": self.args }), + } + } +} diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 72b049e62f9..ef74e3aac20 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -2,6 +2,54 @@ use crate::eval::BenchmarkSpec; use anyhow::{anyhow, Result}; use std::path::Path; +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_026_auth_identity_check { + include!("../benchmarks/auth/t_026_auth_identity_check/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_027_private_vs_public_table { + include!("../benchmarks/auth/t_027_private_vs_public_table/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_041_registered_user_gate { + include!("../benchmarks/auth/t_041_registered_user_gate/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_042_admin_bootstrap { + include!("../benchmarks/auth/t_042_admin_bootstrap/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_043_role_based_access { + include!("../benchmarks/auth/t_043_role_based_access/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_044_ban_list { + include!("../benchmarks/auth/t_044_ban_list/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_045_rate_limit { + include!("../benchmarks/auth/t_045_rate_limit/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_046_shared_document { + include!("../benchmarks/auth/t_046_shared_document/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod basics_t_000_empty_reducers { @@ -74,6 +122,108 @@ mod basics_t_011_helper_function { include!("../benchmarks/basics/t_011_helper_function/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod basics_t_038_schedule_at_time { + include!("../benchmarks/basics/t_038_schedule_at_time/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod basics_t_039_cancel_schedule { + include!("../benchmarks/basics/t_039_cancel_schedule/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod basics_t_040_lifecycle_player { + include!("../benchmarks/basics/t_040_lifecycle_player/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod data_modeling_t_024_event_table { + include!("../benchmarks/data_modeling/t_024_event_table/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod data_modeling_t_025_optional_fields { + include!("../benchmarks/data_modeling/t_025_optional_fields/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod data_modeling_t_028_cascade_delete { + include!("../benchmarks/data_modeling/t_028_cascade_delete/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod data_modeling_t_029_filter_and_aggregate { + include!("../benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod data_modeling_t_030_two_table_join { + include!("../benchmarks/data_modeling/t_030_two_table_join/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod data_modeling_t_031_unique_constraint { + include!("../benchmarks/data_modeling/t_031_unique_constraint/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_022_view_basic { + include!("../benchmarks/queries/t_022_view_basic/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_023_view_per_user { + include!("../benchmarks/queries/t_023_view_per_user/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_032_range_query { + include!("../benchmarks/queries/t_032_range_query/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_033_sort_and_limit { + include!("../benchmarks/queries/t_033_sort_and_limit/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_034_find_first { + include!("../benchmarks/queries/t_034_find_first/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_035_select_distinct { + include!("../benchmarks/queries/t_035_select_distinct/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_036_count_without_collect { + include!("../benchmarks/queries/t_036_count_without_collect/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod queries_t_037_multi_column_filter { + include!("../benchmarks/queries/t_037_multi_column_filter/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod schema_t_012_spacetime_product_type { @@ -145,6 +295,14 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { .ok_or_else(|| anyhow!("missing category name"))?; let ctor = match (category, task) { + ("auth", "t_026_auth_identity_check") => auth_t_026_auth_identity_check::spec, + ("auth", "t_027_private_vs_public_table") => auth_t_027_private_vs_public_table::spec, + ("auth", "t_041_registered_user_gate") => auth_t_041_registered_user_gate::spec, + ("auth", "t_042_admin_bootstrap") => auth_t_042_admin_bootstrap::spec, + ("auth", "t_043_role_based_access") => auth_t_043_role_based_access::spec, + ("auth", "t_044_ban_list") => auth_t_044_ban_list::spec, + ("auth", "t_045_rate_limit") => auth_t_045_rate_limit::spec, + ("auth", "t_046_shared_document") => auth_t_046_shared_document::spec, ("basics", "t_000_empty_reducers") => basics_t_000_empty_reducers::spec, ("basics", "t_001_basic_tables") => basics_t_001_basic_tables::spec, ("basics", "t_002_scheduled_table") => basics_t_002_scheduled_table::spec, @@ -157,6 +315,23 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("basics", "t_009_init") => basics_t_009_init::spec, ("basics", "t_010_connect") => basics_t_010_connect::spec, ("basics", "t_011_helper_function") => basics_t_011_helper_function::spec, + ("basics", "t_038_schedule_at_time") => basics_t_038_schedule_at_time::spec, + ("basics", "t_039_cancel_schedule") => basics_t_039_cancel_schedule::spec, + ("basics", "t_040_lifecycle_player") => basics_t_040_lifecycle_player::spec, + ("data_modeling", "t_024_event_table") => data_modeling_t_024_event_table::spec, + ("data_modeling", "t_025_optional_fields") => data_modeling_t_025_optional_fields::spec, + ("data_modeling", "t_028_cascade_delete") => data_modeling_t_028_cascade_delete::spec, + ("data_modeling", "t_029_filter_and_aggregate") => data_modeling_t_029_filter_and_aggregate::spec, + ("data_modeling", "t_030_two_table_join") => data_modeling_t_030_two_table_join::spec, + ("data_modeling", "t_031_unique_constraint") => data_modeling_t_031_unique_constraint::spec, + ("queries", "t_022_view_basic") => queries_t_022_view_basic::spec, + ("queries", "t_023_view_per_user") => queries_t_023_view_per_user::spec, + ("queries", "t_032_range_query") => queries_t_032_range_query::spec, + ("queries", "t_033_sort_and_limit") => queries_t_033_sort_and_limit::spec, + ("queries", "t_034_find_first") => queries_t_034_find_first::spec, + ("queries", "t_035_select_distinct") => queries_t_035_select_distinct::spec, + ("queries", "t_036_count_without_collect") => queries_t_036_count_without_collect::spec, + ("queries", "t_037_multi_column_filter") => queries_t_037_multi_column_filter::spec, ("schema", "t_012_spacetime_product_type") => schema_t_012_spacetime_product_type::spec, ("schema", "t_013_spacetime_sum_type") => schema_t_013_spacetime_sum_type::spec, ("schema", "t_014_elementary_columns") => schema_t_014_elementary_columns::spec, diff --git a/tools/xtask-llm-benchmark/src/lib.rs b/tools/xtask-llm-benchmark/src/lib.rs index 14ceb381c42..f26ebd26b3f 100644 --- a/tools/xtask-llm-benchmark/src/lib.rs +++ b/tools/xtask-llm-benchmark/src/lib.rs @@ -1,5 +1,6 @@ #![allow(clippy::disallowed_macros)] +pub mod api; pub mod bench; pub mod context; pub mod eval; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs index 4472f8a132b..c7a057c4638 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ anthropic_ctx_limit_tokens, build_anthropic_messages, desired_output_tokens, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, }; -use crate::llm::types::Vendor; +use crate::llm::types::{LlmOutput, Vendor}; use anyhow::{anyhow, bail, Context, Result}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE}; use reqwest::{Client, StatusCode}; @@ -30,7 +30,7 @@ impl AnthropicClient { format!("{}/v1/messages", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); let mut static_prefix = prompt.static_prefix.clone().unwrap_or_default(); @@ -114,10 +114,19 @@ impl AnthropicClient { bail!("POST {} -> {}: {}", url, status, body); } + #[derive(Deserialize)] + struct AnthropicUsage { + #[serde(default)] + input_tokens: Option, + #[serde(default)] + output_tokens: Option, + } #[derive(Deserialize)] struct MsgResp { #[serde(default)] content: Vec, + #[serde(default)] + usage: Option, } #[derive(Deserialize)] struct ContentPart { @@ -128,11 +137,18 @@ impl AnthropicClient { } let parsed: MsgResp = serde_json::from_str(&body).context("parse anthropic resp")?; - parsed + let input_tokens = parsed.usage.as_ref().and_then(|u| u.input_tokens); + let output_tokens = parsed.usage.as_ref().and_then(|u| u.output_tokens); + let text = parsed .content .into_iter() .find_map(|p| p.text) - .ok_or_else(|| anyhow!("no text")) + .ok_or_else(|| anyhow!("no text"))?; + Ok(LlmOutput { + text, + input_tokens, + output_tokens, + }) } async fn post_with_retries( diff --git a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs index 889467deff1..5b7bc7b8636 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs @@ -1,12 +1,13 @@ use anyhow::{anyhow, Context, Result}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use super::http::HttpClient; +use super::oa_compat::OACompatResp; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deepseek_ctx_limit_tokens, deterministic_trim_prefix, estimate_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::Vendor; +use crate::llm::types::{LlmOutput, Vendor}; #[derive(Clone)] pub struct DeepSeekClient { @@ -20,7 +21,7 @@ impl DeepSeekClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); let system = prompt.system.clone(); @@ -82,24 +83,13 @@ impl DeepSeekClient { let auth = HttpClient::bearer(&self.api_key); let body = self.http.post_json(&url, &[auth], &req).await?; let resp: OACompatResp = serde_json::from_str(&body).context("parse deepseek resp")?; - resp.first_text().ok_or_else(|| anyhow!("no content from DeepSeek")) - } -} - -#[derive(Debug, Deserialize)] -struct OACompatResp { - choices: Vec, -} -#[derive(Debug, Deserialize)] -struct Choice { - message: MsgOut, -} -#[derive(Debug, Deserialize)] -struct MsgOut { - content: String, -} -impl OACompatResp { - fn first_text(self) -> Option { - self.choices.into_iter().next().map(|c| c.message.content) + let input_tokens = resp.usage.as_ref().and_then(|u| u.prompt_tokens); + let output_tokens = resp.usage.as_ref().and_then(|u| u.completion_tokens); + let text = resp.first_text().ok_or_else(|| anyhow!("no content from DeepSeek"))?; + Ok(LlmOutput { + text, + input_tokens, + output_tokens, + }) } } diff --git a/tools/xtask-llm-benchmark/src/llm/clients/google.rs b/tools/xtask-llm-benchmark/src/llm/clients/google.rs index 2f6401aca96..ad309a88477 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/google.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/google.rs @@ -8,7 +8,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, gemini_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::Vendor; +use crate::llm::types::{LlmOutput, Vendor}; /// Google uses API key in the query string rather than Authorization header. #[derive(Clone)] @@ -23,7 +23,7 @@ impl GoogleGeminiClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { // ---- Never trim system or dynamic segments ---- let system = prompt.system.clone(); let segs: Vec> = prompt.segments.clone(); @@ -146,6 +146,8 @@ impl GoogleGeminiClient { #[derive(Debug, Deserialize)] struct GeminiResp { candidates: Vec, + #[serde(default, rename = "usageMetadata")] + usage_metadata: Option, } #[derive(Debug, Deserialize)] @@ -164,6 +166,14 @@ impl GoogleGeminiClient { text: Option, } + #[derive(Debug, Deserialize)] + struct GeminiUsageMetadata { + #[serde(default, rename = "promptTokenCount")] + prompt_token_count: Option, + #[serde(default, rename = "candidatesTokenCount")] + candidates_token_count: Option, + } + impl GeminiResp { fn first_text(self) -> Option { self.candidates @@ -174,8 +184,14 @@ impl GoogleGeminiClient { } let resp: GeminiResp = serde_json::from_str(&body).context("parse gemini response")?; - let out = resp.first_text().ok_or_else(|| anyhow!("no text in Gemini response"))?; - Ok(out) + let input_tokens = resp.usage_metadata.as_ref().and_then(|u| u.prompt_token_count); + let output_tokens = resp.usage_metadata.as_ref().and_then(|u| u.candidates_token_count); + let text = resp.first_text().ok_or_else(|| anyhow!("no text in Gemini response"))?; + Ok(LlmOutput { + text, + input_tokens, + output_tokens, + }) } } diff --git a/tools/xtask-llm-benchmark/src/llm/clients/http.rs b/tools/xtask-llm-benchmark/src/llm/clients/http.rs index b432b2215a7..8e061cfae4a 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/http.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/http.rs @@ -15,7 +15,7 @@ impl HttpClient { pub fn new() -> Result { let inner = Client::builder() .user_agent("xtask-llm-benchmark/0.1") - .timeout(Duration::from_secs(120)) + .timeout(Duration::from_secs(660)) // longer than any per-model Tokio timeout .pool_idle_timeout(Duration::from_secs(30)) .build() .context("build reqwest client")?; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs index 4a34ba2a87f..4744361df73 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs @@ -1,12 +1,13 @@ use anyhow::{anyhow, Context, Result}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use super::http::HttpClient; +use super::oa_compat::OACompatResp; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ - deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, + desired_output_tokens, deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::Vendor; +use crate::llm::types::{LlmOutput, Vendor}; #[derive(Clone)] pub struct MetaLlamaClient { @@ -21,7 +22,7 @@ impl MetaLlamaClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); // Build input like other clients @@ -83,8 +84,8 @@ impl MetaLlamaClient { model: wire_model, messages, temperature: 0.0, - top_p: Some(0.9), // avoid creative tangents; deterministic code - max_tokens: None, + top_p: None, + max_tokens: Some(desired_output_tokens().max(1) as u32), }; // Auth only; optional OpenRouter headers can live in HttpClient if desired @@ -96,8 +97,16 @@ impl MetaLlamaClient { .with_context(|| format!("OpenRouter (Meta) POST {}", url))?; let resp: OACompatResp = serde_json::from_str(&body).context("parse OpenRouter (Meta) response")?; - resp.first_text() - .ok_or_else(|| anyhow!("no content from Meta/OpenRouter")) + let input_tokens = resp.usage.as_ref().and_then(|u| u.prompt_tokens); + let output_tokens = resp.usage.as_ref().and_then(|u| u.completion_tokens); + let text = resp + .first_text() + .ok_or_else(|| anyhow!("no content from Meta/OpenRouter"))?; + Ok(LlmOutput { + text, + input_tokens, + output_tokens, + }) } } @@ -124,21 +133,3 @@ fn normalize_meta_model(id: &str) -> &str { other => other, } } - -#[derive(Debug, Deserialize)] -struct OACompatResp { - choices: Vec, -} -#[derive(Debug, Deserialize)] -struct Choice { - message: MsgOut, -} -#[derive(Debug, Deserialize)] -struct MsgOut { - content: String, -} -impl OACompatResp { - fn first_text(self) -> Option { - self.choices.into_iter().next().map(|c| c.message.content) - } -} diff --git a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs index 4c5b1dfe195..172beef8ff8 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs @@ -3,7 +3,9 @@ pub mod deepseek; pub mod google; pub(crate) mod http; pub mod meta; +pub(crate) mod oa_compat; pub mod openai; +pub mod openrouter; pub mod xai; pub use anthropic::AnthropicClient; @@ -11,4 +13,5 @@ pub use deepseek::DeepSeekClient; pub use google::GoogleGeminiClient; pub use meta::MetaLlamaClient; pub use openai::OpenAiClient; +pub use openrouter::OpenRouterClient; pub use xai::XaiGrokClient; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs new file mode 100644 index 00000000000..a199b3b3c0c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs @@ -0,0 +1,45 @@ +//! Shared types for OpenAI-compatible chat completions responses. +//! Used by DeepSeek, Meta/OpenRouter, and OpenRouter clients. + +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +pub struct OACompatResp { + #[serde(default)] + pub choices: Vec, + #[serde(default)] + pub usage: Option, + /// OpenRouter returns an `error` object when the upstream provider fails. + /// Absent (None) for direct vendor APIs. + #[serde(default)] + pub error: Option, +} + +#[derive(Debug, Deserialize)] +pub struct OAError { + pub message: String, +} + +#[derive(Debug, Deserialize)] +pub struct Choice { + pub message: MsgOut, +} + +#[derive(Debug, Deserialize)] +pub struct MsgOut { + pub content: String, +} + +#[derive(Debug, Deserialize)] +pub struct UsageInfo { + #[serde(default)] + pub prompt_tokens: Option, + #[serde(default)] + pub completion_tokens: Option, +} + +impl OACompatResp { + pub fn first_text(self) -> Option { + self.choices.into_iter().next().map(|c| c.message.content) + } +} diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs index c95a44cedc7..9fed933d3fb 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ build_openai_responses_input, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, openai_ctx_limit_tokens, }; -use crate::llm::types::Vendor; +use crate::llm::types::{LlmOutput, Vendor}; use anyhow::{bail, Context, Result}; use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; @@ -29,7 +29,7 @@ impl OpenAiClient { format!("{}/v1/responses", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); @@ -132,6 +132,13 @@ impl OpenAiClient { summary: Option>, } #[derive(Deserialize)] + struct OpenAiUsage { + #[serde(default)] + input_tokens: Option, + #[serde(default)] + output_tokens: Option, + } + #[derive(Deserialize)] struct ResponsesPayload { #[serde(default)] status: Option, @@ -141,14 +148,25 @@ impl OpenAiClient { output_text: Option, #[serde(default)] output: Option>, + #[serde(default)] + usage: Option, } let parsed: ResponsesPayload = serde_json::from_str(&body).context("parse OpenAI response")?; + let input_tokens = parsed.usage.as_ref().and_then(|u| u.input_tokens); + let output_tokens = parsed.usage.as_ref().and_then(|u| u.output_tokens); + + let make_output = |text: String| LlmOutput { + text, + input_tokens, + output_tokens, + }; + if let Some(t) = parsed.output_text.as_ref() && !t.is_empty() { - return Ok(t.clone()); + return Ok(make_output(t.clone())); } if let Some(items) = parsed.output.as_ref() { for it in items { @@ -159,7 +177,7 @@ impl OpenAiClient { if let Some(txt) = &c.text && !txt.is_empty() { - return Ok(txt.clone()); + return Ok(make_output(txt.clone())); } } } @@ -172,7 +190,7 @@ impl OpenAiClient { if let Some(txt) = &c.text && !txt.is_empty() { - return Ok(txt.clone()); + return Ok(make_output(txt.clone())); } } } diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs new file mode 100644 index 00000000000..623570298af --- /dev/null +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -0,0 +1,175 @@ +use anyhow::{anyhow, Context, Result}; +use serde::Serialize; + +use super::http::HttpClient; +use super::oa_compat::OACompatResp; +use crate::llm::prompt::BuiltPrompt; +use crate::llm::segmentation::{ + desired_output_tokens, deterministic_trim_prefix, non_context_reserve_tokens_env, Segment, +}; +use crate::llm::types::{LlmOutput, Vendor}; + +const OPENROUTER_BASE: &str = "https://openrouter.ai/api/v1"; + +/// Default context limit for OpenRouter models (conservative). +/// Per-model overrides can be added to `openrouter_ctx_limit_tokens`. +const DEFAULT_CTX_LIMIT: usize = 128_000; + +#[derive(Clone)] +pub struct OpenRouterClient { + base: String, + api_key: String, + http: HttpClient, +} + +impl OpenRouterClient { + pub fn new(http: HttpClient, api_key: String) -> Self { + Self { + base: OPENROUTER_BASE.to_string(), + api_key, + http, + } + } + + pub fn with_base(http: HttpClient, base: String, api_key: String) -> Self { + Self { base, api_key, http } + } + + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); + + let system = prompt.system.clone(); + let segs: Vec> = prompt.segments.clone(); + + // Trim static prefix against model's context allowance + let mut static_prefix = prompt.static_prefix.clone().unwrap_or_default(); + let ctx_limit = openrouter_ctx_limit_tokens(model); + // Use a generic reserve since we don't know the vendor ahead of time. + // OpenRouter routes to the right vendor, so this is a safe conservative default. + let reserve = non_context_reserve_tokens_env(Vendor::OpenRouter); + let allowance = ctx_limit.saturating_sub(reserve); + static_prefix = deterministic_trim_prefix(&static_prefix, allowance); + + // OpenAI-compatible chat completions schema + #[derive(Serialize)] + struct Req<'a> { + model: &'a str, + messages: Vec>, + temperature: f32, + #[serde(skip_serializing_if = "Option::is_none")] + top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + max_tokens: Option, + } + + #[derive(Serialize)] + struct Msg<'a> { + role: &'a str, + content: &'a str, + } + + let mut messages: Vec = Vec::new(); + + if let Some(sys) = system.as_deref() + && !sys.is_empty() + { + messages.push(Msg { + role: "system", + content: sys, + }); + } + if !static_prefix.is_empty() { + messages.push(Msg { + role: "user", + content: &static_prefix, + }); + } + for s in &segs { + messages.push(Msg { + role: s.role, + content: &s.text, + }); + } + + let max_tokens = desired_output_tokens().max(1) as u32; + let req = Req { + model, + messages, + temperature: 0.0, + top_p: None, + max_tokens: Some(max_tokens), + }; + + let auth = HttpClient::bearer(&self.api_key); + let body = self + .http + .post_json(&url, &[auth], &req) + .await + .with_context(|| format!("OpenRouter POST {}", url))?; + + let resp: OACompatResp = serde_json::from_str(&body).context("parse OpenRouter response")?; + // Check for upstream provider errors returned by OpenRouter. + if let Some(err) = resp.error { + anyhow::bail!("OpenRouter upstream error (model={}): {}", model, err.message); + } + let input_tokens = resp.usage.as_ref().and_then(|u| u.prompt_tokens); + let output_tokens = resp.usage.as_ref().and_then(|u| u.completion_tokens); + let text = resp + .first_text() + .ok_or_else(|| anyhow!("no content from OpenRouter (model={})", model))?; + Ok(LlmOutput { + text, + input_tokens, + output_tokens, + }) + } +} + +/// Context limits for models accessed via OpenRouter. +/// Uses the same limits as direct clients where known, +/// falls back to a conservative default. +pub fn openrouter_ctx_limit_tokens(model: &str) -> usize { + let m = model.to_ascii_lowercase(); + + // Anthropic + if m.contains("claude") { + return 185_000; + } + // OpenAI + if m.contains("gpt-5") || m.contains("gpt-4.1") { + return 400_000; + } + if m.contains("gpt-4o") || m.contains("gpt-4") { + return 128_000; + } + // xAI / Grok — leave ~50 k headroom for segments + output on top of trimmed prefix + if m.contains("grok-code-fast") { + return 200_000; + } + if m.contains("grok-4") { + return 200_000; + } + if m.contains("grok") { + return 90_000; + } + // DeepSeek — hard cap is 131 072 on OpenRouter; leave ~25 k headroom for segments + output + if m.contains("deepseek") { + return 106_000; + } + // Gemini + if m.contains("gemini") { + return 900_000; + } + // Meta / Llama + if m.contains("maverick") { + return 992_000; + } + if m.contains("scout") { + return 320_000; + } + if m.contains("llama") { + return 120_000; + } + + DEFAULT_CTX_LIMIT +} diff --git a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs index 76c8de46e97..be345c82b32 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs @@ -6,7 +6,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, non_context_reserve_tokens_env, xai_ctx_limit_tokens, Segment, }; -use crate::llm::types::Vendor; +use crate::llm::types::{LlmOutput, Vendor}; #[derive(Clone)] pub struct XaiGrokClient { @@ -21,7 +21,7 @@ impl XaiGrokClient { } /// Uses BuiltPrompt (system, static_prefix, segments) and maps to xAI /chat/completions. - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { let url = format!("{}/v1/chat/completions", self.base.trim_end_matches('/')); // Never trim system or dynamic segments @@ -80,13 +80,22 @@ impl XaiGrokClient { let auth = HttpClient::bearer(&self.api_key); let body = self.http.post_json(&url, &[auth], &req).await?; let resp: GrokChatResp = serde_json::from_str(&body).context("parse grok resp")?; - resp.into_first_text().ok_or_else(|| anyhow!("no content from Grok")) + let input_tokens = resp.usage.as_ref().and_then(|u| u.prompt_tokens); + let output_tokens = resp.usage.as_ref().and_then(|u| u.completion_tokens); + let text = resp.into_first_text().ok_or_else(|| anyhow!("no content from Grok"))?; + Ok(LlmOutput { + text, + input_tokens, + output_tokens, + }) } } #[derive(Debug, Deserialize)] struct GrokChatResp { choices: Vec, + #[serde(default)] + usage: Option, } #[derive(Debug, Deserialize)] struct GrokChoice { @@ -96,6 +105,13 @@ struct GrokChoice { struct GrokMsgOut { content: String, } +#[derive(Debug, Deserialize)] +struct GrokUsageInfo { + #[serde(default)] + prompt_tokens: Option, + #[serde(default)] + completion_tokens: Option, +} impl GrokChatResp { fn into_first_text(self) -> Option { diff --git a/tools/xtask-llm-benchmark/src/llm/config.rs b/tools/xtask-llm-benchmark/src/llm/config.rs index 362b3164f54..5eacdf40248 100644 --- a/tools/xtask-llm-benchmark/src/llm/config.rs +++ b/tools/xtask-llm-benchmark/src/llm/config.rs @@ -3,7 +3,7 @@ use std::{env, sync::Arc}; use crate::llm::clients::http::HttpClient; use crate::llm::clients::{ - AnthropicClient, DeepSeekClient, GoogleGeminiClient, MetaLlamaClient, OpenAiClient, XaiGrokClient, + AnthropicClient, DeepSeekClient, GoogleGeminiClient, MetaLlamaClient, OpenAiClient, OpenRouterClient, XaiGrokClient, }; use crate::llm::provider::{LlmProvider, RouterProvider}; use crate::llm::types::Vendor; @@ -16,27 +16,36 @@ fn force_vendor_from_env() -> Option { Some("xai") | Some("grok") => Some(Vendor::Xai), Some("deepseek") => Some(Vendor::DeepSeek), Some("meta") | Some("llama") => Some(Vendor::Meta), + Some("openrouter") | Some("or") => Some(Vendor::OpenRouter), _ => None, } } /// Env vars: +/// - OPENROUTER_API_KEY (unified proxy — routes to any vendor) /// - OPENAI_API_KEY, OPENAI_BASE_URL (default https://api.openai.com) /// - ANTHROPIC_API_KEY, ANTHROPIC_BASE_URL (default https://api.anthropic.com) /// - GOOGLE_API_KEY, GOOGLE_BASE_URL (default https://generativelanguage.googleapis.com) /// - XAI_API_KEY, XAI_BASE_URL (default https://api.x.ai) /// - DEEPSEEK_API_KEY, DEEPSEEK_BASE_URL (default https://api.deepseek.com) /// - META_API_KEY, META_BASE_URL (no default) -/// - LLM_VENDOR: openai|anthropic|google|xai|deepseek|meta +/// - LLM_VENDOR: openai|anthropic|google|xai|deepseek|meta|openrouter +/// +/// When OPENROUTER_API_KEY is set, it acts as a fallback for any vendor that doesn't +/// have its own direct API key configured. This means you can set just OPENROUTER_API_KEY +/// to run all models through OpenRouter, or mix direct keys with OpenRouter fallback. pub fn make_provider_from_env() -> Result> { let http = HttpClient::new()?; - let openai_key = env::var("OPENAI_API_KEY").ok(); - let anth_key = env::var("ANTHROPIC_API_KEY").ok(); - let google_key = env::var("GOOGLE_API_KEY").ok(); - let xai_key = env::var("XAI_API_KEY").ok(); - let deep_key = env::var("DEEPSEEK_API_KEY").ok(); - let meta_key = env::var("META_API_KEY").ok(); + // Filter out empty strings so an empty env var falls through to OpenRouter. + let non_empty = |k: &str| env::var(k).ok().filter(|v| !v.trim().is_empty()); + let openai_key = non_empty("OPENAI_API_KEY"); + let anth_key = non_empty("ANTHROPIC_API_KEY"); + let google_key = non_empty("GOOGLE_API_KEY"); + let xai_key = non_empty("XAI_API_KEY"); + let deep_key = non_empty("DEEPSEEK_API_KEY"); + let meta_key = non_empty("META_API_KEY"); + let openrouter_key = non_empty("OPENROUTER_API_KEY"); // IMPORTANT: no trailing /v1 here; clients append their own versioned paths. let openai_base = env::var("OPENAI_BASE_URL").unwrap_or_else(|_| "https://api.openai.com".to_string()); @@ -72,7 +81,9 @@ pub fn make_provider_from_env() -> Result> { _ => None, }; + let openrouter = openrouter_key.map(|k| OpenRouterClient::new(http.clone(), k)); + let force = force_vendor_from_env(); - let router = RouterProvider::new(openai, anthropic, google, xai, deepseek, meta, force); + let router = RouterProvider::new(openai, anthropic, google, xai, deepseek, meta, openrouter, force); Ok(Arc::new(router)) } diff --git a/tools/xtask-llm-benchmark/src/llm/mod.rs b/tools/xtask-llm-benchmark/src/llm/mod.rs index 5d3e96670a5..4b72185c760 100644 --- a/tools/xtask-llm-benchmark/src/llm/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/mod.rs @@ -10,3 +10,4 @@ pub use config::make_provider_from_env; pub use model_routes::{default_model_routes, ModelRoute}; pub use prompt::PromptBuilder; pub use provider::{LlmProvider, RouterProvider}; +pub use types::LlmOutput; diff --git a/tools/xtask-llm-benchmark/src/llm/model_routes.rs b/tools/xtask-llm-benchmark/src/llm/model_routes.rs index 993a996fcc2..6598e6bf127 100644 --- a/tools/xtask-llm-benchmark/src/llm/model_routes.rs +++ b/tools/xtask-llm-benchmark/src/llm/model_routes.rs @@ -1,80 +1,80 @@ use crate::llm::types::Vendor; use serde::{Deserialize, Serialize}; +use std::sync::LazyLock; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModelRoute { - pub display_name: &'static str, // human-friendly label for reports - pub vendor: Vendor, // which API family to use - pub api_model: &'static str, // model id expected by the vendor API + pub display_name: String, // human-friendly label for reports + pub vendor: Vendor, // which API family to use + pub api_model: String, // model id expected by the vendor's direct API + pub openrouter_model: Option, // OpenRouter model id (if different from api_model) } -pub fn default_model_routes() -> &'static [ModelRoute] { +static DEFAULT_ROUTES: LazyLock> = LazyLock::new(|| { use Vendor::*; - &[ + vec![ // OpenAI: Best GPT-5.2-Codex, Cheaper GPT-5-mini - ModelRoute { - display_name: "GPT-5.2-Codex", - vendor: OpenAi, - api_model: "gpt-5.2-codex", - }, - ModelRoute { - display_name: "GPT-5-mini", - vendor: OpenAi, - api_model: "gpt-5-mini", - }, + ModelRoute::new("GPT-5.2-Codex", OpenAi, "gpt-5.2-codex", Some("openai/gpt-5.2-codex")), + ModelRoute::new("GPT-5-mini", OpenAi, "gpt-5-mini", Some("openai/gpt-5-mini")), // Claude: Best Opus 4.6, Cheaper Sonnet 4.6 - ModelRoute { - display_name: "Claude Opus 4.6", - vendor: Anthropic, - api_model: "claude-opus-4-6", - }, - ModelRoute { - display_name: "Claude Sonnet 4.6", - vendor: Anthropic, - api_model: "claude-sonnet-4-6", - }, + // Direct API uses dashes (claude-opus-4-6); OpenRouter uses dots (claude-opus-4.6) + ModelRoute::new( + "Claude Opus 4.6", + Anthropic, + "claude-opus-4-6", + Some("anthropic/claude-opus-4.6"), + ), + ModelRoute::new( + "Claude Sonnet 4.6", + Anthropic, + "claude-sonnet-4-6", + Some("anthropic/claude-sonnet-4.6"), + ), // Grok: Best Grok 4, Cheaper Grok Code - ModelRoute { - display_name: "Grok 4", - vendor: Xai, - api_model: "grok-4", - }, - ModelRoute { - display_name: "Grok Code", - vendor: Xai, - api_model: "grok-code-fast-1", - }, - ModelRoute { - display_name: "Gemini 3.1 Pro", - vendor: Meta, // uses MetaLlamaClient = OpenRouter - api_model: "google/gemini-3.1-pro-preview", - }, - ModelRoute { - display_name: "Gemini 3 Flash", - vendor: Meta, - api_model: "google/gemini-3-flash-preview", - }, - // Gemini: Best 3.1 Pro, Cheaper 3 Flash - // ModelRoute { - // display_name: "Gemini 3.1 Pro", - // vendor: Google, - // api_model: "gemini-3.1-pro-preview", - // }, - // ModelRoute { - // display_name: "Gemini 3 Flash", - // vendor: Google, - // api_model: "gemini-3-flash-preview", - // }, + // grok-4 → x-ai/grok-4.20-beta on OpenRouter; grok-code-fast-1 not on OpenRouter → x-ai/grok-3 + ModelRoute::new("Grok 4", Xai, "grok-4", Some("x-ai/grok-4.20-beta")), + ModelRoute::new("Grok Code", Xai, "grok-code-fast-1", Some("x-ai/grok-code-fast-1")), + // Gemini: direct via GOOGLE_API_KEY, falls back to OpenRouter if not set + ModelRoute::new( + "Gemini 3.1 Pro", + Google, + "gemini-3.1-pro-preview", + Some("google/gemini-3.1-pro-preview"), + ), + ModelRoute::new( + "Gemini 3 Flash", + Google, + "gemini-3-flash-preview", + Some("google/gemini-3-flash-preview"), + ), // DeepSeek: Reasoner (thinking), Chat (general) - ModelRoute { - display_name: "DeepSeek Reasoner", - vendor: DeepSeek, - api_model: "deepseek-reasoner", - }, - ModelRoute { - display_name: "DeepSeek Chat", - vendor: DeepSeek, - api_model: "deepseek-chat", - }, + // deepseek-reasoner is listed as deepseek-r1 on OpenRouter + ModelRoute::new( + "DeepSeek Reasoner", + DeepSeek, + "deepseek-reasoner", + Some("deepseek/deepseek-r1"), + ), + ModelRoute::new( + "DeepSeek Chat", + DeepSeek, + "deepseek-chat", + Some("deepseek/deepseek-chat"), + ), ] +}); + +impl ModelRoute { + pub fn new(display_name: &str, vendor: Vendor, api_model: &str, openrouter_model: Option<&str>) -> Self { + Self { + display_name: display_name.to_string(), + vendor, + api_model: api_model.to_string(), + openrouter_model: openrouter_model.map(|s| s.to_string()), + } + } +} + +pub fn default_model_routes() -> &'static [ModelRoute] { + &DEFAULT_ROUTES } diff --git a/tools/xtask-llm-benchmark/src/llm/prompt.rs b/tools/xtask-llm-benchmark/src/llm/prompt.rs index 1864587b638..462d6a5946c 100644 --- a/tools/xtask-llm-benchmark/src/llm/prompt.rs +++ b/tools/xtask-llm-benchmark/src/llm/prompt.rs @@ -15,11 +15,14 @@ pub struct BuiltPrompt { pub system: Option, pub static_prefix: Option, pub segments: Vec>, + /// When true, the provider should enable web search (OpenRouter :online). + pub search_enabled: bool, } impl PromptBuilder { - pub fn build_segmented(&self, context: &str) -> BuiltPrompt { + pub fn build_segmented(&self, mode: &str, context: &str) -> BuiltPrompt { let version = "1.6"; + let search_enabled = mode == "search"; // SYSTEM: hygiene-only for Knowledge; hygiene + stricter output rules for Conformance. let system = Some(format!( @@ -32,13 +35,21 @@ Rules:\n\ lang = self.lang )); - let static_prefix = Some(if context.trim().is_empty() { - "<<>>\nNo documentation provided. You must rely on your knowledge of SpacetimeDB syntax and conventions.\n<<>>\n".to_string() + let static_prefix = Some(if search_enabled { + "<<>>\nYou MUST search the web for SpacetimeDB documentation and examples before writing any code. Do not write code until you have searched.\n<<>>\n".to_string() + } else if context.trim().is_empty() { + "<<>>\nUse your knowledge of the latest SpacetimeDB syntax and conventions.\n<<>>\n" + .to_string() } else { - format!( - "<<>>Context:\n{context}\n<<>>\n", - context = context, - ) + let preamble = match mode { + "guidelines" => format!( + "The following are SpacetimeDB {} guidelines. \ + All examples shown are correct patterns to follow.", + self.lang + ), + _ => "Reference documentation:".to_string(), + }; + format!("<<>>\n{preamble}\n\n{context}\n<<>>\n",) }); // TASK: identical in both modes; API details must come from DOCS in Knowledge mode. @@ -62,6 +73,7 @@ HARD CONSTRAINTS:\n\ system, static_prefix, segments: vec![Segment::new("user", dynamic).keep().min_chars(0).weight(8.0)], + search_enabled, } } } diff --git a/tools/xtask-llm-benchmark/src/llm/provider.rs b/tools/xtask-llm-benchmark/src/llm/provider.rs index 01ac930f887..65d587d9526 100644 --- a/tools/xtask-llm-benchmark/src/llm/provider.rs +++ b/tools/xtask-llm-benchmark/src/llm/provider.rs @@ -2,15 +2,15 @@ use anyhow::{Context, Result}; use async_trait::async_trait; use crate::llm::clients::{ - AnthropicClient, DeepSeekClient, GoogleGeminiClient, MetaLlamaClient, OpenAiClient, XaiGrokClient, + AnthropicClient, DeepSeekClient, GoogleGeminiClient, MetaLlamaClient, OpenAiClient, OpenRouterClient, XaiGrokClient, }; use crate::llm::model_routes::ModelRoute; use crate::llm::prompt::BuiltPrompt; -use crate::llm::types::Vendor; +use crate::llm::types::{LlmOutput, Vendor}; #[async_trait] pub trait LlmProvider: Send + Sync { - async fn generate(&self, route: &ModelRoute, prompt: &BuiltPrompt) -> Result; + async fn generate(&self, route: &ModelRoute, prompt: &BuiltPrompt) -> Result; } pub struct RouterProvider { @@ -20,10 +20,14 @@ pub struct RouterProvider { pub xai: Option, pub deepseek: Option, pub meta: Option, + /// OpenRouter client used as a unified fallback when a direct vendor client + /// is not configured. Set via `OPENROUTER_API_KEY`. + pub openrouter: Option, pub force: Option, } impl RouterProvider { + #[allow(clippy::too_many_arguments)] pub fn new( openai: Option, anthropic: Option, @@ -31,6 +35,7 @@ impl RouterProvider { xai: Option, deepseek: Option, meta: Option, + openrouter: Option, force: Option, ) -> Self { Self { @@ -40,6 +45,7 @@ impl RouterProvider { xai, deepseek, meta, + openrouter, force, } } @@ -47,36 +53,113 @@ impl RouterProvider { #[async_trait] impl LlmProvider for RouterProvider { - async fn generate(&self, route: &ModelRoute, prompt: &BuiltPrompt) -> Result { + async fn generate(&self, route: &ModelRoute, prompt: &BuiltPrompt) -> Result { + // Web search mode: route all models through OpenRouter with :online suffix. + // OpenRouter's :online feature adds Bing-powered web search to any model. + if prompt.search_enabled { + let cli = self.openrouter.as_ref().context( + "Search mode requires OPENROUTER_API_KEY — OpenRouter provides unified web search via :online models", + )?; + let base_model = route + .openrouter_model + .clone() + .unwrap_or_else(|| openrouter_model_id(route.vendor, &route.api_model)); + let online_model = format!("{base_model}:online"); + eprintln!( + "[search] {} → OpenRouter :online model '{}'", + route.display_name, online_model + ); + return cli.generate(&online_model, prompt).await; + } + let vendor = self.force.unwrap_or(route.vendor); + + // If vendor is explicitly OpenRouter, or if the direct client isn't configured + // but OpenRouter is available, route through OpenRouter. + if vendor == Vendor::OpenRouter { + let cli = self + .openrouter + .as_ref() + .context("OpenRouter client not configured (set OPENROUTER_API_KEY)")?; + let model = route.openrouter_model.as_deref().unwrap_or(&route.api_model); + return cli.generate(model, prompt).await; + } + + // Try direct client first, fall back to OpenRouter if available. match vendor { - Vendor::OpenAi => { - let cli = self.openai.as_ref().context("OpenAI client not configured")?; - cli.generate(route.api_model, prompt).await - } - Vendor::Anthropic => { - let cli = self.anthropic.as_ref().context("Anthropic client not configured")?; - cli.generate(route.api_model, prompt).await - } - Vendor::Google => { - let cli = self.google.as_ref().context("Google client not configured")?; - cli.generate(route.api_model, prompt).await - } - Vendor::Xai => { - let cli = self.xai.as_ref().context("xAI client not configured")?; - cli.generate(route.api_model, prompt).await - } - Vendor::DeepSeek => { - let cli = self.deepseek.as_ref().context("DeepSeek client not configured")?; - cli.generate(route.api_model, prompt).await - } - Vendor::Meta => { - let c = self - .meta - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Meta Llama not configured"))?; - c.generate(route.api_model, prompt).await + Vendor::OpenAi => match self.openai.as_ref() { + Some(cli) => cli.generate(&route.api_model, prompt).await, + None => self.fallback_openrouter(route, prompt, "OpenAI").await, + }, + Vendor::Anthropic => match self.anthropic.as_ref() { + Some(cli) => cli.generate(&route.api_model, prompt).await, + None => self.fallback_openrouter(route, prompt, "Anthropic").await, + }, + Vendor::Google => match self.google.as_ref() { + Some(cli) => cli.generate(&route.api_model, prompt).await, + None => self.fallback_openrouter(route, prompt, "Google").await, + }, + Vendor::Xai => match self.xai.as_ref() { + Some(cli) => cli.generate(&route.api_model, prompt).await, + None => self.fallback_openrouter(route, prompt, "xAI").await, + }, + Vendor::DeepSeek => match self.deepseek.as_ref() { + Some(cli) => cli.generate(&route.api_model, prompt).await, + None => self.fallback_openrouter(route, prompt, "DeepSeek").await, + }, + Vendor::Meta => match self.meta.as_ref() { + Some(cli) => cli.generate(&route.api_model, prompt).await, + None => self.fallback_openrouter(route, prompt, "Meta").await, + }, + Vendor::OpenRouter => unreachable!("handled above"), + } + } +} + +impl RouterProvider { + /// Fall back to the OpenRouter client when a direct vendor client is not configured. + async fn fallback_openrouter( + &self, + route: &ModelRoute, + prompt: &BuiltPrompt, + vendor_name: &str, + ) -> Result { + match self.openrouter.as_ref() { + Some(cli) => { + let or_model = route + .openrouter_model + .clone() + .unwrap_or_else(|| openrouter_model_id(route.vendor, &route.api_model)); + eprintln!( + "[openrouter] {} client not configured, falling back to OpenRouter for model '{}'", + vendor_name, or_model + ); + cli.generate(&or_model, prompt).await } + None => anyhow::bail!( + "{} client not configured and no OpenRouter fallback available. \ + Set {}_API_KEY or OPENROUTER_API_KEY.", + vendor_name, + vendor_name.to_ascii_uppercase() + ), } } } + +/// Map a vendor + bare model id to the `vendor/model` namespace that OpenRouter requires. +/// If the model id already contains `/` it is returned as-is (e.g. `google/gemini-3.1-pro-preview`). +fn openrouter_model_id(vendor: Vendor, api_model: &str) -> String { + if api_model.contains('/') { + return api_model.to_string(); + } + let prefix = match vendor { + Vendor::Anthropic => "anthropic", + Vendor::OpenAi => "openai", + Vendor::Xai => "x-ai", + Vendor::DeepSeek => "deepseek", + Vendor::Google => "google", + // Meta rows already carry a full `vendor/model` id (caught by the `/` check above). + Vendor::Meta | Vendor::OpenRouter => return api_model.to_string(), + }; + format!("{}/{}", prefix, api_model) +} diff --git a/tools/xtask-llm-benchmark/src/llm/segmentation.rs b/tools/xtask-llm-benchmark/src/llm/segmentation.rs index c8d3e53c55d..2926852ada0 100644 --- a/tools/xtask-llm-benchmark/src/llm/segmentation.rs +++ b/tools/xtask-llm-benchmark/src/llm/segmentation.rs @@ -191,6 +191,7 @@ pub fn headroom_tokens_env(vendor: Vendor) -> usize { Vendor::Xai => std::env::var("XAI_HEADROOM_TOKENS").ok(), Vendor::DeepSeek => std::env::var("DEEPSEEK_HEADROOM_TOKENS").ok(), Vendor::Meta => std::env::var("META_HEADROOM_TOKENS").ok(), + Vendor::OpenRouter => std::env::var("OPENROUTER_HEADROOM_TOKENS").ok(), }; let raw = per_vendor @@ -206,7 +207,7 @@ pub fn headroom_tokens_env(vendor: Vendor) -> usize { fn default_headroom(vendor: Vendor) -> usize { match vendor { // Conservative floors tuned for typical max-context models - Vendor::OpenAi | Vendor::Xai | Vendor::DeepSeek | Vendor::Meta => 3_000, + Vendor::OpenAi | Vendor::Xai | Vendor::DeepSeek | Vendor::Meta | Vendor::OpenRouter => 3_000, Vendor::Anthropic => 5_000, Vendor::Google => 10_000, } @@ -255,6 +256,7 @@ pub fn non_context_reserve_tokens_env(vendor: Vendor) -> usize { Vendor::Xai => "XAI_RESERVE_TOKENS", Vendor::DeepSeek => "DEEPSEEK_RESERVE_TOKENS", Vendor::Meta => "META_RESERVE_TOKENS", + Vendor::OpenRouter => "OPENROUTER_RESERVE_TOKENS", }; if let Ok(v) = std::env::var(key) && let Ok(n) = v.parse::() @@ -276,6 +278,7 @@ pub fn non_context_reserve_tokens_env(vendor: Vendor) -> usize { Vendor::Xai => 12_000, Vendor::DeepSeek => 12_000, Vendor::Meta => 12_000, + Vendor::OpenRouter => 12_000, }; round_500(def) } diff --git a/tools/xtask-llm-benchmark/src/llm/types.rs b/tools/xtask-llm-benchmark/src/llm/types.rs index 206b567afe5..5ccfd14f339 100644 --- a/tools/xtask-llm-benchmark/src/llm/types.rs +++ b/tools/xtask-llm-benchmark/src/llm/types.rs @@ -1,6 +1,14 @@ use serde::{Deserialize, Serialize}; use std::fmt; +/// Output from an LLM generation call, including token usage. +#[derive(Debug, Clone, Default)] +pub struct LlmOutput { + pub text: String, + pub input_tokens: Option, + pub output_tokens: Option, +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] pub enum Vendor { OpenAi, @@ -8,7 +16,8 @@ pub enum Vendor { Google, // Gemini Xai, // Grok DeepSeek, - Meta, // Llama + Meta, // Llama + OpenRouter, // unified proxy — routes to any vendor } impl Vendor { @@ -21,6 +30,7 @@ impl Vendor { Vendor::Xai => "xai", Vendor::DeepSeek => "deepseek", Vendor::Meta => "meta", + Vendor::OpenRouter => "openrouter", } } @@ -33,6 +43,7 @@ impl Vendor { Vendor::Xai => "xAI", Vendor::DeepSeek => "DeepSeek", Vendor::Meta => "Meta", + Vendor::OpenRouter => "OpenRouter", } } @@ -46,6 +57,7 @@ impl Vendor { "xai" | "grok" => Vendor::Xai, "deepseek" => Vendor::DeepSeek, "meta" | "llama" => Vendor::Meta, + "openrouter" | "or" => Vendor::OpenRouter, _ => return None, }) } diff --git a/tools/xtask-llm-benchmark/src/results/io.rs b/tools/xtask-llm-benchmark/src/results/io.rs deleted file mode 100644 index 7dfdca2f6c6..00000000000 --- a/tools/xtask-llm-benchmark/src/results/io.rs +++ /dev/null @@ -1,272 +0,0 @@ -use super::schema::{GoldenAnswer, LangSummary, ModeSummary, ModelSummary, Results, Summary, Totals}; -use crate::bench::results_merge::ensure_lang; -use anyhow::{Context, Result}; -use chrono::{SecondsFormat, Utc}; -use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet}; -use std::collections::BTreeMap; -use std::fs; -use std::path::Path; - -/// Load a Summary from a JSON file. Returns a default Summary if file doesn't exist. -pub fn load_summary>(path: P) -> Result { - use std::io::ErrorKind; - let p = path.as_ref(); - - match fs::read_to_string(p) { - Ok(raw) => { - let v: Summary = serde_json::from_str(&raw).with_context(|| format!("parse {}", p.display()))?; - Ok(v) - } - Err(e) if e.kind() == ErrorKind::NotFound => Ok(Summary { - version: 1, - generated_at: String::new(), - by_language: BTreeMap::new(), - }), - Err(e) => Err(e).with_context(|| format!("read {}", p.display())), - } -} - -fn pct(passed: u32, total: u32) -> f32 { - if total == 0 { - 0.0 - } else { - (passed as f32) * 100.0 / (total as f32) - } -} - -/// Build `Summary` from your in-memory `Results`. -pub fn summary_from_results(results: &Results) -> Summary { - let mut by_language: BTreeMap = BTreeMap::new(); - - for lang_ent in &results.languages { - let lang_key = lang_ent.lang.clone(); - let lang_sum = by_language - .entry(lang_key) - .or_insert_with(|| LangSummary { modes: BTreeMap::new() }); - - for mode_ent in &lang_ent.modes { - let mode_key = mode_ent.mode.clone(); - let hash = mode_ent.hash.clone().unwrap_or_default(); - let mode_sum = lang_sum.modes.entry(mode_key).or_insert_with(|| ModeSummary { - hash: hash.clone(), - models: BTreeMap::new(), - }); - // Update hash if it changed - mode_sum.hash = hash; - - for model_ent in &mode_ent.models { - let model_key = model_ent.name.clone(); - let model_sum = mode_sum.models.entry(model_key).or_insert_with(|| ModelSummary { - categories: BTreeMap::new(), - totals: Totals::default(), - }); - - for t in model_ent.tasks.values() { - let cat_key = t.category.clone().expect("Missing category"); - let cat_sum = model_sum.categories.entry(cat_key).or_default(); - - // counts - cat_sum.tasks += 1; - cat_sum.total_tests += t.total_tests; - cat_sum.passed_tests += t.passed_tests; - - model_sum.totals.tasks += 1; - model_sum.totals.total_tests += t.total_tests; - model_sum.totals.passed_tests += t.passed_tests; - - // per-task partials - let f = frac(t.passed_tests, t.total_tests); - cat_sum.task_pass_equiv += f; - model_sum.totals.task_pass_equiv += f; - } - - // finalize percentages - for v in model_sum.categories.values_mut() { - v.pass_pct = pct(v.passed_tests, v.total_tests); - v.task_pass_pct = if v.tasks == 0 { - 0.0 - } else { - (v.task_pass_equiv / v.tasks as f32) * 100.0 - }; - } - model_sum.totals.pass_pct = pct(model_sum.totals.passed_tests, model_sum.totals.total_tests); - model_sum.totals.task_pass_pct = if model_sum.totals.tasks == 0 { - 0.0 - } else { - (model_sum.totals.task_pass_equiv / model_sum.totals.tasks as f32) * 100.0 - }; - } - } - } - - Summary { - version: 1, - generated_at: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - by_language, - } -} - -fn frac(pt: u32, tt: u32) -> f32 { - if tt == 0 { - 0.0 - } else { - (pt as f32) / (tt as f32) - } -} - -/// Convenience: read the details file **into your Results**, then write summary. -pub fn write_summary_from_details_file, POut: AsRef>( - details_json: PIn, - out_json: POut, -) -> Result<()> { - let data = fs::read_to_string(details_json.as_ref()) - .with_context(|| format!("failed to read {}", details_json.as_ref().display()))?; - let results: Results = - serde_json::from_str(&data).with_context(|| "failed to deserialize Results from details file")?; - let summary = summary_from_results(&results); - let pretty = serde_json::to_string_pretty(&summary)?; - fs::write(out_json.as_ref(), pretty).with_context(|| format!("failed to write {}", out_json.as_ref().display()))?; - Ok(()) -} - -fn collect_seen_tasks_by_lang(root: &Results) -> HashMap<&'static str, HashSet> { - let mut map: HashMap<&'static str, HashSet> = HashMap::new(); - - for le in &root.languages { - let target = match le.lang.as_str() { - "rust" => map.entry("rust").or_default(), - "csharp" => map.entry("csharp").or_default(), - _ => continue, - }; - for me in &le.modes { - for mdl in &me.models { - for tid in mdl.tasks.keys() { - target.insert(tid.to_string()); - } - } - } - } - map -} - -/// Update language-level golden answers from filesystem. -/// - `all=false` : only ingest if task_id exists for that language in JSON -/// - `overwrite=false`: keep existing entries (no clobber) -pub fn update_golden_answers_on_disk( - results_json_path: &Path, - bench_root: &Path, - all: bool, - overwrite: bool, -) -> Result<()> { - // Load current results - let bytes = fs::read(results_json_path).with_context(|| format!("read {}", results_json_path.display()))?; - let mut root: Results = - serde_json::from_slice(&bytes).with_context(|| format!("parse {}", results_json_path.display()))?; - - let seen = collect_seen_tasks_by_lang(&root); - - // benchmarks///answers/{rust.rs|csharp.cs} - let cats = fs::read_dir(bench_root).with_context(|| format!("read_dir {}", bench_root.display()))?; - - for cat_entry in cats { - let cat_entry = match cat_entry { - Ok(e) => e, - Err(_) => continue, - }; - let Ok(cat_ft) = cat_entry.file_type() else { continue }; - if !cat_ft.is_dir() { - continue; - } - let Ok(cat_name) = cat_entry.file_name().into_string() else { - continue; - }; - let cat_path = cat_entry.path(); - - let tasks = match fs::read_dir(&cat_path) { - Ok(rd) => rd, - Err(_) => continue, - }; - for task_entry in tasks { - let task_entry = match task_entry { - Ok(e) => e, - Err(_) => continue, - }; - let Ok(task_ft) = task_entry.file_type() else { continue }; - if !task_ft.is_dir() { - continue; - } - let Ok(task_name) = task_entry.file_name().into_string() else { - continue; - }; - let task_path = task_entry.path(); - - let answers_dir = task_path.join("answers"); - if !answers_dir.is_dir() { - continue; - } - - // IDs: bare (matches runs like "t_004_insert"); alias kept for compatibility. - let task_id_bare = task_name.clone(); - let task_id_alias = format!("{}/{}", cat_name, task_name); - - // rust.rs → lang "rust" - let rust_path = answers_dir.join("rust.rs"); - if rust_path.is_file() - && (all - || seen - .get("rust") - .is_some_and(|s| s.contains(&task_id_bare) || s.contains(&task_id_alias))) - { - let le = ensure_lang(&mut root, "rust"); - let text = fs::read_to_string(&rust_path).with_context(|| format!("read {}", rust_path.display()))?; - if overwrite || !le.golden_answers.contains_key(&task_id_bare) { - le.golden_answers.insert( - task_id_bare.clone(), - GoldenAnswer { - answer: text.clone(), - syntax: Some("rust".into()), - }, - ); - } - // comment out the next line to drop the alias entirely - le.golden_answers.entry(task_id_alias.clone()).or_insert(GoldenAnswer { - answer: text, - syntax: Some("rust".into()), - }); - } - - // csharp.cs → lang "csharp" - let cs_path = answers_dir.join("csharp.cs"); - if cs_path.is_file() - && (all - || seen - .get("csharp") - .is_some_and(|s| s.contains(&task_id_bare) || s.contains(&task_id_alias))) - { - let le = ensure_lang(&mut root, "csharp"); - let text = fs::read_to_string(&cs_path).with_context(|| format!("read {}", cs_path.display()))?; - if overwrite || !le.golden_answers.contains_key(&task_id_bare) { - le.golden_answers.insert( - task_id_bare.clone(), - GoldenAnswer { - answer: text.clone(), - syntax: Some("csharp".into()), - }, - ); - } - // comment out the next line to drop the alias entirely - le.golden_answers.entry(task_id_alias.clone()).or_insert(GoldenAnswer { - answer: text, - syntax: Some("csharp".into()), - }); - } - } - } - - // Save pretty-printed - let mut out = Vec::new(); - serde_json::to_writer_pretty(&mut out, &root)?; - out.push(b'\n'); - fs::write(results_json_path, out).with_context(|| format!("write {}", results_json_path.display()))?; - Ok(()) -} diff --git a/tools/xtask-llm-benchmark/src/results/mod.rs b/tools/xtask-llm-benchmark/src/results/mod.rs index c26eeb4d4f2..1ce7e176664 100644 --- a/tools/xtask-llm-benchmark/src/results/mod.rs +++ b/tools/xtask-llm-benchmark/src/results/mod.rs @@ -1,5 +1 @@ -pub mod io; pub mod schema; - -pub use io::load_summary; -pub use schema::Summary; diff --git a/tools/xtask-llm-benchmark/src/results/schema.rs b/tools/xtask-llm-benchmark/src/results/schema.rs index d0ce8b93801..999ea00c65e 100644 --- a/tools/xtask-llm-benchmark/src/results/schema.rs +++ b/tools/xtask-llm-benchmark/src/results/schema.rs @@ -2,10 +2,10 @@ use crate::bench::RunOutcome; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -// -- RESULTS (details.json) -- - #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Results { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub generated_at: Option, pub languages: Vec, } @@ -13,8 +13,6 @@ pub struct Results { pub struct LangEntry { pub lang: String, pub modes: Vec, - #[serde(default)] - pub golden_answers: BTreeMap, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -30,61 +28,3 @@ pub struct ModelEntry { pub route_api_model: Option, pub tasks: BTreeMap, } - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct GoldenAnswer { - pub answer: String, - #[serde(default)] - pub syntax: Option, // "rust" | "csharp" -} - -// -- SUMMARY (summary.json) -- -#[derive(Debug, Serialize, Deserialize)] -pub struct Summary { - pub version: u32, - pub generated_at: String, - pub by_language: BTreeMap, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct LangSummary { - pub modes: BTreeMap, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct ModeSummary { - pub hash: String, - pub models: BTreeMap, -} - -#[derive(Debug, Serialize, Deserialize, Default, Clone)] -pub struct Totals { - pub tasks: u32, - pub total_tests: u32, - pub passed_tests: u32, - pub pass_pct: f32, - - // sum of (passed_tests / total_tests) across tasks - pub task_pass_equiv: f32, - // task_pass_equiv / tasks * 100 - pub task_pass_pct: f32, -} - -#[derive(Debug, Serialize, Deserialize, Default, Clone)] -pub struct ModelSummary { - pub categories: BTreeMap, - pub totals: Totals, -} - -#[derive(Debug, Serialize, Deserialize, Default, Clone)] -pub struct CategorySummary { - pub tasks: u32, - pub total_tests: u32, - pub passed_tests: u32, - pub pass_pct: f32, - - // sum of (passed_tests / total_tests) for tasks in this category - pub task_pass_equiv: f32, - // task_pass_equiv / tasks * 100 - pub task_pass_pct: f32, -}