feat(scorers): create LLM scorers and classifiers

AGENTS.md

@@ -14,10 +14,10 @@
 
 ## BTQL Safety
 
-- Every BTQL query must include either:
-  - a timestamp filter (for example, `created >= NOW() - INTERVAL ...` or `created >= "<ts>"`), or
-  - a `root_span_id` filter.
-- Do not run BTQL queries that lack both constraints.
+- BTQL queries over `project_logs(...)` or the combined `project(...)` source must include a useful segment-elimination constraint:
+  - a selective range on `created`, `_xact_id`, or `_pagination_key`; or
+  - scoping to specific `root_span_id` or `id` values.
+- This requirement does not apply to other object sources such as `project_functions(...)`, `project_prompts(...)`, `dataset(...)`, or `experiment(...)`.
 
 ## Tooling
 

Cargo.toml

@@ -30,6 +30,7 @@ reqwest = { version = "0.12.7", default-features = false, features = ["json", "r
 serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "1.0.128"
 serde_path_to_error = "0.1.20"
+yaml_serde = "0.10"
 toml = "0.8"
 sha2 = "0.10.8"
 strip-ansi-escapes = "0.2.0"

README.md

@@ -150,9 +150,31 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
 | `bt projects` | Manage projects (list, create, view, delete)                       |
 | `bt datasets` | Manage remote datasets (list, create, update, view, delete)        |
 | `bt prompts`  | Manage prompts (list, view, delete)                                |
+| `bt scorers`  | Manage scorers (list, create, view, invoke, delete)                |
 | `bt sync`     | Synchronize project logs between Braintrust and local NDJSON files |
 | `bt update`   | Update bt in-place                                                 |
 
+## `bt scorers`
+
+Create prompt-based LLM scorers or classifiers in the current project:
+
+```bash
+bt scorers create "Helpfulness" \
+  --model gpt-5.4-nano \
+  --messages @messages.json \
+  --choice-scores '{"A":1,"B":0}'
+
+bt scorers create "Safety label" \
+  --model gpt-5.4-nano \
+  --messages @messages.json \
+  --classifications '["safe","unsafe"]' \
+  --allow-no-match
+```
+
+Use `--if-exists error|ignore|replace` to control slug conflicts. Text and structured input flags accept an inline value, `@PATH`, or `-` for stdin; only one flag per command may read stdin. Use `--template-format mustache|jinja|none`; `nunjucks` and `jinja2` are accepted aliases for Jinja. Model options include `--use-cache[=true|false]` and `--response-format text|json-object|<SOURCE>`; structured output accepts a full `response_format` JSON object inline or from `@PATH`.
+
+For TypeScript and Python code scorers, use the Braintrust SDK and `bt functions push`.
+
 ## `bt eval`
 
 **File selection:**

src/functions/api.rs

@@ -58,27 +58,57 @@ pub struct CodeUploadSlot {
     pub bundle_id: String,
 }
 
+#[derive(Debug, Clone, Deserialize)]
+pub struct InsertedFunctionResult {
+    pub id: String,
+    pub project_id: String,
+    pub slug: String,
+    pub found_existing: bool,
+}
+
 #[derive(Debug, Clone)]
 pub struct InsertFunctionsResult {
     pub ignored_entries: Option<usize>,
+    pub xact_id: Option<String>,
+    pub functions: Vec<InsertedFunctionResult>,
+}
+
+#[derive(Debug, Deserialize)]
+struct InsertFunctionsResponse {
+    #[serde(default)]
+    xact_id: Option<String>,
+    #[serde(default)]
+    functions: Vec<InsertedFunctionResult>,
 }
 
 pub async fn list_functions(
     client: &ApiClient,
     project_id: &str,
     function_type: Option<&str>,
 ) -> Result<Vec<Function>> {
+    let query = list_functions_query(project_id, function_type);
+    let response = client.btql::<Function>(&query).await?;
+
+    Ok(response.data)
+}
+
+fn list_functions_query(project_id: &str, function_type: Option<&str>) -> String {
     let pid = escape_sql(project_id);
-    let query = match function_type {
+    let type_filter = match function_type {
+        // Match the web UI's Scorers tab: label-producing classifiers appear
+        // alongside score-producing scorers, while topic maps do not.
+        Some("scorer") => "function_type IN ('scorer', 'classifier') \
+            AND COALESCE(function_data.type, '') != 'topic_map' \
+            AND (origin IS NULL OR NOT COALESCE(origin.internal, FALSE))"
+            .to_string(),
         Some(ft) => {
             let ft = escape_sql(ft);
-            format!("SELECT * FROM project_functions('{pid}') WHERE function_type = '{ft}'")
+            format!("function_type = '{ft}'")
         }
-        None => format!("SELECT * FROM project_functions('{pid}')"),
+        None => return format!("SELECT * FROM project_functions('{pid}')"),
     };
-    let response = client.btql::<Function>(&query).await?;
 
-    Ok(response.data)
+    format!("SELECT * FROM project_functions('{pid}') WHERE {type_filter}")
 }
 
 pub async fn get_function_by_slug(
@@ -258,8 +288,14 @@ pub async fn insert_functions(
         .await
         .context("failed to insert functions")?;
 
+    let response: InsertFunctionsResponse = serde_json::from_value(raw.clone())
+        .context("unexpected insert-functions response shape")?;
+
     Ok(InsertFunctionsResult {
-        ignored_entries: ignored_count(&raw),
+        ignored_entries: ignored_count(&raw)
+            .or_else(|| ignored_count_from_function_results(&raw, functions)),
+        xact_id: response.xact_id,
+        functions: response.functions,
     })
 }
 
@@ -273,10 +309,48 @@ fn ignored_count(raw: &Value) -> Option<usize> {
         .and_then(|count| usize::try_from(count).ok())
 }
 
+fn ignored_count_from_function_results(raw: &Value, requests: &[Value]) -> Option<usize> {
+    let results = raw.get("functions")?.as_array()?;
+    if results.len() != requests.len() {
+        return None;
+    }
+
+    results
+        .iter()
+        .zip(requests)
+        .try_fold(0usize, |count, (result, request)| {
+            let should_ignore = request.get("if_exists").and_then(Value::as_str) == Some("ignore");
+            if !should_ignore {
+                return Some(count);
+            }
+
+            let found_existing = result.get("found_existing")?.as_bool()?;
+            Some(count + usize::from(found_existing))
+        })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn scorer_list_query_matches_web_ui_filter() {
+        let query = list_functions_query("test-project-id", Some("scorer"));
+
+        assert!(query.contains("function_type IN ('scorer', 'classifier')"));
+        assert!(query.contains("COALESCE(function_data.type, '') != 'topic_map'"));
+        assert!(query.contains("origin IS NULL"));
+        assert!(query.contains("origin.internal"));
+    }
+
+    #[test]
+    fn non_scorer_list_query_keeps_exact_type_filter() {
+        let query = list_functions_query("test-project-id", Some("tool"));
+
+        assert!(query.contains("function_type = 'tool'"));
+        assert!(!query.contains("classifier"));
+    }
+
     #[test]
     fn ignored_count_extracts_canonical_shape() {
         let first = serde_json::json!({ "ignored_count": 3 });
@@ -291,6 +365,61 @@ mod tests {
         assert_eq!(ignored_count(&serde_json::json!({})), None);
     }
 
+    #[test]
+    fn derives_ignored_count_from_found_existing_results() {
+        let requests = vec![
+            serde_json::json!({ "slug": "first", "if_exists": "ignore" }),
+            serde_json::json!({ "slug": "second", "if_exists": "replace" }),
+            serde_json::json!({ "slug": "third", "if_exists": "ignore" }),
+        ];
+        let response = serde_json::json!({
+            "functions": [
+                { "slug": "first", "found_existing": true },
+                { "slug": "second", "found_existing": true },
+                { "slug": "third", "found_existing": false },
+            ]
+        });
+
+        assert_eq!(
+            ignored_count_from_function_results(&response, &requests),
+            Some(1)
+        );
+    }
+
+    #[test]
+    fn ignored_count_fallback_rejects_mismatched_response_length() {
+        let requests = vec![serde_json::json!({
+            "slug": "first",
+            "if_exists": "ignore"
+        })];
+        let response = serde_json::json!({ "functions": [] });
+
+        assert_eq!(
+            ignored_count_from_function_results(&response, &requests),
+            None
+        );
+    }
+
+    #[test]
+    fn parses_insert_function_operation_fields() {
+        let response: InsertFunctionsResponse = serde_json::from_value(serde_json::json!({
+            "xact_id": "1000000000000000001",
+            "functions": [{
+                "id": "fn_test_scorer",
+                "project_id": "test-project",
+                "slug": "test-scorer",
+                "found_existing": true
+            }]
+        }))
+        .expect("insert response");
+
+        assert_eq!(response.xact_id.as_deref(), Some("1000000000000000001"));
+        assert_eq!(response.functions[0].id, "fn_test_scorer");
+        assert_eq!(response.functions[0].project_id, "test-project");
+        assert_eq!(response.functions[0].slug, "test-scorer");
+        assert!(response.functions[0].found_existing);
+    }
+
     #[test]
     fn insert_functions_body_wraps_functions_array() {
         let functions = vec![serde_json::json!({ "slug": "demo" })];