bt scorers create/update

Cargo.toml

@@ -30,6 +30,7 @@ reqwest = { version = "0.12.7", default-features = false, features = ["json", "r
 serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "1.0.128"
 serde_path_to_error = "0.1.20"
+yaml_serde = "0.10"
 toml = "0.8"
 sha2 = "0.10.8"
 strip-ansi-escapes = "0.2.0"

README.md

@@ -135,22 +135,104 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
 
 ## Commands
 
-| Command       | Description                                                        |
-| ------------- | ------------------------------------------------------------------ |
-| `bt init`     | Initialize `.bt/` config directory and link to a project           |
-| `bt login`    | Log in to Braintrust or refresh an OAuth login                     |
-| `bt logout`   | Remove a saved Braintrust login                                    |
-| `bt switch`   | Switch org and project context                                     |
-| `bt status`   | Show current org and project context                               |
-| `bt datasets` | Manage datasets and dataset pipelines                              |
-| `bt eval`     | Run eval files (Unix only)                                         |
-| `bt sql`      | Run SQL queries against Braintrust                                 |
-| `bt view`     | View logs, traces, and spans                                       |
-| `bt projects` | Manage projects (list, create, view, delete)                       |
-| `bt datasets` | Manage remote datasets (list, create, update, view, delete)        |
-| `bt prompts`  | Manage prompts (list, view, delete)                                |
-| `bt sync`     | Synchronize project logs between Braintrust and local NDJSON files |
-| `bt update`   | Update bt in-place                                                 |
+| Command        | Description                                                        |
+| -------------- | ------------------------------------------------------------------ |
+| `bt init`      | Initialize `.bt/` config directory and link to a project           |
+| `bt login`     | Log in to Braintrust or refresh an OAuth login                     |
+| `bt logout`    | Remove a saved Braintrust login                                    |
+| `bt switch`    | Switch org and project context                                     |
+| `bt status`    | Show current org and project context                               |
+| `bt eval`      | Run eval files (Unix only)                                         |
+| `bt sql`       | Run SQL queries against Braintrust                                 |
+| `bt view`      | View logs, traces, and spans                                       |
+| `bt projects`  | Manage projects (list, create, view, delete)                       |
+| `bt datasets`  | Manage remote datasets (list, create, update, view, delete)        |
+| `bt prompts`   | Manage prompts (list, view, update, delete)                        |
+| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete)  |
+| `bt tools`     | Manage tools (list, view, invoke, update, delete)                  |
+| `bt scorers`   | Manage scorers (list, create, view, invoke, update, delete)        |
+| `bt sync`      | Synchronize project logs between Braintrust and local NDJSON files |
+| `bt update`    | Update bt in-place                                                 |
+
+## `bt scorers`
+
+Create and update prompt-based LLM scorers in the current project:
+
+```bash
+bt scorers create "Helpfulness" \
+  --model gpt-5.4-nano \
+  --messages @messages.json \
+  --choice-scores '{"A":1,"B":0}'
+
+bt scorers update helpfulness --messages @messages.json
+bt scorers update helpfulness --model gpt-5.4-nano
+```
+
+`@PATH` and `-` are CLI-only source notation, not scorer settings in the web UI. For example, `--messages @messages.json` reads chat messages from `messages.json`, while `--messages -` reads them from stdin.
+
+LLM scorer configuration mirrors the web UI:
+
+```bash
+bt scorers create "Quality judge" \
+  --model gpt-5.4-nano \
+  --messages @messages.json \
+  --choice-scores '{"pass":1,"fail":0}' \
+  --temperature 0.1 \
+  --max-tokens 512 \
+  --top-p 0.9 \
+  --frequency-penalty 0 \
+  --presence-penalty 0 \
+  --stop-sequence END \
+  --tool-choice auto \
+  --reasoning-effort none \
+  --verbosity low \
+  --template-format mustache \
+  --pass-threshold 0.7 \
+  --metadata @metadata.yaml
+```
+
+Use `--template-format mustache|jinja|none`; `nunjucks` and `jinja2` are accepted aliases for Jinja. Repeat `--stop-sequence` for multiple values. Tool choice accepts `auto`, `none`, `required`, or a function name.
+
+Model parameters are validated against the same model catalog and custom-model metadata used by the web UI, including parameter availability, provider-specific ranges, reasoning options, and output-token limits. Note that the ranges follow the web UI rather than the raw provider APIs — for example `--frequency-penalty` and `--presence-penalty` accept `0` to `1`.
+
+Parameters are also stored under the names each provider's prompt editor expects, so a scorer created by `bt` shows the same populated fields as one created in the web UI. Google models, for example, use `maxOutputTokens` and `topP`.
+
+A model that appears in neither the catalog nor your org's or project's custom models is not assigned capabilities by format, so it receives only provider-independent range checks; a few parameters (notably `--temperature`) are still gated by well-known model-name patterns. The catalog is cached for 24 hours per app URL, org, and project, so most commands validate without any network request; a lookup miss refetches immediately, so a newly added custom model is picked up right away. Pass `--refresh-models` (or set `BRAINTRUST_REFRESH_MODELS=1`) to ignore the cache after editing a custom model in the web UI. If the metadata cannot be loaded at all, `bt` warns that it checked only basic ranges rather than failing or silently skipping the check.
+
+For classification output instead of a numeric score, use classifications in place of choice scores:
+
+```bash
+bt scorers create "Safety label" \
+  --model gpt-5.4-nano \
+  --messages @messages.json \
+  --classifications '["safe","unsafe"]' \
+  --allow-no-match
+```
+
+Use `--if-exists error|ignore|replace` when creating a scorer. Text and structured input flags accept an inline value, `@PATH` to read from a file, or `-` for stdin; only one flag per command may read from stdin. For fields without a dedicated update flag, use `--patch` with a JSON object, which is deep-merged last and therefore wins over any overlapping flag.
+
+`update` only changes the fields you pass. Note that the API replaces `prompt_data` rather than merging into it, so `bt` reads the current definition and sends it back with your changes applied; a concurrent edit to the same scorer can therefore be overwritten.
+
+For code scorers, use the Braintrust SDK for your language and push the source file:
+
+```ts
+// TypeScript
+import { projects } from "braintrust";
+const project = projects.create({ name: "test-project" });
+project.scorers.create({ name: "Test scorer", handler: ({ output }) => 1 });
+```
+
+```python
+# Python
+from braintrust import projects
+project = projects.create("test-project")
+project.scorers.create(name="Test scorer", handler=test_scorer, parameters=ScorerInput)
+```
+
+```bash
+bt functions push scorer.ts
+bt functions push scorer.py
+```
 
 ## `bt eval`
 

scripts/skill-smoke-test.sh

@@ -27,7 +27,7 @@ Options:
 
 Examples:
   scripts/skill-smoke-test.sh --agent codex
-  scripts/skill-smoke-test.sh --agent codex --agent-cmd 'codex run --prompt-file AGENT_TASK.md'
+  scripts/skill-smoke-test.sh --agent codex --agent-cmd 'codex exec - < AGENT_TASK.md'
   scripts/skill-smoke-test.sh --demo-dir /tmp/bt-skill-demo --verify-only
 EOF
 }

src/auth.rs

@@ -519,15 +519,13 @@ pub async fn login(base: &BaseArgs) -> Result<LoginContext> {
     }
     let login = builder.build().await?.wait_for_login().await?;
 
-    let api_url = login
-        .api_url()
-        .or(auth.api_url.clone())
-        .unwrap_or_else(|| DEFAULT_API_URL.to_string());
+    let api_url = resolve_login_api_url(auth.api_url.clone(), login.api_url());
 
     let app_url = auth
         .app_url
         .clone()
         .unwrap_or_else(|| DEFAULT_APP_URL.to_string());
+    let login = normalize_login_state(login, api_key, &api_url, &app_url);
 
     let ctx = LoginContext {
         login,
@@ -539,6 +537,35 @@ pub async fn login(base: &BaseArgs) -> Result<LoginContext> {
     Ok(ctx)
 }
 
+fn resolve_login_api_url(configured: Option<String>, discovered: Option<String>) -> String {
+    // The configured CLI/env/profile URL is the request target. Do not let a
+    // cached or server-returned login URL silently replace it.
+    configured
+        .or(discovered)
+        .unwrap_or_else(|| DEFAULT_API_URL.to_string())
+}
+
+fn normalize_login_state(
+    login: LoginState,
+    api_key: String,
+    api_url: &str,
+    app_url: &str,
+) -> LoginState {
+    // Keep LoginContext's two URL sources consistent. Most commands use
+    // LoginContext::api_url through ApiClient, but SDK-backed paths may inspect
+    // LoginState directly.
+    let normalized = LoginState::new();
+    let did_set = normalized.set(
+        api_key,
+        login.org_id().unwrap_or_default(),
+        login.org_name().unwrap_or_default(),
+        api_url.to_string(),
+        app_url.to_string(),
+    );
+    debug_assert!(did_set, "new login state should be unset");
+    normalized
+}
+
 #[derive(Debug, Deserialize)]
 struct AiProviderSecret {
     #[serde(default)]
@@ -3627,6 +3654,40 @@ mod tests {
         }
     }
 
+    #[test]
+    fn configured_urls_override_discovered_login_state() {
+        let discovered = LoginState::new();
+        assert!(discovered.set(
+            "test-api-key".to_string(),
+            "org_test".to_string(),
+            "test-org".to_string(),
+            DEFAULT_API_URL.to_string(),
+            DEFAULT_APP_URL.to_string(),
+        ));
+        let api_url = resolve_login_api_url(
+            Some("https://api.test.example".to_string()),
+            discovered.api_url(),
+        );
+
+        let normalized = normalize_login_state(
+            discovered,
+            "test-api-key".to_string(),
+            &api_url,
+            "https://app.test.example",
+        );
+
+        assert_eq!(
+            normalized.api_url().as_deref(),
+            Some("https://api.test.example")
+        );
+        assert_eq!(
+            normalized.app_url().as_deref(),
+            Some("https://app.test.example")
+        );
+        assert_eq!(normalized.org_id().as_deref(), Some("org_test"));
+        assert_eq!(normalized.org_name().as_deref(), Some("test-org"));
+    }
+
     fn assert_invalid_api_url<T>(result: Result<T>) {
         assert_err_contains(result, "invalid api_url");
     }

src/datasets/api.rs

@@ -6,13 +6,12 @@ use serde::{Deserialize, Serialize};
 use serde_json::{json, Map, Value};
 use urlencoding::encode;
 
-use crate::http::{ApiClient, HttpError};
+use crate::http::{ApiClient, HttpError, BTQL_EPOCH};
 
 use super::records::DATASET_RECORD_FIELDS;
 
 const MAX_DATASET_ROWS_PAGE_LIMIT: usize = 1000;
 const MAX_DATASET_ROWS_PAGES: usize = 10_000;
-const DATASET_ROWS_SINCE: &str = "1970-01-01T00:00:00Z";
 const MAX_ERROR_RESPONSE_BODY_CHARS: usize = 4000;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -390,7 +389,7 @@ fn build_dataset_rows_query(
         "filter": {
             "op": "ge",
             "left": {"op": "ident", "name": ["created"]},
-            "right": {"op": "literal", "value": DATASET_ROWS_SINCE}
+            "right": {"op": "literal", "value": BTQL_EPOCH}
         },
         "preview_length": preview_length.btql_value(),
         "limit": limit
@@ -427,7 +426,7 @@ fn build_dataset_head_xact_query(dataset_id: &str) -> Value {
         "filter": {
             "op": "ge",
             "left": {"op": "ident", "name": ["created"]},
-            "right": {"op": "literal", "value": DATASET_ROWS_SINCE}
+            "right": {"op": "literal", "value": BTQL_EPOCH}
         },
         "sort": [{
             "expr": {"op": "ident", "name": ["_xact_id"]},

src/functions/api.rs

@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use urlencoding::encode;
 
-use crate::http::ApiClient;
+use crate::http::{ApiClient, BTQL_EPOCH};
 
 fn escape_sql(s: &str) -> String {
     s.replace('\'', "''")
@@ -68,17 +68,26 @@ pub async fn list_functions(
     project_id: &str,
     function_type: Option<&str>,
 ) -> Result<Vec<Function>> {
+    let query = list_functions_query(project_id, function_type);
+    let response = client.btql::<Function>(&query).await?;
+
+    Ok(response.data)
+}
+
+fn list_functions_query(project_id: &str, function_type: Option<&str>) -> String {
     let pid = escape_sql(project_id);
-    let query = match function_type {
+    let type_filter = match function_type {
+        // The Braintrust UI lists score-producing scorers and label-producing
+        // classifiers together in the Scorers section.
+        Some("scorer") => " AND function_type IN ('scorer', 'classifier')".to_string(),
         Some(ft) => {
             let ft = escape_sql(ft);
-            format!("SELECT * FROM project_functions('{pid}') WHERE function_type = '{ft}'")
+            format!(" AND function_type = '{ft}'")
         }
-        None => format!("SELECT * FROM project_functions('{pid}')"),
+        None => String::new(),
     };
-    let response = client.btql::<Function>(&query).await?;
-
-    Ok(response.data)
+    // Definitions have no meaningful time window; see BTQL_EPOCH.
+    format!("SELECT * FROM project_functions('{pid}') WHERE created >= '{BTQL_EPOCH}'{type_filter}")
 }
 
 pub async fn get_function_by_slug(
@@ -144,6 +153,20 @@ pub async fn delete_function(client: &ApiClient, function_id: &str) -> Result<()
     client.delete(&path).await
 }
 
+/// Partially update a function (scorer/tool/prompt/...) by id.
+///
+/// The Braintrust API deep-merges object fields, so callers can send only the
+/// nested fields they want to change (for example `prompt_data.prompt`) without
+/// sending the complete function definition.
+pub async fn patch_function(
+    client: &ApiClient,
+    function_id: &str,
+    body: &serde_json::Value,
+) -> Result<serde_json::Value> {
+    let path = format!("/v1/function/{}", encode(function_id));
+    client.patch(&path, body).await
+}
+
 pub async fn list_functions_page(
     client: &ApiClient,
     query: &FunctionListQuery,
@@ -277,6 +300,14 @@ fn ignored_count(raw: &Value) -> Option<usize> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn scorer_list_query_includes_classifiers_and_a_timestamp_bound() {
+        let query = list_functions_query("test-project-id", Some("scorer"));
+
+        assert!(query.contains("created >= '1970-01-01T00:00:00Z'"));
+        assert!(query.contains("function_type IN ('scorer', 'classifier')"));
+    }
+
     #[test]
     fn ignored_count_extracts_canonical_shape() {
         let first = serde_json::json!({ "ignored_count": 3 });