diff --git a/AGENTS.md b/AGENTS.md index 0c838320..4502de59 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,10 +14,10 @@ ## BTQL Safety -- Every BTQL query must include either: - - a timestamp filter (for example, `created >= NOW() - INTERVAL ...` or `created >= ""`), or - - a `root_span_id` filter. -- Do not run BTQL queries that lack both constraints. +- BTQL queries over `project_logs(...)` or the combined `project(...)` source must include a useful segment-elimination constraint: + - a selective range on `created`, `_xact_id`, or `_pagination_key`; or + - scoping to specific `root_span_id` or `id` values. +- This requirement does not apply to other object sources such as `project_functions(...)`, `project_prompts(...)`, `dataset(...)`, or `experiment(...)`. ## Tooling diff --git a/Cargo.lock b/Cargo.lock index 3af95ea7..07ef953a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -588,6 +588,7 @@ dependencies = [ "unicode-width 0.1.14", "urlencoding", "uuid", + "yaml_serde", ] [[package]] @@ -1888,6 +1889,12 @@ dependencies = [ "libc", ] +[[package]] +name = "libyaml-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" + [[package]] name = "lingua" version = "0.1.0" @@ -4084,6 +4091,19 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "yaml_serde" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c7c1b1a6a7c8a6b2741a6c21a4f8918e51899b111cfa08d1288202656e3975" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "libyaml-rs", + "ryu", + "serde", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index e8cad8b3..265c8352 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ reqwest = { version = "0.12.7", default-features = false, features = ["json", "r serde = { version = "1.0.210", features = ["derive"] } serde_json = "1.0.128" serde_path_to_error = "0.1.20" +yaml_serde = "0.10" toml = "0.8" sha2 = "0.10.8" strip-ansi-escapes = "0.2.0" diff --git a/README.md b/README.md index 3844d5ee..aa927ea6 100644 --- a/README.md +++ b/README.md @@ -150,9 +150,31 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC | `bt projects` | Manage projects (list, create, view, delete) | | `bt datasets` | Manage remote datasets (list, create, update, view, delete) | | `bt prompts` | Manage prompts (list, view, delete) | +| `bt scorers` | Manage scorers (list, create, view, invoke, delete) | | `bt sync` | Synchronize project logs between Braintrust and local NDJSON files | | `bt update` | Update bt in-place | +## `bt scorers` + +Create prompt-based LLM scorers or classifiers in the current project: + +```bash +bt scorers create "Helpfulness" \ + --model gpt-5.4-nano \ + --messages @messages.json \ + --choice-scores '{"A":1,"B":0}' + +bt scorers create "Safety label" \ + --model gpt-5.4-nano \ + --messages @messages.json \ + --classifications '["safe","unsafe"]' \ + --allow-no-match +``` + +Use `--if-exists error|ignore|replace` to control slug conflicts. Text and structured input flags accept an inline value, `@PATH`, or `-` for stdin; only one flag per command may read stdin. Use `--template-format mustache|jinja|none`; `nunjucks` and `jinja2` are accepted aliases for Jinja. Model options include `--use-cache[=true|false]` and `--response-format text|json-object|`; structured output accepts a full `response_format` JSON object inline or from `@PATH`. + +For TypeScript and Python code scorers, use the Braintrust SDK and `bt functions push`. + ## `bt eval` **File selection:** diff --git a/src/functions/api.rs b/src/functions/api.rs index 57f8829c..ea5d6bd9 100644 --- a/src/functions/api.rs +++ b/src/functions/api.rs @@ -58,9 +58,27 @@ pub struct CodeUploadSlot { pub bundle_id: String, } +#[derive(Debug, Clone, Deserialize)] +pub struct InsertedFunctionResult { + pub id: String, + pub project_id: String, + pub slug: String, + pub found_existing: bool, +} + #[derive(Debug, Clone)] pub struct InsertFunctionsResult { pub ignored_entries: Option, + pub xact_id: Option, + pub functions: Vec, +} + +#[derive(Debug, Deserialize)] +struct InsertFunctionsResponse { + #[serde(default)] + xact_id: Option, + #[serde(default)] + functions: Vec, } pub async fn list_functions( @@ -68,17 +86,29 @@ pub async fn list_functions( project_id: &str, function_type: Option<&str>, ) -> Result> { + let query = list_functions_query(project_id, function_type); + let response = client.btql::(&query).await?; + + Ok(response.data) +} + +fn list_functions_query(project_id: &str, function_type: Option<&str>) -> String { let pid = escape_sql(project_id); - let query = match function_type { + let type_filter = match function_type { + // Match the web UI's Scorers tab: label-producing classifiers appear + // alongside score-producing scorers, while topic maps do not. + Some("scorer") => "function_type IN ('scorer', 'classifier') \ + AND COALESCE(function_data.type, '') != 'topic_map' \ + AND (origin IS NULL OR NOT COALESCE(origin.internal, FALSE))" + .to_string(), Some(ft) => { let ft = escape_sql(ft); - format!("SELECT * FROM project_functions('{pid}') WHERE function_type = '{ft}'") + format!("function_type = '{ft}'") } - None => format!("SELECT * FROM project_functions('{pid}')"), + None => return format!("SELECT * FROM project_functions('{pid}')"), }; - let response = client.btql::(&query).await?; - Ok(response.data) + format!("SELECT * FROM project_functions('{pid}') WHERE {type_filter}") } pub async fn get_function_by_slug( @@ -258,8 +288,14 @@ pub async fn insert_functions( .await .context("failed to insert functions")?; + let response: InsertFunctionsResponse = serde_json::from_value(raw.clone()) + .context("unexpected insert-functions response shape")?; + Ok(InsertFunctionsResult { - ignored_entries: ignored_count(&raw), + ignored_entries: ignored_count(&raw) + .or_else(|| ignored_count_from_function_results(&raw, functions)), + xact_id: response.xact_id, + functions: response.functions, }) } @@ -273,10 +309,48 @@ fn ignored_count(raw: &Value) -> Option { .and_then(|count| usize::try_from(count).ok()) } +fn ignored_count_from_function_results(raw: &Value, requests: &[Value]) -> Option { + let results = raw.get("functions")?.as_array()?; + if results.len() != requests.len() { + return None; + } + + results + .iter() + .zip(requests) + .try_fold(0usize, |count, (result, request)| { + let should_ignore = request.get("if_exists").and_then(Value::as_str) == Some("ignore"); + if !should_ignore { + return Some(count); + } + + let found_existing = result.get("found_existing")?.as_bool()?; + Some(count + usize::from(found_existing)) + }) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn scorer_list_query_matches_web_ui_filter() { + let query = list_functions_query("test-project-id", Some("scorer")); + + assert!(query.contains("function_type IN ('scorer', 'classifier')")); + assert!(query.contains("COALESCE(function_data.type, '') != 'topic_map'")); + assert!(query.contains("origin IS NULL")); + assert!(query.contains("origin.internal")); + } + + #[test] + fn non_scorer_list_query_keeps_exact_type_filter() { + let query = list_functions_query("test-project-id", Some("tool")); + + assert!(query.contains("function_type = 'tool'")); + assert!(!query.contains("classifier")); + } + #[test] fn ignored_count_extracts_canonical_shape() { let first = serde_json::json!({ "ignored_count": 3 }); @@ -291,6 +365,61 @@ mod tests { assert_eq!(ignored_count(&serde_json::json!({})), None); } + #[test] + fn derives_ignored_count_from_found_existing_results() { + let requests = vec![ + serde_json::json!({ "slug": "first", "if_exists": "ignore" }), + serde_json::json!({ "slug": "second", "if_exists": "replace" }), + serde_json::json!({ "slug": "third", "if_exists": "ignore" }), + ]; + let response = serde_json::json!({ + "functions": [ + { "slug": "first", "found_existing": true }, + { "slug": "second", "found_existing": true }, + { "slug": "third", "found_existing": false }, + ] + }); + + assert_eq!( + ignored_count_from_function_results(&response, &requests), + Some(1) + ); + } + + #[test] + fn ignored_count_fallback_rejects_mismatched_response_length() { + let requests = vec![serde_json::json!({ + "slug": "first", + "if_exists": "ignore" + })]; + let response = serde_json::json!({ "functions": [] }); + + assert_eq!( + ignored_count_from_function_results(&response, &requests), + None + ); + } + + #[test] + fn parses_insert_function_operation_fields() { + let response: InsertFunctionsResponse = serde_json::from_value(serde_json::json!({ + "xact_id": "1000000000000000001", + "functions": [{ + "id": "fn_test_scorer", + "project_id": "test-project", + "slug": "test-scorer", + "found_existing": true + }] + })) + .expect("insert response"); + + assert_eq!(response.xact_id.as_deref(), Some("1000000000000000001")); + assert_eq!(response.functions[0].id, "fn_test_scorer"); + assert_eq!(response.functions[0].project_id, "test-project"); + assert_eq!(response.functions[0].slug, "test-scorer"); + assert!(response.functions[0].found_existing); + } + #[test] fn insert_functions_body_wraps_functions_array() { let functions = vec![serde_json::json!({ "slug": "demo" })]; diff --git a/src/functions/create.rs b/src/functions/create.rs new file mode 100644 index 00000000..2a6f4cb5 --- /dev/null +++ b/src/functions/create.rs @@ -0,0 +1,483 @@ +use anyhow::{bail, Context, Result}; +use clap::{builder::BoolishValueParser, ArgGroup, Args}; +use dialoguer::Input; +use serde_json::{json, Map, Value}; + +use crate::{ + ui::{is_interactive, print_command_status, with_spinner, CommandStatus}, + utils::{merge_json_objects, read_text_source, read_yaml_object_source}, +}; + +use super::{ + api, + prompt_config::{ + parse_choice_scores_source, parse_classifications_source, validate_unit_interval, + PromptConfigArgs, + }, + IfExistsMode, ResolvedContext, +}; + +/// Create an LLM scorer or classifier. +/// +/// The generated definition matches Braintrust's prompt-function schema with +/// an `llm_classifier` parser. `--choice-scores` produces numeric scores; +/// `--classifications` produces labels. +#[derive(Debug, Clone, Args)] +#[command(group( + ArgGroup::new("output") + .required(true) + .multiple(false) + .args(["choice_scores", "classifications"]) +))] +#[command(after_help = "\ +Examples: + bt scorers create \"Helpfulness\" --model gpt-5.4-nano --messages @messages.json \\ + --choice-scores '{\"A\":1,\"B\":0}' + bt scorers create \"Correctness\" --slug correctness --model gpt-5.4-nano \\ + --messages @messages.json \\ + --choice-scores '{\"correct\":1,\"incorrect\":0}' --use-cot=false + bt scorers create \"Tone\" --model gpt-5.4-nano \\ + --messages @messages.json --choice-scores @scores.json + bt scorers create \"Safety label\" --model gpt-5.4-nano --messages @messages.json \\ + --classifications '[\"safe\",\"unsafe\"]' --template-format jinja + +TypeScript and Python code scorers: + TypeScript: projects.create({ name: \"test-project\" }).scorers.create({...}) + bt functions push scorer.ts + Python: projects.create(\"test-project\").scorers.create(...) + bt functions push scorer.py +")] +pub(crate) struct CreateArgs { + /// Scorer name. + #[arg(value_name = "NAME", conflicts_with = "name")] + name_positional: Option, + + /// Scorer name (alternative to the positional name). + #[arg(long, value_name = "NAME")] + name: Option, + + /// Unique scorer slug. Defaults to a slug generated from the name. + #[arg(long, short = 's')] + slug: Option, + + /// Scorer description. + #[arg(long, short = 'd')] + description: Option, + + /// Chat messages source: inline JSON, @PATH to read from a file, or - for + /// stdin. + #[arg(long, value_name = "SOURCE")] + messages: String, + + /// Model used by the LLM judge. + #[arg(long, short = 'm', value_name = "MODEL")] + model: String, + + #[command(flatten)] + prompt_config: PromptConfigArgs, + + /// Choice-to-score mapping for score output: inline JSON, @PATH to read + /// from a file, or - for stdin. Scores must be between 0 and 1. + #[arg(long, value_name = "SOURCE")] + choice_scores: Option, + + /// Labels for classification output: an inline JSON array, @PATH to read + /// from a file, or - for stdin. This creates an LLM classifier, which is + /// shown alongside scorers in the Braintrust UI. + #[arg(long, value_name = "SOURCE")] + classifications: Option, + + /// Allow a classifier to return no matching classification. + #[arg(long, requires = "classifications")] + allow_no_match: bool, + + /// Whether the scorer should use chain-of-thought reasoning. Defaults to + /// true; pass --use-cot=false to disable it. + #[arg( + long, + num_args = 0..=1, + default_missing_value = "true", + default_value_t = true, + value_parser = BoolishValueParser::new() + )] + use_cot: bool, + + /// Score threshold for passing, between 0 and 1. + #[arg(long, value_name = "NUMBER", conflicts_with = "classifications")] + pass_threshold: Option, + + /// Metadata as inline YAML, @PATH to a YAML file, or - for stdin. + #[arg(long, value_name = "SOURCE")] + metadata: Option, + + /// Behavior when a scorer with the same slug already exists. + #[arg(long, value_enum, default_value = "error")] + if_exists: IfExistsMode, +} + +pub(crate) async fn run(ctx: &ResolvedContext, args: &CreateArgs, json_output: bool) -> Result<()> { + let name = resolve_name(args)?; + let slug = resolve_slug(args, &name)?; + let definition = build_scorer_definition(args, &ctx.project.id, &name, &slug)?; + + let result = match with_spinner( + "Creating scorer...", + api::insert_functions(&ctx.client, std::slice::from_ref(&definition)), + ) + .await + { + Ok(result) => result, + Err(error) => { + print_command_status(CommandStatus::Error, &format!("Failed to create '{name}'")); + return Err(error); + } + }; + + let ignored = result.ignored_entries.is_some_and(|count| count > 0); + + if json_output { + let function = result + .functions + .first() + .context("insert-functions response did not include the scorer identity")?; + println!( + "{}", + serde_json::to_string(&json!({ + "id": function.id, + "project_id": function.project_id, + "slug": function.slug, + "version": result.xact_id, + "found_existing": function.found_existing, + "ignored": ignored, + }))? + ); + return Ok(()); + } + + if ignored { + print_command_status( + CommandStatus::Warning, + &format!("Scorer '{name}' already exists; left it unchanged"), + ); + } else if args.if_exists == IfExistsMode::Replace { + print_command_status(CommandStatus::Success, &format!("Saved '{name}'")); + } else { + print_command_status(CommandStatus::Success, &format!("Created '{name}'")); + } + + Ok(()) +} + +fn resolve_name(args: &CreateArgs) -> Result { + let name = match (&args.name_positional, &args.name) { + (Some(_), Some(_)) => bail!("use either a positional name or --name, not both"), + (Some(name), None) | (None, Some(name)) => name.trim().to_string(), + (None, None) if is_interactive() => Input::::new() + .with_prompt("Scorer name") + .interact_text()? + .trim() + .to_string(), + (None, None) => bail!("scorer name required. Use: bt scorers create ..."), + }; + + if name.is_empty() { + bail!("scorer name cannot be empty"); + } + Ok(name) +} + +fn resolve_slug(args: &CreateArgs, name: &str) -> Result { + let slug = args + .slug + .as_deref() + .map(str::trim) + .map(ToOwned::to_owned) + .unwrap_or_else(|| slugify(name)); + if slug.is_empty() { + bail!("could not generate a slug from the scorer name; pass --slug explicitly"); + } + Ok(slug) +} + +fn slugify(value: &str) -> String { + let mut slug = String::new(); + let mut pending_separator = false; + + for character in value.trim().chars() { + if character.is_alphanumeric() { + if pending_separator && !slug.is_empty() { + slug.push('-'); + } + slug.extend(character.to_lowercase()); + pending_separator = false; + } else if !slug.is_empty() { + pending_separator = true; + } + } + + slug +} + +fn build_scorer_definition( + args: &CreateArgs, + project_id: &str, + name: &str, + slug: &str, +) -> Result { + let prompt = resolve_prompt_block(args)?; + let (function_type, parser) = resolve_output_parser(args)?; + + let mut prompt_data = json!({ + "prompt": prompt, + "parser": parser, + }) + .as_object() + .expect("prompt data is an object") + .clone(); + let prompt_config = args + .prompt_config + .build_prompt_data_patch(Some(&args.model))?; + merge_json_objects(&mut prompt_data, &prompt_config); + + let mut definition = json!({ + "project_id": project_id, + "name": name, + "slug": slug, + "function_data": { + "type": "prompt", + }, + "prompt_data": prompt_data, + "if_exists": args.if_exists.as_str(), + "function_type": function_type, + }); + + if let Some(description) = args.description.as_deref() { + definition["description"] = Value::String(description.to_string()); + } + + let metadata = resolve_metadata(args)?; + if !metadata.is_empty() { + definition["metadata"] = Value::Object(metadata); + } + + Ok(definition) +} + +fn resolve_output_parser(args: &CreateArgs) -> Result<(&'static str, Value)> { + match ( + args.choice_scores.as_deref(), + args.classifications.as_deref(), + ) { + (Some(source), None) => Ok(( + "scorer", + json!({ + "type": "llm_classifier", + "use_cot": args.use_cot, + "choice_scores": parse_choice_scores_source(source)?, + }), + )), + (None, Some(source)) => Ok(( + "classifier", + json!({ + "type": "llm_classifier", + "use_cot": args.use_cot, + "choice": parse_classifications_source(source)?, + "allow_no_match": args.allow_no_match, + }), + )), + (Some(_), Some(_)) => bail!( + "use either --choice-scores for score output or --classifications for classification output, not both" + ), + (None, None) => bail!( + "output choices required. Pass --choice-scores or --classifications " + ), + } +} + +fn resolve_metadata(args: &CreateArgs) -> Result> { + let mut metadata = match args.metadata.as_deref() { + Some(source) => read_yaml_object_source(source, "scorer metadata")?, + None => Map::new(), + }; + if let Some(pass_threshold) = args.pass_threshold { + validate_unit_interval(pass_threshold, "--pass-threshold")?; + metadata.insert("__pass_threshold".to_string(), json!(pass_threshold)); + } + Ok(metadata) +} + +fn resolve_prompt_block(args: &CreateArgs) -> Result { + let raw = read_text_source(&args.messages, "messages")?; + parse_messages(&raw) +} + +fn parse_messages(raw: &str) -> Result { + let messages: Value = serde_json::from_str(raw).context("invalid JSON in scorer messages")?; + match messages { + Value::Array(_) => Ok(json!({ "type": "chat", "messages": messages })), + _ => bail!("scorer messages must be a JSON array"), + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + + #[derive(Debug, Parser)] + struct CreateArgsHarness { + #[command(flatten)] + args: CreateArgs, + } + + fn args() -> CreateArgs { + CreateArgs { + name_positional: Some("Test Helpfulness".to_string()), + name: None, + slug: None, + description: Some("Synthetic test scorer".to_string()), + messages: r#"[{"role":"user","content":"Judge {{output}}."}]"#.to_string(), + model: "gpt-test".to_string(), + prompt_config: PromptConfigArgs::default(), + choice_scores: Some(r#"{"A":1,"B":0}"#.to_string()), + classifications: None, + allow_no_match: false, + use_cot: true, + pass_threshold: None, + metadata: None, + if_exists: IfExistsMode::Error, + } + } + + #[test] + fn use_cot_defaults_to_true() { + let parsed = CreateArgsHarness::try_parse_from([ + "bt-scorers-create", + "Test scorer", + "--model", + "gpt-test", + "--messages", + r#"[{"role":"user","content":"Judge {{output}}"}]"#, + "--choice-scores", + r#"{"yes":1,"no":0}"#, + ]) + .expect("parse create args"); + + assert!(parsed.args.use_cot); + } + + #[test] + fn builds_sdk_compatible_llm_scorer_definition() { + let args = args(); + let body = build_scorer_definition( + &args, + "00000000-0000-0000-0000-000000000001", + "Test Helpfulness", + "test-helpfulness", + ) + .expect("definition"); + + assert_eq!(body["function_data"], json!({ "type": "prompt" })); + assert_eq!(body["function_type"], "scorer"); + assert_eq!(body["prompt_data"]["prompt"]["type"], "chat"); + assert_eq!(body["prompt_data"]["options"]["model"], "gpt-test"); + assert_eq!( + body["prompt_data"]["parser"], + json!({ + "type": "llm_classifier", + "use_cot": true, + "choice_scores": { "A": 1, "B": 0 }, + }) + ); + assert_eq!(body["if_exists"], "error"); + assert_eq!(body["description"], "Synthetic test scorer"); + } + + #[test] + fn builds_chat_prompt_definition() { + let args = args(); + + let body = + build_scorer_definition(&args, "test-project", "Test", "test").expect("definition"); + + assert_eq!(body["prompt_data"]["prompt"]["type"], "chat"); + assert_eq!( + body["prompt_data"]["prompt"]["messages"], + json!([{ "role": "user", "content": "Judge {{output}}." }]) + ); + } + + #[test] + fn rejects_non_array_messages() { + let mut args = args(); + args.messages = r#"{"role":"user","content":"Judge {{output}}"}"#.to_string(); + + let error = build_scorer_definition(&args, "test-project", "Test", "test") + .expect_err("messages should be an array"); + assert!(error.to_string().contains("messages must be a JSON array")); + } + + #[test] + fn rejects_non_numeric_choice_score() { + let mut args = args(); + args.choice_scores = Some(r#"{"A":"one"}"#.to_string()); + + let error = build_scorer_definition(&args, "test-project", "Test", "test") + .expect_err("string score should fail"); + assert!(error.to_string().contains("must be a number")); + } + + #[test] + fn supports_disabling_use_cot() { + let mut args = args(); + args.use_cot = false; + + let body = + build_scorer_definition(&args, "test-project", "Test", "test").expect("definition"); + assert_eq!(body["prompt_data"]["parser"]["use_cot"], false); + } + + #[test] + fn builds_classification_output() { + let mut args = args(); + args.choice_scores = None; + args.classifications = Some(r#"["safe","unsafe"]"#.to_string()); + args.allow_no_match = true; + + let body = + build_scorer_definition(&args, "test-project", "Test", "test").expect("definition"); + + assert_eq!(body["function_type"], "classifier"); + assert_eq!( + body["prompt_data"]["parser"]["choice"], + json!(["safe", "unsafe"]) + ); + assert_eq!(body["prompt_data"]["parser"]["allow_no_match"], true); + assert!(body["prompt_data"]["parser"].get("choice_scores").is_none()); + } + + #[test] + fn builds_metadata_and_pass_threshold() { + let mut args = args(); + args.metadata = Some("owner: test-team".to_string()); + args.pass_threshold = Some(0.7); + + let body = build_scorer_definition(&args, "test-project", "Test scorer", "test-scorer") + .expect("definition"); + + assert_eq!( + body["metadata"], + json!({ "owner": "test-team", "__pass_threshold": 0.7 }) + ); + } + + #[test] + fn slugify_normalizes_name() { + assert_eq!( + slugify(" Test Helpfulness / Judge "), + "test-helpfulness-judge" + ); + assert_eq!(slugify("Already--Separated"), "already-separated"); + } +} diff --git a/src/functions/invoke.rs b/src/functions/invoke.rs index 2815ca0e..e7b294a1 100644 --- a/src/functions/invoke.rs +++ b/src/functions/invoke.rs @@ -42,6 +42,10 @@ impl InvokeArgs { } } +fn resolve_mode(mode: Option<&str>, json_output: bool) -> Option<&str> { + mode.or(json_output.then_some("json")) +} + fn resolve_input(input_arg: &Option) -> Result> { if let Some(raw) = input_arg { let parsed: Value = serde_json::from_str(raw).context("invalid JSON in --input")?; @@ -97,7 +101,7 @@ pub async fn run( .collect(); body["messages"] = json!(messages); } - if let Some(mode) = &args.mode { + if let Some(mode) = resolve_mode(args.mode.as_deref(), json_output) { body["mode"] = json!(mode); } if let Some(version) = &args.version { @@ -118,3 +122,23 @@ pub async fn run( Ok(()) } + +#[cfg(test)] +mod tests { + use super::resolve_mode; + + #[test] + fn json_output_requests_json_invoke_mode() { + assert_eq!(resolve_mode(None, true), Some("json")); + } + + #[test] + fn explicit_invoke_mode_takes_precedence_over_json_output() { + assert_eq!(resolve_mode(Some("text"), true), Some("text")); + } + + #[test] + fn default_output_does_not_set_invoke_mode() { + assert_eq!(resolve_mode(None, false), None); + } +} diff --git a/src/functions/mod.rs b/src/functions/mod.rs index d055c231..a43e2618 100644 --- a/src/functions/mod.rs +++ b/src/functions/mod.rs @@ -13,9 +13,11 @@ use crate::{ }; pub(crate) mod api; +pub(crate) mod create; mod delete; mod invoke; mod list; +pub(crate) mod prompt_config; mod pull; mod push; pub(crate) mod report; @@ -114,6 +116,9 @@ fn build_web_path(function: &Function) -> String { match function.function_type.as_deref() { Some("tool") => format!("tools?pr={}", urlencoding::encode(id)), Some("scorer") => format!("scorers/{}", urlencoding::encode(id)), + Some("classifier") if function.prompt_data.is_some() => { + format!("scorers/{}", urlencoding::encode(id)) + } Some("classifier") => { let xact_id = function._xact_id.as_deref().unwrap_or(""); format!( @@ -177,7 +182,7 @@ pub struct FunctionArgs { } #[derive(Debug, Clone, Subcommand)] -enum FunctionCommands { +pub(crate) enum FunctionCommands { /// List all in the current project List, /// View a function's details @@ -608,8 +613,16 @@ pub(crate) async fn select_function_interactive( } pub async fn run_typed(base: BaseArgs, args: FunctionArgs, kind: FunctionTypeFilter) -> Result<()> { + run_typed_command(base, args.command, kind).await +} + +pub(crate) async fn run_typed_command( + base: BaseArgs, + command: Option, + kind: FunctionTypeFilter, +) -> Result<()> { let ft = Some(kind); - match args.command { + match command { Some(FunctionCommands::View(v)) => match v.selector()? { ViewSelector::Id(id) => { let auth_ctx = resolve_auth_context(&base).await?; @@ -652,6 +665,12 @@ pub async fn run_typed(base: BaseArgs, args: FunctionArgs, kind: FunctionTypeFil } } +pub(crate) async fn run_scorer_create(base: BaseArgs, args: create::CreateArgs) -> Result<()> { + let json_output = base.json; + let ctx = resolve_context(&base).await?; + create::run(&ctx, &args, json_output).await +} + pub async fn run(base: BaseArgs, args: FunctionsArgs) -> Result<()> { let function_type = args.function_type; match args.command { @@ -1085,6 +1104,26 @@ mod tests { assert!(err.to_string().contains("either --id or a slug")); } + #[test] + fn prompt_classifier_web_path_uses_scorers_page() { + let function = Function { + id: "fn_test_classifier".to_string(), + name: "Test classifier".to_string(), + slug: "test-classifier".to_string(), + project_id: "test-project".to_string(), + description: None, + function_type: Some("classifier".to_string()), + prompt_data: Some(serde_json::json!({"parser": {"choice": ["a", "b"]}})), + function_data: Some(serde_json::json!({"type": "prompt"})), + tags: None, + metadata: None, + created: None, + _xact_id: None, + }; + + assert_eq!(build_web_path(&function), "scorers/fn_test_classifier"); + } + #[test] fn function_selection_label_includes_slug_when_name_differs() { let function = Function { diff --git a/src/functions/prompt_config.rs b/src/functions/prompt_config.rs new file mode 100644 index 00000000..2a120a1b --- /dev/null +++ b/src/functions/prompt_config.rs @@ -0,0 +1,497 @@ +use std::collections::HashSet; + +use anyhow::{bail, Context, Result}; +use clap::{builder::BoolishValueParser, Args, ValueEnum}; +use serde_json::{json, Map, Number, Value}; + +use crate::utils::read_text_source; + +#[derive(Debug, Clone, Default, Args)] +pub(crate) struct PromptConfigArgs { + /// Sampling temperature. + #[arg(long, value_name = "NUMBER")] + temperature: Option, + + /// Maximum number of generated tokens. + #[arg(long, value_name = "N")] + max_tokens: Option, + + /// Nucleus sampling probability. + #[arg(long, value_name = "NUMBER")] + top_p: Option, + + /// Frequency penalty. + #[arg(long, value_name = "NUMBER", allow_hyphen_values = true)] + frequency_penalty: Option, + + /// Presence penalty. + #[arg(long, value_name = "NUMBER", allow_hyphen_values = true)] + presence_penalty: Option, + + /// Stop sequence. Repeat this flag to specify multiple sequences. + #[arg(long, value_name = "TEXT", action = clap::ArgAction::Append)] + stop_sequence: Vec, + + /// Tool choice: auto, none, required, or a specific function name. + #[arg(long, value_name = "CHOICE")] + tool_choice: Option, + + /// Reasoning effort for supported models. + #[arg(long, value_enum)] + reasoning_effort: Option, + + /// Response verbosity for supported models. + #[arg(long, value_enum)] + verbosity: Option, + + /// Whether to use Braintrust's completion cache. Pass --use-cache=false to + /// bypass it. + #[arg( + long, + num_args = 0..=1, + default_missing_value = "true", + value_parser = BoolishValueParser::new() + )] + use_cache: Option, + + /// Model response format: text, json-object, or a response_format JSON + /// object supplied inline, from @PATH, or from stdin with -. + #[arg(long, value_name = "FORMAT|SOURCE")] + response_format: Option, + + /// Prompt template syntax. Jinja is stored using Braintrust's `nunjucks` + /// format; `nunjucks` and `jinja2` are accepted aliases. + #[arg(long, value_enum, value_name = "FORMAT")] + template_format: Option, +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum ReasoningEffort { + None, + Minimal, + Low, + Medium, + High, +} + +impl ReasoningEffort { + fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Minimal => "minimal", + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + } + } +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum Verbosity { + Low, + Medium, + High, +} + +impl Verbosity { + fn as_str(self) -> &'static str { + match self { + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + } + } +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum TemplateFormat { + Mustache, + #[value(name = "jinja", alias = "nunjucks", alias = "jinja2")] + Nunjucks, + None, +} + +impl TemplateFormat { + fn as_str(self) -> &'static str { + match self { + Self::Mustache => "mustache", + Self::Nunjucks => "nunjucks", + Self::None => "none", + } + } +} + +impl PromptConfigArgs { + /// Build a partial `prompt_data` object matching the app's prompt schema. + pub(crate) fn build_prompt_data_patch( + &self, + model: Option<&str>, + ) -> Result> { + let mut prompt_data = Map::new(); + let mut options = Map::new(); + let mut params = Map::new(); + + if let Some(model) = model { + let model = model.trim(); + if model.is_empty() { + bail!("--model cannot be empty"); + } + options.insert("model".to_string(), Value::String(model.to_string())); + } + + let temperature = match (self.temperature, self.use_cache) { + (None, Some(true)) => Some(0.0), + (Some(temperature), Some(true)) if temperature != 0.0 => { + bail!("--use-cache=true requires --temperature=0") + } + (temperature, _) => temperature, + }; + insert_optional_number(&mut params, "temperature", temperature)?; + if let Some(max_tokens) = self.max_tokens { + params.insert("max_tokens".to_string(), Value::Number(max_tokens.into())); + } + if let Some(top_p) = self.top_p { + validate_unit_interval(top_p, "--top-p")?; + insert_number(&mut params, "top_p", top_p, "--top-p")?; + } + insert_optional_number(&mut params, "frequency_penalty", self.frequency_penalty)?; + insert_optional_number(&mut params, "presence_penalty", self.presence_penalty)?; + + if !self.stop_sequence.is_empty() { + params.insert( + "stop".to_string(), + Value::Array( + self.stop_sequence + .iter() + .map(|value| Value::String(value.clone())) + .collect(), + ), + ); + } + + if let Some(tool_choice) = self.tool_choice.as_deref() { + let tool_choice = tool_choice.trim(); + if tool_choice.is_empty() { + bail!("--tool-choice cannot be empty"); + } + let value = match tool_choice { + "auto" | "none" | "required" => Value::String(tool_choice.to_string()), + function_name => json!({ + "type": "function", + "function": { "name": function_name }, + }), + }; + params.insert("tool_choice".to_string(), value); + } + + if let Some(reasoning_effort) = self.reasoning_effort { + params.insert( + "reasoning_effort".to_string(), + Value::String(reasoning_effort.as_str().to_string()), + ); + } + if let Some(verbosity) = self.verbosity { + params.insert( + "verbosity".to_string(), + Value::String(verbosity.as_str().to_string()), + ); + } + if let Some(use_cache) = self.use_cache { + params.insert("use_cache".to_string(), Value::Bool(use_cache)); + } + if let Some(source) = self.response_format.as_deref() { + params.insert( + "response_format".to_string(), + parse_response_format_source(source)?, + ); + } + + if !params.is_empty() { + options.insert("params".to_string(), Value::Object(params)); + } + if !options.is_empty() { + prompt_data.insert("options".to_string(), Value::Object(options)); + } + if let Some(template_format) = self.template_format { + prompt_data.insert( + "template_format".to_string(), + Value::String(template_format.as_str().to_string()), + ); + } + + Ok(prompt_data) + } +} + +fn parse_response_format_source(source: &str) -> Result { + let value = match source { + "text" => json!({ "type": "text" }), + "json-object" | "json_object" => json!({ "type": "json_object" }), + _ => { + let raw = read_text_source(source, "response format")?; + serde_json::from_str(&raw) + .context("invalid response format; use text, json-object, or a JSON object")? + } + }; + + let Some(format) = value.as_object() else { + bail!("response format must be a JSON object"); + }; + match format.get("type").and_then(Value::as_str) { + Some("text" | "json_object") => {} + Some("json_schema") => validate_json_schema_response_format(format)?, + Some(other) => bail!( + "unsupported response format type '{other}'; expected text, json_object, or json_schema" + ), + None => bail!("response format must contain a string 'type' field"), + } + + Ok(value) +} + +fn validate_json_schema_response_format(format: &Map) -> Result<()> { + let Some(schema) = format.get("json_schema").and_then(Value::as_object) else { + bail!("json_schema response format must contain a 'json_schema' object"); + }; + match schema.get("name") { + Some(Value::String(_)) => {} + _ => bail!("json_schema response format must contain a string 'json_schema.name' field"), + } + if let Some(value) = schema.get("description") { + if !value.is_string() { + bail!("response format 'json_schema.description' must be a string"); + } + } + if let Some(value) = schema.get("schema") { + if !value.is_object() && !value.is_string() { + bail!("response format 'json_schema.schema' must be an object or template string"); + } + } + if let Some(value) = schema.get("strict") { + if !value.is_boolean() && !value.is_null() { + bail!("response format 'json_schema.strict' must be a boolean or null"); + } + } + Ok(()) +} + +fn insert_optional_number( + target: &mut Map, + key: &str, + value: Option, +) -> Result<()> { + if let Some(value) = value { + insert_number(target, key, value, &format!("--{}", key.replace('_', "-")))?; + } + Ok(()) +} + +fn insert_number( + target: &mut Map, + key: &str, + value: f64, + label: &str, +) -> Result<()> { + let number = + Number::from_f64(value).ok_or_else(|| anyhow::anyhow!("{label} must be finite"))?; + target.insert(key.to_string(), Value::Number(number)); + Ok(()) +} + +pub(crate) fn validate_unit_interval(value: f64, label: &str) -> Result<()> { + if !value.is_finite() || !(0.0..=1.0).contains(&value) { + bail!("{label} must be between 0 and 1"); + } + Ok(()) +} + +pub(crate) fn parse_choice_scores_source(source: &str) -> Result> { + let raw = read_text_source(source, "choice scores")?; + let value: Value = serde_json::from_str(&raw).context("invalid JSON in choice scores")?; + let scores = match value { + Value::Object(scores) => scores, + _ => bail!("choice scores must be a JSON object mapping choices to numeric scores"), + }; + if scores.is_empty() { + bail!("choice scores cannot be empty"); + } + for (choice, score) in &scores { + if choice.trim().is_empty() { + bail!("choice score labels cannot be empty"); + } + let Some(score) = score.as_f64() else { + bail!("score for choice '{choice}' must be a number"); + }; + validate_unit_interval(score, &format!("score for choice '{choice}'"))?; + } + Ok(scores) +} + +pub(crate) fn parse_classifications_source(source: &str) -> Result> { + let raw = read_text_source(source, "classifications")?; + let value: Value = serde_json::from_str(&raw).context("invalid JSON in classifications")?; + let choices = match value { + Value::Array(choices) => choices, + _ => bail!("classifications must be a JSON array of strings"), + }; + if choices.is_empty() { + bail!("classifications cannot be empty"); + } + + let mut seen = HashSet::new(); + choices + .into_iter() + .map(|choice| { + let Value::String(choice) = choice else { + bail!("every classification must be a string"); + }; + let choice = choice.trim(); + if choice.is_empty() { + bail!("classifications cannot contain an empty label"); + } + if !seen.insert(choice.to_string()) { + bail!("classification labels must be unique; found '{choice}' more than once"); + } + Ok(Value::String(choice.to_string())) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + + #[derive(Debug, Parser)] + struct Harness { + #[command(flatten)] + config: PromptConfigArgs, + } + + #[test] + fn builds_web_ui_compatible_prompt_configuration() { + let args = Harness::try_parse_from([ + "test", + "--temperature", + "0.2", + "--max-tokens", + "512", + "--top-p", + "0.9", + "--frequency-penalty", + "-0.5", + "--presence-penalty", + "0.25", + "--stop-sequence", + "END", + "--stop-sequence", + "DONE", + "--tool-choice", + "test_tool", + "--reasoning-effort", + "high", + "--verbosity", + "low", + "--use-cache=false", + "--response-format", + r#"{"type":"json_schema","json_schema":{"name":"test_result","schema":{"type":"object"},"strict":true}}"#, + "--template-format", + "jinja", + ]) + .expect("parse arguments"); + + let patch = args + .config + .build_prompt_data_patch(Some("gpt-test")) + .expect("prompt data"); + assert_eq!(patch["options"]["model"], "gpt-test"); + assert_eq!(patch["options"]["params"]["temperature"], 0.2); + assert_eq!(patch["options"]["params"]["max_tokens"], 512); + assert_eq!(patch["options"]["params"]["top_p"], 0.9); + assert_eq!(patch["options"]["params"]["frequency_penalty"], -0.5); + assert_eq!(patch["options"]["params"]["presence_penalty"], 0.25); + assert_eq!(patch["options"]["params"]["stop"], json!(["END", "DONE"])); + assert_eq!( + patch["options"]["params"]["tool_choice"], + json!({"type": "function", "function": {"name": "test_tool"}}) + ); + assert_eq!(patch["options"]["params"]["reasoning_effort"], "high"); + assert_eq!(patch["options"]["params"]["verbosity"], "low"); + assert_eq!(patch["options"]["params"]["use_cache"], false); + assert_eq!( + patch["options"]["params"]["response_format"], + json!({ + "type": "json_schema", + "json_schema": { + "name": "test_result", + "schema": { "type": "object" }, + "strict": true, + } + }) + ); + assert_eq!(patch["template_format"], "nunjucks"); + } + + #[test] + fn enabling_cache_sets_the_temperature_required_by_the_web_ui() { + let args = Harness::try_parse_from(["test", "--use-cache=true"]).expect("parse arguments"); + + let patch = args + .config + .build_prompt_data_patch(Some("claude-test")) + .expect("prompt data"); + assert_eq!(patch["options"]["params"]["temperature"], 0.0); + assert_eq!(patch["options"]["params"]["use_cache"], true); + } + + #[test] + fn rejects_cache_with_nonzero_temperature() { + let args = Harness::try_parse_from(["test", "--temperature", "0.5", "--use-cache=true"]) + .expect("parse arguments"); + + let error = args + .config + .build_prompt_data_patch(Some("claude-test")) + .expect_err("nonzero temperature should conflict with caching"); + assert!(error + .to_string() + .contains("--use-cache=true requires --temperature=0")); + } + + #[test] + fn supports_response_format_shorthands() { + assert_eq!( + parse_response_format_source("text").expect("text format"), + json!({ "type": "text" }) + ); + assert_eq!( + parse_response_format_source("json-object").expect("JSON object format"), + json!({ "type": "json_object" }) + ); + } + + #[test] + fn rejects_invalid_json_schema_response_format() { + let error = parse_response_format_source(r#"{"type":"json_schema"}"#) + .expect_err("missing json_schema should fail"); + assert!(error.to_string().contains("'json_schema' object")); + } + + #[test] + fn validates_scores_against_api_range() { + let error = parse_choice_scores_source(r#"{"bad":1.5}"#) + .expect_err("out-of-range score should fail"); + assert!(error.to_string().contains("between 0 and 1")); + } + + #[test] + fn parses_unique_classification_labels() { + let choices = + parse_classifications_source(r#"["safe","unsafe"]"#).expect("classifications"); + assert_eq!( + choices, + json!(["safe", "unsafe"]).as_array().unwrap().clone() + ); + } +} diff --git a/src/main.rs b/src/main.rs index 22a05fef..25bf4006 100644 --- a/src/main.rs +++ b/src/main.rs @@ -256,13 +256,21 @@ enum ExitCode { User = 4, } +static JSON_OUTPUT_REQUESTED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + fn main() { let exit_code = match try_main() { Ok(()) => ExitCode::Success, Err(err) => { let missing_credential = crate::auth::is_missing_credential_error(&err); let code = classify_error(&err, missing_credential); - print_error(&err, code, missing_credential); + print_error( + &err, + code, + missing_credential, + JSON_OUTPUT_REQUESTED.load(std::sync::atomic::Ordering::Relaxed), + ); code } }; @@ -315,6 +323,10 @@ fn try_main() -> Result<()> { apply_base_arg_sources(&matches, cli.command.base_mut()); cli.command.base_mut().profile_explicit = has_explicit_profile_arg(&argv); apply_base_output_defaults(&mut cli.command); + JSON_OUTPUT_REQUESTED.store( + cli.command.base().json, + std::sync::atomic::Ordering::Relaxed, + ); configure_output(cli.command.base()); apply_runtime_env_overrides(cli.command.base()); let runtime = tokio::runtime::Builder::new_multi_thread() @@ -433,7 +445,7 @@ fn classify_error(err: &anyhow::Error, missing_credential: bool) -> ExitCode { if let Some(http_error) = find_http_error(err) { let status = http_error.status.as_u16(); - if status == 401 || status == 403 { + if (status == 401 || status == 403) && !is_upstream_provider_auth_error(http_error) { return ExitCode::Auth; } if (400..=499).contains(&status) { @@ -468,6 +480,22 @@ fn find_http_error(err: &anyhow::Error) -> Option<&crate::http::HttpError> { .find_map(|source| source.downcast_ref::()) } +fn is_upstream_provider_auth_error(error: &crate::http::HttpError) -> bool { + let Ok(body) = serde_json::from_str::(&error.body) else { + return false; + }; + let provider_error = body.get("error").unwrap_or(&body); + provider_error.get("code").and_then(|value| value.as_str()) == Some("invalid_api_key") + || provider_error + .get("message") + .and_then(|value| value.as_str()) + .is_some_and(|message| { + let message = message.to_ascii_lowercase(); + message.contains("incorrect api key provided") + || message.contains("llm provider") && message.contains("credential") + }) +} + fn classify_sdk_error(err: &anyhow::Error) -> Option { let sdk_err = err .chain() @@ -519,7 +547,18 @@ fn looks_like_user_error(err: &anyhow::Error) -> bool { || message.contains("invalid") } -fn print_error(err: &anyhow::Error, code: ExitCode, missing_credential: bool) { +fn json_error_payload(err: &anyhow::Error) -> serde_json::Value { + find_http_error(err) + .and_then(|error| serde_json::from_str(&error.body).ok()) + .unwrap_or_else(|| serde_json::json!({ "error": { "message": err.to_string() } })) +} + +fn print_error(err: &anyhow::Error, code: ExitCode, missing_credential: bool, json_output: bool) { + if json_output { + println!("{}", json_error_payload(err)); + return; + } + eprintln!("error: {err}"); if code == ExitCode::Auth && !missing_credential { eprintln!("Your credentials may be expired or invalid. For OAuth profiles, try `bt login --refresh --profile `; if refresh fails, re-run `bt login --oauth --profile `. Run `bt status --all` to inspect profile status."); @@ -687,6 +726,35 @@ mod tests { parts.iter().map(OsString::from).collect() } + #[test] + fn provider_credential_errors_are_not_classified_as_bt_auth_errors() { + let err = anyhow::Error::new(crate::http::HttpError { + status: reqwest::StatusCode::UNAUTHORIZED, + body: serde_json::json!({ + "error": { + "message": "Incorrect API key provided: synthetic-key", + "type": "invalid_request_error", + "code": "invalid_api_key" + }, + "status": 401 + }) + .to_string(), + }); + + assert_eq!(classify_error(&err, false), ExitCode::User); + assert_eq!(json_error_payload(&err)["error"]["code"], "invalid_api_key"); + } + + #[test] + fn bt_unauthorized_errors_remain_auth_errors() { + let err = anyhow::Error::new(crate::http::HttpError { + status: reqwest::StatusCode::UNAUTHORIZED, + body: r#"{"error":"Unauthorized"}"#.to_string(), + }); + + assert_eq!(classify_error(&err, false), ExitCode::Auth); + } + #[test] fn handle_version_json_detects_long_form() { assert!(handle_version_json(&argv(&["bt", "--version", "--json"])).unwrap()); diff --git a/src/scorers.rs b/src/scorers.rs index 842240b3..be8a5c47 100644 --- a/src/scorers.rs +++ b/src/scorers.rs @@ -1,10 +1,121 @@ use anyhow::Result; +use clap::{Args, Subcommand}; use crate::args::BaseArgs; -use crate::functions::{self, FunctionArgs, FunctionTypeFilter}; +use crate::functions::{self, FunctionCommands, FunctionTypeFilter}; -pub type ScorersArgs = FunctionArgs; +#[derive(Debug, Clone, Args)] +#[command(after_help = "\ +Examples: + bt scorers list + bt scorers view my-scorer + bt scorers create \"Helpfulness\" --model gpt-5.4-nano --messages @messages.json \\ + --choice-scores '{\"A\":1,\"B\":0}' + bt scorers delete my-scorer + +TypeScript and Python code scorers: + TypeScript: projects.create({ name: \"test-project\" }).scorers.create({...}) + bt functions push scorer.ts + Python: projects.create(\"test-project\").scorers.create(...) + bt functions push scorer.py +")] +pub struct ScorersArgs { + #[command(subcommand)] + command: Option, +} + +#[derive(Debug, Clone, Subcommand)] +enum ScorersCommands { + /// Create an LLM scorer or classifier + Create(Box), + #[command(flatten)] + Function(FunctionCommands), +} pub async fn run(base: BaseArgs, args: ScorersArgs) -> Result<()> { - functions::run_typed(base, args, FunctionTypeFilter::Scorer).await + match args.command { + Some(ScorersCommands::Create(create)) => functions::run_scorer_create(base, *create).await, + Some(ScorersCommands::Function(command)) => { + functions::run_typed_command(base, Some(command), FunctionTypeFilter::Scorer).await + } + None => functions::run_typed_command(base, None, FunctionTypeFilter::Scorer).await, + } +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + use crate::args::CLIArgs; + + #[derive(Debug, Parser)] + struct ScorersArgsHarness { + #[command(flatten)] + args: ScorersArgs, + } + + #[test] + fn invoke_accepts_global_json_flag() { + #[derive(Debug, Parser)] + struct Harness { + #[command(flatten)] + command: CLIArgs, + } + + let parsed = Harness::try_parse_from(["bt-scorers", "invoke", "test-scorer", "--json"]) + .expect("parse scorer invoke with global JSON output"); + + assert!(parsed.command.base.json); + assert!(matches!( + parsed.command.args.command, + Some(ScorersCommands::Function(FunctionCommands::Invoke(_))) + )); + } + + #[test] + fn parses_create_scorer() { + let parsed = ScorersArgsHarness::try_parse_from([ + "bt-scorers", + "create", + "Test scorer", + "--model", + "gpt-test", + "--messages", + r#"[{"role":"user","content":"Judge {{output}}"}]"#, + "--choice-scores", + r#"{"yes":1,"no":0}"#, + "--use-cot=false", + "--if-exists", + "replace", + ]) + .expect("parse create"); + + assert!(matches!( + parsed.args.command, + Some(ScorersCommands::Create(_)) + )); + } + + #[test] + fn parses_create_classifier() { + let parsed = ScorersArgsHarness::try_parse_from([ + "bt-scorers", + "create", + "Test classifier", + "--model", + "gpt-test", + "--messages", + r#"[{"role":"user","content":"Classify {{output}}"}]"#, + "--classifications", + r#"["safe","unsafe"]"#, + "--allow-no-match", + ]) + .expect("parse create classifier"); + + assert!(matches!( + parsed.args.command, + Some(ScorersCommands::Create(_)) + )); + } } diff --git a/src/utils/json_object.rs b/src/utils/json_object.rs index 20bc850c..4f2ee3d8 100644 --- a/src/utils/json_object.rs +++ b/src/utils/json_object.rs @@ -1,5 +1,18 @@ use serde_json::{Map, Value}; +pub(crate) fn merge_json_objects(target: &mut Map, source: &Map) { + for (key, value) in source { + match (target.get_mut(key), value) { + (Some(Value::Object(target_inner)), Value::Object(source_inner)) => { + merge_json_objects(target_inner, source_inner); + } + _ => { + target.insert(key.clone(), value.clone()); + } + } + } +} + pub(crate) fn lookup_object_path<'a, P>( object: &'a Map, path: &[P], @@ -20,6 +33,27 @@ mod tests { use super::*; + #[test] + fn merge_json_objects_deep_merges_nested_maps() { + let mut target = json!({ + "prompt_data": { "options": { "model": "gpt-test" } } + }) + .as_object() + .expect("object") + .clone(); + let source = json!({ + "prompt_data": { "options": { "params": { "temperature": 0 } } } + }) + .as_object() + .expect("object") + .clone(); + + merge_json_objects(&mut target, &source); + + assert_eq!(target["prompt_data"]["options"]["model"], "gpt-test"); + assert_eq!(target["prompt_data"]["options"]["params"]["temperature"], 0); + } + #[test] fn lookup_object_path_finds_nested_values() { let object = json!({ diff --git a/src/utils/mod.rs b/src/utils/mod.rs index adb19676..1429bebe 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -6,6 +6,8 @@ mod ids; mod json_object; mod plurals; mod profile; +mod structured_source; +mod text_source; pub(crate) use app_url::{app_project_url, app_project_url_with_encoded_path}; pub use duration::parse_duration_to_seconds; @@ -14,6 +16,8 @@ pub use fs_atomic::{ }; pub use git::GitRepo; pub(crate) use ids::new_uuid_id; -pub(crate) use json_object::lookup_object_path; +pub(crate) use json_object::{lookup_object_path, merge_json_objects}; pub use plurals::pluralize; pub(crate) use profile::{profile_author_slug, resolve_profile_info, sanitize_name_segment}; +pub(crate) use structured_source::read_yaml_object_source; +pub(crate) use text_source::read_text_source; diff --git a/src/utils/structured_source.rs b/src/utils/structured_source.rs new file mode 100644 index 00000000..fd5e0818 --- /dev/null +++ b/src/utils/structured_source.rs @@ -0,0 +1,39 @@ +use anyhow::{bail, Context, Result}; +use serde_json::{Map, Value}; + +use super::read_text_source; + +pub(crate) fn read_yaml_object_source( + source: &str, + description: &str, +) -> Result> { + let raw = read_text_source(source, description)?; + let value: Value = + yaml_serde::from_str(&raw).with_context(|| format!("invalid YAML in {description}"))?; + match value { + Value::Object(object) => Ok(object), + _ => bail!("{description} must be a YAML mapping/object"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_inline_yaml_object() { + let value = + read_yaml_object_source("owner: test-team\nsettings:\n enabled: true\n", "metadata") + .expect("metadata"); + + assert_eq!(value["owner"], "test-team"); + assert_eq!(value["settings"]["enabled"], true); + } + + #[test] + fn rejects_yaml_array() { + let error = + read_yaml_object_source("- one\n- two\n", "metadata").expect_err("array should fail"); + assert!(error.to_string().contains("mapping/object")); + } +} diff --git a/src/utils/text_source.rs b/src/utils/text_source.rs new file mode 100644 index 00000000..1f2ce754 --- /dev/null +++ b/src/utils/text_source.rs @@ -0,0 +1,107 @@ +use std::io::Read; +use std::sync::Mutex; + +use anyhow::{bail, Context, Result}; + +/// Which label drained stdin, so a second `-` fails loudly instead of reading "". +static STDIN_READER: Mutex> = Mutex::new(None); + +/// Resolve inline text, an `@PATH` file reference, or `-` for stdin. +/// +/// A leading literal `@` can be escaped as `@@`. Only one source per invocation +/// may read from stdin. +pub(crate) fn read_text_source(value: &str, label: &str) -> Result { + read_text_source_with_stdin_guard(value, label, &STDIN_READER) +} + +fn read_text_source_with_stdin_guard( + value: &str, + label: &str, + stdin_reader: &Mutex>, +) -> Result { + if value == "-" { + // The second reader would otherwise see "" and call it malformed input. + let mut reader = stdin_reader + .lock() + .map_err(|_| anyhow::anyhow!("stdin guard poisoned"))?; + if let Some(previous) = reader.as_deref() { + bail!("stdin was already read for {previous}; only one source can be '-'"); + } + *reader = Some(label.to_string()); + drop(reader); + + let mut content = String::new(); + std::io::stdin() + .read_to_string(&mut content) + .with_context(|| format!("failed to read {label} from stdin"))?; + return Ok(content); + } + + if let Some(literal) = value.strip_prefix("@@") { + return Ok(format!("@{literal}")); + } + + if let Some(path) = value.strip_prefix('@') { + if path.is_empty() { + bail!("{label} file path cannot be empty after '@'"); + } + return std::fs::read_to_string(path) + .with_context(|| format!("failed to read {label} file {path}")); + } + + Ok(value.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn returns_inline_text() { + assert_eq!( + read_text_source("Judge the answer.", "prompt").expect("inline prompt"), + "Judge the answer." + ); + } + + #[test] + fn reads_at_prefixed_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("judge.md"); + std::fs::write(&path, "Judge from a file.\n").expect("write prompt"); + + let source = format!("@{}", path.display()); + assert_eq!( + read_text_source(&source, "prompt").expect("file prompt"), + "Judge from a file.\n" + ); + } + + #[test] + fn double_at_escapes_literal_at() { + assert_eq!( + read_text_source("@@mention", "prompt").expect("literal prompt"), + "@mention" + ); + } + + #[test] + fn rejects_empty_file_reference() { + let error = read_text_source("@", "prompt").expect_err("empty path should fail"); + assert!(error.to_string().contains("cannot be empty")); + } + + #[test] + fn rejects_a_second_stdin_source() { + // A local guard avoids draining the suite's shared stdin; the rejection + // happens before any read. + let guard = Mutex::new(Some("metadata".to_string())); + + let error = read_text_source_with_stdin_guard("-", "patch", &guard) + .expect_err("second stdin source should fail"); + assert_eq!( + error.to_string(), + "stdin was already read for metadata; only one source can be '-'" + ); + } +} diff --git a/tests/cli.rs b/tests/cli.rs index 9c26cdd7..93cb6976 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1190,6 +1190,38 @@ fn trace_setup_keeps_each_agents_persistent_selection_independent() { ); } +#[test] +fn scorers_create_help_includes_llm_judge_configuration() { + bt_command() + .args(["scorers", "create", "--help"]) + .assert() + .success() + .stdout(predicate::str::contains("--messages ")) + .stdout(predicate::str::contains("--model")) + .stdout(predicate::str::contains("--temperature")) + .stdout(predicate::str::contains("--max-tokens")) + .stdout(predicate::str::contains("--top-p")) + .stdout(predicate::str::contains("--frequency-penalty")) + .stdout(predicate::str::contains("--presence-penalty")) + .stdout(predicate::str::contains("--stop-sequence")) + .stdout(predicate::str::contains("--tool-choice")) + .stdout(predicate::str::contains("--reasoning-effort")) + .stdout(predicate::str::contains("--verbosity")) + .stdout(predicate::str::contains("--use-cache")) + .stdout(predicate::str::contains("--response-format")) + .stdout(predicate::str::contains("--template-format")) + .stdout(predicate::str::contains("--choice-scores")) + .stdout(predicate::str::contains("--classifications")) + .stdout(predicate::str::contains("--use-cot")) + .stdout(predicate::str::contains("--pass-threshold")) + .stdout(predicate::str::contains("--metadata")) + .stdout(predicate::str::contains("--if-exists")) + .stdout(predicate::str::contains("TypeScript: projects.create")) + .stdout(predicate::str::contains("Python: projects.create")) + .stdout(predicate::str::contains("bt functions push scorer.ts")) + .stdout(predicate::str::contains("bt functions push scorer.py")); +} + #[test] fn topics_report_help_accepts_global_org_short_conflict_free() { bt_command() diff --git a/tests/functions.rs b/tests/functions.rs index 19e14d42..a23938c2 100644 --- a/tests/functions.rs +++ b/tests/functions.rs @@ -678,9 +678,9 @@ fn functions_push_requires_app_url_with_custom_api_url() { .expect("run push with custom API URL and no app URL"); assert!(!output.status.success()); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(stderr.contains("--app-url or BRAINTRUST_APP_URL")); - assert!(!stderr.contains("https://www.braintrust.dev/api/apikey/login")); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("--app-url or BRAINTRUST_APP_URL")); + assert!(!stdout.contains("https://www.braintrust.dev/api/apikey/login")); } #[test] @@ -725,7 +725,7 @@ fn root_login_refresh_uses_selected_profile() { let output = cmd.output().expect("run bt login --refresh"); assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr) + assert!(String::from_utf8_lossy(&output.stdout) .contains("`bt login --refresh` only applies to oauth profiles")); }