diff --git a/src/cmd/submissions.rs b/src/cmd/submissions.rs index 203642e..4288176 100644 --- a/src/cmd/submissions.rs +++ b/src/cmd/submissions.rs @@ -26,7 +26,7 @@ pub async fn list_submissions( // Print each submission for sub in submissions { - let status = if sub.done { "done" } else { "pending" }; + let status = list_status(sub.done, sub.runs.is_empty()); // Collect all GPU types and best score from runs let gpus: Vec<&str> = sub.runs.iter().map(|r| r.gpu_type.as_str()).collect(); @@ -77,8 +77,16 @@ pub async fn show_submission(cli_id: String, submission_id: i64, no_code: bool) println!("Submitted: {}", sub.submission_time); println!( "Status: {}", - if sub.done { "done" } else { "pending" } + detail_status(sub.done, sub.job.as_ref()) ); + if let Some(error) = sub + .job + .as_ref() + .and_then(|job| job.error.as_deref()) + .filter(|error| !error.is_empty()) + { + println!("Job error: {}", error); + } if !sub.runs.is_empty() { println!("\nRuns:"); @@ -173,6 +181,22 @@ fn format_score(score: Option) -> String { .unwrap_or_else(|| "-".to_string()) } +fn list_status(done: bool, has_no_ranked_runs: bool) -> &'static str { + if !done { + "pending" + } else if has_no_ranked_runs { + "not ranked" + } else { + "done" + } +} + +fn detail_status(done: bool, job: Option<&crate::models::SubmissionJobStatus>) -> String { + job.and_then(|job| job.status.as_deref()) + .unwrap_or(if done { "done" } else { "pending" }) + .to_string() +} + #[cfg(test)] mod tests { use super::*; @@ -203,6 +227,13 @@ mod tests { assert_eq!(format_score(None), "-"); } + #[test] + fn test_list_status_marks_done_without_ranked_runs() { + assert_eq!(list_status(false, true), "pending"); + assert_eq!(list_status(true, true), "not ranked"); + assert_eq!(list_status(true, false), "done"); + } + #[test] fn test_truncate() { assert_eq!(truncate("short", 19), "short"); diff --git a/src/service/mod.rs b/src/service/mod.rs index 66976ca..79d244c 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1090,6 +1090,9 @@ async fn submit_solution_background>( } if details.done { + if let Some(error) = leaderboard_completion_error(&details, submission_mode, gpu) { + return Err(anyhow!(error)); + } return format_submission_details(&details); } @@ -1133,6 +1136,86 @@ fn leaderboard_score_summary(details: &SubmissionDetails) -> Option { } } +fn same_gpu(run_gpu: &str, requested_gpu: &str) -> bool { + run_gpu.eq_ignore_ascii_case(requested_gpu) +} + +fn has_qualifying_leaderboard_result(details: &SubmissionDetails, gpu: &str) -> bool { + let has_public_score = details.runs.iter().any(|run| { + run.mode == "leaderboard" + && !run.secret + && same_gpu(&run.runner, gpu) + && run.passed + && run.score.is_some() + }); + + let has_secret_leaderboard_pass = details.runs.iter().any(|run| { + run.mode == "leaderboard" && run.secret && same_gpu(&run.runner, gpu) && run.passed + }); + + let has_secret_failure = details + .runs + .iter() + .any(|run| run.secret && same_gpu(&run.runner, gpu) && !run.passed); + + has_public_score && has_secret_leaderboard_pass && !has_secret_failure +} + +fn leaderboard_completion_error( + details: &SubmissionDetails, + submission_mode: &str, + gpu: &str, +) -> Option { + if !submission_mode.eq_ignore_ascii_case("leaderboard") { + return None; + } + + if has_qualifying_leaderboard_result(details, gpu) { + return None; + } + + if let Some(error) = details + .job + .as_ref() + .and_then(|job| job.error.as_deref()) + .filter(|error| !error.is_empty()) + { + return Some(format!("Submission {}: {}", details.id, error)); + } + + let secret_failed = details + .runs + .iter() + .any(|run| run.secret && same_gpu(&run.runner, gpu) && !run.passed); + + if secret_failed { + return Some(format!( + "Submission {} failed secret validation on {}; it will not appear on the leaderboard", + details.id, gpu + )); + } + + let has_public_leaderboard_score = details.runs.iter().any(|run| { + run.mode == "leaderboard" + && !run.secret + && same_gpu(&run.runner, gpu) + && run.passed + && run.score.is_some() + }); + + if has_public_leaderboard_score { + Some(format!( + "Submission {} did not pass secret leaderboard validation on {}; it will not appear on the leaderboard", + details.id, gpu + )) + } else { + Some(format!( + "Submission {} completed without a leaderboard score on {}; it will not appear on the leaderboard", + details.id, gpu + )) + } +} + fn format_submission_details(details: &SubmissionDetails) -> Result { let runs: Vec = details .runs @@ -1150,11 +1233,19 @@ fn format_submission_details(details: &SubmissionDetails) -> Result { }) .collect(); + let job = details.job.as_ref().map(|job| { + serde_json::json!({ + "status": job.status, + "error": job.error, + }) + }); + let json = serde_json::to_string_pretty(&serde_json::json!({ "submission_id": details.id, "leaderboard": details.leaderboard_name, "file_name": details.file_name, "done": details.done, + "job": job, "runs": runs, })) .map_err(|e| anyhow!("Failed to format submission result: {}", e))?; @@ -1697,6 +1788,12 @@ mod tests { } } + fn failed_run(mode: &str, secret: bool, score: Option) -> SubmissionRun { + let mut r = run(mode, secret, score); + r.passed = false; + r + } + fn details(runs: Vec) -> SubmissionDetails { SubmissionDetails { id: 1, @@ -1762,4 +1859,57 @@ mod tests { // format_submission_details still works, just without a summary header. assert!(format_submission_details(&d).unwrap().starts_with('{')); } + + #[test] + fn test_leaderboard_completion_error_none_for_qualified_submission() { + let d = details(vec![ + run("leaderboard", false, Some(0.0066)), + run("leaderboard", true, Some(0.0018)), + ]); + + assert!(leaderboard_completion_error(&d, "leaderboard", "B200").is_none()); + } + + #[test] + fn test_leaderboard_completion_error_for_secret_failure() { + let d = details(vec![ + run("leaderboard", false, Some(0.0066)), + failed_run("benchmark", true, None), + ]); + + let error = leaderboard_completion_error(&d, "leaderboard", "B200") + .expect("expected secret validation failure"); + + assert!(error.contains("failed secret validation")); + assert!(error.contains("will not appear on the leaderboard")); + } + + #[test] + fn test_leaderboard_completion_error_for_missing_secret_leaderboard() { + let d = details(vec![run("leaderboard", false, Some(0.0066))]); + + let error = leaderboard_completion_error(&d, "leaderboard", "B200") + .expect("expected missing secret validation failure"); + + assert!(error.contains("did not pass secret leaderboard validation")); + assert!(error.contains("will not appear on the leaderboard")); + } + + #[test] + fn test_leaderboard_completion_error_for_missing_public_score() { + let d = details(vec![run("leaderboard", false, None)]); + + let error = leaderboard_completion_error(&d, "leaderboard", "B200") + .expect("expected missing score failure"); + + assert!(error.contains("completed without a leaderboard score")); + assert!(error.contains("will not appear on the leaderboard")); + } + + #[test] + fn test_leaderboard_completion_error_ignores_non_leaderboard_modes() { + let d = details(vec![run("test", false, None)]); + + assert!(leaderboard_completion_error(&d, "test", "B200").is_none()); + } }