From f6688894e953c9945dc7d4cd5440c35c15afe323 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 08:39:31 -0700 Subject: [PATCH 01/16] First --- .../sample_optimization_job_basic.py | 3 +- .../sample_optimization_job_basic_async.py | 6 +- .../sample_optimization_job_basic_polling.py | 54 +++++++--------- ...le_optimization_job_basic_polling_async.py | 64 ++++++++----------- ...generation_job_simpleqna_for_finetuning.py | 25 +++++++- ...eration_job_simpleqna_with_agent_source.py | 28 +++++++- ...neration_job_simpleqna_with_file_source.py | 29 +++++++-- ...ration_job_simpleqna_with_prompt_source.py | 31 +++++++-- ...et_generation_job_traces_for_evaluation.py | 31 ++++++++- ...et_generation_job_traces_for_finetuning.py | 31 ++++++++- ...rubric_evaluator_generation_all_sources.py | 59 +++++++++++++++-- ...ample_rubric_evaluator_generation_basic.py | 33 ++++++++-- ...ple_rubric_evaluator_generation_iterate.py | 30 +++++++-- ...e_rubric_evaluator_generation_lifecycle.py | 58 +++++++++++++++-- 14 files changed, 363 insertions(+), 119 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py index 4e914f36cfb8..86d398bcf9f7 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py @@ -7,7 +7,8 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to create an agent - optimization job, poll it to completion, and read the results. + optimization job, and use the SDK's built-in polling mechanism to wait for + its completion to get the result. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py index 4be9f2ee07c5..1fc89964fabf 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py @@ -6,9 +6,9 @@ """ DESCRIPTION: - Async version of sample_optimization_job_basic.py. Demonstrates how to - create an agent optimization job, poll it to completion, and read the - results using the async AIProjectClient. + Given an async AIProjectClient, this sample demonstrates how to create an agent + optimization job, and use the SDK's built-in polling mechanism to wait for + its completion to get the result. USAGE: python sample_optimization_job_basic_async.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py index 0e1bdbacc9dc..df264edd0e37 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py @@ -7,7 +7,7 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to create an agent - optimization job and manually poll it to completion. + optimization job and poll its standard LRO to completion. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your @@ -42,7 +42,6 @@ from azure.ai.projects.models import ( OptimizationAgentIdentifier as AgentIdentifier, OptimizationEvaluatorRef as EvaluatorRef, - JobStatus, OptimizationJob, OptimizationJobInputs, OptimizationOptions, @@ -60,23 +59,26 @@ eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") -terminal_statuses = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - with ( DefaultAzureCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): # ------------------------------------------------------------------ - # 1. Create an optimization job without SDK polling. + # 1. Create an optimization job. # ------------------------------------------------------------------ print("Creating optimization job...") - created_jobs: list[OptimizationJob] = [] + latest_lro_response = {} - def capture_created_job(response): - created_jobs.append(OptimizationJob(response.http_response.json())) + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) - project_client.beta.agents.begin_create_optimization_job( + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.agents.begin_create_optimization_job( job=OptimizationJob( inputs=OptimizationJobInputs( agent=AgentIdentifier(agent_name=agent_name), @@ -92,39 +94,27 @@ def capture_created_job(response): ), ) ), - polling=False, - raw_response_hook=capture_created_job, + polling_interval=poll_interval, + raw_response_hook=capture_lro_response, ) - if not created_jobs: - raise RuntimeError("The create operation did not return an optimization job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") # ------------------------------------------------------------------ # 2. Poll the job to completion. # ------------------------------------------------------------------ - while job.status not in terminal_statuses: + while not poller.done(): + print(f"Optimization job status: {poller.status()}") time.sleep(poll_interval) - job = project_client.beta.agents.get_optimization_job(job_id=job.id) - print(f"Job status: {job.status}") - - if job.warnings: - for warning in job.warnings: - print(f"[WARNING] {warning}") - - if job.status == JobStatus.FAILED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Optimization job `{job.id}` failed: {message}") - if job.status == JobStatus.CANCELLED: - raise RuntimeError(f"Optimization job `{job.id}` was cancelled.") + status = poller.status() + print(f"Final optimization job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Optimization job ended with status `{status}`: {message}") # ------------------------------------------------------------------ # 3. Inspect the results. # ------------------------------------------------------------------ - if job.result is None: - raise RuntimeError(f"Optimization job `{job.id}` completed without a result.") - - result = job.result + result = poller.result() print(f"\nBaseline candidate: {result.baseline}") print(f"Best candidate: {result.best}") print(f"Candidates ({len(result.candidates or [])}):") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py index d49bba940b1f..0493e7d38310 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py @@ -7,7 +7,7 @@ """ DESCRIPTION: Given an async AIProjectClient, this sample demonstrates how to create an - agent optimization job and manually poll it to completion. + agent optimization job and poll its standard LRO to completion. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your @@ -38,14 +38,11 @@ from dotenv import load_dotenv -from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse, HttpRequest from azure.identity.aio import DefaultAzureCredential from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( OptimizationAgentIdentifier as AgentIdentifier, OptimizationEvaluatorRef as EvaluatorRef, - JobStatus, OptimizationJob, OptimizationJobInputs, OptimizationOptions, @@ -63,9 +60,6 @@ eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") -terminal_statuses = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - - async def main() -> None: async with ( DefaultAzureCredential() as credential, @@ -73,17 +67,20 @@ async def main() -> None: ): # ------------------------------------------------------------------ - # 1. Create an optimization job without SDK polling. + # 1. Create an optimization job. # ------------------------------------------------------------------ print("Creating optimization job...") - initial_responses: list[PipelineResponse[HttpRequest, AsyncHttpResponse]] = [] + latest_lro_response = {} - def capture_created_job_response( - response: PipelineResponse[HttpRequest, AsyncHttpResponse], - ) -> None: - initial_responses.append(response) + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = json.loads(response.http_response.text()) + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) - await project_client.beta.agents.begin_create_optimization_job( + # Alternatively, await `.result()` on the returned poller to let the SDK handle polling. + poller = await project_client.beta.agents.begin_create_optimization_job( job=OptimizationJob( inputs=OptimizationJobInputs( agent=AgentIdentifier(agent_name=agent_name), @@ -99,39 +96,32 @@ def capture_created_job_response( ), ) ), - polling=False, - raw_response_hook=capture_created_job_response, + polling_interval=poll_interval, + raw_response_hook=capture_lro_response, ) - if not initial_responses: - raise RuntimeError("The create operation did not return an optimization job.") - job = OptimizationJob(json.loads(initial_responses[0].http_response.text())) - print(f"Created job: id={job.id}, status={job.status}") # ------------------------------------------------------------------ # 2. Poll the job to completion. # ------------------------------------------------------------------ - while job.status not in terminal_statuses: + result_task = asyncio.create_task(poller.result()) + while not poller.done(): + print(f"Optimization job status: {poller.status()}") await asyncio.sleep(poll_interval) - job = await project_client.beta.agents.get_optimization_job(job_id=job.id) - print(f"Job status: {job.status}") - - if job.warnings: - for warning in job.warnings: - print(f"[WARNING] {warning}") - - if job.status == JobStatus.FAILED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Optimization job `{job.id}` failed: {message}") - if job.status == JobStatus.CANCELLED: - raise RuntimeError(f"Optimization job `{job.id}` was cancelled.") + status = poller.status() + print(f"Final optimization job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + try: + await result_task + except Exception as exception: # pylint: disable=broad-exception-caught + raise RuntimeError(f"Optimization job ended with status `{status}`: {message}") from exception + raise RuntimeError(f"Optimization job ended with status `{status}`: {message}") # ------------------------------------------------------------------ # 3. Inspect the results. # ------------------------------------------------------------------ - if job.result is None: - raise RuntimeError(f"Optimization job `{job.id}` completed without a result.") - - result = job.result + result = await result_task print(f"\nBaseline candidate: {result.baseline}") print(f"Best candidate: {result.best}") print(f"Candidates ({len(result.candidates or [])}):") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index c75d34c6f65b..c4d8450baf13 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -169,10 +169,31 @@ ), ) print("Create a fine-tuning data generation job and wait for it to complete.") - job_result = project_client.beta.datasets.begin_create_generation_job( + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.datasets.begin_create_generation_job( job=job, polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Data generation job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final data generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") + job_result = poller.result() # ------------------------------------------------------------------ # 3. Inspect the generated fine-tuning file outputs. diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py index 984e7041f228..4dd6a39839a9 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py @@ -53,6 +53,7 @@ """ import os +import time import uuid from datetime import datetime, timezone @@ -158,11 +159,32 @@ output_options=DataGenerationJobOutputOptions(name=output_dataset_name), ), ) - print("Creating data generation job and waiting for completion (polling is handled by the SDK)...") - job_result = project_client.beta.datasets.begin_create_generation_job( + print("Creating data generation job and polling until completion...") + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.datasets.begin_create_generation_job( job=job, polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Data generation job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final data generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") + job_result = poller.result() # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py index 44ee1e5b29fa..e365c8782cd5 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py @@ -153,10 +153,18 @@ # - The File source contributes the source material (the reference # document uploaded above). # - The Prompt source contributes a steering instruction (difficulty). - print( - "Creating multi-source data generation job (File + Prompt) and waiting for completion (polling is handled by the SDK)..." - ) - job_result = project_client.beta.datasets.begin_create_generation_job( + print("Creating multi-source data generation job (File + Prompt) and polling until completion...") + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"simpleqna-multisource-{run_id}", @@ -185,7 +193,18 @@ ), ), polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Data generation job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final data generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") + job_result = poller.result() # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py index 7db76b290cac..6bda15cd21ff 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py @@ -11,8 +11,8 @@ 1. Creates a `DataGenerationJob` (scenario=EVALUATION, type=simple_qna) that synthesizes question/answer pairs from an inline prompt and writes them - to a new versioned Dataset. Uses `begin_create_generation_job` which returns - `LROPoller[DataGenerationJobResult]`; `.result()` polls automatically. + to a new versioned Dataset. Uses `begin_create_generation_job` and + reports the standard LRO poller's status until the operation completes. 2. Resolves the resulting `DatasetVersion` from the job result. 3. Creates an OpenAI evaluation (`client.evals.create`) with builtin Azure AI evaluators. @@ -117,11 +117,32 @@ def main() -> None: ), ), ) - print("Creating data generation job and waiting for completion (polling is handled by the SDK)...") - job_result = project_client.beta.datasets.begin_create_generation_job( + print("Creating data generation job and polling until completion...") + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.datasets.begin_create_generation_job( job=job, polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Data generation job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final data generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") + job_result = poller.result() # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py index 18a5d59582b1..85153a03ff92 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py @@ -138,7 +138,17 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - job = project_client.beta.datasets.begin_create_generation_job( + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-eval-{run_id}-a{attempt}", @@ -157,8 +167,23 @@ ), ), polling_interval=POLL_INTERVAL_SECONDS, - ).result() - print(f"Data generation job succeeded.") + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Data generation job status: {poller.status()}") + time.sleep(POLL_INTERVAL_SECONDS) + status = poller.status() + print(f"Final data generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = ( + error.get("message", "") + if isinstance(error, dict) + else "" + ) + raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") + job = poller.result() + print("Data generation job succeeded.") break except Exception as e: # pylint: disable=broad-except if attempt == MAX_JOB_ATTEMPTS: diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py index 114e6c99fa35..cbb1d6bc0071 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py @@ -139,7 +139,17 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - job = project_client.beta.datasets.begin_create_generation_job( + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-ft-{run_id}-a{attempt}", @@ -160,8 +170,23 @@ ), ), polling_interval=POLL_INTERVAL_SECONDS, - ).result() - print(f"Data generation job succeeded.") + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Data generation job status: {poller.status()}") + time.sleep(POLL_INTERVAL_SECONDS) + status = poller.status() + print(f"Final data generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = ( + error.get("message", "") + if isinstance(error, dict) + else "" + ) + raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") + job = poller.result() + print("Data generation job succeeded.") break except Exception as e: # pylint: disable=broad-except if attempt == MAX_JOB_ATTEMPTS: diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 99e4b7496962..345f392c063a 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -48,6 +48,7 @@ """ import os +import time import uuid from datetime import datetime, timedelta, timezone from typing import List @@ -124,9 +125,19 @@ else: print("Skipping Dataset source (FOUNDRY_REFERENCE_DATASET_NAME / _VERSION not set).") - print("Waiting for multi-source job to complete (polling is handled by the SDK)...") + print("Waiting for multi-source job to complete...") try: - evaluator = project_client.beta.evaluators.begin_create_generation_job( + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_multi_source_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -138,7 +149,18 @@ ), operation_id=f"rubric-multi-{short}", polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_multi_source_lro_response, + ) + while not poller.done(): + print(f"Multi-source job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final multi-source job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Multi-source job ended with status `{status}`: {message}") + evaluator = poller.result() # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) @@ -160,9 +182,19 @@ start_time = now - timedelta(days=traces_window_days) end_time = now + timedelta(seconds=600) # small padding for clock skew - print("Waiting for traces job to complete (polling is handled by the SDK)...") + print("Waiting for traces job to complete...") try: - evaluator = project_client.beta.evaluators.begin_create_generation_job( + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_traces_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -185,7 +217,22 @@ ), operation_id=f"rubric-traces-{short}", polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_traces_lro_response, + ) + while not poller.done(): + print(f"Traces job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final traces job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = ( + error.get("message", "") + if isinstance(error, dict) + else "" + ) + raise RuntimeError(f"Traces job ended with status `{status}`: {message}") + evaluator = poller.result() # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index 4375842e523f..3b54f746d11b 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -13,8 +13,8 @@ 1. Creates an `EvaluatorGenerationJob` whose only source is an inline natural-language description of the application's purpose, capabilities, and tools. The service synthesizes a rubric tailored to that application. - 2. Calls `begin_create_generation_job` which returns an `LROPoller[EvaluatorVersion]`; - `.result()` polls automatically and returns the generated `EvaluatorVersion`. + 2. Calls `begin_create_generation_job` and reports the standard LRO + poller's status until it returns the generated `EvaluatorVersion`. 3. Creates an OpenAI evaluation referencing the generated evaluator as a testing criterion. 4. Runs the evaluation against inline JSONL sample data. @@ -87,10 +87,18 @@ project_client.get_openai_client() as openai_client, ): # 1. Generate an evaluator from a single `Prompt` source. - # The LRO polls automatically; `.result()` blocks until the job reaches a terminal state - # and returns the produced EvaluatorVersion directly. - print("Waiting for generation job to complete (polling is handled by the SDK)...") - evaluator = project_client.beta.evaluators.begin_create_generation_job( + print("Waiting for generation job to complete...") + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -119,7 +127,18 @@ # `operation_id` makes the call idempotent - re-submitting the same id attaches to the existing job. operation_id=f"rubric-eval-basic-{short}", polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Generation job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Generation job ended with status `{status}`: {message}") + evaluator = poller.result() # On success, the evaluator is automatically saved as version 1. # `isinstance` narrows the discriminated `definition` to the rubric subtype. diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index 0ca337d5bbfe..7328021a0c81 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -37,6 +37,7 @@ """ import os +import time import uuid from datetime import datetime, timezone @@ -69,10 +70,18 @@ AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): # 1. Generate v1 of the evaluator from a single `Prompt` source. - # The LRO polls automatically; `.result()` blocks until the job reaches a terminal state - # and returns the produced EvaluatorVersion directly. - print("Waiting for generation job to complete (polling is handled by the SDK)...") - v1 = project_client.beta.evaluators.begin_create_generation_job( + print("Waiting for generation job to complete...") + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -94,7 +103,18 @@ ), operation_id=f"rubric-iterate-{short}", polling_interval=poll_interval_seconds, - ).result() + raw_response_hook=capture_lro_response, + ) + while not poller.done(): + print(f"Generation job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Generation job ended with status `{status}`: {message}") + v1 = poller.result() # `isinstance` narrows the discriminated `definition` to the rubric subtype. v1_definition = v1.definition diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index d48a007729de..3c2fc3619888 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -10,8 +10,8 @@ jobs. The sample exercises: * `begin_create_generation_job` with `operation_id` for idempotent re-submits; - returns `LROPoller[EvaluatorVersion]` — the SDK polls automatically and - `.result()` blocks until the job reaches a terminal state. + returns `LROPoller[EvaluatorVersion]`, whose status is reported until + the job reaches a terminal state. * `list_generation_jobs` to enumerate recent jobs in the project. * `delete_generation_job` to remove a finished job record. * `delete_version` to remove the persisted evaluator that the job produced. @@ -41,6 +41,7 @@ import os import itertools +import time import uuid from datetime import datetime, timezone from typing import cast @@ -93,19 +94,53 @@ ): # 1. Start the generation job LRO. `operation_id` makes the call idempotent - # re-submitting with the same id returns a poller attached to the existing job. + latest_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_lro_response.clear() + latest_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. poller = project_client.beta.evaluators.begin_create_generation_job( - job=job_body, operation_id=operation_id, polling_interval=poll_interval_seconds + job=job_body, + operation_id=operation_id, + polling_interval=poll_interval_seconds, + raw_response_hook=capture_lro_response, ) print("Generation job started; LRO polling in progress.") # Idempotency: a second call with the same operation_id attaches to the same job. + latest_replay_lro_response = {} + + # Optionally capture LRO responses to extract an error message if the job fails. + def capture_replay_lro_response(response): + body = response.http_response.json() + if isinstance(body, dict) and "status" in body: + latest_replay_lro_response.clear() + latest_replay_lro_response.update(body) + + # Alternatively, append `.result()` to block while the SDK handles polling. replay_poller = project_client.beta.evaluators.begin_create_generation_job( - job=job_body, operation_id=operation_id, polling_interval=poll_interval_seconds + job=job_body, + operation_id=operation_id, + polling_interval=poll_interval_seconds, + raw_response_hook=capture_replay_lro_response, ) - # 2. Block until the LRO finishes. The SDK polls automatically; `.result()` returns - # the produced EvaluatorVersion once the job reaches a terminal state. - print("Waiting for the generation job to complete (polling is handled by the SDK)...") + # 2. Poll until the LRO finishes, then retrieve the produced EvaluatorVersion. + print("Waiting for the generation job to complete...") + while not poller.done(): + print(f"Generation job status: {poller.status()}") + time.sleep(poll_interval_seconds) + status = poller.status() + print(f"Final generation job status: `{status}`.") + if status.lower() != "succeeded": + error = latest_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Generation job ended with status `{status}`: {message}") evaluator: EvaluatorVersion = poller.result() print( f"Generated evaluator `{evaluator.name}` version `{evaluator.version}` " @@ -113,6 +148,15 @@ ) # Verify the idempotency: the replay poller resolves to the same underlying job. + while not replay_poller.done(): + print(f"Replay job status: {replay_poller.status()}") + time.sleep(poll_interval_seconds) + replay_status = replay_poller.status() + print(f"Final replay job status: `{replay_status}`.") + if replay_status.lower() != "succeeded": + error = latest_replay_lro_response.get("error") + message = error.get("message", "") if isinstance(error, dict) else "" + raise RuntimeError(f"Replay job ended with status `{replay_status}`: {message}") replay_evaluator: EvaluatorVersion = replay_poller.result() assert replay_evaluator.generation_job_id == evaluator.generation_job_id From b20ec5960f570dc0c07b6367ca88db9f38c08674 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:01:55 -0700 Subject: [PATCH 02/16] Address some CoPilot code review comments --- .../agents/optimization/sample_optimization_job_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py index 86d398bcf9f7..b89765a1b935 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py @@ -7,7 +7,7 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to create an agent - optimization job, and use the SDK's built-in polling mechanism to wait for + optimization job and use the SDK's built-in polling mechanism to wait for its completion to get the result. Agent optimization automatically improves an agent's system prompt, model From 94e131f9a2f81d71cff981f29ba390379f1143fc Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:05:54 -0700 Subject: [PATCH 03/16] More --- .../agents/optimization/sample_optimization_job_basic_async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py index 1fc89964fabf..6d467ba8ddd2 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py @@ -7,7 +7,7 @@ """ DESCRIPTION: Given an async AIProjectClient, this sample demonstrates how to create an agent - optimization job, and use the SDK's built-in polling mechanism to wait for + optimization job and use the SDK's built-in polling mechanism to wait for its completion to get the result. USAGE: From 64304860560a87dfb0c70ea7e6deb13aada269d8 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:03:44 -0700 Subject: [PATCH 04/16] Update Agent Optimization polling samples --- .../sample_optimization_job_basic_polling.py | 59 ++++++++++------ ...le_optimization_job_basic_polling_async.py | 67 +++++++++++-------- 2 files changed, 76 insertions(+), 50 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py index df264edd0e37..c28c2fc27769 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py @@ -40,6 +40,7 @@ from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient from azure.ai.projects.models import ( + JobStatus, OptimizationAgentIdentifier as AgentIdentifier, OptimizationEvaluatorRef as EvaluatorRef, OptimizationJob, @@ -59,26 +60,25 @@ eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + with ( DefaultAzureCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): # ------------------------------------------------------------------ - # 1. Create an optimization job. + # 1. Create an optimization job without SDK polling. # ------------------------------------------------------------------ print("Creating optimization job...") - latest_lro_response = {} + created_jobs: list[OptimizationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + # Since `polling=False` is set below, it is guaranteed that `capture_created_job` will be + # invoked once on the initial "201 Created" response, and `response` is of type `OptimizationJob`. + created_jobs.append(OptimizationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.agents.begin_create_optimization_job( + project_client.beta.agents.begin_create_optimization_job( job=OptimizationJob( inputs=OptimizationJobInputs( agent=AgentIdentifier(agent_name=agent_name), @@ -94,27 +94,42 @@ def capture_lro_response(response): ), ) ), - polling_interval=poll_interval, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) + if not created_jobs: + raise RuntimeError("The create operation did not return an optimization job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") # ------------------------------------------------------------------ - # 2. Poll the job to completion. + # 2. Poll until the job reaches a terminal state. # ------------------------------------------------------------------ - while not poller.done(): - print(f"Optimization job status: {poller.status()}") + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval) - status = poller.status() - print(f"Final optimization job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Optimization job ended with status `{status}`: {message}") + job = project_client.beta.agents.get_optimization_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.warnings: + for warning in job.warnings: + print(f"[WARNING] {warning}") + + if job.status == JobStatus.FAILED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Optimization job `{job.id}` failed: {message}") + if job.status == JobStatus.CANCELLED: + raise RuntimeError(f"Optimization job `{job.id}` was cancelled.") # ------------------------------------------------------------------ # 3. Inspect the results. # ------------------------------------------------------------------ - result = poller.result() + if job.result is None: + raise RuntimeError(f"Optimization job `{job.id}` completed without a result.") + + result = job.result print(f"\nBaseline candidate: {result.baseline}") print(f"Best candidate: {result.best}") print(f"Candidates ({len(result.candidates or [])}):") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py index 0493e7d38310..d216a3e35f1e 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py @@ -7,7 +7,7 @@ """ DESCRIPTION: Given an async AIProjectClient, this sample demonstrates how to create an - agent optimization job and poll its standard LRO to completion. + agent optimization job and poll it to completion. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your @@ -41,6 +41,7 @@ from azure.identity.aio import DefaultAzureCredential from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( + JobStatus, OptimizationAgentIdentifier as AgentIdentifier, OptimizationEvaluatorRef as EvaluatorRef, OptimizationJob, @@ -60,6 +61,9 @@ eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + + async def main() -> None: async with ( DefaultAzureCredential() as credential, @@ -67,20 +71,17 @@ async def main() -> None: ): # ------------------------------------------------------------------ - # 1. Create an optimization job. + # 1. Create an optimization job without SDK polling. # ------------------------------------------------------------------ print("Creating optimization job...") - latest_lro_response = {} + created_jobs: list[OptimizationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = json.loads(response.http_response.text()) - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + # Since `polling=False` is set below, it is guaranteed that `capture_created_job` will be + # invoked once on the initial "201 Created" response, and `response` is of type `OptimizationJob`. + created_jobs.append(OptimizationJob(json.loads(response.http_response.text()))) - # Alternatively, await `.result()` on the returned poller to let the SDK handle polling. - poller = await project_client.beta.agents.begin_create_optimization_job( + await project_client.beta.agents.begin_create_optimization_job( job=OptimizationJob( inputs=OptimizationJobInputs( agent=AgentIdentifier(agent_name=agent_name), @@ -96,32 +97,42 @@ def capture_lro_response(response): ), ) ), - polling_interval=poll_interval, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) + if not created_jobs: + raise RuntimeError("The create operation did not return an optimization job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") # ------------------------------------------------------------------ - # 2. Poll the job to completion. + # 2. Poll until the job reaches a terminal state. # ------------------------------------------------------------------ - result_task = asyncio.create_task(poller.result()) - while not poller.done(): - print(f"Optimization job status: {poller.status()}") + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: await asyncio.sleep(poll_interval) - status = poller.status() - print(f"Final optimization job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - try: - await result_task - except Exception as exception: # pylint: disable=broad-exception-caught - raise RuntimeError(f"Optimization job ended with status `{status}`: {message}") from exception - raise RuntimeError(f"Optimization job ended with status `{status}`: {message}") + job = await project_client.beta.agents.get_optimization_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.warnings: + for warning in job.warnings: + print(f"[WARNING] {warning}") + + if job.status == JobStatus.FAILED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Optimization job `{job.id}` failed: {message}") + if job.status == JobStatus.CANCELLED: + raise RuntimeError(f"Optimization job `{job.id}` was cancelled.") # ------------------------------------------------------------------ # 3. Inspect the results. # ------------------------------------------------------------------ - result = await result_task + if job.result is None: + raise RuntimeError(f"Optimization job `{job.id}` completed without a result.") + + result = job.result print(f"\nBaseline candidate: {result.baseline}") print(f"Best candidate: {result.best}") print(f"Candidates ({len(result.candidates or [])}):") From 5503b86f3e0719f50fd7293a33626e9f3f7b7798 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:24:27 -0700 Subject: [PATCH 05/16] More --- ...generation_job_simpleqna_for_finetuning.py | 46 +++++---- ...eration_job_simpleqna_with_agent_source.py | 46 +++++---- ...neration_job_simpleqna_with_file_source.py | 46 +++++---- ...ration_job_simpleqna_with_prompt_source.py | 46 +++++---- ...et_generation_job_traces_for_evaluation.py | 54 +++++------ ...et_generation_job_traces_for_finetuning.py | 54 +++++------ ...rubric_evaluator_generation_all_sources.py | 93 ++++++++++--------- ...ample_rubric_evaluator_generation_basic.py | 46 +++++---- ...ple_rubric_evaluator_generation_iterate.py | 46 +++++---- ...e_rubric_evaluator_generation_lifecycle.py | 89 +++++++----------- 10 files changed, 300 insertions(+), 266 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index c4d8450baf13..a208d78d0736 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -61,6 +61,7 @@ DataGenerationModelOptions, FileDataGenerationJobOutput, FileDataGenerationJobSource, + JobStatus, SimpleQnADataGenerationJobOptions, SimpleQnAFineTuningQuestionType, ) @@ -72,6 +73,8 @@ dataset_name = os.environ.get("DATASET_NAME", "simpleqna-finetuning-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + # Unique per-run output name so repeated runs do not collide. # Output names are capped at 50 characters by the service. run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" @@ -169,31 +172,36 @@ ), ) print("Create a fine-tuning data generation job and wait for it to complete.") - latest_lro_response = {} + created_jobs: list[DataGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.datasets.begin_create_generation_job( + project_client.beta.datasets.begin_create_generation_job( job=job, - polling_interval=poll_interval_seconds, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Data generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a data generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final data generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") - job_result = poller.result() + job = project_client.beta.datasets.get_generation_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") + job_result = job.result # ------------------------------------------------------------------ # 3. Inspect the generated fine-tuning file outputs. diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py index 4dd6a39839a9..4f12ff247629 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py @@ -70,6 +70,7 @@ DataGenerationModelOptions, DatasetDataGenerationJobOutput, DatasetVersion, + JobStatus, PromptAgentDefinition, SimpleQnADataGenerationJobOptions, ) @@ -81,6 +82,8 @@ dataset_name = os.environ.get("DATASET_NAME", "simpleqna-agent-source-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + # Unique per-run names so repeated runs do not collide. # Output names are capped at 50 characters by the service. run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" @@ -160,31 +163,36 @@ ), ) print("Creating data generation job and polling until completion...") - latest_lro_response = {} + created_jobs: list[DataGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.datasets.begin_create_generation_job( + project_client.beta.datasets.begin_create_generation_job( job=job, - polling_interval=poll_interval_seconds, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Data generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a data generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final data generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") - job_result = poller.result() + job = project_client.beta.datasets.get_generation_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") + job_result = job.result # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py index e365c8782cd5..9c1816d68f99 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py @@ -66,6 +66,7 @@ DatasetDataGenerationJobOutput, DatasetVersion, FileDataGenerationJobSource, + JobStatus, PromptDataGenerationJobSource, SimpleQnADataGenerationJobOptions, ) @@ -77,6 +78,8 @@ dataset_name = os.environ.get("DATASET_NAME", "simpleqna-file-source-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + # Unique per-run resource names so repeated runs do not collide. # Output names are capped at 50 characters by the service. run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" @@ -154,17 +157,13 @@ # document uploaded above). # - The Prompt source contributes a steering instruction (difficulty). print("Creating multi-source data generation job (File + Prompt) and polling until completion...") - latest_lro_response = {} + created_jobs: list[DataGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.datasets.begin_create_generation_job( + project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"simpleqna-multisource-{run_id}", @@ -192,19 +191,28 @@ def capture_lro_response(response): ), ), ), - polling_interval=poll_interval_seconds, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Data generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a data generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final data generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") - job_result = poller.result() + job = project_client.beta.datasets.get_generation_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") + job_result = job.result # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py index 6bda15cd21ff..5f3995f1c7ce 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py @@ -67,6 +67,7 @@ DataGenerationModelOptions, DatasetDataGenerationJobOutput, DatasetVersion, + JobStatus, PromptDataGenerationJobSource, SimpleQnADataGenerationJobOptions, TestingCriterionAzureAIEvaluator, @@ -79,6 +80,8 @@ dataset_name = os.environ.get("DATASET_NAME", "dataset-generation-eval-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + def main() -> None: with ( @@ -118,31 +121,36 @@ def main() -> None: ), ) print("Creating data generation job and polling until completion...") - latest_lro_response = {} + created_jobs: list[DataGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.datasets.begin_create_generation_job( + project_client.beta.datasets.begin_create_generation_job( job=job, - polling_interval=poll_interval_seconds, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Data generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a data generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final data generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") - job_result = poller.result() + job = project_client.beta.datasets.get_generation_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") + job_result = job.result # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py index 85153a03ff92..f8700fefe527 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py @@ -54,6 +54,7 @@ DataGenerationJobScenario, DatasetDataGenerationJobOutput, DatasetVersion, + JobStatus, PromptAgentDefinition, TracesDataGenerationJobOptions, TracesDataGenerationJobSource, @@ -81,6 +82,7 @@ model_deployment = os.environ["FOUNDRY_MODEL_NAME"] DATASET_NAME = "traces-eval-sample" POLL_INTERVAL_SECONDS = 10 +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} INITIAL_INGEST_WAIT_SECONDS = 60 MAX_JOB_ATTEMPTS = 5 RETRY_WAIT_SECONDS = 60 @@ -138,17 +140,13 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - latest_lro_response = {} + created_jobs: list[DataGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.datasets.begin_create_generation_job( + project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-eval-{run_id}-a{attempt}", @@ -166,23 +164,25 @@ def capture_lro_response(response): output_options=DataGenerationJobOutputOptions(name=output_dataset_name), ), ), - polling_interval=POLL_INTERVAL_SECONDS, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Data generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a data generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(POLL_INTERVAL_SECONDS) - status = poller.status() - print(f"Final data generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = ( - error.get("message", "") - if isinstance(error, dict) - else "" - ) - raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") - job = poller.result() + job = project_client.beta.datasets.get_generation_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") print("Data generation job succeeded.") break except Exception as e: # pylint: disable=broad-except @@ -192,9 +192,9 @@ def capture_lro_response(response): time.sleep(RETRY_WAIT_SECONDS) # 3. Resolve the generated dataset. - if job is None: + if job is None or job.result is None: raise RuntimeError("The data generation job did not return a result.") - outputs = job.outputs or [] + outputs = job.result.outputs or [] dataset_output = next((o for o in outputs if isinstance(o, DatasetDataGenerationJobOutput)), None) if dataset_output is None or not dataset_output.name or not dataset_output.version: raise RuntimeError("The data generation job did not produce a dataset output.") @@ -204,8 +204,8 @@ def capture_lro_response(response): f"Generated dataset: name=`{created_dataset.name}` " f"version=`{created_dataset.version}` id=`{created_dataset.id}`" ) - if job.generated_samples is not None: - print(f"Generated samples: {job.generated_samples}") + if job.result.generated_samples is not None: + print(f"Generated samples: {job.result.generated_samples}") finally: # Best-effort cleanup, outputs -> producers (dataset, job, conversations, agent). diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py index cbb1d6bc0071..955375c35b1a 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py @@ -53,6 +53,7 @@ DataGenerationJobOutputOptions, DataGenerationJobScenario, FileDataGenerationJobOutput, + JobStatus, PromptAgentDefinition, TracesDataGenerationJobOptions, TracesDataGenerationJobSource, @@ -83,6 +84,7 @@ model_deployment = os.environ["FOUNDRY_MODEL_NAME"] DATASET_NAME = "traces-ft-sample" POLL_INTERVAL_SECONDS = 10 +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} INITIAL_INGEST_WAIT_SECONDS = 60 MAX_JOB_ATTEMPTS = 5 RETRY_WAIT_SECONDS = 60 @@ -139,17 +141,13 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - latest_lro_response = {} + created_jobs: list[DataGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.datasets.begin_create_generation_job( + project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-ft-{run_id}-a{attempt}", @@ -169,23 +167,25 @@ def capture_lro_response(response): output_options=DataGenerationJobOutputOptions(name=output_name), ), ), - polling_interval=POLL_INTERVAL_SECONDS, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Data generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a data generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(POLL_INTERVAL_SECONDS) - status = poller.status() - print(f"Final data generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = ( - error.get("message", "") - if isinstance(error, dict) - else "" - ) - raise RuntimeError(f"Data generation job ended with status `{status}`: {message}") - job = poller.result() + job = project_client.beta.datasets.get_generation_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") print("Data generation job succeeded.") break except Exception as e: # pylint: disable=broad-except @@ -195,9 +195,9 @@ def capture_lro_response(response): time.sleep(RETRY_WAIT_SECONDS) # 3. Resolve generated fine-tuning files. - if job is None: + if job is None or job.result is None: raise RuntimeError("The data generation job did not return a result.") - outputs = job.outputs or [] + outputs = job.result.outputs or [] file_outputs = [o for o in outputs if isinstance(o, FileDataGenerationJobOutput)] if not file_outputs: raise RuntimeError("The data generation job did not produce any file outputs.") @@ -209,8 +209,8 @@ def capture_lro_response(response): created_file_ids.append(output.id) file_info = openai_client.files.retrieve(file_id=output.id) print(f" - filename=`{file_info.filename}` id=`{output.id}` bytes={file_info.bytes}") - if job.generated_samples is not None: - print(f"Generated samples: {job.generated_samples}") + if job.result.generated_samples is not None: + print(f"Generated samples: {job.result.generated_samples}") finally: # Best-effort cleanup, outputs -> producers (files, job, conversations, agent). diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 345f392c063a..128b8f013dc9 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -63,6 +63,7 @@ EvaluatorGenerationInputs, EvaluatorGenerationJob, EvaluatorGenerationJobSource, + JobStatus, PromptEvaluatorGenerationJobSource, RubricBasedEvaluatorDefinition, TracesEvaluatorGenerationJobSource, @@ -78,6 +79,8 @@ traces_window_days = int(os.environ.get("FOUNDRY_TRACES_WINDOW_DAYS", "7")) poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + # Unique per-run suffix so repeated runs do not collide on evaluator name. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -127,17 +130,13 @@ print("Waiting for multi-source job to complete...") try: - latest_lro_response = {} + created_jobs: list[EvaluatorGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_multi_source_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.evaluators.begin_create_generation_job( + project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -148,19 +147,28 @@ def capture_multi_source_lro_response(response): ), ), operation_id=f"rubric-multi-{short}", - polling_interval=poll_interval_seconds, - raw_response_hook=capture_multi_source_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Multi-source job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final multi-source job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Multi-source job ended with status `{status}`: {message}") - evaluator = poller.result() + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Multi-source job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Multi-source job `{job.id}` completed without a result.") + evaluator = job.result # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) @@ -184,17 +192,13 @@ def capture_multi_source_lro_response(response): print("Waiting for traces job to complete...") try: - latest_lro_response = {} + created_jobs = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_traces_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.evaluators.begin_create_generation_job( + project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -216,23 +220,28 @@ def capture_traces_lro_response(response): ), ), operation_id=f"rubric-traces-{short}", - polling_interval=poll_interval_seconds, - raw_response_hook=capture_traces_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Traces job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final traces job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = ( - error.get("message", "") - if isinstance(error, dict) - else "" - ) - raise RuntimeError(f"Traces job ended with status `{status}`: {message}") - evaluator = poller.result() + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Traces job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Traces job `{job.id}` completed without a result.") + evaluator = job.result # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index 3b54f746d11b..af93994a65eb 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -63,6 +63,7 @@ from azure.ai.projects.models import ( EvaluatorGenerationInputs, EvaluatorGenerationJob, + JobStatus, PromptEvaluatorGenerationJobSource, RubricBasedEvaluatorDefinition, TestingCriterionAzureAIEvaluator, @@ -74,6 +75,8 @@ model_name = os.environ["FOUNDRY_MODEL_NAME"] poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + # Unique per-run name so repeated runs do not collide. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -88,17 +91,13 @@ ): # 1. Generate an evaluator from a single `Prompt` source. print("Waiting for generation job to complete...") - latest_lro_response = {} + created_jobs: list[EvaluatorGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.evaluators.begin_create_generation_job( + project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -126,19 +125,28 @@ def capture_lro_response(response): ), # `operation_id` makes the call idempotent - re-submitting the same id attaches to the existing job. operation_id=f"rubric-eval-basic-{short}", - polling_interval=poll_interval_seconds, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Generation job ended with status `{status}`: {message}") - evaluator = poller.result() + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Generation job `{job.id}` completed without a result.") + evaluator = job.result # On success, the evaluator is automatically saved as version 1. # `isinstance` narrows the discriminated `definition` to the rubric subtype. diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index 7328021a0c81..21850eb59ba2 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -50,6 +50,7 @@ EvaluatorDefinitionType, EvaluatorGenerationInputs, EvaluatorGenerationJob, + JobStatus, PromptEvaluatorGenerationJobSource, RubricBasedEvaluatorDefinition, ) @@ -60,6 +61,8 @@ model_name = os.environ["FOUNDRY_MODEL_NAME"] poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + # Unique per-run name so repeated runs do not collide. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -71,17 +74,13 @@ ): # 1. Generate v1 of the evaluator from a single `Prompt` source. print("Waiting for generation job to complete...") - latest_lro_response = {} + created_jobs: list[EvaluatorGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.evaluators.begin_create_generation_job( + project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -102,19 +101,28 @@ def capture_lro_response(response): ), ), operation_id=f"rubric-iterate-{short}", - polling_interval=poll_interval_seconds, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - while not poller.done(): - print(f"Generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Generation job ended with status `{status}`: {message}") - v1 = poller.result() + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Generation job `{job.id}` completed without a result.") + v1 = job.result # `isinstance` narrows the discriminated `definition` to the rubric subtype. v1_definition = v1.definition diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index 3c2fc3619888..41dc5f720dae 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -66,6 +66,8 @@ model_name = os.environ["FOUNDRY_MODEL_NAME"] poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + # Unique per-run name so repeated runs do not collide. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -92,74 +94,49 @@ DefaultAzureCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): - # 1. Start the generation job LRO. `operation_id` makes the call idempotent - - # re-submitting with the same id returns a poller attached to the existing job. - latest_lro_response = {} + # 1. Create the generation job. `operation_id` makes the call idempotent - + # re-submitting with the same id returns the existing job. + created_jobs: list[EvaluatorGenerationJob] = [] - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_lro_response.clear() - latest_lro_response.update(body) + def capture_created_job(response): + created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. - poller = project_client.beta.evaluators.begin_create_generation_job( + project_client.beta.evaluators.begin_create_generation_job( job=job_body, operation_id=operation_id, - polling_interval=poll_interval_seconds, - raw_response_hook=capture_lro_response, + polling=False, + raw_response_hook=capture_created_job, ) - print("Generation job started; LRO polling in progress.") - - # Idempotency: a second call with the same operation_id attaches to the same job. - latest_replay_lro_response = {} - - # Optionally capture LRO responses to extract an error message if the job fails. - def capture_replay_lro_response(response): - body = response.http_response.json() - if isinstance(body, dict) and "status" in body: - latest_replay_lro_response.clear() - latest_replay_lro_response.update(body) - - # Alternatively, append `.result()` to block while the SDK handles polling. - replay_poller = project_client.beta.evaluators.begin_create_generation_job( - job=job_body, - operation_id=operation_id, - polling_interval=poll_interval_seconds, - raw_response_hook=capture_replay_lro_response, - ) - - # 2. Poll until the LRO finishes, then retrieve the produced EvaluatorVersion. - print("Waiting for the generation job to complete...") - while not poller.done(): - print(f"Generation job status: {poller.status()}") + if not created_jobs: + raise RuntimeError("The create operation did not return a generation job.") + job = created_jobs[0] + print(f"Created job: id={job.id}, status={job.status}") + + # Idempotency: a second call with the same operation_id returns the same job. + replay_job = project_client.beta.evaluators.get_generation_job(job.id) + assert replay_job.id == job.id + + # 2. Poll until the job reaches a terminal state. + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval_seconds) - status = poller.status() - print(f"Final generation job status: `{status}`.") - if status.lower() != "succeeded": - error = latest_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Generation job ended with status `{status}`: {message}") - evaluator: EvaluatorVersion = poller.result() + job = project_client.beta.evaluators.get_generation_job(job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Generation job `{job.id}` completed without a result.") + evaluator: EvaluatorVersion = job.result print( f"Generated evaluator `{evaluator.name}` version `{evaluator.version}` " f"(job `{evaluator.generation_job_id}`)." ) - # Verify the idempotency: the replay poller resolves to the same underlying job. - while not replay_poller.done(): - print(f"Replay job status: {replay_poller.status()}") - time.sleep(poll_interval_seconds) - replay_status = replay_poller.status() - print(f"Final replay job status: `{replay_status}`.") - if replay_status.lower() != "succeeded": - error = latest_replay_lro_response.get("error") - message = error.get("message", "") if isinstance(error, dict) else "" - raise RuntimeError(f"Replay job ended with status `{replay_status}`: {message}") - replay_evaluator: EvaluatorVersion = replay_poller.result() - assert replay_evaluator.generation_job_id == evaluator.generation_job_id - # 3. List the 5 most recent generation jobs in this project. # `limit` controls the page size; use `itertools.islice` to cap the total. print("Recent generation jobs:") From 3b8e30fb4f5a8dbbc2b4a5acaec0f397d4e8d690 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:43:58 -0700 Subject: [PATCH 06/16] fix alignment --- .../sample_rubric_evaluator_generation_lifecycle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index 41dc5f720dae..7be242444026 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -10,8 +10,8 @@ jobs. The sample exercises: * `begin_create_generation_job` with `operation_id` for idempotent re-submits; - returns `LROPoller[EvaluatorVersion]`, whose status is reported until - the job reaches a terminal state. + returns `LROPoller[EvaluatorVersion]`, whose status is reported until + the job reaches a terminal state. * `list_generation_jobs` to enumerate recent jobs in the project. * `delete_generation_job` to remove a finished job record. * `delete_version` to remove the persisted evaluator that the job produced. From 392f5b63e23fa75cf8087da4db8c5c00296e81e4 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:44:35 -0700 Subject: [PATCH 07/16] more fixes --- .../sample_optimization_job_basic_polling.py | 7 ++++--- ...ple_optimization_job_basic_polling_async.py | 18 +++++++++--------- .../sample_optimization_job_cancel.py | 5 +++-- ..._generation_job_simpleqna_for_finetuning.py | 5 +++-- ...neration_job_simpleqna_with_agent_source.py | 5 +++-- ...eneration_job_simpleqna_with_file_source.py | 5 +++-- ...eration_job_simpleqna_with_prompt_source.py | 5 +++-- ...set_generation_job_traces_for_evaluation.py | 5 +++-- ...set_generation_job_traces_for_finetuning.py | 5 +++-- ..._rubric_evaluator_generation_all_sources.py | 10 ++++++---- ...sample_rubric_evaluator_generation_basic.py | 5 +++-- ...mple_rubric_evaluator_generation_iterate.py | 5 +++-- ...le_rubric_evaluator_generation_lifecycle.py | 5 +++-- 13 files changed, 49 insertions(+), 36 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py index c28c2fc27769..70cd279d8313 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py @@ -73,9 +73,10 @@ print("Creating optimization job...") created_jobs: list[OptimizationJob] = [] - def capture_created_job(response): - # Since `polling=False` is set below, it is guaranteed that `capture_created_job` will be + def raw_response_hook(response): + # Since `polling=False` is set below, it is guaranteed that `raw_response_hook` will be # invoked once on the initial "201 Created" response, and `response` is of type `OptimizationJob`. + response.http_response.read() created_jobs.append(OptimizationJob(response.http_response.json())) project_client.beta.agents.begin_create_optimization_job( @@ -95,7 +96,7 @@ def capture_created_job(response): ) ), polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return an optimization job.") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py index d216a3e35f1e..e98bec729abb 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py @@ -33,7 +33,6 @@ """ import asyncio -import json import os from dotenv import load_dotenv @@ -74,12 +73,13 @@ async def main() -> None: # 1. Create an optimization job without SDK polling. # ------------------------------------------------------------------ print("Creating optimization job...") - created_jobs: list[OptimizationJob] = [] + # The raw_response_hook is called synchronously before the generated LRO method + # awaits read() on the initial response. Capture the pipeline response object here + # and parse the body afterwards, when read() has already been awaited. + pipeline_responses = [] - def capture_created_job(response): - # Since `polling=False` is set below, it is guaranteed that `capture_created_job` will be - # invoked once on the initial "201 Created" response, and `response` is of type `OptimizationJob`. - created_jobs.append(OptimizationJob(json.loads(response.http_response.text()))) + def raw_response_hook(response): + pipeline_responses.append(response) await project_client.beta.agents.begin_create_optimization_job( job=OptimizationJob( @@ -98,11 +98,11 @@ def capture_created_job(response): ) ), polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) - if not created_jobs: + if not pipeline_responses: raise RuntimeError("The create operation did not return an optimization job.") - job = created_jobs[0] + job = OptimizationJob(pipeline_responses[0].http_response.json()) print(f"Created job: id={job.id}, status={job.status}") # ------------------------------------------------------------------ diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py index 0a80c5bc41fd..de3e831e1588 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py @@ -65,7 +65,8 @@ print("Creating optimization job...") created_jobs: list[OptimizationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(OptimizationJob(response.http_response.json())) project_client.beta.agents.begin_create_optimization_job( @@ -85,7 +86,7 @@ def capture_created_job(response): ) ), polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return an optimization job.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index a208d78d0736..f6cac268621f 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -174,14 +174,15 @@ print("Create a fine-tuning data generation job and wait for it to complete.") created_jobs: list[DataGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=job, polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py index 4f12ff247629..9673c258d4c7 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py @@ -165,14 +165,15 @@ print("Creating data generation job and polling until completion...") created_jobs: list[DataGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=job, polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py index 9c1816d68f99..d5d0c9c71c05 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py @@ -159,7 +159,8 @@ print("Creating multi-source data generation job (File + Prompt) and polling until completion...") created_jobs: list[DataGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -192,7 +193,7 @@ def capture_created_job(response): ), ), polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py index 5f3995f1c7ce..1f1da7d43f3a 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py @@ -123,14 +123,15 @@ def main() -> None: print("Creating data generation job and polling until completion...") created_jobs: list[DataGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=job, polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py index f8700fefe527..403deb9dab81 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py @@ -142,7 +142,8 @@ try: created_jobs: list[DataGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -165,7 +166,7 @@ def capture_created_job(response): ), ), polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py index 955375c35b1a..8b98234e65bf 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py @@ -143,7 +143,8 @@ try: created_jobs: list[DataGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -168,7 +169,7 @@ def capture_created_job(response): ), ), polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 128b8f013dc9..11ed5bd395f2 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -132,7 +132,8 @@ try: created_jobs: list[EvaluatorGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -148,7 +149,7 @@ def capture_created_job(response): ), operation_id=f"rubric-multi-{short}", polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") @@ -194,7 +195,8 @@ def capture_created_job(response): try: created_jobs = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -221,7 +223,7 @@ def capture_created_job(response): ), operation_id=f"rubric-traces-{short}", polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index af93994a65eb..50ec17bff251 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -93,7 +93,8 @@ print("Waiting for generation job to complete...") created_jobs: list[EvaluatorGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -126,7 +127,7 @@ def capture_created_job(response): # `operation_id` makes the call idempotent - re-submitting the same id attaches to the existing job. operation_id=f"rubric-eval-basic-{short}", polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index 21850eb59ba2..30cecb6711da 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -76,7 +76,8 @@ print("Waiting for generation job to complete...") created_jobs: list[EvaluatorGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -102,7 +103,7 @@ def capture_created_job(response): ), operation_id=f"rubric-iterate-{short}", polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index 7be242444026..ee7227c8cb85 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -98,7 +98,8 @@ # re-submitting with the same id returns the existing job. created_jobs: list[EvaluatorGenerationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) # Alternatively, append `.result()` to block while the SDK handles polling. @@ -106,7 +107,7 @@ def capture_created_job(response): job=job_body, operation_id=operation_id, polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") From b050cafe20817cd3cfb6259248e134b881923fa7 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:23:17 -0700 Subject: [PATCH 08/16] More --- ...le_optimization_job_basic_polling_async.py | 8 +- ...generation_job_simpleqna_for_finetuning.py | 4 +- ...tion_job_simpleqna_for_finetuning_async.py | 245 ++++++++++++++++++ ...eration_job_simpleqna_with_agent_source.py | 2 +- ...neration_job_simpleqna_with_file_source.py | 2 +- ...ration_job_simpleqna_with_prompt_source.py | 2 +- ...et_generation_job_traces_for_evaluation.py | 2 +- ...et_generation_job_traces_for_finetuning.py | 2 +- ...rubric_evaluator_generation_all_sources.py | 4 +- ...ample_rubric_evaluator_generation_basic.py | 2 +- ...ple_rubric_evaluator_generation_iterate.py | 2 +- ...e_rubric_evaluator_generation_lifecycle.py | 2 +- 12 files changed, 261 insertions(+), 16 deletions(-) create mode 100644 sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py index e98bec729abb..a01c9012bfdc 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py @@ -18,7 +18,7 @@ Before running the sample: - pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv + pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv aiohttp Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found @@ -73,12 +73,12 @@ async def main() -> None: # 1. Create an optimization job without SDK polling. # ------------------------------------------------------------------ print("Creating optimization job...") - # The raw_response_hook is called synchronously before the generated LRO method - # awaits read() on the initial response. Capture the pipeline response object here - # and parse the body afterwards, when read() has already been awaited. pipeline_responses = [] def raw_response_hook(response): + # The raw_response_hook is called synchronously before the generated LRO method + # awaits read() on the initial response. Capture the pipeline response object here + # and parse the body afterwards, when read() has already been awaited. pipeline_responses.append(response) await project_client.beta.agents.begin_create_optimization_job( diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index f6cac268621f..b0b2dc3ff472 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -144,7 +144,7 @@ # ------------------------------------------------------------------ # 2. Submit a fine-tuning data generation job that consumes the file. # ------------------------------------------------------------------ - print("Create a fine-tuning data generation job from the Azure OpenAI file.") + job = DataGenerationJob( inputs=DataGenerationJobInputs( name=f"simpleqna-finetuning-{run_id}", @@ -178,12 +178,12 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=job, polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py new file mode 100644 index 000000000000..e94eb0bd987c --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py @@ -0,0 +1,245 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Generates supervised fine-tuning data from a Markdown reference document + uploaded as an Azure OpenAI File. The sample: + + 1. Uploads a short reference document via the Azure OpenAI Files API + (`purpose=user_data`) so it can be referenced by file id. + 2. Creates a `DataGenerationJob` (scenario=SUPERVISED_FINETUNING, + type=simple_qna) that synthesizes short-answer and long-answer + question / answer pairs from the file content and emits them as + training and validation JSONL files. + 3. Polls the job to completion and prints every generated file output. + 4. Cleans up the generated fine-tuning files and the Azure OpenAI input file. + + `simple_qna` REQUIRES `model_options` — the service uses the configured LLM + to synthesize the QnA pairs. Setting `train_split` triggers a split of + the generated samples into two Azure OpenAI output files. + +USAGE: + python sample_dataset_generation_job_simpleqna_for_finetuning_async.py + + Before running the sample: + + pip install "azure-ai-projects>=2.2.0" azure-identity openai python-dotenv aiohttp + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry project. + 2) FOUNDRY_MODEL_NAME - Required. The name of an Azure OpenAI model + deployment used to synthesize the QnA samples. For `simple_qna` fine-tuning, + the deployment must support the chat completions API (e.g. `gpt-4o`, `gpt-4.1`). + 3) DATASET_NAME - Optional. Name to assign to the generated output files + (used as the file name prefix). Defaults to `simpleqna-finetuning-sample`. + The service caps the rendered output name at 50 characters, so keep + custom values short — the sample appends a unique run id suffix. + 4) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status + polls for the data generation job. Defaults to 10. +""" + +import asyncio +import io +import os +import uuid +from datetime import datetime, timezone + +from dotenv import load_dotenv + +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import ( + DataGenerationJob, + DataGenerationJobInputs, + DataGenerationJobOutputOptions, + DataGenerationJobScenario, + DataGenerationModelOptions, + FileDataGenerationJobOutput, + FileDataGenerationJobSource, + JobStatus, + SimpleQnADataGenerationJobOptions, + SimpleQnAFineTuningQuestionType, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_name = os.environ["FOUNDRY_MODEL_NAME"] +dataset_name = os.environ.get("DATASET_NAME", "simpleqna-finetuning-sample") +poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} + +# Unique per-run output name so repeated runs do not collide. +# Output names are capped at 50 characters by the service. +run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" +output_name = f"{dataset_name}-{run_id}" +if len(output_name) > 50: + raise ValueError( + f"Output name `{output_name}` exceeds the 50-character service limit. " + f"Lower DATASET_NAME (currently `{dataset_name}`) so that `-` fits within 50 characters." + ) + +# Reference document the sample uploads as an Azure OpenAI file. The service +# requires the file to contain at least 1 KB of content to generate QnA from. +SEED_REFERENCE_DOCUMENT = """# Widgets and Gizmos Reference + +## Products +- Widget: blue, manufactured at Factory 7 in Acme, carbon-fiber, rated to 80 C, sold in packs of 4, 250 g each. +- Gizmo: red, manufactured at Factory 12 in Bedrock, carbon-fiber, rated to 80 C, sold individually, 1.2 kg each. +- Sprocket: green, manufactured at Factory 3 in Acme, stainless steel, rated to 200 C, sold individually, 500 g each. + +## Operations +- Factory operates weekdays 0700-1900 local time. +- Closed on public holidays, except for the annual maintenance run on December 27. +- ISO 9001 certified; audited annually by an independent third party. +- Quality control samples every 100th unit and runs full destructive testing on every 5000th unit. + +## Customer support +- Warranty claims: email support@example.com with the serial number printed on the underside of the product. +- Returns: accepted within 30 days if unopened; opened items are eligible for repair only. +- Bulk orders (50+ units): contact sales@example.com for volume pricing and an extended 90-day return window. +- Replacement parts: orderable directly from the support portal using the original order number. + +## Pricing and SLAs +- Widget pack: USD 24.99 per 4-pack; free shipping on orders over USD 75. +- Gizmo unit: USD 49.99; free shipping on orders over USD 75. +- Sprocket unit: USD 14.99; ships from regional warehouses in 1-2 business days. +- Standard support response: within one business day. Priority support response: within four hours. +""" + + +async def main() -> None: + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + project_client.get_openai_client() as openai_client, + ): + + # ------------------------------------------------------------------ + # 1. Upload the seed reference document as an Azure OpenAI file. + # ------------------------------------------------------------------ + seed_filename = f"widgets-gizmos-seed-{run_id}.md" + print(f"Upload the seed reference document as Azure OpenAI file `{seed_filename}`.") + seed_file = await openai_client.files.create( + file=(seed_filename, io.BytesIO(SEED_REFERENCE_DOCUMENT.encode("utf-8"))), + purpose="user_data", + ) + print(f"Uploaded Azure OpenAI file (id: {seed_file.id}).") + + # Wait for the file to finish processing — the data generation service + # rejects references to files that are not yet in the `processed` state. + print("Wait for the Azure OpenAI file to be processed.", end="", flush=True) + while seed_file.status not in ("processed", "error"): + await asyncio.sleep(2) + seed_file = await openai_client.files.retrieve(file_id=seed_file.id) + print(".", end="", flush=True) + print() + if seed_file.status != "processed": + raise RuntimeError(f"Azure OpenAI file `{seed_file.id}` failed to process: status=`{seed_file.status}`.") + + # ------------------------------------------------------------------ + # 2. Submit a fine-tuning data generation job that consumes the file. + # ------------------------------------------------------------------ + + job = DataGenerationJob( + inputs=DataGenerationJobInputs( + name=f"simpleqna-finetuning-{run_id}", + scenario=DataGenerationJobScenario.SUPERVISED_FINETUNING, + sources=[ + FileDataGenerationJobSource( + description="Widgets & Gizmos product / operations reference (Azure OpenAI file).", + id=seed_file.id, + ), + ], + options=SimpleQnADataGenerationJobOptions( + # Service requires max_samples to be between 15 and 1000. + max_samples=15, + # `simple_qna` REQUIRES model_options. + model_options=DataGenerationModelOptions(model=model_name), + # Split generated samples 80% training / 20% validation. + train_split=0.8, + # Ask for both short-answer and long-answer questions. + question_types=[ + SimpleQnAFineTuningQuestionType.SHORT_ANSWER, + SimpleQnAFineTuningQuestionType.LONG_ANSWER, + ], + ), + output_options=DataGenerationJobOutputOptions(name=output_name), + ), + ) + print("Create a fine-tuning data generation job and wait for it to complete.") + pipeline_responses = [] + + def raw_response_hook(response): + # The raw_response_hook is called synchronously before the generated LRO method + # awaits read() on the initial response. Capture the pipeline response object here + # and parse the body afterwards, when read() has already been awaited. + pipeline_responses.append(response) + + await project_client.beta.datasets.begin_create_generation_job( + job=job, + polling=False, + raw_response_hook=raw_response_hook, + ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. + if not pipeline_responses: + raise RuntimeError("The create operation did not return a data generation job.") + job = DataGenerationJob(pipeline_responses[0].http_response.json()) + print(f"Created job: id={job.id}, status={job.status}") + + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: + await asyncio.sleep(poll_interval_seconds) + job = await project_client.beta.datasets.get_generation_job(job_id=job.id) + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") + + if job.status != JobStatus.SUCCEEDED: + message = job.error.message if job.error else "" + raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") + if job.result is None: + raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") + job_result = job.result + + # ------------------------------------------------------------------ + # 3. Inspect the generated fine-tuning file outputs. + # ------------------------------------------------------------------ + # `train_split=0.8` produces two Azure OpenAI files: a training partition + # and a validation partition. Both are emitted as FileDataGenerationJobOutput + # entries in `job_result.outputs`. + file_outputs = [ + output for output in (job_result.outputs or []) if isinstance(output, FileDataGenerationJobOutput) + ] + if not file_outputs: + raise RuntimeError("The data generation job did not produce any file outputs.") + + print(f"Generated {len(file_outputs)} fine-tuning file(s):") + for output in file_outputs: + if not output.id: + raise RuntimeError("A file output was returned without an id.") + # Resolve the Azure OpenAI file to surface its real filename and size. + file_info = await openai_client.files.retrieve(file_id=output.id) + print(f" - filename=`{file_info.filename}` id=`{output.id}` bytes={file_info.bytes}") + if job_result.generated_samples is not None: + print(f"Generated samples: {job_result.generated_samples}") + + # ------------------------------------------------------------------ + # 4. Clean up. + # ------------------------------------------------------------------ + for output in file_outputs: + print(f"Delete the generated Azure OpenAI file `{output.id}`.") + await openai_client.files.delete(file_id=output.id) + + print(f"Delete the Azure OpenAI input file `{seed_file.id}`.") + await openai_client.files.delete(file_id=seed_file.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py index 9673c258d4c7..b917bdcecef4 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py @@ -169,12 +169,12 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=job, polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py index d5d0c9c71c05..5f6f33ece43c 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py @@ -163,7 +163,6 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( @@ -195,6 +194,7 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py index 1f1da7d43f3a..55c3fa8f01ab 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py @@ -127,12 +127,12 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=job, polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py index 403deb9dab81..a67e8564289f 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py @@ -146,7 +146,6 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( @@ -168,6 +167,7 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py index 8b98234e65bf..7c6fe5de07d2 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py @@ -147,7 +147,6 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(DataGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( @@ -171,6 +170,7 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a data generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 11ed5bd395f2..21157a504529 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -136,7 +136,6 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( @@ -151,6 +150,7 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") job = created_jobs[0] @@ -199,7 +199,6 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( @@ -225,6 +224,7 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index 50ec17bff251..61d9412e3948 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -97,7 +97,6 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( @@ -129,6 +128,7 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index 30cecb6711da..a4284b713dcb 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -80,7 +80,6 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( @@ -105,6 +104,7 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") job = created_jobs[0] diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index ee7227c8cb85..e7b21e3f174a 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -102,13 +102,13 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - # Alternatively, append `.result()` to block while the SDK handles polling. project_client.beta.evaluators.begin_create_generation_job( job=job_body, operation_id=operation_id, polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. if not created_jobs: raise RuntimeError("The create operation did not return a generation job.") job = created_jobs[0] From 1b3e83a8e260aa6d7a546b8cee58647dd6c51dc9 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:13:45 -0700 Subject: [PATCH 09/16] More --- ..._optimization_job_advanced_app_polling.py} | 2 +- ...ization_job_advanced_app_polling_async.py} | 4 +- .../sample_optimization_job_app_polling.py | 114 +++++++++++++++++ ...mple_optimization_job_app_polling_async.py | 120 ++++++++++++++++++ ...tion_job_simpleqna_for_finetuning_async.py | 3 +- 5 files changed, 240 insertions(+), 3 deletions(-) rename sdk/ai/azure-ai-projects/samples/agents/optimization/{sample_optimization_job_basic_polling.py => sample_optimization_job_advanced_app_polling.py} (99%) rename sdk/ai/azure-ai-projects/samples/agents/optimization/{sample_optimization_job_basic_polling_async.py => sample_optimization_job_advanced_app_polling_async.py} (96%) create mode 100644 sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py similarity index 99% rename from sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py rename to sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py index 70cd279d8313..f38debb73490 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py @@ -14,7 +14,7 @@ training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic_polling.py + python sample_optimization_job_advanced_app_polling.py Before running the sample: diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py similarity index 96% rename from sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py rename to sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py index a01c9012bfdc..b58cbbafbd4f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py @@ -14,7 +14,7 @@ training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic_polling_async.py + python sample_optimization_job_advanced_app_polling_async.py Before running the sample: @@ -100,6 +100,8 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) + # Alternatively, have the SDK handle polling by removing `polling=False`, assigning the awaited call + # to a poller, and then awaiting `poller.result()`. if not pipeline_responses: raise RuntimeError("The create operation did not return an optimization job.") job = OptimizationJob(pipeline_responses[0].http_response.json()) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py new file mode 100644 index 000000000000..bf7ff1a576b9 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py @@ -0,0 +1,114 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an AIProjectClient, this sample demonstrates how to create an agent + optimization job, observe the SDK poller until it is done, and then get + the result. + + Agent optimization automatically improves an agent's system prompt, model + choice, or tool definitions by running candidate variants against your + training dataset and scoring them with the evaluators you specify. + +USAGE: + python sample_optimization_job_app_polling.py + + Before running the sample: + + pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry portal. + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". +""" + +import os +import time + +from dotenv import load_dotenv + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import ( + OptimizationAgentIdentifier as AgentIdentifier, + OptimizationEvaluatorRef as EvaluatorRef, + OptimizationJob, + OptimizationJobInputs, + OptimizationOptions, + OptimizationReferenceDatasetInput as ReferenceDatasetInput, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ["FOUNDRY_AGENT_NAME"] +dataset_name = os.environ["DATASET_NAME"] +evaluator_name = os.environ["EVALUATOR_NAME"] +dataset_version = os.environ.get("DATASET_VERSION", "1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") +optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") + +with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, +): + + # ------------------------------------------------------------------ + # 1. Create an optimization job and observe the SDK-managed poller. + # ------------------------------------------------------------------ + print("Creating optimization job...") + poller = project_client.beta.agents.begin_create_optimization_job( + job=OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ) + ), + polling_interval=poll_interval, + ) + + print("SDK is polling the optimization job to completion") + while not poller.done(): + time.sleep(poll_interval) + print(f"status=`{poller.status()}`") + + # done() is true, so result() returns without waiting further and propagates + # any exception raised by the SDK's LRO polling operation. + result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + + # ------------------------------------------------------------------ + # 2. Inspect the results. + # ------------------------------------------------------------------ + print(f"\nBaseline candidate: {result.baseline}") + print(f"Best candidate: {result.best}") + print(f"Candidates ({len(result.candidates or [])}):") + for candidate in result.candidates or []: + print( + f" - {candidate.name}" + f" | avg_score={candidate.avg_score:.4f}" + f" | avg_tokens={candidate.avg_tokens:.0f}" + ) + if candidate.eval_id: + print(f" eval_id={candidate.eval_id}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py new file mode 100644 index 000000000000..5a9f99db0fdb --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py @@ -0,0 +1,120 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Given an async AIProjectClient, this sample demonstrates how to create an + agent optimization job, observe the SDK poller until it is done, and then + get the result. + + Agent optimization automatically improves an agent's system prompt, model + choice, or tool definitions by running candidate variants against your + training dataset and scoring them with the evaluators you specify. + +USAGE: + python sample_optimization_job_app_polling_async.py + + Before running the sample: + + pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv aiohttp + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry portal. + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". +""" + +import asyncio +import os + +from dotenv import load_dotenv + +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import ( + OptimizationAgentIdentifier as AgentIdentifier, + OptimizationEvaluatorRef as EvaluatorRef, + OptimizationJob, + OptimizationJobInputs, + OptimizationOptions, + OptimizationReferenceDatasetInput as ReferenceDatasetInput, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +agent_name = os.environ["FOUNDRY_AGENT_NAME"] +dataset_name = os.environ["DATASET_NAME"] +evaluator_name = os.environ["EVALUATOR_NAME"] +dataset_version = os.environ.get("DATASET_VERSION", "1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) +eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") +optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") + + +async def main() -> None: + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + ): + + # ------------------------------------------------------------------ + # 1. Create an optimization job and observe the SDK-managed poller. + # ------------------------------------------------------------------ + print("Creating optimization job...") + poller = await project_client.beta.agents.begin_create_optimization_job( + job=OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ) + ), + polling_interval=poll_interval, + ) + + print("SDK is polling the optimization job to completion") + while not poller.done(): + await asyncio.sleep(poll_interval) + print(f"status=`{poller.status()}`") + + # done() is true, so awaiting result() returns without waiting further and + # propagates any exception raised by the SDK's LRO polling operation. + result = await poller.result() + print(f"Final LRO status: `{poller.status()}`.") + + # ------------------------------------------------------------------ + # 2. Inspect the results. + # ------------------------------------------------------------------ + print(f"\nBaseline candidate: {result.baseline}") + print(f"Best candidate: {result.best}") + print(f"Candidates ({len(result.candidates or [])}):") + for candidate in result.candidates or []: + print( + f" - {candidate.name}" + f" | avg_score={candidate.avg_score:.4f}" + f" | avg_tokens={candidate.avg_tokens:.0f}" + ) + if candidate.eval_id: + print(f" eval_id={candidate.eval_id}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py index e94eb0bd987c..da51a54aefa4 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py @@ -187,7 +187,8 @@ def raw_response_hook(response): polling=False, raw_response_hook=raw_response_hook, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. + # Alternatively, have the SDK handle polling by removing `polling=False`, assigning the awaited call + # to a poller, and then awaiting `poller.result()`. if not pipeline_responses: raise RuntimeError("The create operation did not return a data generation job.") job = DataGenerationJob(pipeline_responses[0].http_response.json()) From 8a340df534ba181892a4dcd6cbecc176fe3c5fc6 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:33:21 -0700 Subject: [PATCH 10/16] More --- ...e_optimization_job_advanced_app_polling.py | 32 ++--- ...mization_job_advanced_app_polling_async.py | 32 ++--- .../sample_optimization_job_app_polling.py | 114 ----------------- ...mple_optimization_job_app_polling_async.py | 120 ------------------ .../sample_optimization_job_basic.py | 70 +++++----- .../sample_optimization_job_basic_async.py | 76 ++++++----- .../sample_optimization_job_cancel.py | 63 +++++---- ...generation_job_simpleqna_for_finetuning.py | 45 ++----- ...tion_job_simpleqna_for_finetuning_async.py | 47 ++----- 9 files changed, 183 insertions(+), 416 deletions(-) delete mode 100644 sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py delete mode 100644 sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py index f38debb73490..62331c1317b5 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py @@ -79,22 +79,24 @@ def raw_response_hook(response): response.http_response.read() created_jobs.append(OptimizationJob(response.http_response.json())) - project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, ) - ), + ) + ) + + project_client.beta.agents.begin_create_optimization_job( + job=job, polling=False, raw_response_hook=raw_response_hook, ) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py index b58cbbafbd4f..e4517530e811 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py @@ -81,22 +81,24 @@ def raw_response_hook(response): # and parse the body afterwards, when read() has already been awaited. pipeline_responses.append(response) - await project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, ) - ), + ) + ) + + await project_client.beta.agents.begin_create_optimization_job( + job=job, polling=False, raw_response_hook=raw_response_hook, ) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py deleted file mode 100644 index bf7ff1a576b9..000000000000 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling.py +++ /dev/null @@ -1,114 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Given an AIProjectClient, this sample demonstrates how to create an agent - optimization job, observe the SDK poller until it is done, and then get - the result. - - Agent optimization automatically improves an agent's system prompt, model - choice, or tool definitions by running candidate variants against your - training dataset and scoring them with the evaluators you specify. - -USAGE: - python sample_optimization_job_app_polling.py - - Before running the sample: - - pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv - - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found - in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. - 3) DATASET_NAME - Required. The name of the registered training dataset. - 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. - 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. - 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". -""" - -import os -import time - -from dotenv import load_dotenv - -from azure.identity import DefaultAzureCredential -from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import ( - OptimizationAgentIdentifier as AgentIdentifier, - OptimizationEvaluatorRef as EvaluatorRef, - OptimizationJob, - OptimizationJobInputs, - OptimizationOptions, - OptimizationReferenceDatasetInput as ReferenceDatasetInput, -) - -load_dotenv() - -endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] -agent_name = os.environ["FOUNDRY_AGENT_NAME"] -dataset_name = os.environ["DATASET_NAME"] -evaluator_name = os.environ["EVALUATOR_NAME"] -dataset_version = os.environ.get("DATASET_VERSION", "1") -poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") -optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") - -with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential) as project_client, -): - - # ------------------------------------------------------------------ - # 1. Create an optimization job and observe the SDK-managed poller. - # ------------------------------------------------------------------ - print("Creating optimization job...") - poller = project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) - ), - polling_interval=poll_interval, - ) - - print("SDK is polling the optimization job to completion") - while not poller.done(): - time.sleep(poll_interval) - print(f"status=`{poller.status()}`") - - # done() is true, so result() returns without waiting further and propagates - # any exception raised by the SDK's LRO polling operation. - result = poller.result() - print(f"Final LRO status: `{poller.status()}`.") - - # ------------------------------------------------------------------ - # 2. Inspect the results. - # ------------------------------------------------------------------ - print(f"\nBaseline candidate: {result.baseline}") - print(f"Best candidate: {result.best}") - print(f"Candidates ({len(result.candidates or [])}):") - for candidate in result.candidates or []: - print( - f" - {candidate.name}" - f" | avg_score={candidate.avg_score:.4f}" - f" | avg_tokens={candidate.avg_tokens:.0f}" - ) - if candidate.eval_id: - print(f" eval_id={candidate.eval_id}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py deleted file mode 100644 index 5a9f99db0fdb..000000000000 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_app_polling_async.py +++ /dev/null @@ -1,120 +0,0 @@ -# pylint: disable=line-too-long,useless-suppression -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Given an async AIProjectClient, this sample demonstrates how to create an - agent optimization job, observe the SDK poller until it is done, and then - get the result. - - Agent optimization automatically improves an agent's system prompt, model - choice, or tool definitions by running candidate variants against your - training dataset and scoring them with the evaluators you specify. - -USAGE: - python sample_optimization_job_app_polling_async.py - - Before running the sample: - - pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv aiohttp - - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found - in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. - 3) DATASET_NAME - Required. The name of the registered training dataset. - 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. - 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. - 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". -""" - -import asyncio -import os - -from dotenv import load_dotenv - -from azure.identity.aio import DefaultAzureCredential -from azure.ai.projects.aio import AIProjectClient -from azure.ai.projects.models import ( - OptimizationAgentIdentifier as AgentIdentifier, - OptimizationEvaluatorRef as EvaluatorRef, - OptimizationJob, - OptimizationJobInputs, - OptimizationOptions, - OptimizationReferenceDatasetInput as ReferenceDatasetInput, -) - -load_dotenv() - -endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] -agent_name = os.environ["FOUNDRY_AGENT_NAME"] -dataset_name = os.environ["DATASET_NAME"] -evaluator_name = os.environ["EVALUATOR_NAME"] -dataset_version = os.environ.get("DATASET_VERSION", "1") -poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") -optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") - - -async def main() -> None: - async with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential) as project_client, - ): - - # ------------------------------------------------------------------ - # 1. Create an optimization job and observe the SDK-managed poller. - # ------------------------------------------------------------------ - print("Creating optimization job...") - poller = await project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) - ), - polling_interval=poll_interval, - ) - - print("SDK is polling the optimization job to completion") - while not poller.done(): - await asyncio.sleep(poll_interval) - print(f"status=`{poller.status()}`") - - # done() is true, so awaiting result() returns without waiting further and - # propagates any exception raised by the SDK's LRO polling operation. - result = await poller.result() - print(f"Final LRO status: `{poller.status()}`.") - - # ------------------------------------------------------------------ - # 2. Inspect the results. - # ------------------------------------------------------------------ - print(f"\nBaseline candidate: {result.baseline}") - print(f"Best candidate: {result.best}") - print(f"Candidates ({len(result.candidates or [])}):") - for candidate in result.candidates or []: - print( - f" - {candidate.name}" - f" | avg_score={candidate.avg_score:.4f}" - f" | avg_tokens={candidate.avg_tokens:.0f}" - ) - if candidate.eval_id: - print(f" eval_id={candidate.eval_id}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py index b89765a1b935..fcde99282dfd 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py @@ -7,15 +7,15 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to create an agent - optimization job and use the SDK's built-in polling mechanism to wait for - its completion to get the result. + optimization job, observe the SDK poller until it is done, and then get + the result. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic.py + python sample_optimization_job_app_polling.py Before running the sample: @@ -24,16 +24,17 @@ Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. - 3) DATASET_NAME - Required. The name of the registered training dataset. - 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. - 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. - 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". """ import os +import time from dotenv import load_dotenv @@ -65,28 +66,39 @@ ): # ------------------------------------------------------------------ - # 1. Create an optimization job. + # 1. Create an optimization job and observe the SDK-managed poller. # ------------------------------------------------------------------ - print("Creating optimization job...") - result = project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), ), + ) + + print("Begin creating an agent optimization job.") + poller = project_client.beta.agents.begin_create_optimization_job( + job=job, polling_interval=poll_interval, - ).result() - print("Optimization job completed.") + ) + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): + time.sleep(poll_interval) + print(f"status=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") # ------------------------------------------------------------------ # 2. Inspect the results. diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py index 6d467ba8ddd2..f17bce07cc96 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py @@ -6,27 +6,31 @@ """ DESCRIPTION: - Given an async AIProjectClient, this sample demonstrates how to create an agent - optimization job and use the SDK's built-in polling mechanism to wait for - its completion to get the result. + Given an async AIProjectClient, this sample demonstrates how to create an + agent optimization job, observe the SDK poller until it is done, and then + get the result. + + Agent optimization automatically improves an agent's system prompt, model + choice, or tool definitions by running candidate variants against your + training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic_async.py + python sample_optimization_job_app_polling_async.py Before running the sample: - pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv + pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv aiohttp Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. - 3) DATASET_NAME - Required. The name of the registered training dataset. - 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. - 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. - 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". """ import asyncio @@ -52,9 +56,9 @@ dataset_name = os.environ["DATASET_NAME"] evaluator_name = os.environ["EVALUATOR_NAME"] dataset_version = os.environ.get("DATASET_VERSION", "1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") -poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) async def main() -> None: @@ -64,29 +68,39 @@ async def main() -> None: ): # ------------------------------------------------------------------ - # 1. Create an optimization job. + # 1. Create an optimization job and observe the SDK-managed poller. # ------------------------------------------------------------------ - print("Creating optimization job...") - poller = await project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), ), + ) + + print("Begin creating an agent optimization job.") + poller = await project_client.beta.agents.begin_create_optimization_job( + job=job, polling_interval=poll_interval, ) + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): + await asyncio.sleep(poll_interval) + print(f"status=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. result = await poller.result() - print("Optimization job completed.") + print(f"Final LRO status: `{poller.status()}`.") # ------------------------------------------------------------------ # 2. Inspect the results. diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py index de3e831e1588..23750ec9be02 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py @@ -19,16 +19,18 @@ Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. 3) DATASET_NAME - Required. The name of the registered training dataset. 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 7) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". """ import os +import time from dotenv import load_dotenv @@ -50,6 +52,7 @@ dataset_name = os.environ["DATASET_NAME"] evaluator_name = os.environ["EVALUATOR_NAME"] dataset_version = os.environ.get("DATASET_VERSION", "1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") @@ -60,42 +63,50 @@ ): # ------------------------------------------------------------------ - # 1. Create a job. + # 1. Create an optimization job and retain the SDK-managed poller. # ------------------------------------------------------------------ - print("Creating optimization job...") + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ), + ) + created_jobs: list[OptimizationJob] = [] def raw_response_hook(response): response.http_response.read() created_jobs.append(OptimizationJob(response.http_response.json())) - project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) - ), - polling=False, + print("Begin creating an agent optimization job.") + poller = project_client.beta.agents.begin_create_optimization_job( + job=job, + polling_interval=poll_interval, raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return an optimization job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") + created_job = created_jobs[0] + print(f"Created job: id={created_job.id}, status={created_job.status}") # ------------------------------------------------------------------ # 2. Cancel it immediately. # ------------------------------------------------------------------ - print(f"Cancelling job {job.id}...") - cancelled = project_client.beta.agents.cancel_optimization_job(job_id=job.id) + print(f"Cancelling job {created_job.id}...") + cancelled = project_client.beta.agents.cancel_optimization_job(job_id=created_job.id) print(f"Job {cancelled.id} status: {cancelled.status}") + + print("Wait for the SDK poller to observe the cancellation.") + while not poller.done(): + time.sleep(poll_interval) + print(f"status=`{poller.status()}`") + print(f"Final LRO status: `{poller.status()}`.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index b0b2dc3ff472..a15da87b1ccf 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -61,7 +61,6 @@ DataGenerationModelOptions, FileDataGenerationJobOutput, FileDataGenerationJobSource, - JobStatus, SimpleQnADataGenerationJobOptions, SimpleQnAFineTuningQuestionType, ) @@ -73,8 +72,6 @@ dataset_name = os.environ.get("DATASET_NAME", "simpleqna-finetuning-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run output name so repeated runs do not collide. # Output names are capped at 50 characters by the service. run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" @@ -171,38 +168,22 @@ output_options=DataGenerationJobOutputOptions(name=output_name), ), ) - print("Create a fine-tuning data generation job and wait for it to complete.") - created_jobs: list[DataGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(DataGenerationJob(response.http_response.json())) - - project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=job, - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a data generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.datasets.get_generation_job(job_id=job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") - job_result = job.result + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # ------------------------------------------------------------------ # 3. Inspect the generated fine-tuning file outputs. diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py index da51a54aefa4..be042ff552ca 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py @@ -61,7 +61,6 @@ DataGenerationModelOptions, FileDataGenerationJobOutput, FileDataGenerationJobSource, - JobStatus, SimpleQnADataGenerationJobOptions, SimpleQnAFineTuningQuestionType, ) @@ -73,8 +72,6 @@ dataset_name = os.environ.get("DATASET_NAME", "simpleqna-finetuning-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run output name so repeated runs do not collide. # Output names are capped at 50 characters by the service. run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" @@ -173,41 +170,23 @@ async def main() -> None: output_options=DataGenerationJobOutputOptions(name=output_name), ), ) - print("Create a fine-tuning data generation job and wait for it to complete.") - pipeline_responses = [] - - def raw_response_hook(response): - # The raw_response_hook is called synchronously before the generated LRO method - # awaits read() on the initial response. Capture the pipeline response object here - # and parse the body afterwards, when read() has already been awaited. - pipeline_responses.append(response) - await project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = await project_client.beta.datasets.begin_create_generation_job( job=job, - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False`, assigning the awaited call - # to a poller, and then awaiting `poller.result()`. - if not pipeline_responses: - raise RuntimeError("The create operation did not return a data generation job.") - job = DataGenerationJob(pipeline_responses[0].http_response.json()) - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): await asyncio.sleep(poll_interval_seconds) - job = await project_client.beta.datasets.get_generation_job(job_id=job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") - job_result = job.result + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = await poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # ------------------------------------------------------------------ # 3. Inspect the generated fine-tuning file outputs. From 01f1715737c0484c71f6cf3ec2405e8d47edb846 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:34:08 -0700 Subject: [PATCH 11/16] MOre --- .../sample_optimization_job_advanced_app_polling.py | 2 +- .../sample_optimization_job_advanced_app_polling_async.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py index 62331c1317b5..ab72b614aeb9 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py @@ -91,7 +91,7 @@ def raw_response_hook(response): max_candidates=3, eval_model=eval_model, optimization_model=optimization_model, - ) + ), ) ) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py index e4517530e811..7a8599ecb48b 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py @@ -93,7 +93,7 @@ def raw_response_hook(response): max_candidates=3, eval_model=eval_model, optimization_model=optimization_model, - ) + ), ) ) From 500616edaa58ad730819e8193dbb405ffea432f8 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:16:49 -0700 Subject: [PATCH 12/16] More --- ...generation_job_simpleqna_for_finetuning.py | 1 + ...eration_job_simpleqna_with_agent_source.py | 45 +++------ ...neration_job_simpleqna_with_file_source.py | 93 ++++++++----------- ...ration_job_simpleqna_with_prompt_source.py | 41 +++----- ...et_generation_job_traces_for_evaluation.py | 51 ++++------ ...et_generation_job_traces_for_finetuning.py | 51 ++++------ 6 files changed, 99 insertions(+), 183 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index a15da87b1ccf..18847d51aeb9 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -168,6 +168,7 @@ output_options=DataGenerationJobOutputOptions(name=output_name), ), ) + print("Begin creating a dataset generation job.") poller = project_client.beta.datasets.begin_create_generation_job( job=job, diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py index b917bdcecef4..46a61ac09fad 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py @@ -70,7 +70,6 @@ DataGenerationModelOptions, DatasetDataGenerationJobOutput, DatasetVersion, - JobStatus, PromptAgentDefinition, SimpleQnADataGenerationJobOptions, ) @@ -82,8 +81,6 @@ dataset_name = os.environ.get("DATASET_NAME", "simpleqna-agent-source-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run names so repeated runs do not collide. # Output names are capped at 50 characters by the service. run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" @@ -162,38 +159,22 @@ output_options=DataGenerationJobOutputOptions(name=output_dataset_name), ), ) - print("Creating data generation job and polling until completion...") - created_jobs: list[DataGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(DataGenerationJob(response.http_response.json())) - - project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=job, - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a data generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.datasets.get_generation_job(job_id=job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") - job_result = job.result + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py index 5f6f33ece43c..bd9baad90940 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py @@ -66,7 +66,6 @@ DatasetDataGenerationJobOutput, DatasetVersion, FileDataGenerationJobSource, - JobStatus, PromptDataGenerationJobSource, SimpleQnADataGenerationJobOptions, ) @@ -78,8 +77,6 @@ dataset_name = os.environ.get("DATASET_NAME", "simpleqna-file-source-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run resource names so repeated runs do not collide. # Output names are capped at 50 characters by the service. run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" @@ -156,64 +153,50 @@ # - The File source contributes the source material (the reference # document uploaded above). # - The Prompt source contributes a steering instruction (difficulty). - print("Creating multi-source data generation job (File + Prompt) and polling until completion...") - created_jobs: list[DataGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(DataGenerationJob(response.http_response.json())) - - project_client.beta.datasets.begin_create_generation_job( - job=DataGenerationJob( - inputs=DataGenerationJobInputs( - name=f"simpleqna-multisource-{run_id}", - scenario=DataGenerationJobScenario.EVALUATION, - sources=[ - FileDataGenerationJobSource( - description="Widgets & Gizmos product / operations reference (Azure OpenAI file).", - id=seed_file.id, - ), - PromptDataGenerationJobSource( - description="Specifies the question difficulty for SimpleQnA generation.", - prompt="Generate expert-level questions of high difficulty.", - ), - ], - options=SimpleQnADataGenerationJobOptions( - # Service requires max_samples to be between 15 and 1000. - max_samples=15, - # `simple_qna` REQUIRES model_options. - model_options=DataGenerationModelOptions(model=model_name), + job = DataGenerationJob( + inputs=DataGenerationJobInputs( + name=f"simpleqna-multisource-{run_id}", + scenario=DataGenerationJobScenario.EVALUATION, + sources=[ + FileDataGenerationJobSource( + description="Widgets & Gizmos product / operations reference (Azure OpenAI file).", + id=seed_file.id, ), - output_options=DataGenerationJobOutputOptions( - name=output_dataset_name, - description=EXPECTED_OUTPUT_DESCRIPTION, - tags=EXPECTED_OUTPUT_TAGS, + PromptDataGenerationJobSource( + description="Specifies the question difficulty for SimpleQnA generation.", + prompt="Generate expert-level questions of high difficulty.", ), + ], + options=SimpleQnADataGenerationJobOptions( + # Service requires max_samples to be between 15 and 1000. + max_samples=15, + # `simple_qna` REQUIRES model_options. + model_options=DataGenerationModelOptions(model=model_name), + ), + output_options=DataGenerationJobOutputOptions( + name=output_dataset_name, + description=EXPECTED_OUTPUT_DESCRIPTION, + tags=EXPECTED_OUTPUT_TAGS, ), ), - polling=False, - raw_response_hook=raw_response_hook, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a data generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( + job=job, + polling_interval=poll_interval_seconds, + ) + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.datasets.get_generation_job(job_id=job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") - job_result = job.result + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py index 55c3fa8f01ab..c5f29c30bbbc 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py @@ -67,7 +67,6 @@ DataGenerationModelOptions, DatasetDataGenerationJobOutput, DatasetVersion, - JobStatus, PromptDataGenerationJobSource, SimpleQnADataGenerationJobOptions, TestingCriterionAzureAIEvaluator, @@ -80,8 +79,6 @@ dataset_name = os.environ.get("DATASET_NAME", "dataset-generation-eval-sample") poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - def main() -> None: with ( @@ -120,38 +117,22 @@ def main() -> None: ), ), ) - print("Creating data generation job and polling until completion...") - created_jobs: list[DataGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(DataGenerationJob(response.http_response.json())) - - project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=job, - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a data generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.datasets.get_generation_job(job_id=job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") + print(f"\tstatus=`{poller.status()}`") - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Data generation job `{job.id}` completed without a result.") - job_result = job.result + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py index a67e8564289f..40595c2629c7 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py @@ -54,7 +54,6 @@ DataGenerationJobScenario, DatasetDataGenerationJobOutput, DatasetVersion, - JobStatus, PromptAgentDefinition, TracesDataGenerationJobOptions, TracesDataGenerationJobSource, @@ -82,7 +81,6 @@ model_deployment = os.environ["FOUNDRY_MODEL_NAME"] DATASET_NAME = "traces-eval-sample" POLL_INTERVAL_SECONDS = 10 -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} INITIAL_INGEST_WAIT_SECONDS = 60 MAX_JOB_ATTEMPTS = 5 RETRY_WAIT_SECONDS = 60 @@ -131,7 +129,7 @@ start_time = seed_start - timedelta(minutes=5) - job = None + job_result = None for attempt in range(1, MAX_JOB_ATTEMPTS + 1): end_time = datetime.now(tz=timezone.utc) print( @@ -140,13 +138,8 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - created_jobs: list[DataGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(DataGenerationJob(response.http_response.json())) - - project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-eval-{run_id}-a{attempt}", @@ -164,27 +157,19 @@ def raw_response_hook(response): output_options=DataGenerationJobOutputOptions(name=output_dataset_name), ), ), - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=POLL_INTERVAL_SECONDS, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a data generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(POLL_INTERVAL_SECONDS) - job = project_client.beta.datasets.get_generation_job(job_id=job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") - print("Data generation job succeeded.") + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") break except Exception as e: # pylint: disable=broad-except if attempt == MAX_JOB_ATTEMPTS: @@ -193,9 +178,9 @@ def raw_response_hook(response): time.sleep(RETRY_WAIT_SECONDS) # 3. Resolve the generated dataset. - if job is None or job.result is None: + if job_result is None: raise RuntimeError("The data generation job did not return a result.") - outputs = job.result.outputs or [] + outputs = job_result.outputs or [] dataset_output = next((o for o in outputs if isinstance(o, DatasetDataGenerationJobOutput)), None) if dataset_output is None or not dataset_output.name or not dataset_output.version: raise RuntimeError("The data generation job did not produce a dataset output.") @@ -205,8 +190,8 @@ def raw_response_hook(response): f"Generated dataset: name=`{created_dataset.name}` " f"version=`{created_dataset.version}` id=`{created_dataset.id}`" ) - if job.result.generated_samples is not None: - print(f"Generated samples: {job.result.generated_samples}") + if job_result.generated_samples is not None: + print(f"Generated samples: {job_result.generated_samples}") finally: # Best-effort cleanup, outputs -> producers (dataset, job, conversations, agent). diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py index 7c6fe5de07d2..b3bd5431e100 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py @@ -53,7 +53,6 @@ DataGenerationJobOutputOptions, DataGenerationJobScenario, FileDataGenerationJobOutput, - JobStatus, PromptAgentDefinition, TracesDataGenerationJobOptions, TracesDataGenerationJobSource, @@ -84,7 +83,6 @@ model_deployment = os.environ["FOUNDRY_MODEL_NAME"] DATASET_NAME = "traces-ft-sample" POLL_INTERVAL_SECONDS = 10 -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} INITIAL_INGEST_WAIT_SECONDS = 60 MAX_JOB_ATTEMPTS = 5 RETRY_WAIT_SECONDS = 60 @@ -132,7 +130,7 @@ start_time = seed_start - timedelta(minutes=5) - job = None + job_result = None for attempt in range(1, MAX_JOB_ATTEMPTS + 1): end_time = datetime.now(tz=timezone.utc) print( @@ -141,13 +139,8 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - created_jobs: list[DataGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(DataGenerationJob(response.http_response.json())) - - project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-ft-{run_id}-a{attempt}", @@ -167,27 +160,19 @@ def raw_response_hook(response): output_options=DataGenerationJobOutputOptions(name=output_name), ), ), - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=POLL_INTERVAL_SECONDS, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a data generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(POLL_INTERVAL_SECONDS) - job = project_client.beta.datasets.get_generation_job(job_id=job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Data generation job `{job.id}` ended with status `{job.status}`: {message}") - print("Data generation job succeeded.") + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") break except Exception as e: # pylint: disable=broad-except if attempt == MAX_JOB_ATTEMPTS: @@ -196,9 +181,9 @@ def raw_response_hook(response): time.sleep(RETRY_WAIT_SECONDS) # 3. Resolve generated fine-tuning files. - if job is None or job.result is None: + if job_result is None: raise RuntimeError("The data generation job did not return a result.") - outputs = job.result.outputs or [] + outputs = job_result.outputs or [] file_outputs = [o for o in outputs if isinstance(o, FileDataGenerationJobOutput)] if not file_outputs: raise RuntimeError("The data generation job did not produce any file outputs.") @@ -210,8 +195,8 @@ def raw_response_hook(response): created_file_ids.append(output.id) file_info = openai_client.files.retrieve(file_id=output.id) print(f" - filename=`{file_info.filename}` id=`{output.id}` bytes={file_info.bytes}") - if job.result.generated_samples is not None: - print(f"Generated samples: {job.result.generated_samples}") + if job_result.generated_samples is not None: + print(f"Generated samples: {job_result.generated_samples}") finally: # Best-effort cleanup, outputs -> producers (files, job, conversations, agent). From d3f91ac74b98b28ea69003f1eef5ba5f14c01670 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Thu, 30 Jul 2026 20:20:38 -0700 Subject: [PATCH 13/16] More --- ...rubric_evaluator_generation_all_sources.py | 79 ++++++------------- ...ample_rubric_evaluator_generation_basic.py | 45 +++-------- ...ple_rubric_evaluator_generation_iterate.py | 45 +++-------- ...e_rubric_evaluator_generation_lifecycle.py | 63 ++++++--------- 4 files changed, 71 insertions(+), 161 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 21157a504529..1436624dd00c 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -63,7 +63,6 @@ EvaluatorGenerationInputs, EvaluatorGenerationJob, EvaluatorGenerationJobSource, - JobStatus, PromptEvaluatorGenerationJobSource, RubricBasedEvaluatorDefinition, TracesEvaluatorGenerationJobSource, @@ -79,8 +78,6 @@ traces_window_days = int(os.environ.get("FOUNDRY_TRACES_WINDOW_DAYS", "7")) poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run suffix so repeated runs do not collide on evaluator name. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -128,15 +125,9 @@ else: print("Skipping Dataset source (FOUNDRY_REFERENCE_DATASET_NAME / _VERSION not set).") - print("Waiting for multi-source job to complete...") + print("Begin creating an evaluator generation job.") try: - created_jobs: list[EvaluatorGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - - project_client.beta.evaluators.begin_create_generation_job( + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -147,29 +138,19 @@ def raw_response_hook(response): ), ), operation_id=f"rubric-multi-{short}", - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") + print(f"\tstatus=`{poller.status()}`") - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Multi-source job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Multi-source job `{job.id}` completed without a result.") - evaluator = job.result + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + evaluator = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) @@ -191,15 +172,9 @@ def raw_response_hook(response): start_time = now - timedelta(days=traces_window_days) end_time = now + timedelta(seconds=600) # small padding for clock skew - print("Waiting for traces job to complete...") + print("Begin creating an evaluator generation job.") try: - created_jobs = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - - project_client.beta.evaluators.begin_create_generation_job( + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -221,29 +196,19 @@ def raw_response_hook(response): ), ), operation_id=f"rubric-traces-{short}", - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") + print(f"\tstatus=`{poller.status()}`") - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Traces job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Traces job `{job.id}` completed without a result.") - evaluator = job.result + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + evaluator = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index 61d9412e3948..8e25af0a23ea 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -63,7 +63,6 @@ from azure.ai.projects.models import ( EvaluatorGenerationInputs, EvaluatorGenerationJob, - JobStatus, PromptEvaluatorGenerationJobSource, RubricBasedEvaluatorDefinition, TestingCriterionAzureAIEvaluator, @@ -75,8 +74,6 @@ model_name = os.environ["FOUNDRY_MODEL_NAME"] poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run name so repeated runs do not collide. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -90,14 +87,8 @@ project_client.get_openai_client() as openai_client, ): # 1. Generate an evaluator from a single `Prompt` source. - print("Waiting for generation job to complete...") - created_jobs: list[EvaluatorGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - - project_client.beta.evaluators.begin_create_generation_job( + print("Begin creating an evaluator generation job.") + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -125,29 +116,19 @@ def raw_response_hook(response): ), # `operation_id` makes the call idempotent - re-submitting the same id attaches to the existing job. operation_id=f"rubric-eval-basic-{short}", - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Generation job `{job.id}` completed without a result.") - evaluator = job.result + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + evaluator = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") # On success, the evaluator is automatically saved as version 1. # `isinstance` narrows the discriminated `definition` to the rubric subtype. diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index a4284b713dcb..9d51114abe41 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -50,7 +50,6 @@ EvaluatorDefinitionType, EvaluatorGenerationInputs, EvaluatorGenerationJob, - JobStatus, PromptEvaluatorGenerationJobSource, RubricBasedEvaluatorDefinition, ) @@ -61,8 +60,6 @@ model_name = os.environ["FOUNDRY_MODEL_NAME"] poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run name so repeated runs do not collide. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -73,14 +70,8 @@ AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): # 1. Generate v1 of the evaluator from a single `Prompt` source. - print("Waiting for generation job to complete...") - created_jobs: list[EvaluatorGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - - project_client.beta.evaluators.begin_create_generation_job( + print("Begin creating an evaluator generation job.") + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -101,29 +92,19 @@ def raw_response_hook(response): ), ), operation_id=f"rubric-iterate-{short}", - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Generation job `{job.id}` completed without a result.") - v1 = job.result + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + v1 = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {v1}") # `isinstance` narrows the discriminated `definition` to the rubric subtype. v1_definition = v1.definition diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index e7b21e3f174a..05ed41819983 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -66,8 +66,6 @@ model_name = os.environ["FOUNDRY_MODEL_NAME"] poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) -TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} - # Unique per-run name so repeated runs do not collide. ts = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") short = uuid.uuid4().hex[:6] @@ -94,51 +92,36 @@ DefaultAzureCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): - # 1. Create the generation job. `operation_id` makes the call idempotent - - # re-submitting with the same id returns the existing job. - created_jobs: list[EvaluatorGenerationJob] = [] - - def raw_response_hook(response): - response.http_response.read() - created_jobs.append(EvaluatorGenerationJob(response.http_response.json())) - - project_client.beta.evaluators.begin_create_generation_job( + # 1. Create the generation job. `operation_id` makes the call idempotent. + print("Begin creating an evaluator generation job.") + poller = project_client.beta.evaluators.begin_create_generation_job( job=job_body, operation_id=operation_id, - polling=False, - raw_response_hook=raw_response_hook, + polling_interval=poll_interval_seconds, ) - # Alternatively, have the SDK handle polling by removing `polling=False` and appending `.result()` to the above call. - if not created_jobs: - raise RuntimeError("The create operation did not return a generation job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") - - # Idempotency: a second call with the same operation_id returns the same job. - replay_job = project_client.beta.evaluators.get_generation_job(job.id) - assert replay_job.id == job.id - - # 2. Poll until the job reaches a terminal state. - print(f"Polling job `{job.id}` to completion...", end="", flush=True) - while job.status not in TERMINAL_STATUSES: + + print("Optional: While SDK is polling, periodically print the job status until the job is complete") + while not poller.done(): time.sleep(poll_interval_seconds) - job = project_client.beta.evaluators.get_generation_job(job.id) - print(".", end="", flush=True) - print() - print(f"Final job status: `{job.status}`.") - - if job.status != JobStatus.SUCCEEDED: - message = job.error.message if job.error else "" - raise RuntimeError(f"Generation job `{job.id}` ended with status `{job.status}`: {message}") - if job.result is None: - raise RuntimeError(f"Generation job `{job.id}` completed without a result.") - evaluator: EvaluatorVersion = job.result + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + evaluator: EvaluatorVersion = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") print( f"Generated evaluator `{evaluator.name}` version `{evaluator.version}` " f"(job `{evaluator.generation_job_id}`)." ) - # 3. List the 5 most recent generation jobs in this project. + # Retrieve the persisted generation job using the id returned in the LRO result. + if evaluator.generation_job_id is None: + raise RuntimeError("The generated evaluator did not include a generation job id.") + replay_job = project_client.beta.evaluators.get_generation_job(evaluator.generation_job_id) + assert replay_job.id == evaluator.generation_job_id + + # 2. List the 5 most recent generation jobs in this project. # `limit` controls the page size; use `itertools.islice` to cap the total. print("Recent generation jobs:") for entry in itertools.islice( @@ -147,10 +130,10 @@ def raw_response_hook(response): entry_name = entry.inputs.evaluator_name if entry.inputs is not None else "" print(f" - id=`{entry.id}` status=`{cast(JobStatus, entry.status).value}` evaluator_name=`{entry_name}`") - # 4. Cancel a running job (not exercised here; the job above already completed). + # 3. Cancel a running job (not exercised here; the job above already completed). # cancelled = project_client.beta.evaluators.cancel_generation_job(some_running_job_id) - # 5. Clean up. `delete_version` cascades to the generation job record, so + # 4. Clean up. `delete_version` cascades to the generation job record, so # the explicit delete below may return 404. print("Cleaning up.") project_client.beta.evaluators.delete_version(name=evaluator.name, version=evaluator.version) From fd4f2ff6d0199cc969fcbcf9ed43f80894fa8f02 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:30:32 -0700 Subject: [PATCH 14/16] Update print --- .../agents/optimization/sample_optimization_job_basic.py | 3 ++- .../optimization/sample_optimization_job_basic_async.py | 3 ++- ...ample_dataset_generation_job_simpleqna_for_finetuning.py | 3 ++- ...dataset_generation_job_simpleqna_for_finetuning_async.py | 3 ++- ...le_dataset_generation_job_simpleqna_with_agent_source.py | 3 ++- ...ple_dataset_generation_job_simpleqna_with_file_source.py | 3 ++- ...e_dataset_generation_job_simpleqna_with_prompt_source.py | 3 ++- .../sample_dataset_generation_job_traces_for_evaluation.py | 3 ++- .../sample_dataset_generation_job_traces_for_finetuning.py | 3 ++- .../sample_rubric_evaluator_generation_all_sources.py | 6 ++++-- .../evaluations/sample_rubric_evaluator_generation_basic.py | 3 ++- .../sample_rubric_evaluator_generation_iterate.py | 3 ++- .../sample_rubric_evaluator_generation_lifecycle.py | 3 ++- 13 files changed, 28 insertions(+), 14 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py index fcde99282dfd..42b0e9883374 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py @@ -90,7 +90,8 @@ polling_interval=poll_interval, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval) print(f"status=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py index f17bce07cc96..74c90b3f0e5f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py @@ -92,7 +92,8 @@ async def main() -> None: polling_interval=poll_interval, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): await asyncio.sleep(poll_interval) print(f"status=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index 18847d51aeb9..9c72ee3e8e9f 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -175,7 +175,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py index be042ff552ca..218bd335f662 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py @@ -177,7 +177,8 @@ async def main() -> None: polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): await asyncio.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py index 46a61ac09fad..a190400a3d15 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py @@ -165,7 +165,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py index bd9baad90940..62c17c93da10 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py @@ -187,7 +187,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py index c5f29c30bbbc..898bc6594069 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py @@ -123,7 +123,8 @@ def main() -> None: polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py index 40595c2629c7..617de180a35c 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py @@ -160,7 +160,8 @@ polling_interval=POLL_INTERVAL_SECONDS, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(POLL_INTERVAL_SECONDS) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py index b3bd5431e100..783a54a78c3f 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py @@ -163,7 +163,8 @@ polling_interval=POLL_INTERVAL_SECONDS, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(POLL_INTERVAL_SECONDS) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 1436624dd00c..1ddb24f1b728 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -141,7 +141,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") @@ -199,7 +200,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index 8e25af0a23ea..375d215501c7 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -119,7 +119,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index 9d51114abe41..1e6dd892fe69 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -95,7 +95,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index 05ed41819983..90d273ed60cd 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -100,7 +100,8 @@ polling_interval=poll_interval_seconds, ) - print("Optional: While SDK is polling, periodically print the job status until the job is complete") + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") while not poller.done(): time.sleep(poll_interval_seconds) print(f"\tstatus=`{poller.status()}`") From 3ddf00c32617593c01a4fe0edda695416aeb4a30 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:01:59 -0700 Subject: [PATCH 15/16] Two options for async poller --- ...tion_job_simpleqna_for_finetuning_async.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py index 218bd335f662..ac604ad9eb2f 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py @@ -177,18 +177,25 @@ async def main() -> None: polling_interval=poll_interval_seconds, ) - # Optional: While SDK is polling, periodically print the job status until the job is complete - print("Periodically check job status:") - while not poller.done(): - await asyncio.sleep(poll_interval_seconds) - print(f"\tstatus=`{poller.status()}`") - - # Since done() is true, result() returns the final deserialized job result without - # waiting further. It also propagates any LRO polling exception. - job_result = await poller.result() - print(f"Final LRO status: `{poller.status()}`.") - print(f"Data generation result: {job_result}") - + # Optionally, print the job status periodically while the SDK polls in the background. + print_poller_status = False + + if print_poller_status: + # Start SDK polling in the background and periodically print the job status until it is complete. + result_task = asyncio.create_task(poller.result()) + print("Periodically check job status:") + while not result_task.done(): + await asyncio.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Awaiting the task returns the final deserialized result and propagates any LRO polling exception. + job_result = await result_task + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") + else: + job_result = await poller.result() + print(f"Data generation result: {job_result}") + # ------------------------------------------------------------------ # 3. Inspect the generated fine-tuning file outputs. # ------------------------------------------------------------------ From 29bd10dbf4fa19484c2969fef2ca5f2ee62c8984 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:08:14 -0700 Subject: [PATCH 16/16] Exclude from tests --- sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py index 00e84fd43ee1..f1ccc1fd7cea 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py @@ -167,6 +167,7 @@ async def test_models_samples(self, sample_path: str, **kwargs) -> None: "datasets", samples_to_skip=[ "sample_datasets_async.py", # Skipped until re-enabled and recorded on Foundry endpoint that supports the new versioning schema + "sample_dataset_generation_job_simpleqna_for_finetuning_async.py", # Need to add recordings ], ), )