diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py similarity index 77% rename from sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py rename to sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py index 0e1bdbacc9dc..ab72b614aeb9 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling.py @@ -7,14 +7,14 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to create an agent - optimization job and manually poll it to completion. + optimization job and poll its standard LRO to completion. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic_polling.py + python sample_optimization_job_advanced_app_polling.py Before running the sample: @@ -40,9 +40,9 @@ from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient from azure.ai.projects.models import ( + JobStatus, OptimizationAgentIdentifier as AgentIdentifier, OptimizationEvaluatorRef as EvaluatorRef, - JobStatus, OptimizationJob, OptimizationJobInputs, OptimizationOptions, @@ -60,7 +60,7 @@ eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") -terminal_statuses = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} with ( DefaultAzureCredential() as credential, @@ -73,27 +73,32 @@ print("Creating optimization job...") created_jobs: list[OptimizationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + # Since `polling=False` is set below, it is guaranteed that `raw_response_hook` will be + # invoked once on the initial "201 Created" response, and `response` is of type `OptimizationJob`. + response.http_response.read() created_jobs.append(OptimizationJob(response.http_response.json())) + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ) + ) + project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) - ), + job=job, polling=False, - raw_response_hook=capture_created_job, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return an optimization job.") @@ -101,12 +106,15 @@ def capture_created_job(response): print(f"Created job: id={job.id}, status={job.status}") # ------------------------------------------------------------------ - # 2. Poll the job to completion. + # 2. Poll until the job reaches a terminal state. # ------------------------------------------------------------------ - while job.status not in terminal_statuses: + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: time.sleep(poll_interval) job = project_client.beta.agents.get_optimization_job(job_id=job.id) - print(f"Job status: {job.status}") + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") if job.warnings: for warning in job.warnings: diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py similarity index 72% rename from sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py rename to sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py index d49bba940b1f..7a8599ecb48b 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_polling_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_advanced_app_polling_async.py @@ -7,18 +7,18 @@ """ DESCRIPTION: Given an async AIProjectClient, this sample demonstrates how to create an - agent optimization job and manually poll it to completion. + agent optimization job and poll it to completion. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic_polling_async.py + python sample_optimization_job_advanced_app_polling_async.py Before running the sample: - pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv + pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv aiohttp Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found @@ -33,19 +33,16 @@ """ import asyncio -import json import os from dotenv import load_dotenv -from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse, HttpRequest from azure.identity.aio import DefaultAzureCredential from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( + JobStatus, OptimizationAgentIdentifier as AgentIdentifier, OptimizationEvaluatorRef as EvaluatorRef, - JobStatus, OptimizationJob, OptimizationJobInputs, OptimizationOptions, @@ -63,7 +60,7 @@ eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") -terminal_statuses = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} +TERMINAL_STATUSES = {JobStatus.SUCCEEDED, JobStatus.FAILED, JobStatus.CANCELLED} async def main() -> None: @@ -76,44 +73,52 @@ async def main() -> None: # 1. Create an optimization job without SDK polling. # ------------------------------------------------------------------ print("Creating optimization job...") - initial_responses: list[PipelineResponse[HttpRequest, AsyncHttpResponse]] = [] - - def capture_created_job_response( - response: PipelineResponse[HttpRequest, AsyncHttpResponse], - ) -> None: - initial_responses.append(response) + pipeline_responses = [] + + def raw_response_hook(response): + # The raw_response_hook is called synchronously before the generated LRO method + # awaits read() on the initial response. Capture the pipeline response object here + # and parse the body afterwards, when read() has already been awaited. + pipeline_responses.append(response) + + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ) + ) await project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) - ), + job=job, polling=False, - raw_response_hook=capture_created_job_response, + raw_response_hook=raw_response_hook, ) - if not initial_responses: + # Alternatively, have the SDK handle polling by removing `polling=False`, assigning the awaited call + # to a poller, and then awaiting `poller.result()`. + if not pipeline_responses: raise RuntimeError("The create operation did not return an optimization job.") - job = OptimizationJob(json.loads(initial_responses[0].http_response.text())) + job = OptimizationJob(pipeline_responses[0].http_response.json()) print(f"Created job: id={job.id}, status={job.status}") # ------------------------------------------------------------------ - # 2. Poll the job to completion. + # 2. Poll until the job reaches a terminal state. # ------------------------------------------------------------------ - while job.status not in terminal_statuses: + print(f"Polling job `{job.id}` to completion...", end="", flush=True) + while job.status not in TERMINAL_STATUSES: await asyncio.sleep(poll_interval) job = await project_client.beta.agents.get_optimization_job(job_id=job.id) - print(f"Job status: {job.status}") + print(".", end="", flush=True) + print() + print(f"Final job status: `{job.status}`.") if job.warnings: for warning in job.warnings: diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py index 4e914f36cfb8..42b0e9883374 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic.py @@ -7,14 +7,15 @@ """ DESCRIPTION: Given an AIProjectClient, this sample demonstrates how to create an agent - optimization job, poll it to completion, and read the results. + optimization job, observe the SDK poller until it is done, and then get + the result. Agent optimization automatically improves an agent's system prompt, model choice, or tool definitions by running candidate variants against your training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic.py + python sample_optimization_job_app_polling.py Before running the sample: @@ -23,16 +24,17 @@ Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. - 3) DATASET_NAME - Required. The name of the registered training dataset. - 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. - 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. - 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". """ import os +import time from dotenv import load_dotenv @@ -64,28 +66,40 @@ ): # ------------------------------------------------------------------ - # 1. Create an optimization job. + # 1. Create an optimization job and observe the SDK-managed poller. # ------------------------------------------------------------------ - print("Creating optimization job...") - result = project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), ), + ) + + print("Begin creating an agent optimization job.") + poller = project_client.beta.agents.begin_create_optimization_job( + job=job, polling_interval=poll_interval, - ).result() - print("Optimization job completed.") + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval) + print(f"status=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") # ------------------------------------------------------------------ # 2. Inspect the results. diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py index 4be9f2ee07c5..74c90b3f0e5f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_basic_async.py @@ -6,27 +6,31 @@ """ DESCRIPTION: - Async version of sample_optimization_job_basic.py. Demonstrates how to - create an agent optimization job, poll it to completion, and read the - results using the async AIProjectClient. + Given an async AIProjectClient, this sample demonstrates how to create an + agent optimization job, observe the SDK poller until it is done, and then + get the result. + + Agent optimization automatically improves an agent's system prompt, model + choice, or tool definitions by running candidate variants against your + training dataset and scoring them with the evaluators you specify. USAGE: - python sample_optimization_job_basic_async.py + python sample_optimization_job_app_polling_async.py Before running the sample: - pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv + pip install "azure-ai-projects>=2.4.0" azure-identity python-dotenv aiohttp Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. - 3) DATASET_NAME - Required. The name of the registered training dataset. - 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. - 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. - 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 3) DATASET_NAME - Required. The name of the registered training dataset. + 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. + 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". """ import asyncio @@ -52,9 +56,9 @@ dataset_name = os.environ["DATASET_NAME"] evaluator_name = os.environ["EVALUATOR_NAME"] dataset_version = os.environ.get("DATASET_VERSION", "1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") -poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) async def main() -> None: @@ -64,29 +68,40 @@ async def main() -> None: ): # ------------------------------------------------------------------ - # 1. Create an optimization job. + # 1. Create an optimization job and observe the SDK-managed poller. # ------------------------------------------------------------------ - print("Creating optimization job...") - poller = await project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), ), + ) + + print("Begin creating an agent optimization job.") + poller = await project_client.beta.agents.begin_create_optimization_job( + job=job, polling_interval=poll_interval, ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + await asyncio.sleep(poll_interval) + print(f"status=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. result = await poller.result() - print("Optimization job completed.") + print(f"Final LRO status: `{poller.status()}`.") # ------------------------------------------------------------------ # 2. Inspect the results. diff --git a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py index 0a80c5bc41fd..23750ec9be02 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py +++ b/sdk/ai/azure-ai-projects/samples/agents/optimization/sample_optimization_job_cancel.py @@ -19,16 +19,18 @@ Set these environment variables with your own values: 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your Microsoft Foundry portal. - 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. + 2) FOUNDRY_AGENT_NAME - Required. The name of the agent to optimize. 3) DATASET_NAME - Required. The name of the registered training dataset. 4) EVALUATOR_NAME - Required. The name of a registered project evaluator. 5) DATASET_VERSION - Optional. Version of the training dataset. Defaults to "1". - 6) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". - 7) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". + 6) POLL_INTERVAL_SECONDS - Optional. Seconds between status polls. Defaults to 10. + 7) EVAL_MODEL - Optional. The model used for evaluation. Defaults to "gpt-4o". + 8) OPTIMIZATION_MODEL - Optional. The model used for optimization. Defaults to "gpt-5.1". """ import os +import time from dotenv import load_dotenv @@ -50,6 +52,7 @@ dataset_name = os.environ["DATASET_NAME"] evaluator_name = os.environ["EVALUATOR_NAME"] dataset_version = os.environ.get("DATASET_VERSION", "1") +poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) eval_model = os.environ.get("EVAL_MODEL", "gpt-4o") optimization_model = os.environ.get("OPTIMIZATION_MODEL", "gpt-5.1") @@ -60,41 +63,50 @@ ): # ------------------------------------------------------------------ - # 1. Create a job. + # 1. Create an optimization job and retain the SDK-managed poller. # ------------------------------------------------------------------ - print("Creating optimization job...") + job = OptimizationJob( + inputs=OptimizationJobInputs( + agent=AgentIdentifier(agent_name=agent_name), + train_dataset=ReferenceDatasetInput( + name=dataset_name, + version=dataset_version, + ), + evaluators=[EvaluatorRef(name=evaluator_name)], + options=OptimizationOptions( + max_candidates=3, + eval_model=eval_model, + optimization_model=optimization_model, + ), + ), + ) + created_jobs: list[OptimizationJob] = [] - def capture_created_job(response): + def raw_response_hook(response): + response.http_response.read() created_jobs.append(OptimizationJob(response.http_response.json())) - project_client.beta.agents.begin_create_optimization_job( - job=OptimizationJob( - inputs=OptimizationJobInputs( - agent=AgentIdentifier(agent_name=agent_name), - train_dataset=ReferenceDatasetInput( - name=dataset_name, - version=dataset_version, - ), - evaluators=[EvaluatorRef(name=evaluator_name)], - options=OptimizationOptions( - max_candidates=3, - eval_model=eval_model, - optimization_model=optimization_model, - ), - ) - ), - polling=False, - raw_response_hook=capture_created_job, + print("Begin creating an agent optimization job.") + poller = project_client.beta.agents.begin_create_optimization_job( + job=job, + polling_interval=poll_interval, + raw_response_hook=raw_response_hook, ) if not created_jobs: raise RuntimeError("The create operation did not return an optimization job.") - job = created_jobs[0] - print(f"Created job: id={job.id}, status={job.status}") + created_job = created_jobs[0] + print(f"Created job: id={created_job.id}, status={created_job.status}") # ------------------------------------------------------------------ # 2. Cancel it immediately. # ------------------------------------------------------------------ - print(f"Cancelling job {job.id}...") - cancelled = project_client.beta.agents.cancel_optimization_job(job_id=job.id) + print(f"Cancelling job {created_job.id}...") + cancelled = project_client.beta.agents.cancel_optimization_job(job_id=created_job.id) print(f"Job {cancelled.id} status: {cancelled.status}") + + print("Wait for the SDK poller to observe the cancellation.") + while not poller.done(): + time.sleep(poll_interval) + print(f"status=`{poller.status()}`") + print(f"Final LRO status: `{poller.status()}`.") diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py index c75d34c6f65b..9c72ee3e8e9f 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning.py @@ -141,7 +141,7 @@ # ------------------------------------------------------------------ # 2. Submit a fine-tuning data generation job that consumes the file. # ------------------------------------------------------------------ - print("Create a fine-tuning data generation job from the Azure OpenAI file.") + job = DataGenerationJob( inputs=DataGenerationJobInputs( name=f"simpleqna-finetuning-{run_id}", @@ -168,11 +168,24 @@ output_options=DataGenerationJobOutputOptions(name=output_name), ), ) - print("Create a fine-tuning data generation job and wait for it to complete.") - job_result = project_client.beta.datasets.begin_create_generation_job( + + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=job, polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # ------------------------------------------------------------------ # 3. Inspect the generated fine-tuning file outputs. diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py new file mode 100644 index 000000000000..ac604ad9eb2f --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_for_finetuning_async.py @@ -0,0 +1,233 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Generates supervised fine-tuning data from a Markdown reference document + uploaded as an Azure OpenAI File. The sample: + + 1. Uploads a short reference document via the Azure OpenAI Files API + (`purpose=user_data`) so it can be referenced by file id. + 2. Creates a `DataGenerationJob` (scenario=SUPERVISED_FINETUNING, + type=simple_qna) that synthesizes short-answer and long-answer + question / answer pairs from the file content and emits them as + training and validation JSONL files. + 3. Polls the job to completion and prints every generated file output. + 4. Cleans up the generated fine-tuning files and the Azure OpenAI input file. + + `simple_qna` REQUIRES `model_options` — the service uses the configured LLM + to synthesize the QnA pairs. Setting `train_split` triggers a split of + the generated samples into two Azure OpenAI output files. + +USAGE: + python sample_dataset_generation_job_simpleqna_for_finetuning_async.py + + Before running the sample: + + pip install "azure-ai-projects>=2.2.0" azure-identity openai python-dotenv aiohttp + + Set these environment variables with your own values: + 1) FOUNDRY_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found + in the overview page of your Microsoft Foundry project. + 2) FOUNDRY_MODEL_NAME - Required. The name of an Azure OpenAI model + deployment used to synthesize the QnA samples. For `simple_qna` fine-tuning, + the deployment must support the chat completions API (e.g. `gpt-4o`, `gpt-4.1`). + 3) DATASET_NAME - Optional. Name to assign to the generated output files + (used as the file name prefix). Defaults to `simpleqna-finetuning-sample`. + The service caps the rendered output name at 50 characters, so keep + custom values short — the sample appends a unique run id suffix. + 4) POLL_INTERVAL_SECONDS - Optional. Number of seconds to sleep between status + polls for the data generation job. Defaults to 10. +""" + +import asyncio +import io +import os +import uuid +from datetime import datetime, timezone + +from dotenv import load_dotenv + +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.models import ( + DataGenerationJob, + DataGenerationJobInputs, + DataGenerationJobOutputOptions, + DataGenerationJobScenario, + DataGenerationModelOptions, + FileDataGenerationJobOutput, + FileDataGenerationJobSource, + SimpleQnADataGenerationJobOptions, + SimpleQnAFineTuningQuestionType, +) + +load_dotenv() + +endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] +model_name = os.environ["FOUNDRY_MODEL_NAME"] +dataset_name = os.environ.get("DATASET_NAME", "simpleqna-finetuning-sample") +poll_interval_seconds = int(os.environ.get("POLL_INTERVAL_SECONDS", "10")) + +# Unique per-run output name so repeated runs do not collide. +# Output names are capped at 50 characters by the service. +run_id = f"{datetime.now(tz=timezone.utc).strftime('%y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}" +output_name = f"{dataset_name}-{run_id}" +if len(output_name) > 50: + raise ValueError( + f"Output name `{output_name}` exceeds the 50-character service limit. " + f"Lower DATASET_NAME (currently `{dataset_name}`) so that `-` fits within 50 characters." + ) + +# Reference document the sample uploads as an Azure OpenAI file. The service +# requires the file to contain at least 1 KB of content to generate QnA from. +SEED_REFERENCE_DOCUMENT = """# Widgets and Gizmos Reference + +## Products +- Widget: blue, manufactured at Factory 7 in Acme, carbon-fiber, rated to 80 C, sold in packs of 4, 250 g each. +- Gizmo: red, manufactured at Factory 12 in Bedrock, carbon-fiber, rated to 80 C, sold individually, 1.2 kg each. +- Sprocket: green, manufactured at Factory 3 in Acme, stainless steel, rated to 200 C, sold individually, 500 g each. + +## Operations +- Factory operates weekdays 0700-1900 local time. +- Closed on public holidays, except for the annual maintenance run on December 27. +- ISO 9001 certified; audited annually by an independent third party. +- Quality control samples every 100th unit and runs full destructive testing on every 5000th unit. + +## Customer support +- Warranty claims: email support@example.com with the serial number printed on the underside of the product. +- Returns: accepted within 30 days if unopened; opened items are eligible for repair only. +- Bulk orders (50+ units): contact sales@example.com for volume pricing and an extended 90-day return window. +- Replacement parts: orderable directly from the support portal using the original order number. + +## Pricing and SLAs +- Widget pack: USD 24.99 per 4-pack; free shipping on orders over USD 75. +- Gizmo unit: USD 49.99; free shipping on orders over USD 75. +- Sprocket unit: USD 14.99; ships from regional warehouses in 1-2 business days. +- Standard support response: within one business day. Priority support response: within four hours. +""" + + +async def main() -> None: + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential) as project_client, + project_client.get_openai_client() as openai_client, + ): + + # ------------------------------------------------------------------ + # 1. Upload the seed reference document as an Azure OpenAI file. + # ------------------------------------------------------------------ + seed_filename = f"widgets-gizmos-seed-{run_id}.md" + print(f"Upload the seed reference document as Azure OpenAI file `{seed_filename}`.") + seed_file = await openai_client.files.create( + file=(seed_filename, io.BytesIO(SEED_REFERENCE_DOCUMENT.encode("utf-8"))), + purpose="user_data", + ) + print(f"Uploaded Azure OpenAI file (id: {seed_file.id}).") + + # Wait for the file to finish processing — the data generation service + # rejects references to files that are not yet in the `processed` state. + print("Wait for the Azure OpenAI file to be processed.", end="", flush=True) + while seed_file.status not in ("processed", "error"): + await asyncio.sleep(2) + seed_file = await openai_client.files.retrieve(file_id=seed_file.id) + print(".", end="", flush=True) + print() + if seed_file.status != "processed": + raise RuntimeError(f"Azure OpenAI file `{seed_file.id}` failed to process: status=`{seed_file.status}`.") + + # ------------------------------------------------------------------ + # 2. Submit a fine-tuning data generation job that consumes the file. + # ------------------------------------------------------------------ + + job = DataGenerationJob( + inputs=DataGenerationJobInputs( + name=f"simpleqna-finetuning-{run_id}", + scenario=DataGenerationJobScenario.SUPERVISED_FINETUNING, + sources=[ + FileDataGenerationJobSource( + description="Widgets & Gizmos product / operations reference (Azure OpenAI file).", + id=seed_file.id, + ), + ], + options=SimpleQnADataGenerationJobOptions( + # Service requires max_samples to be between 15 and 1000. + max_samples=15, + # `simple_qna` REQUIRES model_options. + model_options=DataGenerationModelOptions(model=model_name), + # Split generated samples 80% training / 20% validation. + train_split=0.8, + # Ask for both short-answer and long-answer questions. + question_types=[ + SimpleQnAFineTuningQuestionType.SHORT_ANSWER, + SimpleQnAFineTuningQuestionType.LONG_ANSWER, + ], + ), + output_options=DataGenerationJobOutputOptions(name=output_name), + ), + ) + + print("Begin creating a dataset generation job.") + poller = await project_client.beta.datasets.begin_create_generation_job( + job=job, + polling_interval=poll_interval_seconds, + ) + + # Optionally, print the job status periodically while the SDK polls in the background. + print_poller_status = False + + if print_poller_status: + # Start SDK polling in the background and periodically print the job status until it is complete. + result_task = asyncio.create_task(poller.result()) + print("Periodically check job status:") + while not result_task.done(): + await asyncio.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Awaiting the task returns the final deserialized result and propagates any LRO polling exception. + job_result = await result_task + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") + else: + job_result = await poller.result() + print(f"Data generation result: {job_result}") + + # ------------------------------------------------------------------ + # 3. Inspect the generated fine-tuning file outputs. + # ------------------------------------------------------------------ + # `train_split=0.8` produces two Azure OpenAI files: a training partition + # and a validation partition. Both are emitted as FileDataGenerationJobOutput + # entries in `job_result.outputs`. + file_outputs = [ + output for output in (job_result.outputs or []) if isinstance(output, FileDataGenerationJobOutput) + ] + if not file_outputs: + raise RuntimeError("The data generation job did not produce any file outputs.") + + print(f"Generated {len(file_outputs)} fine-tuning file(s):") + for output in file_outputs: + if not output.id: + raise RuntimeError("A file output was returned without an id.") + # Resolve the Azure OpenAI file to surface its real filename and size. + file_info = await openai_client.files.retrieve(file_id=output.id) + print(f" - filename=`{file_info.filename}` id=`{output.id}` bytes={file_info.bytes}") + if job_result.generated_samples is not None: + print(f"Generated samples: {job_result.generated_samples}") + + # ------------------------------------------------------------------ + # 4. Clean up. + # ------------------------------------------------------------------ + for output in file_outputs: + print(f"Delete the generated Azure OpenAI file `{output.id}`.") + await openai_client.files.delete(file_id=output.id) + + print(f"Delete the Azure OpenAI input file `{seed_file.id}`.") + await openai_client.files.delete(file_id=seed_file.id) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py index 984e7041f228..a190400a3d15 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_agent_source.py @@ -53,6 +53,7 @@ """ import os +import time import uuid from datetime import datetime, timezone @@ -158,11 +159,23 @@ output_options=DataGenerationJobOutputOptions(name=output_dataset_name), ), ) - print("Creating data generation job and waiting for completion (polling is handled by the SDK)...") - job_result = project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=job, polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py index 44ee1e5b29fa..62c17c93da10 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_file_source.py @@ -153,39 +153,51 @@ # - The File source contributes the source material (the reference # document uploaded above). # - The Prompt source contributes a steering instruction (difficulty). - print( - "Creating multi-source data generation job (File + Prompt) and waiting for completion (polling is handled by the SDK)..." - ) - job_result = project_client.beta.datasets.begin_create_generation_job( - job=DataGenerationJob( - inputs=DataGenerationJobInputs( - name=f"simpleqna-multisource-{run_id}", - scenario=DataGenerationJobScenario.EVALUATION, - sources=[ - FileDataGenerationJobSource( - description="Widgets & Gizmos product / operations reference (Azure OpenAI file).", - id=seed_file.id, - ), - PromptDataGenerationJobSource( - description="Specifies the question difficulty for SimpleQnA generation.", - prompt="Generate expert-level questions of high difficulty.", - ), - ], - options=SimpleQnADataGenerationJobOptions( - # Service requires max_samples to be between 15 and 1000. - max_samples=15, - # `simple_qna` REQUIRES model_options. - model_options=DataGenerationModelOptions(model=model_name), + job = DataGenerationJob( + inputs=DataGenerationJobInputs( + name=f"simpleqna-multisource-{run_id}", + scenario=DataGenerationJobScenario.EVALUATION, + sources=[ + FileDataGenerationJobSource( + description="Widgets & Gizmos product / operations reference (Azure OpenAI file).", + id=seed_file.id, ), - output_options=DataGenerationJobOutputOptions( - name=output_dataset_name, - description=EXPECTED_OUTPUT_DESCRIPTION, - tags=EXPECTED_OUTPUT_TAGS, + PromptDataGenerationJobSource( + description="Specifies the question difficulty for SimpleQnA generation.", + prompt="Generate expert-level questions of high difficulty.", ), + ], + options=SimpleQnADataGenerationJobOptions( + # Service requires max_samples to be between 15 and 1000. + max_samples=15, + # `simple_qna` REQUIRES model_options. + model_options=DataGenerationModelOptions(model=model_name), + ), + output_options=DataGenerationJobOutputOptions( + name=output_dataset_name, + description=EXPECTED_OUTPUT_DESCRIPTION, + tags=EXPECTED_OUTPUT_TAGS, ), ), + ) + + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( + job=job, polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py index 7db76b290cac..898bc6594069 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_simpleqna_with_prompt_source.py @@ -11,8 +11,8 @@ 1. Creates a `DataGenerationJob` (scenario=EVALUATION, type=simple_qna) that synthesizes question/answer pairs from an inline prompt and writes them - to a new versioned Dataset. Uses `begin_create_generation_job` which returns - `LROPoller[DataGenerationJobResult]`; `.result()` polls automatically. + to a new versioned Dataset. Uses `begin_create_generation_job` and + reports the standard LRO poller's status until the operation completes. 2. Resolves the resulting `DatasetVersion` from the job result. 3. Creates an OpenAI evaluation (`client.evals.create`) with builtin Azure AI evaluators. @@ -117,11 +117,23 @@ def main() -> None: ), ), ) - print("Creating data generation job and waiting for completion (polling is handled by the SDK)...") - job_result = project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=job, polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") # Locate the Dataset output produced by the job. output_name: str = "" diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py index 18a5d59582b1..617de180a35c 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_evaluation.py @@ -129,7 +129,7 @@ start_time = seed_start - timedelta(minutes=5) - job = None + job_result = None for attempt in range(1, MAX_JOB_ATTEMPTS + 1): end_time = datetime.now(tz=timezone.utc) print( @@ -138,7 +138,8 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - job = project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-eval-{run_id}-a{attempt}", @@ -157,8 +158,19 @@ ), ), polling_interval=POLL_INTERVAL_SECONDS, - ).result() - print(f"Data generation job succeeded.") + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(POLL_INTERVAL_SECONDS) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") break except Exception as e: # pylint: disable=broad-except if attempt == MAX_JOB_ATTEMPTS: @@ -167,9 +179,9 @@ time.sleep(RETRY_WAIT_SECONDS) # 3. Resolve the generated dataset. - if job is None: + if job_result is None: raise RuntimeError("The data generation job did not return a result.") - outputs = job.outputs or [] + outputs = job_result.outputs or [] dataset_output = next((o for o in outputs if isinstance(o, DatasetDataGenerationJobOutput)), None) if dataset_output is None or not dataset_output.name or not dataset_output.version: raise RuntimeError("The data generation job did not produce a dataset output.") @@ -179,8 +191,8 @@ f"Generated dataset: name=`{created_dataset.name}` " f"version=`{created_dataset.version}` id=`{created_dataset.id}`" ) - if job.generated_samples is not None: - print(f"Generated samples: {job.generated_samples}") + if job_result.generated_samples is not None: + print(f"Generated samples: {job_result.generated_samples}") finally: # Best-effort cleanup, outputs -> producers (dataset, job, conversations, agent). diff --git a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py index 114e6c99fa35..783a54a78c3f 100644 --- a/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py +++ b/sdk/ai/azure-ai-projects/samples/datasets/sample_dataset_generation_job_traces_for_finetuning.py @@ -130,7 +130,7 @@ start_time = seed_start - timedelta(minutes=5) - job = None + job_result = None for attempt in range(1, MAX_JOB_ATTEMPTS + 1): end_time = datetime.now(tz=timezone.utc) print( @@ -139,7 +139,8 @@ f"window: {start_time.isoformat()} .. {end_time.isoformat()})." ) try: - job = project_client.beta.datasets.begin_create_generation_job( + print("Begin creating a dataset generation job.") + poller = project_client.beta.datasets.begin_create_generation_job( job=DataGenerationJob( inputs=DataGenerationJobInputs( name=f"traces-ft-{run_id}-a{attempt}", @@ -160,8 +161,19 @@ ), ), polling_interval=POLL_INTERVAL_SECONDS, - ).result() - print(f"Data generation job succeeded.") + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(POLL_INTERVAL_SECONDS) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + job_result = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Data generation result: {job_result}") break except Exception as e: # pylint: disable=broad-except if attempt == MAX_JOB_ATTEMPTS: @@ -170,9 +182,9 @@ time.sleep(RETRY_WAIT_SECONDS) # 3. Resolve generated fine-tuning files. - if job is None: + if job_result is None: raise RuntimeError("The data generation job did not return a result.") - outputs = job.outputs or [] + outputs = job_result.outputs or [] file_outputs = [o for o in outputs if isinstance(o, FileDataGenerationJobOutput)] if not file_outputs: raise RuntimeError("The data generation job did not produce any file outputs.") @@ -184,8 +196,8 @@ created_file_ids.append(output.id) file_info = openai_client.files.retrieve(file_id=output.id) print(f" - filename=`{file_info.filename}` id=`{output.id}` bytes={file_info.bytes}") - if job.generated_samples is not None: - print(f"Generated samples: {job.generated_samples}") + if job_result.generated_samples is not None: + print(f"Generated samples: {job_result.generated_samples}") finally: # Best-effort cleanup, outputs -> producers (files, job, conversations, agent). diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py index 99e4b7496962..1ddb24f1b728 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_all_sources.py @@ -48,6 +48,7 @@ """ import os +import time import uuid from datetime import datetime, timedelta, timezone from typing import List @@ -124,9 +125,9 @@ else: print("Skipping Dataset source (FOUNDRY_REFERENCE_DATASET_NAME / _VERSION not set).") - print("Waiting for multi-source job to complete (polling is handled by the SDK)...") + print("Begin creating an evaluator generation job.") try: - evaluator = project_client.beta.evaluators.begin_create_generation_job( + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -138,7 +139,19 @@ ), operation_id=f"rubric-multi-{short}", polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + evaluator = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) @@ -160,9 +173,9 @@ start_time = now - timedelta(days=traces_window_days) end_time = now + timedelta(seconds=600) # small padding for clock skew - print("Waiting for traces job to complete (polling is handled by the SDK)...") + print("Begin creating an evaluator generation job.") try: - evaluator = project_client.beta.evaluators.begin_create_generation_job( + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -185,7 +198,19 @@ ), operation_id=f"rubric-traces-{short}", polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + evaluator = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") # `isinstance` narrows the discriminated `definition` to the rubric subtype. definition = evaluator.definition assert isinstance(definition, RubricBasedEvaluatorDefinition) diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py index 4375842e523f..375d215501c7 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_basic.py @@ -13,8 +13,8 @@ 1. Creates an `EvaluatorGenerationJob` whose only source is an inline natural-language description of the application's purpose, capabilities, and tools. The service synthesizes a rubric tailored to that application. - 2. Calls `begin_create_generation_job` which returns an `LROPoller[EvaluatorVersion]`; - `.result()` polls automatically and returns the generated `EvaluatorVersion`. + 2. Calls `begin_create_generation_job` and reports the standard LRO + poller's status until it returns the generated `EvaluatorVersion`. 3. Creates an OpenAI evaluation referencing the generated evaluator as a testing criterion. 4. Runs the evaluation against inline JSONL sample data. @@ -87,10 +87,8 @@ project_client.get_openai_client() as openai_client, ): # 1. Generate an evaluator from a single `Prompt` source. - # The LRO polls automatically; `.result()` blocks until the job reaches a terminal state - # and returns the produced EvaluatorVersion directly. - print("Waiting for generation job to complete (polling is handled by the SDK)...") - evaluator = project_client.beta.evaluators.begin_create_generation_job( + print("Begin creating an evaluator generation job.") + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -119,7 +117,19 @@ # `operation_id` makes the call idempotent - re-submitting the same id attaches to the existing job. operation_id=f"rubric-eval-basic-{short}", polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + evaluator = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") # On success, the evaluator is automatically saved as version 1. # `isinstance` narrows the discriminated `definition` to the rubric subtype. diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py index 0ca337d5bbfe..1e6dd892fe69 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_iterate.py @@ -37,6 +37,7 @@ """ import os +import time import uuid from datetime import datetime, timezone @@ -69,10 +70,8 @@ AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): # 1. Generate v1 of the evaluator from a single `Prompt` source. - # The LRO polls automatically; `.result()` blocks until the job reaches a terminal state - # and returns the produced EvaluatorVersion directly. - print("Waiting for generation job to complete (polling is handled by the SDK)...") - v1 = project_client.beta.evaluators.begin_create_generation_job( + print("Begin creating an evaluator generation job.") + poller = project_client.beta.evaluators.begin_create_generation_job( job=EvaluatorGenerationJob( inputs=EvaluatorGenerationInputs( model=model_name, @@ -94,7 +93,19 @@ ), operation_id=f"rubric-iterate-{short}", polling_interval=poll_interval_seconds, - ).result() + ) + + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") + + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. + v1 = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {v1}") # `isinstance` narrows the discriminated `definition` to the rubric subtype. v1_definition = v1.definition diff --git a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py index d48a007729de..90d273ed60cd 100644 --- a/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py +++ b/sdk/ai/azure-ai-projects/samples/evaluations/sample_rubric_evaluator_generation_lifecycle.py @@ -10,8 +10,8 @@ jobs. The sample exercises: * `begin_create_generation_job` with `operation_id` for idempotent re-submits; - returns `LROPoller[EvaluatorVersion]` — the SDK polls automatically and - `.result()` blocks until the job reaches a terminal state. + returns `LROPoller[EvaluatorVersion]`, whose status is reported until + the job reaches a terminal state. * `list_generation_jobs` to enumerate recent jobs in the project. * `delete_generation_job` to remove a finished job record. * `delete_version` to remove the persisted evaluator that the job produced. @@ -41,6 +41,7 @@ import os import itertools +import time import uuid from datetime import datetime, timezone from typing import cast @@ -91,32 +92,37 @@ DefaultAzureCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential) as project_client, ): - # 1. Start the generation job LRO. `operation_id` makes the call idempotent - - # re-submitting with the same id returns a poller attached to the existing job. + # 1. Create the generation job. `operation_id` makes the call idempotent. + print("Begin creating an evaluator generation job.") poller = project_client.beta.evaluators.begin_create_generation_job( - job=job_body, operation_id=operation_id, polling_interval=poll_interval_seconds + job=job_body, + operation_id=operation_id, + polling_interval=poll_interval_seconds, ) - print("Generation job started; LRO polling in progress.") - # Idempotency: a second call with the same operation_id attaches to the same job. - replay_poller = project_client.beta.evaluators.begin_create_generation_job( - job=job_body, operation_id=operation_id, polling_interval=poll_interval_seconds - ) + # Optional: While SDK is polling, periodically print the job status until the job is complete + print("Periodically check job status:") + while not poller.done(): + time.sleep(poll_interval_seconds) + print(f"\tstatus=`{poller.status()}`") - # 2. Block until the LRO finishes. The SDK polls automatically; `.result()` returns - # the produced EvaluatorVersion once the job reaches a terminal state. - print("Waiting for the generation job to complete (polling is handled by the SDK)...") + # Since done() is true, result() returns the final deserialized job result without + # waiting further. It also propagates any LRO polling exception. evaluator: EvaluatorVersion = poller.result() + print(f"Final LRO status: `{poller.status()}`.") + print(f"Evaluator generation result: {evaluator}") print( f"Generated evaluator `{evaluator.name}` version `{evaluator.version}` " f"(job `{evaluator.generation_job_id}`)." ) - # Verify the idempotency: the replay poller resolves to the same underlying job. - replay_evaluator: EvaluatorVersion = replay_poller.result() - assert replay_evaluator.generation_job_id == evaluator.generation_job_id + # Retrieve the persisted generation job using the id returned in the LRO result. + if evaluator.generation_job_id is None: + raise RuntimeError("The generated evaluator did not include a generation job id.") + replay_job = project_client.beta.evaluators.get_generation_job(evaluator.generation_job_id) + assert replay_job.id == evaluator.generation_job_id - # 3. List the 5 most recent generation jobs in this project. + # 2. List the 5 most recent generation jobs in this project. # `limit` controls the page size; use `itertools.islice` to cap the total. print("Recent generation jobs:") for entry in itertools.islice( @@ -125,10 +131,10 @@ entry_name = entry.inputs.evaluator_name if entry.inputs is not None else "" print(f" - id=`{entry.id}` status=`{cast(JobStatus, entry.status).value}` evaluator_name=`{entry_name}`") - # 4. Cancel a running job (not exercised here; the job above already completed). + # 3. Cancel a running job (not exercised here; the job above already completed). # cancelled = project_client.beta.evaluators.cancel_generation_job(some_running_job_id) - # 5. Clean up. `delete_version` cascades to the generation job record, so + # 4. Clean up. `delete_version` cascades to the generation job record, so # the explicit delete below may return 404. print("Cleaning up.") project_client.beta.evaluators.delete_version(name=evaluator.name, version=evaluator.version) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py index 00e84fd43ee1..f1ccc1fd7cea 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples_async.py @@ -167,6 +167,7 @@ async def test_models_samples(self, sample_path: str, **kwargs) -> None: "datasets", samples_to_skip=[ "sample_datasets_async.py", # Skipped until re-enabled and recorded on Foundry endpoint that supports the new versioning schema + "sample_dataset_generation_job_simpleqna_for_finetuning_async.py", # Need to add recordings ], ), )