diff --git a/eng/common/pipelines/ai-evals-tests.yml b/eng/common/pipelines/ai-evals-tests.yml deleted file mode 100644 index 43a7ae5926d6..000000000000 --- a/eng/common/pipelines/ai-evals-tests.yml +++ /dev/null @@ -1,48 +0,0 @@ -trigger: - branches: - include: - - main - paths: - include: - - .github/copilot-instructions.md - - eng/common/instructions/azsdk-tools/** - -pr: none - -parameters: - - name: EvalProject - type: string - default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations' - - name: OpenAIEndPoint - type: string - default: 'https://openai-shared.openai.azure.com/' - - name: Model - type: string - default: 'gpt-5' - - name: EvalRepoOwner - type: string - default: 'Azure' - - name: EvalRepoName - type: string - default: 'azure-sdk-tools' - - name: EvalRepoCommit - type: string - default: 'main' - -variables: - TargetRepoOwner: ${{ split(variables['Build.Repository.Name'], '/')[0] }} - TargetRepoName: ${{ split(variables['Build.Repository.Name'], '/')[1] }} - TargetRepoCommit: ${{ variables['Build.SourceVersion'] }} - -extends: - template: /eng/common/pipelines/templates/jobs/ai-eval-job.yml - parameters: - EvalProject: ${{ parameters.EvalProject }} - OpenAIEndPoint: ${{ parameters.OpenAIEndPoint }} - Model: ${{ parameters.Model }} - EvalRepoOwner: ${{ parameters.EvalRepoOwner }} - EvalRepoName: ${{ parameters.EvalRepoName }} - EvalRepoCommit: ${{ parameters.EvalRepoCommit }} - TargetRepoOwner: $(TargetRepoOwner) - TargetRepoName: $(TargetRepoName) - TargetRepoCommit: $(TargetRepoCommit) diff --git a/eng/common/pipelines/templates/jobs/ai-eval-job.yml b/eng/common/pipelines/templates/jobs/ai-eval-job.yml deleted file mode 100644 index b6e4ceeb7d1e..000000000000 --- a/eng/common/pipelines/templates/jobs/ai-eval-job.yml +++ /dev/null @@ -1,100 +0,0 @@ -parameters: - - name: EvalProject - type: string - default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Evaluations' - - name: OpenAIEndPoint - type: string - default: 'https://openai-shared.openai.azure.com/' - - name: Model - type: string - default: 'gpt-5' -# Repository where evals are located - - name: EvalRepoOwner - type: string - default: 'Azure' - - name: EvalRepoName - type: string - default: 'azure-sdk-tools' - - name: EvalRepoCommit - type: string - default: 'main' -# Target repository for copilot instructions - - name: TargetRepoOwner - type: string - - name: TargetRepoName - type: string - - name: TargetRepoCommit - type: string - -jobs: - - job: Run_Eval - variables: - - template: /eng/pipelines/templates/variables/globals.yml - - template: /eng/pipelines/templates/variables/image.yml - displayName: 'Run AI Eval' - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - steps: - - checkout: none - - task: UseDotNet@2 - displayName: "Use .NET SDK 9.0.x" - retryCountOnTaskFailure: 3 - inputs: - packageType: sdk - version: 9.0.x - performMultiLevelLookup: true - - - task: UseDotNet@2 - displayName: "Use .NET SDK 8.0.x" - retryCountOnTaskFailure: 3 - inputs: - packageType: sdk - version: 8.0.x - performMultiLevelLookup: true - - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: ${{ parameters.EvalRepoOwner }}/${{ parameters.EvalRepoName }} - Commitish: ${{ parameters.EvalRepoCommit }} - WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }} - - Name: ${{ parameters.TargetRepoOwner }}/${{ parameters.TargetRepoName }} - Commitish: ${{ parameters.TargetRepoCommit }} - WorkingDirectory: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }} - Paths: - - 'tools/**' - - 'eng/common/**' - - '.github/**' - - - task: AzureCLI@2 - displayName: 'Run eval' - condition: and(succeeded(), ne(variables['Skip.Eval'], 'true')) - inputs: - azureSubscription: opensource-api-connection - scriptType: 'bash' - scriptLocation: 'inlineScript' - workingDirectory: '$(System.DefaultWorkingDirectory)/${{ parameters.EvalRepoName }}/${{ parameters.EvalProject }}' - inlineScript: | - echo "Logged in to Azure" - echo "Running eval" - dotnet test --logger trx - env: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - DOTNET_MULTILEVEL_LOOKUP: 0 - AZURE_OPENAI_MODEL_DEPLOYMENT_NAME: ${{ parameters.Model }} - AZURE_OPENAI_ENDPOINT: ${{ parameters.OpenAIEndPoint }} - REPOSITORY_NAME: ${{ parameters.TargetRepoName }} - COPILOT_INSTRUCTIONS_PATH_MCP_EVALS: $(System.DefaultWorkingDirectory)/${{ parameters.TargetRepoName }}/.github/copilot-instructions.md - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/*.trx' - testRunTitle: $(System.JobDisplayName) - testResultsFormat: 'VSTest' - mergeTestResults: true