diff --git a/.cspell.json b/.cspell.json index 6de99c14f..b2ab0eff8 100644 --- a/.cspell.json +++ b/.cspell.json @@ -172,6 +172,7 @@ "gqlgen", "gradlew", "Grafana", + "gescazz", "gremcos", "gremgo", "Gruber", @@ -263,6 +264,9 @@ "mkdir", "mkdocs", "mlops", + "msal", + "MSAL", + "microsoftonline", "msrc", "MSRC", "MTBF", @@ -334,6 +338,7 @@ "pwsh", "pycodestyle", "pydocstyle", + "pygments", "Pyflakes", "pylint", "Pylint", diff --git a/.gitattributes b/.gitattributes index 344edca86..58f89fd8f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,4 +16,6 @@ *.{md,[mM][dD]} text eol=lf *.{markdown,[mM][aA][rR][kK][dD][oO][wW][nN]} text eol=lf *.{markdn,[mM][aA][rR][kK][dD][nN]} text eol=lf -*.{mdown,[mM][dD][oO][wW][nN]} text eol=lf \ No newline at end of file +*.{mdown,[mM][dD][oO][wW][nN]} text eol=lf + +.github/workflows/*.lock.yml linguist-generated=true merge=ours \ No newline at end of file diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index 1dafb518b..6cc6a6870 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -15,12 +15,12 @@ jobs: pages: write pull-requests: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.13" architecture: "x64" @@ -32,7 +32,7 @@ jobs: - name: Build site run: mkdocs build - name: Deploy - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./site diff --git a/.gitignore b/.gitignore index 8b0acb5a8..d03dbe4e1 100644 --- a/.gitignore +++ b/.gitignore @@ -345,3 +345,6 @@ _site site/ mkdocs/ megalinter-reports/ + +# Copilot agent tracking artifacts +.copilot-tracking/ diff --git a/.mega-linter.yml b/.mega-linter.yml index 6bf22fe3f..588627273 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -12,7 +12,9 @@ ENABLE_LINTERS: - SPELL_CSPELL - YAML_PRETTIER - YAML_YAMLLINT - - SPELL_LYCHEE + # SPELL_LYCHEE intentionally not enabled yet: the link checker surfaces a large + # backlog of pre-existing broken/blocked external links. Re-enable once those + # are triaged. The lychee.toml config is kept ready for when it's turned on. SPELL_CSPELL_DISABLE_ERRORS: true SHOW_ELAPSED_TIME: true diff --git a/docs/CI-CD/README.md b/docs/CI-CD/README.md index 9a377d668..61a57d15d 100644 --- a/docs/CI-CD/README.md +++ b/docs/CI-CD/README.md @@ -62,10 +62,19 @@ AppVeyor is another free CI service for open source projects which also supports AI tools can accelerate writing CI/CD pipeline YAML, jobs, and scripting snippets, but they must be used with explicit guardrails. +For AI-powered applications, CI/CD also needs to validate the release artifacts that shape model behavior. We treat those artifacts as deployable units with owners, review history, rollback paths, and release gates, rather than as loose configuration. The artifacts we version this way include: + +- Prompts and safety policies +- Model configuration and grounding indexes +- Evaluation datasets and tool permission manifests + +Use the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared review, security, and traceability expectations when AI assistance is used to draft pipeline or release automation changes. + Suggested workflow: - Use AI to draft CI/CD pipeline templates or job steps as a starting point (for example, generating a minimal GitHub Actions workflow). - Run the draft pipeline in a safe non-production environment or CI sandbox to validate syntax and basic behavior. - Require a human reviewer to validate generated steps for correctness, idempotence, and security implications (especially around secrets, permissions, and external actions). +- For generative AI or agentic features, add automated evaluation, safety, grounding, and tool-permission checks before deployment to shared environments. - Add tests or smoke checks to the pipeline so changes can be validated automatically when the pipeline runs. - Promote approved templates into a central location (for example, `.github/workflows/` or a shared pipeline template repository) so teams reuse vetted, audited pipelines. @@ -74,9 +83,12 @@ Guardrails and checklist (before merging AI-generated pipeline changes): - [ ] No secrets or credentials are hard-coded - [ ] Required linting and syntax checks pass locally and in CI - [ ] Security and license scans run and report no critical issues +- [ ] Prompt, model, retrieval, safety, and tool-permission changes have evaluation evidence in the PR or release record +- [ ] AI-enabled systems include evaluation gates for prompt, model, retrieval, and agent behavior, plus safety and regression checks, where applicable - [ ] Pipeline steps are idempotent and have clear rollback strategies where applicable - [ ] Generated content is annotated in the PR description (e.g., "AI-assisted draft") so reviewers know to apply extra scrutiny Notes: - AI-generated pipelines are excellent for reducing boilerplate and accelerating iteration, but they do not replace domain knowledge and security review. - Maintain a small set of vetted pipeline templates to reduce risk and improve reproducibility. +- Use the [generative AI and agentic systems](../ml-and-ai-projects/generative-ai-and-agentic-systems.md#review-the-system-across-disciplines) guidance when a delivery pipeline changes prompts, grounding data, model settings, safety controls, or agent tools. diff --git a/docs/CI-CD/continuous-integration.md b/docs/CI-CD/continuous-integration.md index 6d80d491a..798bcdbb4 100644 --- a/docs/CI-CD/continuous-integration.md +++ b/docs/CI-CD/continuous-integration.md @@ -28,6 +28,10 @@ A robust build automation pipeline will: ### Code / Manifest Artifacts Required to Build Your Project Should be Maintained Within Your Projects Git Repository - CI provider-specific build pipeline definition(s) should reside within your project(s) git repository(s). +- For generative AI and agentic applications, prompts, evaluation datasets, model configuration, grounding index definitions, safety policies, orchestration settings, and tool permission manifests should be maintained as versioned artifacts when they affect product behavior. +- AI application artifacts should have the same review, test, rollback, and promotion expectations as application code and infrastructure templates. +- Build validation should fail when prompt, model, retrieval, safety, or tool-permission changes are missing required evaluation evidence or schema checks. +- Generated indexes, embeddings, and caches do not always belong in git, but the source data selection, transformation logic, index configuration, and release record should be traceable. ## Build Automation @@ -82,6 +86,7 @@ Manage as much of the following as possible, as code: - Configuration Files - Configuration Management(ie environment variable automation via [terraform](https://github.com/microsoft/cobalt/blob/master/infra/modules/providers/azure/app-service/main.tf#L49)) - Secret Management(ie creating Azure secrets via [terraform](https://github.com/microsoft/cobalt/blob/master/infra/templates/az-isolated-service-single-region/app.tf#L84)) +- AI application release artifacts, including prompt templates, eval suites, model settings, grounding index definitions, safety policies, and tool permission manifests - Cloud Resource Provisioning - Role Assignments - Load Test Scenarios @@ -180,7 +185,7 @@ The schema has 30+ [validators](https://json-schema.org/tools?query=#validator) ## Integration Validation -An effective way to identify bugs in your build at a rapid pace is to invest early into a reliable suite of automated tests that validate the baseline functionality of the system: +An effective way to identify bugs in your build at a rapid pace is to invest early into a reliable suite of automated tests that validate the baseline functionality of the system. For the unit, integration, and end-to-end testing taxonomy and when to apply each, see the [automated testing fundamentals](../automated-testing/README.md#the-fundamentals) and the [end-to-end testing guide](../automated-testing/e2e-testing/README.md); the points below focus on wiring those tests into CI. ### End-to-End Integration Tests @@ -216,6 +221,8 @@ An effective way to identify bugs in your build at a rapid pace is to invest ear ### Branch Policy Enforcement +The canonical branch and pull request workflow lives in [source control](../source-control/README.md#creating-a-new-repository) and the [pull request guidance](../code-reviews/pull-requests.md). The CI-specific enforcement below ensures builds gate reviews and merges: + - Protected [branch policies](https://help.github.com/en/github/administering-a-repository/about-protected-branches) should be setup on the main branch to ensure that CI stage(s) have passed prior to starting a code review. Code review approvers will only start reviewing a pull request once the CI pipeline run passes for the latest pushed git commit. - Broken builds should block pull request reviews. - Prevent commits directly into main branch. @@ -233,7 +240,7 @@ In the spirit of transparency and embracing frequent communication across a dev ### Everyone Commits to the Git Repository Each Day - End of day checked-in code should contain unit tests at the minimum. -- Run the build locally before checking in to avoid CI pipeline failure saturation. You should verify what caused the error, and try to solve it as soon as possible instead of committing your code. We encourage developers to follow a [lean SDLC principles](https://leankit.com/learn/lean/principles-of-lean-development/). +- Run the build locally before checking in to avoid CI pipeline failure saturation. You should verify what caused the error, and try to solve it as soon as possible instead of committing your code. We encourage developers to follow a [lean SDLC principles](https://www.planview.com/resources/guide/lean-principles-101/). - Isolate work into small chunks which ties directly to business value and refactor incrementally. ## Isolated Environments diff --git a/docs/UI-UX/README.md b/docs/UI-UX/README.md index 26eb0cb0c..2d5f9b7b2 100644 --- a/docs/UI-UX/README.md +++ b/docs/UI-UX/README.md @@ -10,6 +10,17 @@ The goal of the **User Interface** section is to provide guidance on developing Keep in mind that like all software, there is no "right way" to build a user interface application. Leverage and trust your team's or your customer's experience and expertise for the best development experience. +## AI-assisted UI and UX work + +AI-generated prototypes, personas, copy, and flows are exploration artifacts, not validated user research. Review them with designers, engineers, product owners, and affected users before treating them as product direction. + +1. Review generated personas, content, alt text, captions, and flows for accessibility, inclusion, bias, plain language, and cognitive load. +1. For AI features in the user experience, design disclosure, user control, fallback, appeal, and human escalation. +1. Include UX evaluation for trust calibration, error recovery, overreliance, and user understanding of AI limitations. +1. Confirm which AI tools are approved for this project and data type before uploading designs, screenshots, transcripts, or customer context. + +Use the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared accessibility, privacy, and governance practices. + ## General Guidance The state of web platform engineering is fast moving. There is no one-size-fits-all solution. For any team to be successful in building a UI, they need to have an understanding of the higher-level aspects of all UI project. @@ -63,7 +74,7 @@ Design Ops, short for Design Operations, is a practice that focuses on optimizin - Creating and maintaining a design system that includes reusable components, style guides, and design tokens. - Promoting consistency and efficiency by using shared design assets. - For most projects within ISE we use [Fluent UI](https://developer.microsoft.com/en-us/fluentui#/controls/webcomponents) to handle most projects, this enables rapid development that allow for web application re-use on non-customer engagements or _white label_ applications. - - Other Design Systems used by customers include: [Google's Material Design](https://mui.com/material-ui/), + - Other Design Systems used by customers include: [Google's Material Design](https://mui.com/material-ui/). 4. **Documentation**: - Documenting design decisions, guidelines, and best practices. @@ -77,7 +88,7 @@ Design Ops, short for Design Operations, is a practice that focuses on optimizin 6. **Metrics and KPIs**: - Defining key performance indicators (KPIs) to measure the effectiveness of design processes. - Using metrics to identify areas for improvement and track progress over time. - - __For long-term projects:__ Incorporate _A/B testing_ for better user experiences, and enhancements to the solution. + - **For long-term projects:** Incorporate _A/B testing_ for better user experiences, and enhancements to the solution. ### Benefits of Design Ops for Software Engineers and Product Owners @@ -90,12 +101,12 @@ By integrating Design Ops into the development process, software engineers can w ## Establishing a web application's architecture -The benefit of building software applications is that there are truly infinite ways to build something. A team can use the latest shiny tools, or they can utilize the tried-and-tested ones. It is for this reason that focussing completely on the user until a solution is defined is better than obsessing over technology choices. +The benefit of building software applications is that there are truly infinite ways to build something. A team can use the latest shiny tools, or they can utilize the tried-and-tested ones. It is for this reason that focussing completely on the user until a solution is defined is better than obsessing over technology choices. When choosing a front-end framework or library, consider the project's complexity, performance, and scalability needs. Evaluate the team's expertise with potential options to ensure efficient development. Assess long-term maintainability and community support. Conduct [Trade Studies](./../design/design-reviews/trade-studies/README.md) to weigh pros and cons, focusing on alignment with project goals and user experience. This thorough analysis helps balance innovation with practicality. -### Some platforms/frameworks to consider when planning a project: +### Some platforms/frameworks to consider when planning a project 1. HTML/CSS/JavaScript - Back to the basics! Start with a single **index.html**, include a popular CSS framework such as [Bootstrap](https://getbootstrap.com/) using their CDN link, and start prototyping! @@ -112,7 +123,7 @@ When choosing a front-end framework or library, consider the project's complexit - A robust framework for building client-side applications. [Angular](https://angular.dev/) provides a comprehensive solution with built-in features like dependency injection, routing, and state management. - Ideal for large-scale applications where maintainability and scalability are crucial. - Better Unit Testing support out of the box than React. - - __Angular has no support for Fluent UI, and has tight integration with Google Cloud Platform.__ + - **Angular has no support for Fluent UI, and has tight integration with Google Cloud Platform.** 1. Blazor - A framework for building interactive web UIs using C# instead of JavaScript. [Blazor](https://dotnet.microsoft.com/en-us/apps/aspnet/web-apps/blazor) allows you to share code between the client and server. - Great for teams already familiar with the .NET ecosystem. @@ -125,7 +136,7 @@ When choosing a front-end framework or library, consider the project's complexit - A React-based framework that enables server-side rendering and static site generation. [Next.js](https://nextjs.org/) improves performance and SEO. - Great for building fast, scalable web applications with a focus on developer experience. - Provides built-in support for API routes, making it easier to build full-stack applications. - - __Recommended by Meta going forward rather than use create-react-app.__ + - **Recommended by Meta going forward rather than use create-react-app.** - Next.js is very heavy in terms of features, and it doesn't allow for re-use in non-Next.js applications like create-react-app, React-Native, or Electron based apps. 1. Svelte - [Svelte](https://svelte.dev/) is a modern framework that shifts much of the work to compile time, resulting in highly optimized and performant applications. @@ -146,5 +157,5 @@ When choosing a front-end framework or library, consider the project's complexit ### Further information > For more information of utilizing any of these frameworks/platforms, read the [Recommended Technologies](./recommended-technologies.md) document. -> +> > Continue reading the [Trade Study](./../design/design-reviews/trade-studies/README.md) section of this site for more information on completing this step in the design process. diff --git a/docs/UI-UX/recommended-technologies.md b/docs/UI-UX/recommended-technologies.md index b537c116b..b2a55b528 100644 --- a/docs/UI-UX/recommended-technologies.md +++ b/docs/UI-UX/recommended-technologies.md @@ -3,32 +3,32 @@ ## Table of Contents - [TypeScript](#typescript) - - [Guidance on types and interfaces](#guidance-on-types-and-interfaces) - - [Example](#example) + - [Guidance on types and interfaces](#guidance-on-types-and-interfaces) + - [Example](#example) - [Bootstrapping Web Projects](#bootstrapping-web-projects) - - [Vite](#vite) - - [Documentation & Recommended Resources](#documentation--recommended-resources) - - [Installing Vite](#installing-vite) - - [Bootstrapping a project](#bootstrapping-a-project) + - [Vite](#vite) + - [Documentation & Recommended Resources](#documentation--recommended-resources) + - [Installing Vite](#installing-vite) + - [Bootstrapping a project](#bootstrapping-a-project) - [React](#react) - - [Quick note on create-react-app](#quick-note-on-create-react-app) - - [Documentation & Using older React coding patterns](#documentation--using-older-react-coding-patterns) - - [Best Practices to Organize Your React Codebase](#best-practices-to-organize-your-react-codebase) - - [Project Structure](#project-structure) - - [Key Directories and Files](#key-directories-and-files) - - [Directory Best Practices](#directory-best-practices) - - [Using React with Entra ID](#using-react-with-entra-id) - - [Resources](#resources) + - [Quick note on create-react-app](#quick-note-on-create-react-app) + - [Documentation & Using older React coding patterns](#documentation--using-older-react-coding-patterns) + - [Best Practices to Organize Your React Codebase](#best-practices-to-organize-your-react-codebase) + - [Project Structure](#project-structure) + - [Key Directories and Files](#key-directories-and-files) + - [Directory Best Practices](#directory-best-practices) + - [Using React with Entra ID](#using-react-with-entra-id) + - [Resources](#resources) - [Managing Secrets with Environmental Variables in React Projects Using Vite](#managing-secrets-with-environmental-variables-in-react-projects-using-vite) - - [Setting Up Environmental Variables](#setting-up-environmental-variables) - - [Best Security Practices](#best-security-practices) - - [Resources](#resources-1) + - [Setting Up Environmental Variables](#setting-up-environmental-variables) + - [Best Security Practices](#best-security-practices) + - [Resources](#resources-1) - [Common Security Pitfalls in React Applications](#common-security-pitfalls-in-react-applications) - - [Resources](#resources-2) + - [Resources](#resources-2) - [Incorporating Axios for REST API Calls in React with Bearer Tokens and MSAL](#incorporating-axios-for-rest-api-calls-in-react-with-bearer-tokens-and-msal) - - [Setting Up Axios](#setting-up-axios) - - [Making API Calls](#making-api-calls) - - [Resources](#resources-3) + - [Setting Up Axios](#setting-up-axios) + - [Making API Calls](#making-api-calls) + - [Resources](#resources-3) --- @@ -37,7 +37,7 @@ The purpose of this page is to review the commonly selected technology options w > Keep in mind that like all software, there is no "right way" to build a user interface application. Leverage and trust your team's or your customer's experience and expertise for the best development experience. -Additionally, while some of these technologies are presented as alternate options, many can be combined together. For example, you can use React in a basic HTML/CSS/JS workflow by inline-importing React along with Babel. See the [Add React to a Website](https://reactjs.org/docs/add-react-to-a-website.html) for more details. Similarly, any [Fast](https://www.fast.design/) web component can be [integrated into any existing React application](https://fast.design/docs/integrations#react). And of course, every JavaScript technology can also be used with TypeScript! +Additionally, while some of these technologies are presented as alternate options, many can be combined together. For example, you can use React in a basic HTML/CSS/JS workflow by inline-importing React along with Babel. See the [Add React to a Website](https://reactjs.org/docs/add-react-to-a-website.html) for more details. Similarly, any [Fast](https://www.fast.design/) web component can be [integrated into any existing React application](https://www.fast.design/docs/3.x/introduction). And of course, every JavaScript technology can also be used with TypeScript! ## TypeScript @@ -187,28 +187,28 @@ my-react-app/ #### Key Directories and Files -- **public/**: Contains static assets like HTML and images. -- **src/**: Contains the source code of your application. - - **assets/**: Contains global assets like styles and images. - - **components/**: Contains reusable UI components, each in its own folder. - - **hooks/**: Contains custom React hooks. - - **pages/**: Contains page components, each in its own folder. - - **services/**: Contains service modules for API calls and other business logic. - - **types/**: Contains TypeScript type definitions. - - **utils/**: Contains utility functions. - - **App.tsx**: The root component of your application. - - **index.tsx**: The entry point of your application. - - **vite-env.d.ts**: TypeScript definitions for Vite-specific features. +- __public/__: Contains static assets like HTML and images. +- __src/__: Contains the source code of your application. + - __assets/__: Contains global assets like styles and images. + - __components/__: Contains reusable UI components, each in its own folder. + - __hooks/__: Contains custom React hooks. + - __pages/__: Contains page components, each in its own folder. + - __services/__: Contains service modules for API calls and other business logic. + - __types/__: Contains TypeScript type definitions. + - __utils/__: Contains utility functions. + - __App.tsx__: The root component of your application. + - __index.tsx__: The entry point of your application. + - __vite-env.d.ts__: TypeScript definitions for Vite-specific features. #### Directory Best Practices -1. **Component-Based Structure**: Organize your code into small, reusable components. -2. **Type Safety**: Use TypeScript to define types and interfaces for your components and data structures. -3. **Custom Hooks**: Encapsulate reusable logic in custom hooks. -4. **Service Layer**: Abstract API calls and business logic into service modules. -5. **Consistent Naming**: Follow a consistent naming convention for files and directories. -6. **Environment Variables**: Use environment variables to manage configuration settings. -7. **Testing**: Write unit tests for your components, hooks, and services. +1. __Component-Based Structure__: Organize your code into small, reusable components. +2. __Type Safety__: Use TypeScript to define types and interfaces for your components and data structures. +3. __Custom Hooks__: Encapsulate reusable logic in custom hooks. +4. __Service Layer__: Abstract API calls and business logic into service modules. +5. __Consistent Naming__: Follow a consistent naming convention for files and directories. +6. __Environment Variables__: Use environment variables to manage configuration settings. +7. __Testing__: Write unit tests for your components, hooks, and services. ### Using React with Entra ID @@ -217,12 +217,12 @@ Integrating [React](https://reactjs.org/) applications with __Entra ID__ (former To get started with Entra ID in your React application, follow these steps: -1. **Install MSAL**: +1. __Install MSAL__: ```sh npm install @azure/msal-browser @azure/msal-react ``` -2. **Configure MSAL**: +2. __Configure MSAL__: Create a configuration file to set up your MSAL instance with your Entra ID credentials. ```typescript @@ -240,7 +240,7 @@ To get started with Entra ID in your React application, follow these steps: ``` -3. **Wrap Your Application**: +3. __Wrap Your Application__: Use the `MsalProvider` to wrap your React application. ```typescript @@ -264,7 +264,7 @@ When developing React applications, it's crucial to manage sensitive information ### Setting Up Environmental Variables -1. **Create an `.env` File**: +1. __Create an `.env` File__: In the root of your project, create a file named `.env`. This file will store your environmental variables. ```plaintext @@ -272,7 +272,7 @@ When developing React applications, it's crucial to manage sensitive information VITE_API_URL=[sample URL] ``` -2. **Access Environmental Variables in Your Code**: +2. __Access Environmental Variables in Your Code__: Vite prefixes all environmental variables with `VITE_` to ensure they are exposed to your client-side code. You can access these variables using `import.meta.env`. ```typescript @@ -291,7 +291,7 @@ When developing React applications, it's crucial to manage sensitive information ### Best Security Practices -- **Do Not Commit `.env` Files**: +- __Do Not Commit `.env` Files__: Ensure your `.env` file is added to `.gitignore` to prevent it from being committed to your version control system. ```plaintext @@ -299,7 +299,7 @@ When developing React applications, it's crucial to manage sensitive information .env ``` -- **Use Environment-Specific Files**: +- __Use Environment-Specific Files__: Vite supports multiple environment files for different stages of development. For example, you can create `.env.development` and `.env.production` files to manage variables for development and production environments, respectively. ```plaintext @@ -310,21 +310,21 @@ When developing React applications, it's crucial to manage sensitive information VITE_API_URL=[example value] ``` -- **Validate Environmental Variables**: - Consider using a validation library like `zod` ( https://github.com/colinhacks/zod ) or `joi` ( joi.dev ) to validate your environmental variables at runtime, ensuring they meet the required format and constraints. +- __Validate Environmental Variables__: + Consider using a validation library like `zod` ( ) or `joi` ( joi.dev ) to validate your environmental variables at runtime, ensuring they meet the required format and constraints. ## Common Security Pitfalls in React Applications When developing React applications, it's crucial to follow best practices to safeguard your application from common security vulnerabilities. Here are some key things to avoid: -### 1. **Do Not Store Sensitive Data in Local Storage** +### 1. __Do Not Store Sensitive Data in Local Storage__ Local storage is accessible from any JavaScript code running on the same domain, making it a potential target for attackers. Avoid storing sensitive information like tokens or user data in local storage. -**Bearer Tokens in sessionStorage and localStorage should be strictly avoided!** +__Bearer Tokens in sessionStorage and localStorage should be strictly avoided!__ -### 2. **Avoid Using `dangerouslySetInnerHTML`** +### 2. __Avoid Using `dangerouslySetInnerHTML`__ Using `dangerouslySetInnerHTML` can expose your application to XSS attacks. Always sanitize any HTML content before rendering it. @@ -339,15 +339,15 @@ Using `dangerouslySetInnerHTML` can expose your application to XSS attacks. Alwa
``` -### 3. **Do Not Expose API Keys in the Frontend** +### 3. __Do Not Expose API Keys in the Frontend__ Exposing API keys in your frontend code can lead to unauthorized access to your backend services. Use environment variables and server-side code to keep your API keys secure. -### 4. **Do Not Ignore Security Updates** +### 4. __Do Not Ignore Security Updates__ Regularly update your dependencies to include the latest security patches. Use tools like `npm audit` to identify and fix vulnerabilities in your project. -### 5. **Avoid Using Untrusted Third-Party Libraries** +### 5. __Avoid Using Untrusted Third-Party Libraries__ Only use third-party libraries from trusted sources. Review the library's code and check for any known vulnerabilities before including it in your project. @@ -357,12 +357,12 @@ When building React applications that interact with REST APIs, [Axios](https://a ### Setting Up Axios -1. **Install Axios**: +1. __Install Axios__: ```sh npm install axios ``` -2. **Create an Axios Instance**: +2. __Create an Axios Instance__: Configure an Axios instance to include the Bearer token in the Authorization header for all requests. ```typescript diff --git a/docs/agile-development/README.md b/docs/agile-development/README.md index 7022c5ee9..70baf4a00 100644 --- a/docs/agile-development/README.md +++ b/docs/agile-development/README.md @@ -33,6 +33,8 @@ Teams are increasingly adapting traditional Scrum practices to take advantage of These are patterns, not prescriptions — adapt them to fit your engagement and stakeholder needs. +Use the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide as the shared baseline for tool approval, human review, data handling, and traceability. The [Engineering Fundamentals Checklist](../engineering-fundamentals-checklist.md#ai-assisted-engineering) includes AI readiness prompts that can be reflected in the team agreement, definition of ready, and definition of done when relevant. The [Project Kickoff Checklist](../start-here/project-kickoff-checklist.md) sequences these readiness checks and the kickoff session at the start of an engagement. + ## References - [What Is Scrum?](https://www.scrum.org/resources/what-is-scrum) diff --git a/docs/agile-development/branching-and-cicd.md b/docs/agile-development/branching-and-cicd.md index ce16b5096..a0dacb080 100644 --- a/docs/agile-development/branching-and-cicd.md +++ b/docs/agile-development/branching-and-cicd.md @@ -4,15 +4,13 @@ Purpose: Provide a concise, practical policy and examples teams can adopt for in ## Recommended approach -- Prefer trunk-based development where possible for new projects. Use short-lived feature branches when necessary and merge frequently into the default integration branch (commonly `main` or `trunk`). -- Use branch protection rules on the integration branch to enforce quality gates (required passing CI, required code reviews, status checks). +- Prefer trunk-based development where possible for new projects. Use short-lived feature branches when necessary and merge frequently into the default integration branch (commonly `main` or `trunk`). The [source control git guidance](../source-control/git-guidance/README.md#branching) owns the canonical feature-branch workflow. +- Use branch protection rules on the integration branch to enforce quality gates (required passing CI, required code reviews, status checks). The [pull request guidance](../code-reviews/pull-requests.md) owns the canonical pull request and merge policy. - Keep releases simple: use tags/releases from the integration branch and keep release process documented separately. -## Example branch protection rules +## Branch protection and merge policy -- Require at least one approving reviewer for pull requests. -- Require successful CI pipeline status checks before merge. -- Require up-to-date branch before merge if your policy prefers. +Enforce the merge gate through branch protection rules rather than restating it here. The [pull request guidance](../code-reviews/pull-requests.md) and [code review evidence and measures](../code-reviews/evidence-and-measures/README.md) own the canonical policy: changes to the integration branch flow through a pull request that requires at least one approving reviewer, passing CI status checks, a linked work item, and updated documentation. ## Sample minimal GitHub Actions CI gate (example) @@ -34,13 +32,6 @@ jobs: run: npm test ``` -## Merge policy checklist (suggested) - -- [ ] Code compiles and automated tests pass in CI -- [ ] At least one approving reviewer has reviewed the change -- [ ] The change has an associated work item or issue -- [ ] Documentation updated where applicable - ## Tips - Keep feature branches short-lived; frequent merges reduce integration risk. @@ -49,10 +40,6 @@ jobs: ## CI/CD guidance -This page complements the central [CI/CD guidance](../CI-CD/README.md). Key expectations teams should follow: +This page complements the central [CI/CD guidance](../CI-CD/README.md). For the shared expectations — quality pipeline on every PR, infrastructure-as-code provisioning, automated deployment to non-production, and repeatable release and rollback — follow the [CI/CD fundamentals](../CI-CD/README.md#the-fundamentals). -- The integration (main) branch should be continuously shippable and stable — at any point we should be able to deploy a build from `main` to production if needed. -- Run a quality pipeline (linting, unit tests, basic integration tests) on each PR and on merges to the integration branch. -- Provision cloud resources and environment configuration via infrastructure-as-code (for example Terraform, Bicep, Pulumi) and exercise them in non-production environments. -- Deploy release candidates automatically to a non-production environment to validate integration and operational concerns. -- Automate release and rollback procedures so releases are repeatable and auditable. \ No newline at end of file +One branching-specific expectation reinforces those fundamentals: the integration (main) branch should stay continuously shippable and stable, so a build from `main` can be deployed to production at any point if needed. \ No newline at end of file diff --git a/docs/agile-development/team-agreements/definition-of-done.md b/docs/agile-development/team-agreements/definition-of-done.md index a1963e6af..65ab9cf3e 100644 --- a/docs/agile-development/team-agreements/definition-of-done.md +++ b/docs/agile-development/team-agreements/definition-of-done.md @@ -4,6 +4,8 @@ To close a user story, a sprint, or a milestone it is important to verify that t The development team should decide together what their Definition of Done is and document this in the project. Below are some examples of checks to verify that the user story, sprint, task is completed. +The test types referenced below (unit, integration, performance, and end-to-end) follow the shared [automated testing fundamentals](../../automated-testing/README.md#the-fundamentals). + ## Feature/User Story - [ ] Acceptance criteria are met diff --git a/docs/ai-assisted-engineering/README.md b/docs/ai-assisted-engineering/README.md new file mode 100644 index 000000000..cac871ea8 --- /dev/null +++ b/docs/ai-assisted-engineering/README.md @@ -0,0 +1,139 @@ +# AI-Assisted Engineering + +AI tools help us explore ideas, draft code, write tests, summarize context, and improve documentation. They work best when they are part of normal engineering practice: a responsible person still owns the work, the team still validates the output, and our decisions still follow the customer and team context. + +Use this guide as the shared baseline for AI-assisted engineering. Domain guides such as [Developer Experience](../developer-experience/README.md), [Automated Testing](../automated-testing/README.md), [Security](../security/README.md), [Observability](../observability/README.md), [Source Control](../source-control/README.md), [Code Reviews](../code-reviews/README.md), and [Documentation](../documentation/README.md) provide more specific guidance. + +## When to Use AI Assistance + +AI assistance pays off most when an early draft or explanation gives us something concrete to react to, so we reach for it on work where a candidate implementation accelerates learning and delivery: + +- Drafting boilerplate, examples, tests, and documentation, along with supporting artifacts like diagrams and review notes +- Explaining unfamiliar code, logs, errors, and dependencies, including infrastructure templates +- Generating implementation options, refactoring ideas, threat-model prompts, and test scenarios +- Summarizing requirements, backlog items, architecture notes, and design tradeoffs +- Exploring alternatives before the team makes a decision + +We avoid leaning on AI alone where the work demands accountability, project-specific judgment, or policy interpretation: + +- Architecture decisions, security-sensitive logic, privacy decisions, and release approvals +- Consequential decisions that affect people, access, eligibility, or legal status — and anything touching finance, health, or safety +- Customer data handling, regulated data handling, retention decisions, and disclosure obligations +- Production incident response, destructive operations, and changes to cloud or production resources +- Claims about compliance, licensing, or vulnerability status — and any claim about performance or model behavior — made without evidence + +## Human Oversight + +AI can produce confident-looking work that is subtly wrong, so we treat its output as draft material until a responsible person has reviewed, tested, and accepted it. + +- Keep a named human owner for every AI-assisted pull request, design decision, deployment, and customer-facing artifact. +- Review AI-generated code and text with the same or higher care as hand-written work. +- Check generated claims against source material, product documentation, requirements, and customer context. +- Reject generated output that is unrelated, unexplainable, or unverifiable — or that is over-broad or inconsistent with the team architecture. +- Preserve our normal engineering controls — peer review, branch protection, and CI checks — along with security review, accessibility review, and rollback planning. + +## Data and Context Hygiene + +Every prompt is a place data can leak, so we give AI tools only the minimum context they need to complete the task. + +- Confirm which AI tools are approved for the project, customer, data classification, and engagement model before work starts. +- Do not paste secrets, credentials, or private customer data into AI tools — and keep regulated data and unnecessary proprietary context out as well. +- Prefer synthetic, representative, anonymized, or redacted examples when exploring prompts or reproducing issues. +- Strip access tokens, connection strings, and personal data — along with customer identifiers and confidential business details — from any prompts, screenshots, logs, or attachments. +- Understand how a tool handles retention, logging, and telemetry — and its training and data-residency behavior — before feeding it project or customer context. +- Keep prompts and retrieved context scoped to the task. Extra context can increase leakage risk and produce less accurate output. + +## Prompt and Repository Hygiene + +AI tools are only as good as the context they can find, so keeping our repositories well-documented directly improves the results we get. + +- Keep the common build, test, and lint commands easy to find in repository documentation, along with setup and run instructions. +- Document project conventions, architectural boundaries, dependency choices, and coding standards where tools and people can discover them. +- Use repository instruction files, prompt files, or agent configuration only when the team has agreed on their purpose and maintenance owner. +- Avoid instructions that ask AI tools to bypass tests, ignore security review, hide generated changes, or make broad edits outside the task. +- Keep reusable prompts factual and current. Remove outdated assumptions when architecture, tooling, or customer constraints change. + +## Security + +A model has no stake in our security, so we treat its output as untrusted input and review it accordingly. + +- Review generated code for the usual high-risk flaws — injection, authorization, and authentication — and for cryptography, dependency, logging, and error-handling issues. +- Do not execute generated commands or scripts until a responsible person understands their effect. +- Scope AI agent tools, credentials, and file access to the least privilege needed for the task — and do the same for cloud permissions and network access. +- Add threat-model coverage for AI-specific risks using the canonical [AI systems threat-modeling considerations](../security/threat-modelling.md#ai-systems-threat-modeling-considerations). +- Validate generated dependency, container, infrastructure, and CI/CD changes against the normal security review path. + +## Testing and Evaluation + +AI assistance changes how we produce code, but not our need for evidence that it works. + +- Review generated tests for meaningful assertions, realistic setup, negative cases, and failure modes. +- Add deterministic automated tests for ordinary software behavior. +- Add AI-specific evaluations whenever product behavior depends on prompts, retrieval, models, or agents — or on generated content, ranking, summarization, or recommendations. +- Where relevant, evaluate quality, groundedness, safety, and bias, along with robustness, tool-call correctness, regression behavior, and fallback behavior. +- Keep evaluation datasets, rubrics, expected outcomes, and known limitations versioned with the system or linked from the project documentation. + +## Observability + +AI-enabled systems can drift in ways traditional ones do not, so we capture enough telemetry to debug behavior and detect that drift while respecting privacy and policy constraints. + +- Track the signals that let us explain a response after the fact: + - Versions in play: model, configuration, and prompt template versions + - What the system did: retrieval sources, tool calls, and fallback paths + - Cost and performance: latency and token or cost signals + - Outcomes: errors, safety outcomes, evaluation scores, and human escalation +- Avoid logging raw prompts or completions, and keep secrets, personal data, customer data, and sensitive retrieval content out of logs unless the project has an explicit, approved logging design. +- Connect production signals to incident response, rollback, model or prompt change review, and customer support workflows. +- Review observability data for quality regression, unexpected tool use, abuse patterns, and changes in model behavior over time. + +## Authorship and Traceability + +When a decision is questioned later, we want to know where AI materially shaped the work, so we keep that influence visible. + +- Follow the team's convention for documenting material AI assistance in pull requests, commit metadata, work items, or decision records. +- Keep generated changes small enough to review and explain. +- Back AI-assisted decisions with the same evidence we expect for any other decision — requirements, tests, and evaluations, plus design notes, customer constraints, and review comments. +- Do not use AI-generated summaries as the only record of customer decisions or stakeholder approval. +- Make sure generated comments, release notes, and documentation do not claim behavior that has not been implemented and validated. + +## Accessibility and Inclusion + +AI can help us spot accessibility and inclusion issues, but it does not replace user research, accessibility testing, or assistive-technology review. + +- Review AI-generated UI, content, flows, and images for accessibility and inclusive language, as well as cognitive load, localization, and cultural context. +- Validate generated UI with the same accessibility requirements used for the rest of the product. +- Test important experiences manually with keyboard navigation, screen readers, color contrast checks, and other assistive technology where appropriate. +- Include affected users, domain experts, and accessibility reviewers when AI-enabled behavior changes the user experience. + +## Governance + +No single policy fits every engagement, so we adapt this guidance to each project's risk, customer policy, and system impact. + +- Decide early which tools are approved, which data they can access, who owns review, and how exceptions are handled. +- Add AI policy and setup questions to project onboarding, first-week planning, team working agreements, and definitions of done where relevant — the [Project Kickoff Checklist](../start-here/project-kickoff-checklist.md) is where a team decides its AI usage model. +- Route higher-risk AI work through the reviews that fit the risk — Responsible AI assessment, privacy review, and security review, plus design, accessibility, and legal or compliance review — before seeking stakeholder approval. +- Document material limitations, known failure modes, monitoring expectations, and human escalation paths for AI-enabled features. +- Revisit the guidance when tools, customer constraints, model capabilities, or production usage change. + +## Related Playbook Areas + +- [Developer Experience](../developer-experience/README.md) and [Copilots](../developer-experience/copilots.md) for AI tool setup and attribution patterns +- [Automated Testing](../automated-testing/README.md) for test strategy and evaluation practices +- [Security](../security/README.md) for secure engineering practices and threat modeling +- [Observability](../observability/README.md) for telemetry and operational readiness +- [Source Control](../source-control/README.md) and [Code Reviews](../code-reviews/README.md) for review, traceability, and change control +- [Data Handling](../non-functional-requirements/privacy/data-handling.md) for privacy and data protection practices +- [Accessibility](../non-functional-requirements/accessibility.md) for inclusive and accessible delivery +- [Responsible AI](../ml-and-ai-projects/responsible-ai.md) for AI system impact assessment + +## External References + +These external resources expand on the practices above. Review them against your project's customer, data, and policy constraints before adopting them. + +- [microsoft/hve-core](https://github.com/microsoft/hve-core) — open-source Hypervelocity Engineering (HVE) accelerator: reusable Copilot agents, prompts, instructions, and skills built around a Research → Plan → Implement (RPI) workflow. +- [HVE Core documentation](https://microsoft.github.io/hve-core/) — guidance for AI-assisted development across the lifecycle. +- [HVE Guide — project lifecycle](https://microsoft.github.io/hve-core/docs/hve-guide/) and [RPI workflow](https://microsoft.github.io/hve-core/docs/rpi/) — explore and specify before coding, with fast iteration loops. +- [HVE project-planning collection](https://microsoft.github.io/hve-core/docs/getting-started/collections) — reusable requirements and ADR templates. +- [Responsible AI in Azure Workloads](https://learn.microsoft.com/en-us/azure/well-architected/ai/responsible-ai) — Well-Architected Responsible AI guardrails and governance per lifecycle stage. +- [Best practices for using GitHub Copilot](https://docs.github.com/en/copilot/get-started/best-practices) — prompt hygiene, iteration, and human oversight. +- [GitHub Copilot Fundamentals](https://learn.microsoft.com/en-us/training/paths/copilot/) — adoption and SDLC use cases. diff --git a/docs/automated-testing/README.md b/docs/automated-testing/README.md index b40faa5b7..ed3b5ee74 100644 --- a/docs/automated-testing/README.md +++ b/docs/automated-testing/README.md @@ -15,6 +15,16 @@ - We write our tests early and block any further code merging if tests fail. - We run load tests/performance tests where appropriate to validate that the system performs under stress +## Testing AI-assisted and AI-enabled changes + +AI can help draft tests, fixtures, and test ideas, but a human reviewer still owns every assertion and expected outcome. The [AI-Assisted Engineering](../ai-assisted-engineering/README.md#testing-and-evaluation) guide owns the shared "treat output as draft" baseline; the points below cover the testing-specific angle. + +- We add regression tests for any AI-generated code paths, covering the edge cases the tool is most likely to miss. +- We review generated tests for realistic setup, negative cases, clear assertions, and useful failure messages. +- For LLM or agent features, we define evaluation datasets, expected-behavior rubrics, safety tests, and threshold gates. +- We include adversarial and malformed inputs such as prompt injection and sensitive-data probes, along with refusal, fallback, and human-escalation scenarios where they apply. +- We track evaluation failures as defects, capturing reproducible prompts, context, model or version, and expected behavior. + ## Build for Testing Testing is a critical part of the development process. It is important to build your application with testing in mind. Here are some tips to help you build for testing: diff --git a/docs/automated-testing/test-planning.md b/docs/automated-testing/test-planning.md index 39e92f226..0fbfd2b3f 100644 --- a/docs/automated-testing/test-planning.md +++ b/docs/automated-testing/test-planning.md @@ -40,6 +40,23 @@ Going through this exercise doesn't only produce test cases, it also helps clari - Review the test cases with peers or stakeholders to ensure that we have covered everything we want to cover, and that expected outcomes are correct - Refine the test cases based on feedback +### AI Evaluation Planning + +When a story uses generative AI, RAG, or tool-calling agents, plan evaluation cases alongside functional test cases. AI behavior can change when prompts, models, grounding data, tool schemas, or safety settings change, so acceptance criteria should name the quality, safety, and operational evidence needed for release. + +Include these evaluation patterns when they fit the story risk: + +- Golden datasets with representative requests, expected outcomes, allowed variation, and known failure examples +- Prompt regression suites for system instructions, developer prompts, prompt templates, and canary prompts +- Scenario tests that cover happy paths, ambiguous requests, out-of-scope requests, refusal behavior, and recovery after a failed answer or action +- RAG checks for retrieval relevance, groundedness, faithfulness to sources, citation quality, security trimming, and source freshness +- Safety and adversarial tests for jailbreak attempts, indirect prompt injection, harmful content, sensitive disclosure, and misuse cases +- Tool-call contract tests for schema validity, argument validation, authorization, idempotency, dry-run behavior, write-path controls, and error handling +- Human-graded rubrics for judgment-heavy outputs, with stakeholder-approved scoring thresholds and examples of acceptable and unacceptable responses +- Drift monitoring plans that connect production feedback, safety signals, retrieval changes, prompt changes, and model updates back to the evaluation suite + +Define release gates before implementation begins. A gate can require minimum quality scores, no critical safety failures, passing tool-call contract tests, acceptable latency and cost, reviewed red-team findings, and documented follow-up for known limitations. Map the evidence back to story acceptance criteria so product owners, engineers, and QA can decide whether the AI behavior is ready to ship. + ### Examples of Test Cases using the Given-When-Then Format | User Login | (Positive Test Case) | diff --git a/docs/automated-testing/unit-testing/README.md b/docs/automated-testing/unit-testing/README.md index c615ffd8e..482791de9 100644 --- a/docs/automated-testing/unit-testing/README.md +++ b/docs/automated-testing/unit-testing/README.md @@ -97,11 +97,11 @@ An example of using dependency injection can be found [here](./authoring-example #### Test-Driven Development -Test-Driven Development (TDD) is a technique for writing your code that will lead you to a testable design from the start. +Test-Driven Development (TDD) is a technique for writing your code that will lead you to a testable design from the start. The basic premise of test-driven development is that you come up with a list behaviors you want your system to have. You then take one behavior from the list, write the test, and then modify the system to make the test pass. Then you move on to the next behavior on your list and repeat this process. -Once you've exhausted your list, you're done! This approach has the benefit of guaranteeing a testable design is built into +Once you've exhausted your list, you're done! This approach has the benefit of guaranteeing a testable design is built into the system since the test was written first. For more information on TDD and an example, see the page on [Test-Driven Development](./tdd-example.md) diff --git a/docs/code-reviews/README.md b/docs/code-reviews/README.md index 67937b5ec..79661f9d7 100644 --- a/docs/code-reviews/README.md +++ b/docs/code-reviews/README.md @@ -10,6 +10,16 @@ Code review is a way to have a conversation about the code where participants wi - **Learn and grow** by having others review the code, we get exposed to unfamiliar design patterns or languages among other topics, and even break some bad habits. - **Shared understanding** between the developers over the project's code. +## Reviewing AI-assisted changes + +We review AI-generated code as untrusted code, confirming the change satisfies the work item, avoids unrelated generated code, and meets the same expectations as any other pull request. + +- Inspect logic, edge cases, error handling, security, performance, accessibility, and maintainability. +- Require meaningful tests or evaluations for generated behavior. +- Check generated comments, documentation, and PR text for unsupported claims. + +The [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide owns the shared baseline for human ownership, data and context hygiene, and authorship traceability. + ## Resources - [Code review tools](./tools.md) diff --git a/docs/code-reviews/process-guidance/reviewer-guidance.md b/docs/code-reviews/process-guidance/reviewer-guidance.md index 9c58e7dbd..c19a271a2 100644 --- a/docs/code-reviews/process-guidance/reviewer-guidance.md +++ b/docs/code-reviews/process-guidance/reviewer-guidance.md @@ -57,6 +57,31 @@ Code reviews play a critical role in product quality and it should not represent - Do the interactions of the various pieces of code in the PR make sense? - Does the code recognize and incorporate architectures and coding patterns? +## AI-Assisted and Agent-Authored Changes + +Review AI-assisted changes as untrusted contribution. A human author or supervising engineer is still accountable for correctness, maintainability, security, and validation before merge. + +Ask for disclosure when material AI or agent assistance affects review context. This is especially useful when an agent authored commits, a prompt or instruction file shaped the change, generated code makes up most of the PR, or the PR changes AI prompts, model configuration, retrieval behavior, tool permissions, eval datasets, or generated documentation. + +During review, check that: + +- The PR description explains what was generated or agent-assisted when that context helps reviewers. +- The changed lines, tests, configuration, prompts, documentation, and generated text were read by a human owner. +- Validation evidence matches the risk of the change, including linting, unit tests, integration tests, smoke tests, manual checks, or screenshots where appropriate. +- New dependencies, generated lockfile updates, external actions, model providers, MCP servers, or tool integrations were checked for license, supply-chain, and security impact. +- Generated code does not call invented APIs, bypass existing abstractions, duplicate stale documentation, broaden permissions, hard-code secrets, or ignore error handling. +- Tests are meaningful and fail for the right reasons, rather than only asserting generated implementation details or shallow happy paths. +- Security-sensitive behavior still follows least privilege, input validation, authorization, logging, and privacy expectations. + +For generative AI or agentic product features, also review evaluation evidence: + +- Prompt, model, retrieval, tool, safety, and evaluation artifacts are versioned or otherwise traceable. +- The PR includes results from relevant golden datasets, prompt regression tests, groundedness checks, safety or jailbreak tests, and tool-call contract tests. +- Known limitations, excluded scenarios, and follow-up work are documented in the PR or linked work item. +- Human approval gates, rollback paths, telemetry, and incident response controls are present for high-impact agent actions. + +Use the [Copilots](../../developer-experience/copilots.md#team-operating-model-for-ai-assisted-delivery), [AI evaluation planning](../../automated-testing/test-planning.md#ai-evaluation-planning), and [AI threat modeling](../../security/threat-modelling.md#ai-systems-threat-modeling-considerations) guidance when a review needs deeper evidence. + ## Code Quality Pass ### Complexity diff --git a/docs/code-reviews/pull-request-template.md b/docs/code-reviews/pull-request-template.md index 6cd77a568..5d0178036 100644 --- a/docs/code-reviews/pull-request-template.md +++ b/docs/code-reviews/pull-request-template.md @@ -30,6 +30,7 @@ For more information about how to contribute to this repo, visit this [page](htt - [ ] My code follows the code style of this project. - [ ] I ran the lint checks which produced no new errors nor warnings for my changes. - [ ] I have checked to ensure there aren't other open Pull Requests for the same update/change. +- [ ] If AI assistance materially shaped this change, I disclosed it and included validation, dependency/license/security, and eval/test evidence where applicable. ## Does This Introduce a Breaking Change? diff --git a/docs/design/design-patterns/non-functional-requirements-capture-guide.md b/docs/design/design-patterns/non-functional-requirements-capture-guide.md index 55949f540..eafef2260 100644 --- a/docs/design/design-patterns/non-functional-requirements-capture-guide.md +++ b/docs/design/design-patterns/non-functional-requirements-capture-guide.md @@ -46,12 +46,26 @@ To support the process of capturing a project's _comprehensive_ non-functional r | [Performance](../../non-functional-requirements/performance.md) | Define the expected response times, throughput, and resource usage of the solution. | - Response time: The application must load and respond to user interactions within 500 ms for button clicks.
- Throughput: The application must be able to handle 100 concurrent users or 500 transactions per second.
- Resource utilization: The application must use less than 80% of CPU and 1 GB of memory.
- Error rates: The application must have an error rate less than 1% of all requests, and be able to handle and recover from errors gracefully, without impacting user experience or data integrity. | | [Scalability](../../non-functional-requirements/scalability.md) | Determine how the system will handle increased user loads or larger datasets over time. | - Load Balancing: The application must be able to handle a minimum of 250 concurrent users and support load balancing across at least 3 servers to handle peak traffic.
- Database Scalability: The application's database must be able to handle at least 1 million records and support partitioning or sharding to ensure efficient storage and retrieval of data.
- Cloud-Based Infrastructure: The application must be deployed on cloud-based infrastructure that can handle at least 100,000 requests per hour, and be able to scale up or down to meet changing demand.
Microservices Architecture: The application must be designed using a microservices architecture that allows for easy scaling of individual services, and be able to handle at least 500 requests per second.
- Caching: The application must be able to cache at least 10,000 records, with a cache hit rate of 95%, and support caching across multiple servers to ensure high availability. | +### Generative AI and Agentic Systems Requirements + +Use measurable requirements for AI features because model quality, retrieval quality, safety behavior, cost, and fallback behavior can change between releases. + +| Quality Attribute | Description | Common Metrics | +| -- | -- | -- | +| Latency | The time required to return a useful response across model calls, retrieval, orchestration, and tool execution. | - The p95 response time for common chat requests must be less than 5 seconds.
- The p95 response time for tool-using workflows must be less than 15 seconds, excluding user approval time.
- The system must stream the first useful token or progress update within 2 seconds for long-running operations. | +| Cost | The budget consumed by prompts, completions, embeddings, retrieval, evaluations, tracing, and retries. | - Average model and retrieval cost must stay below the agreed budget per successful task.
- The system must alert when daily token or model spend exceeds 80% of budget.
- Retry and fallback paths must not increase per-request cost by more than the agreed threshold. | +| Safety | The system's ability to prevent harmful, abusive, unauthorized, or policy-violating outputs and actions. | - Safety evals must pass the release threshold for jailbreak, prompt-injection, harmful-content, and unsafe-tool-call scenarios.
- High-impact actions must require explicit confirmation or human approval before execution.
- Unsafe requests must receive a refusal or safe completion path in 100% of defined test cases. | +| Privacy | The protection of prompts, retrieved content, generated outputs, telemetry, memory, and tool data. | - Prompts, logs, traces, embeddings, indexes, caches, and memory stores must have documented data classification and retention periods.
- Sensitive content must be redacted or minimized before telemetry is stored, unless there is an approved exception.
- Deletion requests must cover generated summaries, stored memory, caches, indexes, and logs where policy requires it. | +| Reliability | The ability to produce bounded behavior when model calls, retrieval, tools, or safety controls fail. | - The service must meet the agreed availability target for AI endpoints and supporting retrieval or tool services.
- The system must degrade to a documented fallback when the model, retrieval index, or required tool is unavailable.
- Circuit breakers must stop repeated tool failures, runaway retries, and budget-exhaustion loops. | +| Explainability | The ability for users, reviewers, and operators to understand why an output or action occurred. | - User-facing answers must include citations or source references when claims depend on retrieved knowledge.
- Audit traces must correlate prompt version, model version, retrieval source IDs, tool calls, approvals, and final outcome.
- Consequential recommendations must expose confidence, uncertainty, or review status in terms users can act on. | +| Fallback | The alternate user and system experience when AI output is uncertain, unsafe, unavailable, or not sufficient. | - The product must provide a human review, deterministic workflow, or support path for defined high-risk scenarios.
- Low-confidence or ungrounded answers must avoid presenting unsupported claims as facts.
- Users must be able to correct, contest, or report AI output when it affects their work or decisions. | + ### Security and Compliance Requirements | Quality Attribute | Description | Common Metrics | | -- |-- |-- | | [Compliance](../../non-functional-requirements/compliance.md) | Adherence to legal, regulatory, and industry standards and requirements. | See [Microsoft Purview Compliance Manager](https://aka.ms/ComplianceManager) | -| [Privacy](../../privacy/README.md) | Protection of sensitive information and compliance with privacy regulations. | - Compliance with Privacy Regulations: Achieve full compliance with GDPR, CCPA and HIPAA.
- Data Anonymization: Implement anonymization techniques in protecting individual privacy while still allowing for data analysis.
- Data Encryption: Ensure that sensitive data is encrypted according to encryption standards and best practices.
- User Privacy Preferences: The ability to respect and accommodate user privacy preferences regarding data collection, processing, and sharing. | +| [Privacy](../../non-functional-requirements/privacy/README.md) | Protection of sensitive information and compliance with privacy regulations. | - Compliance with Privacy Regulations: Achieve full compliance with GDPR, CCPA and HIPAA.
- Data Anonymization: Implement anonymization techniques in protecting individual privacy while still allowing for data analysis.
- Data Encryption: Ensure that sensitive data is encrypted according to encryption standards and best practices.
- User Privacy Preferences: The ability to respect and accommodate user privacy preferences regarding data collection, processing, and sharing. | | [Security](../../security/README.md) | Establish the security requirements of the system, such as authentication, authorization, encryption, and compliance with industry or legal regulations. | See [Threat Modeling Tool](https://aka.ms/tmt) | | [Sustainability](../sustainability/README.md) | Ability to operate over an extended period while minimizing environmental impact and resource consumption. | - Energy Efficiency: Kilowatt-hours/Transaction.
- Carbon Footprint: Tons of CO2 emissions per year. | diff --git a/docs/design/design-reviews/recipes/high-level-design-recipe.md b/docs/design/design-reviews/recipes/high-level-design-recipe.md index 0389a5a49..28e5a9786 100644 --- a/docs/design/design-reviews/recipes/high-level-design-recipe.md +++ b/docs/design/design-reviews/recipes/high-level-design-recipe.md @@ -14,6 +14,14 @@ Design at macroscopic level shows the interactions between systems and services * Generic box diagrams depicting data flow or call origination/destination are useful. However, the title should clearly define what the arrows show indicate. In most cases, a diagram will show either data flow or call directions but not both. * Visualize the contrasting aspects of the system/diagram for ease of communication. e.g. differing technologies employed, modified vs. untouched components, or internet vs. local cloud components. Colors, grouping boxes, and iconography can be used for differentiating. * Prefer ease-of-understanding for communicating ideas over strict UML correctness. +* When the design includes generative AI, retrieval-augmented generation, or tool-using agents, make these decisions visible in the review artifact: + * Which model, provider, deployment, or model router is used, and what latency, cost, quality, privacy, or availability trade-off drove that choice? + * What sources ground the answers, how are retrieval permissions enforced, and how are citations or source identifiers returned to users and operators? + * Which tools can the system call, what side effects can each tool create, and how are tool schemas, inputs, outputs, and errors validated? + * What conversation history, memory, cache, or generated summary is stored, how long is it retained, and how can it be deleted? + * Which actions require user confirmation, human approval, policy checks, or a second system validation before execution? + * What prompt, model, retrieval, safety-filter, tool-call, latency, cost, and user-feedback telemetry is captured without exposing sensitive content? + * How does the system respond when the model is unavailable, retrieval quality is low, content is unsafe, costs exceed budget, or a tool call fails partway through? * Design reviews should be lightweight and should not feel like an additional process overhead. ## Examples diff --git a/docs/developer-experience/README.md b/docs/developer-experience/README.md index e71a66ae8..3c1038cec 100644 --- a/docs/developer-experience/README.md +++ b/docs/developer-experience/README.md @@ -21,6 +21,12 @@ This document makes several references to running a solution end-to-end (aka E2E - Minimize the amount of time spent manual setup and configuration of tooling - Minimize regressions and new defects by making end-to-end testing easy +## AI-assisted developer experience + +AI tools can improve the inner development loop when the team treats their output as draft engineering work. Use the [Copilots](copilots.md) guidance for tool setup, prompting, and attribution patterns, and use the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared expectations around data handling, review, testing, security, accessibility, and traceability. + +When defining the project's DevEx expectations, include approved AI tools, allowed data and context, repository instructions, prompt files, review responsibilities, and validation commands alongside build, test, start, and debug tasks. + ## Impact Developer experience can have a significant impact on the efficiency of the day-to-day execution of the team. A positive experience can pay dividends throughout the lifetime of the project; especially as new developers join the team. diff --git a/docs/developer-experience/copilots.md b/docs/developer-experience/copilots.md index d09b31762..fd5f02b3e 100644 --- a/docs/developer-experience/copilots.md +++ b/docs/developer-experience/copilots.md @@ -42,6 +42,8 @@ You can also use configuration files and extensions to customize and extend Copi - **MCP Servers.** You can use [MCP servers](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) to expand the capabilities of Copilot by providing it access to external tools or services, such as connecting to a project management system (such as Jira or Azure DevOps) or custom APIs (such as the Azure resource manager API). +- **Reusable prompt and agent libraries.** Instead of building these configuration files from scratch, a team can adopt a curated, open-source library of prompts, agents, and instruction files — such as [microsoft/hve-core](https://github.com/microsoft/hve-core), a Hypervelocity Engineering (HVE) accelerator — to standardize AI-assisted workflows across a repository. + ### GitHub Copilot Coding Agent The [GitHub Copilot Coding Agent](https://docs.github.com/en/copilot/how-tos/agents/copilot-coding-agent) is an AI software development agent that can be assigned to work on issues within the GitHub work management system such as fixing bugs or implementing new features. Once an issue has been assigned to the Copilot Agent, the bot analyzes the work item, creates a branch, starts a virtual environment to execute and test the code iteratively, authors commits, and opens a pull request for review. @@ -62,6 +64,73 @@ Chat AI tools are only as good as the prompts you give them. The quality and app If you make use of AI tools, it is important to understand how the data (including private or commercial code) might be used by the system. You can read more about how GitHub copilot handles your data and code at the [GitHub Copilot Privacy FAQ](https://github.com/features/copilot/#faq) and how to [manage Copilot policies](https://docs.github.com/en/copilot/how-tos/manage-your-account/manage-policies). Many organizations have policies that restrict the use of AI tools when working with proprietary code or data, so it is important to check for applicable policies before using these tools. +For cross-cutting expectations that apply beyond Copilot-specific tooling, see the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide. + +## Team Operating Model for AI-Assisted Delivery + +AI assistants and coding agents work best when we use them inside the same engineering controls as human-authored changes. As a team, we agree on the expected workflow before pointing an agent at production code, customer data, infrastructure, or security-sensitive changes. + +We reach for the lightest tool that fits the task, so the workflow stays proportional to the risk of the change: + +- Use IDE suggestions for small edits, refactoring help, unit-test scaffolding, explanations, and documentation drafts where the developer stays in the loop. +- Use chat-based editing when the task spans a few files, needs repository context, or benefits from an explicit prompt and review loop. +- Use a coding agent when the work can be described as a bounded issue with acceptance criteria, validation commands, and clear out-of-scope areas. +- Prefer direct human implementation for ambiguous product decisions, high-risk security changes, incident response, code involving sensitive data, or work that requires judgment the team has not captured in instructions. + +We give the assistant the same context we would give a new teammate, so the change it proposes is reviewable rather than a guess: + +- Problem statement and user impact +- Acceptance criteria and expected behavior +- Relevant files, existing patterns, and design constraints +- Validation commands, test data, and environment assumptions +- Security, privacy, licensing, dependency, and accessibility constraints +- Out-of-scope work and areas the assistant should not modify +- Links to work items, design notes, and team conventions when they exist + +We treat prompt and context hygiene as a shared team habit, because anything we paste into a tool can leave the approved environment: + +- Keep prompts specific, bounded, and tied to the work item. +- Include the minimum code and data needed for the task. +- Do not paste secrets, credentials, customer data, personal data, private keys, production logs with sensitive content, or data that policy prohibits from leaving the approved environment. +- Treat custom instructions, prompt files, and reusable prompts as project artifacts that need review when they influence production code, tests, deployment, or security posture. +- Record important assumptions or prompts in the work item or pull request when they affect how reviewers should inspect the change. + +Before we enable agents or MCP servers, we set tool and execution boundaries so an assistant only reaches as far as the task needs: + +- Use only approved tools, extensions, models, and trusted MCP servers for the repository. +- Confirm what each MCP server can read, write, execute, and send outside the local environment. +- Run destructive commands, production operations, migrations, and write-path cloud operations only with explicit human approval. +- Limit agent access to the repositories, branches, work items, credentials, and environments needed for the task. +- Review generated dependency, package, license, and infrastructure changes with the same scrutiny as manually authored changes. + +Before we merge a pull request, one of us stays accountable for the result, however the change was produced: + +- Read the changed code and generated text as untrusted contribution, even when the output looks plausible. +- Verify that tests, documentation, and migration notes match the change. +- Run or review the validation evidence needed by the team, including linting, unit tests, integration tests, security checks, and manual verification when applicable. +- For generative AI or agentic features, include evaluation evidence for prompts, model configuration, retrieval, safety behavior, and tool-call behavior. Use the [generative AI and agentic systems](../ml-and-ai-projects/generative-ai-and-agentic-systems.md) and [test planning](../automated-testing/test-planning.md#ai-evaluation-planning) guidance. +- Disclose material AI or agent assistance in the pull request when it affects review context, authorship, risk, or validation expectations. + +We lean on the rest of the playbook as the control plane for AI-assisted delivery, rather than inventing a parallel process: + +- Use [code reviews](../code-reviews/process-guidance/reviewer-guidance.md#ai-assisted-and-agent-authored-changes) to inspect generated code, tests, prompts, tool configuration, and dependencies. +- Use [source control](../source-control/README.md) and [commit guidance](../source-control/git-guidance/README.md#commit-best-practices) to keep AI-assisted changes small, traceable, and attributable. +- Use [CI/CD](../CI-CD/README.md#ai-assisted-cicd-authoring) to validate generated pipeline, deployment, and infrastructure changes before they can affect shared environments. +- Use [documentation](../documentation/README.md) to keep generated docs accurate and connected to the code they describe. +- Use [threat modeling](../security/threat-modelling.md#ai-systems-threat-modeling-considerations) when assistants, agents, prompts, retrieval, model providers, or tools influence a production system. + +## Validating AI-Assisted Work + +The [AI-Assisted Engineering](../ai-assisted-engineering/README.md#human-oversight) guide owns the shared rule: treat AI output as draft material until a responsible person reviews, tests, and accepts it. Before we merge or publish AI-assisted work, we confirm that: + +- Reviewers can explain the generated code, configuration, documentation, or tests. +- Relevant unit, integration, end-to-end, or AI evaluation checks have been added or run. +- Security-sensitive code, including authentication, authorization, cryptography, secrets, permissions, data access, CI/CD, and infrastructure changes, has received extra scrutiny. +- Generated documentation is source-grounded, current, and verified against the implementation, commands, examples, and product behavior it describes. +- Generated UI, content, images, alt text, captions, and flows meet the project's accessibility expectations. + +Use the [Testing](../automated-testing/README.md), [Code Reviews](../code-reviews/README.md), [Security](../security/README.md), [Documentation](../documentation/README.md), and [Accessibility](../non-functional-requirements/accessibility.md) guides for the domain-specific review path. + ## Attributing AI-Assisted Code Authorship When using AI tools to generate code, it can be beneficial to maintain transparency about authorship for accountability, code review, and auditing purposes. This can be done easily by using [Git trailers](https://git-scm.com/docs/git-interpret-trailers) that append structured metadata to the end of commit messages. diff --git a/docs/documentation/README.md b/docs/documentation/README.md index 78061f63c..6be529ad1 100644 --- a/docs/documentation/README.md +++ b/docs/documentation/README.md @@ -10,6 +10,18 @@ Documentation shouldn't be an afterthought. Different written documents and mate - Improve communication and collaboration between teams (especially when distributed across time zones). - Improve the transition of the project to another team. +## AI-assisted documentation + +AI tools can help us draft outlines, explain code, summarize decisions, and produce examples, but we validate generated documentation before it becomes project guidance. + +- Ground AI-assisted drafts in source material such as code, tests, work items, design notes, decisions, and product documentation. +- Ask subject-matter experts to review generated explanations for accuracy, missing context, and project-specific constraints. +- Verify generated commands, configuration examples, API behavior, screenshots, diagrams, and troubleshooting steps before publishing. +- Remove invented behavior, unsupported claims, outdated assumptions, and content that exposes secrets, customer data, or unnecessary proprietary context. +- Keep generated documentation linked to the same review, traceability, and update process used for other project documentation. + +The [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide covers the shared review, data handling, authorship, and accessibility expectations for AI-assisted content. + ## Challenges When working in an engineering project, we typically encounter one or more of these challenges related to documentation (including some examples): @@ -29,6 +41,8 @@ When working in an engineering project, we typically encounter one or more of th - Missing settings in the "how to run this" document that are required to run the application. - **Inaccurate**. - Documents not updated along with the code, so they don't mention the right folders, settings, etc. + - AI-assisted documentation that describes commands, APIs, configuration, or product behavior without checking the current source, tests, or generated reference output. + - Generated explanations that sound plausible but hide uncertainty, omit prerequisites, or invent links, parameters, environment variables, screenshots, metrics, or limitations. - **Obsolete**. - Design documents that don't apply anymore, sitting next to valid documents. Which one shows the latest decisions? - **Out of order (subject / date)**. @@ -39,6 +53,9 @@ When working in an engineering project, we typically encounter one or more of th - **Afterthought**. - Key documents created several weeks into the project: onboarding, how to run the app, etc. - Documents created last minute just before the end of a project, forgetting that they also help the team while working on the project. +- **Unreviewed generated content**. + - Generated sections that fill space without a clear reader need, source-backed claim, or owner. + - Rephrased content that drifts from the original technical meaning, softens warnings, or drops accessibility, security, or privacy details. ## What Documentation Should Exist diff --git a/docs/documentation/best-practices/automation.md b/docs/documentation/best-practices/automation.md index f77b66a84..a414bac63 100644 --- a/docs/documentation/best-practices/automation.md +++ b/docs/documentation/best-practices/automation.md @@ -2,6 +2,17 @@ You can document how to set up your dev machine with the right version of the framework required to run the code, which extensions are useful to develop the application with your editor, or how to configure your editor to launch and debug the application. If it is possible, a better solution is to provide the means to automate tool installs, application startup, etc., instead. +Prefer automation over generated setup prose when the reader needs a repeatable result. AI-assisted instructions can help draft context, explain trade-offs, or summarize what a script does, but they should not replace executable setup, validation, or recovery steps that the team can test. + +Automation is usually the better artifact when: + +- The same commands must run on every developer machine, build agent, or test environment. +- The instructions include version pinning, environment variables, credentials setup, generated files, or ordered prerequisites. +- A failure should stop early with a clear error instead of relying on a reader to notice a missed step. +- The process must stay current as dependencies, toolchains, containers, or cloud resources change. + +Use documentation to explain intent, boundaries, troubleshooting, and when not to run the automation. Keep generated prose source-backed by the script, task, container definition, or pipeline it describes. + Some examples are provided below: ## Dev Containers in Visual Studio Code diff --git a/docs/documentation/guidance/code.md b/docs/documentation/guidance/code.md index dc3e44ce2..1f434881f 100644 --- a/docs/documentation/guidance/code.md +++ b/docs/documentation/guidance/code.md @@ -12,6 +12,16 @@ There can be single line and multi-line comments (e.g., [C# Comments](https://le Now, the use of these comments is often considered a code smell. If you need to clarify your code, that may mean the code is too complex. So you should work towards the removal of the clarification by making the code simpler, easier to read, and understand. Still, these comments can be useful to give overviews of the code, or provide additional context information that is not available in the code itself. +Be especially careful with generated comments and generated code explanations. Treat them as drafts that need the same review as code because they can repeat a wrong assumption with confident wording. Before keeping generated commentary, check that it: + +- Explains why the code exists or what constraint it protects, not what each line already says. +- Matches the current implementation, tests, public API, error behavior, and security or privacy boundary. +- Does not invent performance claims, compatibility guarantees, architectural intent, or future work. +- Does not hide unclear code that should be renamed, decomposed, tested, or removed. +- Does not expose secrets, customer data, internal incident details, or prompt content that should not be documented. + +When an AI tool generates a code explanation for documentation or review, anchor the final text in source files, tests, specifications, or design decisions that the team can verify. + Examples of useful comments: - Single line comment in C# that explains **why** that piece of code is there (from a private method in [System.Text.Json.JsonSerializer](https://github.com/dotnet/runtime/blob/main/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.String.cs)): diff --git a/docs/engineering-feedback/README.md b/docs/engineering-feedback/README.md index f6a66148a..1d82c5eb3 100644 --- a/docs/engineering-feedback/README.md +++ b/docs/engineering-feedback/README.md @@ -14,6 +14,18 @@ Capturing and providing high-quality actionable Engineering Feedback is an integ You should jot down the details of the feedback close to the time when you encounter the specific blockers, challenges, and friction since that is when it is freshest in your mind. The project team can then decide how to prioritize and when to submit the feedback into the official CSE Feedback system (accessible to ISE team members) during each sprint. +## AI Tooling Feedback + +AI tooling feedback should be captured when it creates repeatable friction, safety concerns, or product gaps for the team or customer. Useful categories include: + +* Hallucinated or unsupported answers that mislead users or require costly rework +* Unsafe output, insecure code suggestions, excessive permissions, or risky tool actions +* Poor suggestion quality for the project's languages, frameworks, architecture, or domain +* Privacy, data handling, policy, retention, or customer-data friction that blocks approved use +* Product limitations, missing integrations, accessibility barriers, evaluation gaps, or operational issues + +When you submit AI tooling feedback, include a reproducible prompt or scenario with sanitized context, describe the expected and actual behavior, and note the impact, any workaround, and the tool name and configuration. Follow the project's [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guidance when deciding what context can be shared. + ## What is Good and High-quality Engineering Feedback Good engineering feedback provides enough information for those who are not part of the code-with engagement to understand the customer pain, the associated product issues, the impact and priority of these issues, and any potential workarounds that exist to minimize that impact. diff --git a/docs/engineering-fundamentals-checklist.md b/docs/engineering-fundamentals-checklist.md index bc6e336bf..33a89f7bd 100644 --- a/docs/engineering-fundamentals-checklist.md +++ b/docs/engineering-fundamentals-checklist.md @@ -37,6 +37,19 @@ More details on [automated testing](./automated-testing/README.md) More details on [continuous integration](./CI-CD/continuous-integration.md) and [continuous delivery](./CI-CD/continuous-delivery.md) +## AI-Assisted Engineering + +- [ ] Confirm which AI tools are approved for this project and data type. +- [ ] Shared AI configuration (custom instructions, prompts, rules) is version-controlled and consistent across the team. +- [ ] AI-assisted changes go through normal human review, PR policy, tests, and security checks. +- [ ] Prompts and AI context exclude secrets, credentials, private customer data, and unnecessary sensitive details. +- [ ] AI-enabled features have quality, safety, security, and regression evaluations. +- [ ] AI-enabled features have observability for model behavior, tool calls, safety outcomes, cost, latency, and failures. +- [ ] Team members are ramped up on applying engineering fundamentals with approved AI tools. +- [ ] Record material AI assistance according to team convention. + +More details on [AI-assisted engineering](./ai-assisted-engineering/README.md) + ## Security - [ ] Access is only granted on an as-needed basis diff --git a/docs/ml-and-ai-projects/README.md b/docs/ml-and-ai-projects/README.md index 50725dbc1..cd2472c60 100644 --- a/docs/ml-and-ai-projects/README.md +++ b/docs/ml-and-ai-projects/README.md @@ -2,6 +2,8 @@ This guideline documents the Machine Learning (ML) practices in ISE. ISE works with customers on ML/AI projects and putting them in production, with an emphasis on engineering and research best practices throughout the project's life cycle. +Modern ML/AI projects may also include generative AI, retrieval-augmented generation, tool-calling agents, or AI-assisted workflows. These systems still need the ML lifecycle guidance in this section, but they also require explicit controls for prompts, grounding data, evaluations, tool permissions, telemetry, and Responsible AI review. + ## Goals * Provide a set of ML practices to follow in an ML project. @@ -13,6 +15,15 @@ This guideline documents the Machine Learning (ML) practices in ISE. ISE works w * If you are starting a new ML project, consider reading through the [general guidance documents](#general-guidance). * For specific aspects of an ML project, refer to the guidelines for different [project phases](#ml-project-phases). +## AI-enabled project entry points + +Generative AI, LLM, and agent projects need the same ML fundamentals plus additional engineering controls for prompts, retrieval, tool use, autonomy, safety, and monitoring. + +* Start with the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared expectations around tool use, review, data handling, testing, security, observability, and traceability. +* Use [Responsible AI](./responsible-ai.md) early for impact assessment, stakeholder analysis, fallback paths, and ongoing review. +* Plan evaluation datasets, rubrics, safety checks, and regression gates with the [Testing](../automated-testing/README.md#testing-ai-assisted-and-ai-enabled-changes) guidance. +* Include AI-specific threat modeling and operational telemetry from the [Security](../security/README.md#ai-and-agent-security) and [Observability](../observability/README.md#ai-observability) guides. + ## ML Project Phases The diagram below shows different phases in an ideal ML project. Due to practical constraints and requirements, it might not always be possible to have a project structured in such a manner, however best practices should be followed for each individual phase. @@ -34,6 +45,7 @@ The diagram below shows different phases in an ideal ML project. Due to practica * [Testing Data Science and ML Ops code](./testing-data-science-and-mlops-code.md) * [Profiling Machine Learning and ML Ops code](./profiling-ml-and-mlops-code.md) * [Responsible AI](./responsible-ai.md) +* [Generative AI and agentic systems](./generative-ai-and-agentic-systems.md) * [Program Management for ML projects](./tpm-considerations-for-ml-projects.md) ## Resources diff --git a/docs/ml-and-ai-projects/generative-ai-and-agentic-systems.md b/docs/ml-and-ai-projects/generative-ai-and-agentic-systems.md new file mode 100644 index 000000000..f50b3cf82 --- /dev/null +++ b/docs/ml-and-ai-projects/generative-ai-and-agentic-systems.md @@ -0,0 +1,89 @@ +# Generative AI and Agentic Systems + +Generative AI systems introduce engineering concerns that are less common in classical ML systems. The same user request can produce different outputs over time, retrieved content can change the answer, prompts become part of the product behavior, and tool-using agents can take actions outside the model runtime. Treat prompts, models, grounding data, tool permissions, evaluations, and telemetry as production artifacts. + +Use this guide when a solution uses large language models, retrieval-augmented generation (RAG), tool calling, autonomous or semi-autonomous agents, model routing, generated content, or AI-assisted workflows that affect users or production systems. + +## Choose the Right Architecture + +Prefer the simplest design that can satisfy the user need, quality bar, and risk profile. + +- Use direct model calls when the task is low risk, does not require private or fast-changing facts, and can be validated with clear output checks. +- Use RAG when answers must be grounded in approved sources, citations are expected, or the knowledge changes more often than the model can be updated. +- Use deterministic orchestration when the process needs predictable steps, explicit branching, repeatable validation, or integration with existing business rules. +- Use tool-calling agents when the system must inspect state, call APIs, modify records, or coordinate work, and when each tool can be scoped, validated, audited, and rolled back. +- Use multi-agent systems only when distinct roles, state boundaries, or review loops reduce complexity enough to justify the extra coordination and observability cost. +- Use model routers when different models serve clearly different latency, cost, quality, privacy, or availability needs, and routing decisions can be tested and monitored. +- Use fine-tuning when examples show that prompt design, RAG, or orchestration cannot meet the quality target, and when the team can own training data quality, evaluation, safety review, and model lifecycle operations. + +Do not use an agent to hide unclear requirements. If a workflow is not understood well enough to describe its tools, permissions, failure states, and acceptance criteria, start with a deterministic workflow or a human-assisted prototype. + +## Workload Readiness Checklist + +Before the team commits to building or operating an AI workload, confirm that the following are explicit and reviewable: + +- [ ] The intended use and measurable user outcome are documented. +- [ ] Prohibited uses, excluded user groups, and unsafe operating conditions are documented. +- [ ] Impacted users, operators, and downstream stakeholders are identified. +- [ ] Input data, grounding data, prompts, responses, logs, and memory stores have data classifications. +- [ ] The architecture choice explains why direct calls, RAG, orchestration, agents, routing, or fine-tuning fit the problem. +- [ ] The evaluation plan covers quality, safety, groundedness, refusal behavior, latency, cost, and regression cases. +- [ ] The observability plan captures enough telemetry to debug behavior while protecting sensitive data. +- [ ] The cost budget covers model calls, embeddings, retrieval, evaluations, tracing, red-team runs, and expected growth. +- [ ] The support model identifies owners for prompts, grounding data, tools, model configuration, evals, and incidents. +- [ ] The incident response plan covers disabling model features, blocking unsafe tools, rolling back prompts or indexes, and communicating user impact. + +Use the [Responsible AI](./responsible-ai.md), [model experimentation](./model-experimentation.md), and [test planning](../automated-testing/test-planning.md) guidance to turn these readiness items into project artifacts. + +## RAG and Grounding Checklist + +RAG systems move part of the product quality bar into the data and retrieval layer. Review grounding content with the same care as application code. + +- [ ] Source systems are authoritative for the questions the product will answer. +- [ ] Retrieval respects user authorization and does not expose documents the user cannot access directly. +- [ ] Chunking, metadata, embedding, and ranking choices are evaluated against realistic user questions. +- [ ] Citations identify the source, section, or record used to generate the answer. +- [ ] The system handles stale, missing, conflicting, or low-quality retrieved content safely. +- [ ] Deletion, retention, and right-to-be-forgotten requirements apply to indexes, caches, prompts, logs, and generated summaries. +- [ ] Index updates, emergency rollback, and source freshness checks are part of release and operations plans. + +## Agentic Systems Checklist + +Agents need stronger controls because they can combine model output, user context, memory, and tools into actions. + +- [ ] Each agent has a clear identity, purpose, owner, and operating boundary. +- [ ] User context is passed only when needed and is validated before use in prompts, tools, and retrieval. +- [ ] The tool inventory lists each tool, data scope, side effects, rate limits, and failure behavior. +- [ ] Tool permissions follow least privilege and are separated by environment, tenant, and user role. +- [ ] Tool schemas are narrow, versioned, validated, and resilient to malformed model output. +- [ ] High-impact actions require approval, confirmation, or human review before execution. +- [ ] Memory and state retention rules cover what is stored, where it is stored, who can read it, and how it is deleted. +- [ ] Audit logs connect user request, prompt or instruction version, retrieval results, tool calls, approvals, and final outcome. +- [ ] Circuit breakers can stop repeated failures, unsafe outputs, runaway cost, excessive retries, and risky tool sequences. +- [ ] Rollback procedures cover prompts, tools, model configuration, grounding indexes, memory, and external side effects. + +## Review the System Across Disciplines + +Generative AI and agentic systems should be reviewed through the same engineering fundamentals as the rest of the system, with extra attention to non-determinism, data boundaries, and tool authority. + +- Lean on [Responsible AI](./responsible-ai.md) to review user impact, harms, transparency, accountability, human oversight, and post-deployment review cadence. +- Turn to [model experimentation](./model-experimentation.md) to version prompts, datasets, model settings, retrieval configuration, and evaluation results. +- Let [test planning](../automated-testing/test-planning.md) define golden datasets, adversarial cases, prompt regressions, groundedness checks, and tool-call contract tests. +- Run [threat modeling](../security/threat-modelling.md) to analyze prompt injection, indirect prompt injection, tool-output injection, data exfiltration, sensitive disclosure, model/provider trust, and excessive agency. +- Plan with [ML observability](../observability/ml-observability.md) for traces of prompts, model versions, retrieval source IDs, citations, safety filters, tool calls, latency, tokens, cost, and user feedback. +- Treat the [CI/CD](../CI-CD/README.md) pipeline as where prompts, eval datasets, grounding indexes, model configuration, safety settings, and tool permission manifests become versioned release artifacts. +- Reach for [Copilots](../developer-experience/copilots.md) when AI assistants or coding agents help author code, tests, documentation, or pull requests. +- Check [privacy fundamentals](../non-functional-requirements/privacy/README.md) for prompt data, logs, memory, telemetry, retention, consent, and deletion handling. +- Revisit [accessibility](../non-functional-requirements/accessibility.md) for generated content, chat experiences, citations, multimodal input and output, fallback paths, and user control. +- Bring [code reviews](../code-reviews/README.md) in to inspect AI-generated code, prompt changes, tool contracts, eval evidence, dependencies, and operational controls. + +## Definition of Done + +A generative AI or agentic feature is not ready because it responds convincingly in a demo. It is ready when the team can show evidence that the feature is useful, bounded, testable, observable, secure, and supportable. + +- [ ] The feature has acceptance criteria for correct, incorrect, uncertain, unsafe, and out-of-scope requests. +- [ ] Evaluation results are reviewed before release and compared with a previous baseline. +- [ ] Human review paths are defined for high-impact decisions or actions. +- [ ] Users understand when they are interacting with AI and how to challenge, correct, or report an output. +- [ ] Operations teams can identify the prompt, model, retrieval data, tool calls, and safety controls involved in an incident. +- [ ] The team has a rollback path that does not require retraining a model under incident pressure. diff --git a/docs/ml-and-ai-projects/model-experimentation.md b/docs/ml-and-ai-projects/model-experimentation.md index 2be0ba604..18f7c0c10 100644 --- a/docs/ml-and-ai-projects/model-experimentation.md +++ b/docs/ml-and-ai-projects/model-experimentation.md @@ -141,6 +141,23 @@ When deciding on the evaluation of the ML model/process, consider the following - [ ] Evaluation code is unit-tested and reviewed by all team members. - [ ] Evaluation flow facilitates further results and error analysis. +### Generative AI and Agent Evaluation + +Generative AI and agentic systems need evaluation artifacts that are versioned and reviewed with the same care as code, datasets, and model parameters. Track prompts, model versions, retrieval configuration, tool schemas, safety settings, evaluation datasets, and scoring rubrics so teams can compare behavior across experiments and releases. + +Use offline evaluations before release to measure: + +- Output quality against golden datasets, scenario tests, and human-graded rubrics +- Prompt regressions across system instructions, developer prompts, templates, and canary prompts +- RAG retrieval relevance, groundedness, faithfulness to retrieved sources, citation quality, security trimming, and freshness +- Safety behavior for jailbreak attempts, prompt injection, indirect prompt injection, harmful content, sensitive disclosure, and misuse cases +- Tool-call correctness, including schema conformance, argument validation, authorization, dry-run behavior, write-path controls, and recovery from tool failures +- Operational targets such as latency, token or compute cost, refusal rate, fallback rate, and failure clustering + +Record evaluation outcomes in the experiment tracker with the artifact versions that produced them. Each run should identify the prompt or instruction version, model or deployment version, grounding dataset or index version, tool contract version, evaluator version, metric thresholds, reviewer notes, and release decision. + +Use release gates when an experiment affects user-facing behavior or agent authority. A gate should state the required scores, the safety failures that block release, the human approvals required for high-impact actions, and the production monitoring signals that will detect drift after release. Re-run the relevant evaluation suite after prompt, model, retrieval, tool, safety policy, or user-population changes. + ## Evaluation Development Process Outcomes 1. Evaluation strategy is agreed upon all stakeholders diff --git a/docs/ml-and-ai-projects/responsible-ai.md b/docs/ml-and-ai-projects/responsible-ai.md index ce8952590..3b5d52f9b 100644 --- a/docs/ml-and-ai-projects/responsible-ai.md +++ b/docs/ml-and-ai-projects/responsible-ai.md @@ -34,4 +34,60 @@ The process begins as soon as we start a prospective project. We start to comple At this point we research available [tools and resources](https://www.microsoft.com/en-us/ai/responsible-ai-resources), such as [InterpretML](https://interpret.ml/) or [Fairlearn](https://github.com/fairlearn/fairlearn), that we may use on the project. We may change the project scope or re-define the [ML problem definition](./envisioning-and-problem-formulation.md) if necessary. +## Generative AI and agent review prompts + +Generative AI and agentic systems need additional Responsible AI review because they can produce open-ended content, use grounding data, remember context, call tools, and take actions on behalf of users. Use these prompts with the [generative AI and agentic systems](./generative-ai-and-agentic-systems.md) guide when reviewing feasibility, development, production readiness, deployment, and monitoring. For shared engineering controls across these areas, see the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide. + +### Grounding and generated content + +We ask these questions early, while we are still deciding how the system should source and present its answers: + +- What sources ground the answer, and are they authoritative for the intended use? +- How does the system handle stale, missing, contradictory, or low-quality retrieved content? +- Are citations or source references available when users need to verify generated answers? +- What harms could occur if the system hallucinates, overstates confidence, or produces misinformation? +- What content safety controls, refusal behaviors, and fallback paths are tested before release? + +### Prompt, retrieval, and tool abuse + +Whenever the system accepts untrusted input or calls tools, we walk through these abuse scenarios: + +- How could a user, retrieved document, web page, file, or tool response inject instructions that override the intended behavior? +- Are system instructions, developer prompts, and tool schemas reviewed and versioned as product artifacts? +- What validation prevents tool-output injection, unsafe command construction, or untrusted content from being treated as instructions? +- Which misuse and abuse cases were red-teamed, and what evidence shows the mitigations were retested? + +Use [threat modeling](../security/threat-modelling.md) to turn these questions into threats, mitigations, and validation evidence. + +### Agent actions and human oversight + +Before we let an agent act on a user's behalf, we get clear on what it can do and who signs off: + +- What actions can the agent take, and which users, roles, tenants, and environments can authorize them? +- Which actions require explicit user approval, human review, or a second system check before execution? +- Can privileged actions be paused, blocked, rate limited, or rolled back during an incident? +- How are failures, retries, partial completion, and external side effects communicated to users and operators? + +### Memory, logs, and retention + +As we decide what to store and for how long, we check these data-handling questions: + +- What conversation history, prompts, retrieved content, generated outputs, tool calls, and user feedback are stored? +- Does stored memory contain personal, confidential, regulated, or customer-owned data? +- How are consent, retention, deletion, and right-to-be-forgotten requirements handled across prompts, logs, indexes, caches, and memories? +- Can telemetry support debugging and audit needs without exposing sensitive content unnecessarily? + +Use [privacy fundamentals](../non-functional-requirements/privacy/README.md) and [ML observability](../observability/ml-observability.md) when designing these controls. + +### Production monitoring and review cadence + +Once the system is live, we keep asking these questions so the review does not stop at launch: + +- What evaluation suite tracks groundedness, harmful content, jailbreak resistance, tool-call accuracy, and regression from previous releases? +- What user feedback, safety filter outcomes, incident signals, and cost or latency metrics are reviewed after deployment? +- Who owns prompt updates, model configuration, grounding data, eval datasets, safety thresholds, and tool permissions? +- How often will the team re-run Responsible AI review after model, prompt, retrieval, data, policy, or user population changes? + +Use [model experimentation](./model-experimentation.md) and [test planning](../automated-testing/test-planning.md) to define measurable evaluation evidence. + The Responsible AI review documents remain living documents that we re-visit and update throughout project development, through the [feasibility study](./feasibility-studies.md), as the model is developed and prepared for production, and new information unfolds. The documents can be used and expanded once the model is deployed, and monitored in production. diff --git a/docs/non-functional-requirements/accessibility.md b/docs/non-functional-requirements/accessibility.md index ab90f489d..493b1eabc 100644 --- a/docs/non-functional-requirements/accessibility.md +++ b/docs/non-functional-requirements/accessibility.md @@ -44,6 +44,30 @@ If you are looking to add automated testing to your Azure Pipelines, you may wan It's important to keep in mind that automated tooling alone is not enough - make sure to augment your automated tests with manual ones. Accessibility Insights (linked above) can guide users through some manual testing steps. +### AI Experiences + +Generative AI and agentic systems need accessibility review because they can create dynamic content, conversational interfaces, citations, multimodal inputs and outputs, and fallback paths that change during a session. Use these prompts with the [Responsible AI](../ml-and-ai-projects/responsible-ai.md) and [generative AI and agentic systems](../ml-and-ai-projects/generative-ai-and-agentic-systems.md) guidance. + +- Can screen readers and keyboard users follow generated content as it appears, including streaming responses, loading states, edits, refusals, errors, and tool progress? +- Does the chat or agent interface preserve focus order, expose status changes through accessible announcements, and avoid trapping users in long-running workflows? +- Are citations, source links, confidence cues, warnings, and safety messages available as text and associated with the generated claims they support? +- Are generated images, diagrams, audio, video, and document summaries reviewed for captions, transcripts, alt text, language metadata, and clear alternatives? +- Can users choose or avoid modalities, such as voice, image upload, camera input, file upload, or generated audio, without losing access to the core task? +- Does the system avoid presenting hallucinated, low-confidence, or unsupported output in ways that users cannot inspect, correct, copy, report, or retry? +- Is there a human fallback, deterministic path, or support route when AI output affects access, eligibility, safety, financial, health, legal, or employment-related decisions? +- Are evaluation datasets and manual tests representative of users with different assistive technologies, languages, literacy levels, cognitive needs, and sensory or motor abilities? + +### AI-assisted accessibility work + +AI can help draft alt text, captions, documentation, test ideas, and candidate UI changes, but it does not replace manual accessibility validation or assistive technology review. + +- Manually validate AI-generated content, UI, alt text, captions, documentation, and user flows for accuracy and usefulness. +- Check generated UI components for semantic structure, keyboard support, focus order, contrast, labels, and screen-reader behavior. +- Review generated content for plain language, inclusive language, cognitive accessibility, localization, and cultural context. +- Confirm which AI tools are approved for this project and data type before uploading screenshots, transcripts, user research, or customer context. + +Use the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared accessibility and inclusion practices. + ### Code and Documentation Basics Before you get to testing, you can make some small changes in how you write code and documentation. diff --git a/docs/non-functional-requirements/maintainability.md b/docs/non-functional-requirements/maintainability.md index 97fdfdabc..f564f7238 100644 --- a/docs/non-functional-requirements/maintainability.md +++ b/docs/non-functional-requirements/maintainability.md @@ -23,5 +23,5 @@ Implementing maintainability in software systems involves adopting practices, to - Code Reviews: Conduct regular [code reviews](../code-reviews/README.md) to ensure adherence to standards and to share knowledge among team members. - External Documentation: Maintain up-to-date documentation, including design documents, user manuals, and [API references](../documentation/guidance/rest-apis.md). There are tools to assist with that like Swagger or Postman. - README Files: Provide README files in repositories to guide new developers on setup, usage, and contribution guidelines. -- Automated Testing: Provide unit test, end-to-end tests, smoke and integration tests as well as continuous integration practices. +- Automated Testing: Provide a layered test suite — see the [automated testing fundamentals](../automated-testing/README.md#the-fundamentals) for the unit, integration, end-to-end, and smoke test taxonomy — alongside continuous integration practices. - Code Refactoring: Regularly refactor code to improve its structure, readability, and maintainability without changing its external behavior. Implementing pre-commit hooks in the pipelines to automate the monitoring of code refactoring tasks, like forcing coding standards, run static code analysis, linting, etc. diff --git a/docs/non-functional-requirements/privacy/README.md b/docs/non-functional-requirements/privacy/README.md index 5bbd99786..cfaf77033 100644 --- a/docs/non-functional-requirements/privacy/README.md +++ b/docs/non-functional-requirements/privacy/README.md @@ -3,4 +3,17 @@ Private data handling and protection requires both the proper design of software, systems and databases, as well as the implementation of organizational processes and procedures. -In general, developers working on [ISE](../../ISE.md) projects should adhere to Microsoft's recommended standard practices and regulations on Privacy and Data Handling. \ No newline at end of file +In general, developers working on [ISE](../../ISE.md) projects should adhere to Microsoft's recommended standard practices and regulations on Privacy and Data Handling. + +## AI Privacy Review Prompts + +Generative AI and agentic systems can move private data through many surfaces — user-facing inputs like prompts and retrieved content, the systems that process them such as model providers and tool calls, and what the system retains afterward in telemetry, memory, and generated summaries. Use these prompts with the [Responsible AI](../../ml-and-ai-projects/responsible-ai.md) and [generative AI and agentic systems](../../ml-and-ai-projects/generative-ai-and-agentic-systems.md) guidance before release and after material model, prompt, retrieval, tool, or telemetry changes. + +- What personal, confidential, regulated, or customer-owned data can appear in user prompts, system prompts, uploaded files, retrieved documents, tool responses, generated outputs, logs, traces, embeddings, caches, or memory stores? +- Are users told what data they should not enter or upload, and does the product block or warn on risky upload types, excessive data scope, or unsupported sensitive data? +- Which model providers, hosting regions, retrieval stores, tool APIs, and telemetry systems process the data, and what contractual, residency, retention, and access controls apply? +- Is telemetry minimized, redacted, aggregated, or sampled so operators can debug quality, safety, cost, and reliability without storing unnecessary sensitive content? +- What conversation memory, generated summaries, personalization state, or agent state is stored, who can read it, how long is it retained, and how can a user or operator delete it? +- Do retrieval indexes enforce the same authorization as the source systems, and do deletion or right-to-be-forgotten workflows cover indexes, embeddings, caches, summaries, and citations? +- Are prompts, prompt templates, system instructions, tool schemas, and safety policies reviewed for accidental disclosure of secrets, internal-only details, or private data handling assumptions? +- Can the team disable memory, tracing, retrieval, or model calls during an incident without losing the ability to communicate user impact and recover safely? diff --git a/docs/non-functional-requirements/privacy/data-handling.md b/docs/non-functional-requirements/privacy/data-handling.md index c5bc7dc6a..0f8beaa35 100644 --- a/docs/non-functional-requirements/privacy/data-handling.md +++ b/docs/non-functional-requirements/privacy/data-handling.md @@ -18,6 +18,23 @@ Developers working on ISE projects should implement best practices and guidance - [Limited Data Protection Addendum](https://aka.ms/mpsldpa) - [Professional Services Data Protection Addendum](https://www.microsoft.com/licensing/docs/view/Microsoft-Products-and-Services-Data-Protection-Addendum-DPA) +## AI tool data handling + +Anything we feed to or get back from an AI tool is project data, and we handle it with the same care as the rest of the engagement's data. That includes: + +- prompts, context files, and embeddings we send to the tool; +- transcripts and generated output we get back; +- screenshots and evaluation datasets we use to test it. + +Before we use customer, proprietary, regulated, or security-sensitive context, we confirm which AI tools are approved for this project and data type. + +- We keep production data, secrets, regulated data, and unnecessary proprietary context out of unapproved AI tools. +- We prefer de-identified, redacted, representative, or synthetic examples for prompting and evaluation wherever we can. +- We agree up front on the retention, logging, access, deletion, and export expectations for AI tool inputs and outputs. +- We apply the same data-minimization and customer-controlled-environment principles to AI prompts, retrieval indexes, and evaluation data. + +Use the [AI-Assisted Engineering](../../ai-assisted-engineering/README.md) guide for shared context hygiene and governance practices. + ## 5 W's of Data Handling When working on an engagement it is important to address the following 5 **W**'s: diff --git a/docs/observability/README.md b/docs/observability/README.md index 8143bbdf1..f765c9474 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -34,6 +34,20 @@ Building observable systems enables development teams at ISE to measure how well - [Observability in Azure Databricks](./observability-databricks.md) - [Recipes](./recipes-observability.md) +## AI observability + +AI-enabled systems need telemetry that helps teams debug behavior, detect drift, and control cost while respecting project privacy and retention rules. + +- Capture request context: model, provider, and version; prompt template version; retrieval source IDs; and tool calls. +- Capture operational signals: latency, token or cost signals, error rates, and fallback paths. +- Capture safety signals: refusal rates, safety outcomes, and escalation rates. +- Use traces to follow multi-step agent decisions and tool invocations. +- Apply project data classification and retention rules before logging prompts, completions, documents, embeddings, or user conversations. +- Run scheduled quality and safety evaluations to detect behavior drift. +- Alert when safety, quality, latency, cost, or fallback metrics cross agreed thresholds. + +Use the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared observability and privacy considerations. + ## Resources - [Non-Functional Requirements Guidance](../design/design-patterns/non-functional-requirements-capture-guide.md) diff --git a/docs/observability/best-practices.md b/docs/observability/best-practices.md index d4bfff8b6..acbc8c4b8 100644 --- a/docs/observability/best-practices.md +++ b/docs/observability/best-practices.md @@ -9,6 +9,9 @@ 1. **Faults, crashes, and failures** are logged as discrete events. This helps engineers identify problem area(s) during failures. 1. Ensure logging configuration (eg: setting logging to "verbose") can be controlled without code changes. 1. Ensure that **metrics** around latency and duration are collected and can be aggregated. +1. For AI workloads, correlate the user request, prompt or template version, model deployment, retrieval configuration, tool-call spans, safety checks, token usage, cost, and user feedback under the same trace or correlation id. This helps teams explain why an answer or action occurred without depending on raw prompt or response logs. +1. Version prompts, model settings, grounding indexes, and tool schemas as observable production artifacts. Include those versions in logs, traces, dashboards, and incident records so teams can compare behavior before and after a release. +1. Capture tool-call traces for agentic systems with the requested tool, authorization context, approval state, side-effect status, retry count, and failure category. Avoid logging tool inputs or outputs verbatim unless they have been classified and redacted. 1. Start small and add where there is customer impact. [Avoiding metric fatigue](./pitfalls.md#metric-fatigue) is very crucial to collecting actionable data. 1. It is important that every data that is collected contains relevant and rich context. 1. Personally Identifiable Information or any other customer sensitive information should never be logged. Special attention should be paid to any local privacy data regulations and collected data must adhere to those. (ex: GDPR) diff --git a/docs/observability/correlation-id.md b/docs/observability/correlation-id.md index b46143318..cacafb504 100644 --- a/docs/observability/correlation-id.md +++ b/docs/observability/correlation-id.md @@ -27,7 +27,7 @@ A Correlation ID is a unique identifier that is added to the very first interact 6. Add it to an outgoing response where possible. 7. Based on the use case, there can be additional correlation IDs that may be needed. For instance, tracking logs based on both Session ID and User ID may be required. While adding multiple correlation ID, remember to propagate them through the components. ->Consider using [OpenTelemetry](./tools/OpenTelemetry.md) as it implements open-source cross-platform context propagation for end-to-end distributed transactions over heterogeneous components out-of-the-box. It takes care of automatically creating and managing the "Correlation-id", called TraceId. +>[OpenTelemetry](./tools/OpenTelemetry.md) can manage this for you: it automatically creates and propagates the Correlation ID (called `TraceId`) across distributed transactions, so you do not have to wire it up by hand. ## Use Cases diff --git a/docs/observability/logs-privacy.md b/docs/observability/logs-privacy.md index 5a36c2cd1..ee0f3f7a9 100644 --- a/docs/observability/logs-privacy.md +++ b/docs/observability/logs-privacy.md @@ -13,6 +13,10 @@ This includes customer's sensitive, Personal Identifiable Information (PII), and 4. Use Structured Logging and include a deny-list for sensitive properties. 5. Put an extra effort on spotting logging statements with sensitive data during code review, as it is common for reviewers to skip reading logging statements. This can be added as an additional checkbox if you're using Pull Request Templates. 6. Include mechanisms to detect sensitive data in logs, on your organizational pipelines for QA or Automated Testing. +7. For generative AI systems, avoid retaining raw prompts, responses, retrieved snippets, tool outputs, and agent memory unless the team has an explicit operational need, data classification, retention period, access control, and deletion path. +8. Store prompt, response, retrieval, memory, and trace metadata as stable identifiers or classifications where possible. Use prompt template IDs, model versions, retrieval source IDs, citation IDs, tool names, policy outcomes, and redacted hashes instead of full text. +9. Redact or tokenize sensitive content before it enters logs, traces, analytics events, prompt stores, vector indexes, replay datasets, and support exports. Apply the same controls to sampled traces used for evaluation or incident review. +10. Treat agent memory and tool traces as user data when they include user intent, private context, retrieved records, or external side effects. Review who can read them, how long they are retained, and how they are removed when a user or policy requires deletion. ### Tools and Implementation Methods diff --git a/docs/observability/microservices.md b/docs/observability/microservices.md index 260fbf48b..78ccaad7f 100644 --- a/docs/observability/microservices.md +++ b/docs/observability/microservices.md @@ -52,4 +52,4 @@ Finally, microservice C, reads the value for the incoming trace-id and sets as h A number of Application Monitoring (APM) technology products already supports most of this Correlation Propagation. The most popular is [OpenZipkin/B3-Propagation](https://github.com/openzipkin/b3-propagation). W3C already proposed a recommendation for the [W3C Trace Context](https://www.w3.org/blog/2019/12/trace-context-enters-proposed-recommendation/), where you can see what SDK and frameworks already support this functionality. It's important to correctly implement the propagation specially when there are different teams that used different technology stacks in the same project. ->Consider using [OpenTelemetry](./tools/OpenTelemetry.md) as it implements open-source cross-platform context propagation for end-to-end distributed transactions over heterogeneous components out-of-the-box. It takes care of automatically creating and managing the Trace Context object among a full stack of microservices implemented across different technical stacks. +>[OpenTelemetry](./tools/OpenTelemetry.md) handles this Trace Context propagation automatically across a full stack of microservices implemented on different technology stacks. diff --git a/docs/observability/ml-observability.md b/docs/observability/ml-observability.md index 4821bde7f..27c6b9fe1 100644 --- a/docs/observability/ml-observability.md +++ b/docs/observability/ml-observability.md @@ -42,6 +42,22 @@ An extensive comparison of the four tools can be found as follows: The trained model can be deployed to production as container. Azure Machine Learning service provides SDK to deploy model as Azure Container Instance and publishes REST endpoint. You can monitor it using microservice observability methods( for more details -refer to [Recipes](./README.md) section). MLFLow is an alternative way to deploy ML model as a service. +### LLM, RAG, and Agent Telemetry + +Generative AI and agentic systems need traces that explain the path from user request to model output, retrieval result, tool action, and final response. Capture enough context to debug quality, safety, cost, and latency without storing sensitive prompt or response content by default. + +For LLM, RAG, and agent workloads, consider tracking: + +* Prompt or template identifier and version, plus system, developer, and user instruction versions where they are managed separately. +* Model provider, model name, deployment version, routing decision, generation settings, and fallback model if used. +* Retrieval query metadata, index version, source document or record IDs, citation IDs, freshness signals, and authorization-trimming outcome. +* Tool-call spans with tool name, schema version, input and output classification, latency, retry count, approval status, side-effect status, and error category. +* Safety filter, policy, jailbreak, content moderation, refusal, and grounding-check outcomes. +* Token counts, request cost, embedding cost, cache hits, queue time, model latency, retrieval latency, tool latency, and end-to-end latency. +* User feedback, escalation reason, correction reason, human-review outcome, and incident or support ticket link when feedback becomes operational work. + +Use stable IDs, classifications, hashes, or sampled redacted payloads instead of raw prompts, responses, retrieval snippets, memory contents, or tool outputs unless there is an explicit approved debugging need. + ## Training and Re-Training To automatically retrain the model you can use AML Pipelines or Azure Databricks. diff --git a/docs/observability/pillars/tracing.md b/docs/observability/pillars/tracing.md index 8a1af63d2..59a6d73df 100644 --- a/docs/observability/pillars/tracing.md +++ b/docs/observability/pillars/tracing.md @@ -19,4 +19,4 @@ Produces the information required to observe series of correlated operations in - [Jaeger Tracing](https://www.jaegertracing.io) - Open source, end-to-end distributed tracing. - [Grafana](https://grafana.com) - Open source dashboard & visualization tool. Supports Log, Metrics and Distributed tracing data sources. ->Consider using [OpenTelemetry](../tools/OpenTelemetry.md) as it implements open-source cross-platform context propagation for end-to-end distributed transactions over heterogeneous components out-of-the-box. It takes care of automatically creating and managing the Trace Context object among a full stack of microservices implemented across different technical stacks. +>[OpenTelemetry](../tools/OpenTelemetry.md) provides cross-platform, end-to-end distributed tracing across a full stack of microservices implemented on different technology stacks. diff --git a/docs/observability/recipes-observability.md b/docs/observability/recipes-observability.md index 2a18ea69f..fb5f9d678 100644 --- a/docs/observability/recipes-observability.md +++ b/docs/observability/recipes-observability.md @@ -32,7 +32,7 @@ This dashboard recipe provides observability for AzDO pipelines by displaying va ## Python Logger Class for Application Insights using OpenCensus -The Azure SDK for Python contains an [Azure Monitor Opentelemetry Distro client library for Python ](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry). You can view samples of how to use the library in this [GitHub Repo](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples). With this library you can easily collect traces, metrics, and logs. +The Azure SDK for Python contains an [Azure Monitor Opentelemetry Distro client library for Python](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry). You can view samples of how to use the library in this [GitHub Repo](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry/samples). With this library you can easily collect traces, metrics, and logs. ## Java OpenTelemetry Examples diff --git a/docs/security/README.md b/docs/security/README.md index 2c98169d7..180baa390 100644 --- a/docs/security/README.md +++ b/docs/security/README.md @@ -8,13 +8,25 @@ Developers working on projects should adhere to industry-recommended standard pr When requesting a security review for your application, please make sure you have familiarized yourself with the [Rules of Engagement](./rules-of-engagement.md). This will help you to prepare the application for testing, as well as understand the scope limits of the test. +## AI and agent security + +AI-enabled systems and coding agents need the same secure engineering discipline as other software, plus threat coverage for model behavior, prompts, context, and tool use. Treat model output as untrusted input before rendering, executing, storing, or sending it to another system. + +- Extend threat models using the canonical [AI systems threat-modeling considerations](./threat-modelling.md#ai-systems-threat-modeling-considerations) for prompt injection, tool authority, model and data poisoning, and related AI risks. +- Scope agent tools, credentials, file access, cloud permissions, and network access to the least privilege needed for the task. +- Confirm which AI tools are approved for this project and data type before they access customer data, source code, cloud resources, or production systems. +- Add security tests for malicious prompts, tool-call abuse, data exfiltration, unsafe generated commands, and untrusted retrieved content. +- Validate generated dependency, container, infrastructure, and CI/CD changes through the normal security review path. + +Use the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared data, review, and governance practices. + ## Quick Resources - [Secure Coding Practices Quick Reference](https://owasp.org/www-pdf-archive/OWASP_SCP_Quick_Reference_Guide_v2.pdf) - [Web Application Security Quick Reference](https://owasp.org/www-pdf-archive//OWASP_Web_Application_Security_Quick_Reference_Guide_0.3.pdf) - [Security Mindset/Creating a Security Program Quick Start](https://github.com/OWASP/Quick-Start-Guide/blob/master/OWASP%20Quick%20Start%20Guide.pdf?raw=true) - [Credential Scanning / Secret Detection](../CI-CD/dev-sec-ops/secrets-management/credential_scanning.md) -- [Threat Modelling](./threat-modelling.md) +- [Threat Modelling](./threat-modelling.md), including AI-specific guidance for prompts, retrieval, model dependencies, tool use, agent authority, and human approval gates ## Azure DevOps Security diff --git a/docs/security/threat-modelling.md b/docs/security/threat-modelling.md index 2f38c72b8..d2dfc967f 100644 --- a/docs/security/threat-modelling.md +++ b/docs/security/threat-modelling.md @@ -16,6 +16,49 @@ Threat modeling is an effective way to help secure your systems, applications, n Example of these phases is covered in the [threat modelling example.](./threat-modelling-example.md) More details about these phases can be found at [Threat Modeling Security Fundamentals.](https://learn.microsoft.com/en-us/training/paths/tm-threat-modeling-fundamentals/) +## AI Systems Threat Modeling Considerations + +Generative AI, retrieval-augmented generation (RAG), and agentic systems should go through the same threat modeling phases as other systems, with extra attention to untrusted instructions, grounding data, model dependencies, and tool authority. Include AI assets and trust boundaries in the data-flow diagram instead of treating the model as a black box. + +As we model AI systems, we give extra attention to the risk categories that classical systems rarely face: + +- Prompt injection +- Insecure output handling +- Sensitive information disclosure +- Excessive agency +- Unsafe tool calls +- Model or data poisoning +- Model theft +- AI supply chain risk + +We also make sure the data-flow diagram names the AI assets and trust boundaries that an attacker would target: + +- User prompts, uploaded files, system and developer instructions, prompt templates, and prompt stores +- Retrieval corpora, embeddings, vector indexes, source documents, citation metadata, and stale or poisoned content +- Model providers, hosted model deployments, model configuration, content filters, and fallback models +- Tool APIs, function schemas, queues, file systems, shells, external services, and write paths +- Agent memory, state stores, conversation history, secrets, delegated user permissions, telemetry, evaluation datasets, and feedback data +- Human approval, escalation, incident response, rollback, and red-team retesting points + +As we walk each boundary, we ask questions like these to turn it into concrete threats: + +- How could a prompt, retrieved document, uploaded file, web page, or tool response inject instructions that override the intended behavior? +- Can the system distinguish untrusted content from instructions, tool arguments, or policy decisions? +- Could the model or agent disclose sensitive data from prompts, retrieval results, memory, telemetry, evaluation data, or tool outputs? +- Are retrieval results security trimmed, fresh, attributable to approved sources, and resilient to poisoned or misleading content? +- Does the agent have more authority than the user, task, tenant, or environment requires? +- Can tool calls exfiltrate data, mutate records, execute commands, trigger external side effects, or bypass normal authorization checks? +- What happens if a model provider, hosted model deployment, embedding model, dependency, or safety service changes behavior or becomes unavailable? +- How are excessive token use, runaway retries, unbounded tool loops, cost spikes, and denial-of-wallet scenarios detected and stopped? +- Could model inversion, model stealing, training data leakage, or improper output handling expose protected information? + +Mitigations should be specific, testable, and tied to owners. We group the controls we reach for most often by theme: + +- Input and output handling: prompt and tool schema review, strict input and output validation, security-trimmed retrieval, and sensitive data redaction. +- Least privilege: least-privilege tools, user-context authorization, and human approval for privileged or irreversible actions. +- Operational safety: rate limits, circuit breakers, dry-run modes for write paths, and audit logs. +- Recovery and follow-up: rollback paths for prompts and indexes, and retesting after red-team findings are fixed. + ## Threat Modeling Example [Here is an example](./threat-modelling-example.md) of a threat modeling document which talks about the architecture and different phases involved in the threat modeling. This document can be used as reference template for creating threat modeling documents. diff --git a/docs/source-control/README.md b/docs/source-control/README.md index ff7c362be..8414fc776 100644 --- a/docs/source-control/README.md +++ b/docs/source-control/README.md @@ -12,6 +12,14 @@ There are many options when working with Source Control. In [ISE](../ISE.md) we Consistency is important, so agree to the approach as a team before starting to code. Treat this as a design decision, so include a design proposal and review, in the same way as you would document all design decisions (see [Working Agreements](../agile-development/team-agreements/working-agreement.md) and [Design Reviews](../design/design-reviews/README.md)). +When AI assistants or coding agents help author changes, use the team's normal source-control conventions for branch ownership, review, commit messages, and traceability. Cross-link to the [Copilots team operating model](../developer-experience/copilots.md#team-operating-model-for-ai-assisted-delivery) and [AI-assisted authorship guidance](../developer-experience/copilots.md#attributing-ai-assisted-code-authorship) instead of creating separate attribution rules in each repository. + +AI assistance does not relax branch protection, pull request review, tests, or work item linkage. Keep AI-assisted changes small enough for a human reviewer to understand and link them to the same evidence expected for other work. + +* Require AI agent branches and pull requests to link to a work item, tests, and a human reviewer. +* Do not commit prompts, transcripts, or generated artifacts that contain secrets or customer data. +* Use [Git guidance](./git-guidance/README.md#ai-assisted-change-traceability) and the [AI-Assisted Engineering](../ai-assisted-engineering/README.md) guide for shared authorship and traceability practices, including how to record material AI assistance. + ## Creating a New Repository When creating a new repository, the team should at least do the following @@ -21,6 +29,7 @@ When creating a new repository, the team should at least do the following * Lock the default branch and merge using [pull requests (PRs)](../code-reviews/pull-requests.md) * Agree on [branch naming](./naming-branches.md) (e.g. `user/your_alias/feature_name`) * Establish [branch/PR policies](../code-reviews/pull-requests.md) +* Agree how agent-authored branches and PRs identify the supervising human owner * For public repositories the default branch should contain the following files: * LICENSE * README.md diff --git a/docs/source-control/git-guidance/README.md b/docs/source-control/git-guidance/README.md index 9970b5e49..dd0414650 100644 --- a/docs/source-control/git-guidance/README.md +++ b/docs/source-control/git-guidance/README.md @@ -272,6 +272,12 @@ For more information on commit message conventions, see: * [On commit messages](http://who-t.blogspot.com/2009/12/on-commit-messages.html) * [Git trailers](https://git-scm.com/docs/git-interpret-trailers) +### AI-assisted change traceability + +AI-assisted commits and pull requests follow the same branch, review, test, and work item practices as other changes. Record material AI assistance according to team convention, such as a pull request note, work item comment, or commit trailer when the team adopts one. + +Do not commit prompts, transcripts, or generated artifacts that contain secrets, customer data, or unnecessary sensitive context. See the [AI-Assisted Engineering](../../ai-assisted-engineering/README.md) guide for shared data, review, and authorship considerations. + ## Managing Remotes A local git repository can have one or more backing remote repositories. You can list the remote repositories using `git remote` - by default, the remote repository you cloned from will be called origin diff --git a/docs/source-control/naming-branches.md b/docs/source-control/naming-branches.md index 792bd28b1..91943ac86 100644 --- a/docs/source-control/naming-branches.md +++ b/docs/source-control/naming-branches.md @@ -17,3 +17,5 @@ dickinson/feature/271_add_more_cowbell ``` The example above is just that - an example. The team can choose to omit or add parts. Choosing a branch convention can depend on the development model (e.g. [trunk-based development](https://trunkbaseddevelopment.com/)), [versioning](component-versioning.md) model, tools used in managing source control, matter of taste etc. Focus on simplicity and reducing ambiguity; a good branch naming strategy allows the team to understand the purpose and ownership of each branch in the repository. + +If coding agents can create branches, decide whether the existing convention is enough or whether agent-authored work needs an explicit owner marker. A simple option is to keep the supervising human alias and work item in the branch name, such as `alias/agent/1234_update_validation`, so reviewers know who owns follow-up, secrets handling, and merge decisions. Tool-generated prefixes are useful only when they remain short, searchable, and tied to the same PR review expectations as human-authored branches. diff --git a/docs/source-control/secrets-management.md b/docs/source-control/secrets-management.md index 685f7ccad..5f1b1a95e 100644 --- a/docs/source-control/secrets-management.md +++ b/docs/source-control/secrets-management.md @@ -10,4 +10,6 @@ E.g. the following pattern will exclude all files with the extension `.private.c For more details on proper management of credentials and secrets in source control, and handling an accidental commit of secrets to source control, please refer to the [Secrets Management](../CI-CD/dev-sec-ops/secrets-management/README.md) document which has further information, split by language as well. +AI prompts, tool configuration, agent transcripts, retrieval traces, and model logs can also capture secrets or sensitive operational details. Do not commit prompts or logs that include credentials, tokens, customer data, private keys, production incident details, or tool outputs that policy would not allow in source control. Review reusable prompt files, MCP or tool manifests, and evaluation fixtures with the same secret-scanning expectations as scripts and configuration. + As an extra security measure, apply [credential scanning](../CI-CD/dev-sec-ops/secrets-management/credential_scanning.md) in your CI/CD pipeline. diff --git a/requirements-docs.txt b/requirements-docs.txt index 7fa7f29c0..8f0dd8d2c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,6 +1,7 @@ mkdocs==1.3.* mkdocs-material==8.2.* pymdown-extensions>=10.0 +pygments<2.20 markdown==3.3.* mdx_truly_sane_lists==1.2 mkdocs-git-revision-date-localized-plugin==1.0.*