diff --git a/.github/actions/configure-package-indexes/action.yml b/.github/actions/configure-package-indexes/action.yml new file mode 100644 index 000000000000..1c40250ca3f8 --- /dev/null +++ b/.github/actions/configure-package-indexes/action.yml @@ -0,0 +1,11 @@ +name: 'Configure Package Indexes' +description: 'Configure pip/uv package indexes to use Azure SDK CFS feed' + +runs: + using: 'composite' + steps: + - name: Configure package indexes + run: | + echo "PIP_INDEX_URL=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" >> "$GITHUB_ENV" + echo "UV_DEFAULT_INDEX=https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" >> "$GITHUB_ENV" + shell: bash diff --git a/.github/workflows/azure-sdk-tools.yml b/.github/workflows/azure-sdk-tools.yml index ed1a6ed97da9..50308f6ac842 100644 --- a/.github/workflows/azure-sdk-tools.yml +++ b/.github/workflows/azure-sdk-tools.yml @@ -19,6 +19,9 @@ jobs: with: python-version: 3.13 + - name: Configure package indexes + uses: ./.github/actions/configure-package-indexes + - name: Install azure-sdk-tools run: | python -m pip install -e eng/tools/azure-sdk-tools[ghtools,conda] @@ -41,6 +44,9 @@ jobs: with: python-version: 3.13 + - name: Configure package indexes + uses: ./.github/actions/configure-package-indexes + - name: Install azure-sdk-tools run: | python -m pip install -e eng/tools/azure-sdk-tools[ghtools,conda] @@ -63,6 +69,9 @@ jobs: with: python-version: 3.13 + - name: Configure package indexes + uses: ./.github/actions/configure-package-indexes + - name: Install uv run: | curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8fade23c59f6..914bf10c8c6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,6 +97,75 @@ SDK performance testing is supported via the custom `perfstress` framework. For We maintain an [additional document](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/eng_sys_checks.md) that has a ton of detail as to what is actually _happening_ in these executions. +### Package Index Configuration + +This repo uses Central Feed Services (CFS) as the default package source for CI, and prefers it for local development over PyPI. + + A repo-root `uv.toml` configures uv to use the CFS feed automatically. + + To use the feed with standard `pip`, use the `--index-url` flag. For example: + + ```bash + pip install --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/ + ``` + + #### `azpysdk` Configuration + + The `azpysdk` tool automatically configures pip and uv subprocesses to use the CFS feed by setting the `PIP_INDEX_URL` and `UV_DEFAULT_INDEX` environment variables. If you have already set these variables to a different feed, `azpysdk` will respect your configuration and won't override them. + +If you wish to bypass CFS and install directly from PyPI, use the `--pypi` flag to pull from PyPI: + +``` +azpysdk --pypi +``` + +#### Authentication for upstream pull-through +When installing a package version not yet cached in the CFS feed, it needs to be pulled through from PyPI upstream, which requires authentication. + +Below are options for authenticating: + +**Authenticate in terminal (with uv):** + +This requires the Azure CLI. First run: + +```bash +az login +``` + +Then: + +```bash +export UV_INDEX_AZURE_SDK_USERNAME=x +export UV_INDEX_AZURE_SDK_PASSWORD=$(az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv) +``` + +**Create a PAT in Azure DevOps:** + +1. Go to [https://dev.azure.com/azure-sdk](https://dev.azure.com/azure-sdk) +2. Click the user settings button in the top right corner next to your profile picture +3. Select "Personal access tokens" +4. Click "New Token" +5. Give it a name (e.g., "Azure SDK Python Feed Access") +6. Set expiration as needed +7. Under "Scopes" -> "Packaging", select "Read, write, & manage" +8. Click "Create" +9. Copy the token and use it in your environment: + +```bash +export UV_INDEX_AZURE_SDK_USERNAME=x +export UV_INDEX_AZURE_SDK_PASSWORD= +``` + +Or for pip: +```bash +export PIP_INDEX_URL="https://your-azure-username:your-pat-token@pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" +``` + +**To override the CFS index and use PyPI with uv:** +```bash +uv pip install --index-url https://pypi.org/simple/ +``` + ### Dev Feed Daily dev build version of Azure sdk packages for python are available and are uploaded to Azure devops feed daily. Below is the link to Azure devops feed. [`https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-python`](https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-python) diff --git a/doc/tool_usage_guide.md b/doc/tool_usage_guide.md index 3c3e03155619..32e34ffd625c 100644 --- a/doc/tool_usage_guide.md +++ b/doc/tool_usage_guide.md @@ -67,6 +67,23 @@ To utilize this feature, add `--isolate` to any `azpysdk` invocation: - The monorepo requires a minimum of `python 3.9`, but `>=3.11` is required for the `sphinx` check due to compatibility constraints with external processes. - You may optionally use the ["uv"](https://docs.astral.sh/uv/) tool, which is fast and handles Python version and venv creation automatically. +## Package Index (CFS) + +This repo defaults to using Central Feed Services (CFS) as the package source instead of PyPI directly. This provides a security layer between the repo and PyPI for all package installs. + +**How it works:** +- A repo-root `uv.toml` configures uv to use the CFS feed automatically for all `uv pip install` / `uv sync` commands run within the repo. +- `azpysdk` sets `PIP_INDEX_URL` and `UV_DEFAULT_INDEX` environment variables so pip and uv subprocesses also use the CFS feed. If you have already set these variables to a different feed, `azpysdk` will respect your configuration and won't override them. + +**Authentication for upstream pull-through:** +When a package version is not yet cached in the CFS feed, uv/pip pulls it through from PyPI upstream, which requires authentication. See [CONTRIBUTING.md](https://github.com/Azure/azure-sdk-for-python/blob/main/CONTRIBUTING.md#authentication-for-upstream-pull-through) for details. + +**Bypassing CFS:** +If you need to install directly from PyPI (e.g., for a package not yet in the feed and auth isn't set up), use: +```bash +azpysdk --pypi +``` + ## Initial setup - Go to the folder of the package you're working on, for instance `sdk/contoso/azure-contoso` diff --git a/eng/tools/azure-sdk-tools/azpysdk/main.py b/eng/tools/azure-sdk-tools/azpysdk/main.py index 00f03d5fd9c1..ee4ef6b45e06 100644 --- a/eng/tools/azure-sdk-tools/azpysdk/main.py +++ b/eng/tools/azure-sdk-tools/azpysdk/main.py @@ -41,12 +41,13 @@ from .devtest import devtest from .optional import optional from .update_snippet import update_snippet - from ci_tools.logging import configure_logging, logger __all__ = ["main", "build_parser"] __version__ = "0.0.0" +CFS_INDEX_URL = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + def build_parser() -> argparse.ArgumentParser: """Create and return the top-level ArgumentParser for the CLI.""" @@ -56,6 +57,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--isolate", action="store_true", default=False, help="If set, run in an isolated virtual environment." ) + parser.add_argument( + "--pypi", + action="store_true", + default=False, + help="Use PyPI directly instead of the CFS (Central Feed Services) feed.", + ) parser.add_argument( "--python", default=None, @@ -155,6 +162,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int: Exit code to return to the OS. """ original_cwd = os.getcwd() + # Save original env vars so we can restore them when azpysdk finishes + original_pip_index = os.environ.get("PIP_INDEX_URL") + original_uv_index = os.environ.get("UV_DEFAULT_INDEX") parser = build_parser() args = parser.parse_args(argv) @@ -171,6 +181,23 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if uv_path: os.environ["TOX_PIP_IMPL"] = "uv" + # default to CFS feed unless --pypi is specified, but allow explicit env var override (e.g. for CI) + if args.pypi: + # Explicitly set PyPI URLs to override uv.toml CFS default + os.environ["PIP_INDEX_URL"] = "https://pypi.org/simple/" + os.environ["UV_DEFAULT_INDEX"] = "https://pypi.org/simple/" + logger.info("Installing from PyPI (--pypi flag set)") + else: + using_cfs = False + if not os.environ.get("PIP_INDEX_URL"): + os.environ["PIP_INDEX_URL"] = CFS_INDEX_URL + using_cfs = True + if not os.environ.get("UV_DEFAULT_INDEX"): + os.environ["UV_DEFAULT_INDEX"] = CFS_INDEX_URL + using_cfs = True + + if using_cfs: + logger.info("Installing from CFS feed: %s", CFS_INDEX_URL) # --python requires both --isolate and uv python_version = getattr(args, "python_version", None) if python_version: @@ -196,6 +223,15 @@ def main(argv: Optional[Sequence[str]] = None) -> int: return 2 finally: os.chdir(original_cwd) + # Restore original env vars (or remove them if they weren't set before) + if original_pip_index is None: + os.environ.pop("PIP_INDEX_URL", None) + else: + os.environ["PIP_INDEX_URL"] = original_pip_index + if original_uv_index is None: + os.environ.pop("UV_DEFAULT_INDEX", None) + else: + os.environ["UV_DEFAULT_INDEX"] = original_uv_index if __name__ == "__main__": diff --git a/eng/tools/azure-sdk-tools/tests/test_pypi_client.py b/eng/tools/azure-sdk-tools/tests/test_pypi_client.py index 186837dd0349..c93fe7db93f3 100644 --- a/eng/tools/azure-sdk-tools/tests/test_pypi_client.py +++ b/eng/tools/azure-sdk-tools/tests/test_pypi_client.py @@ -133,7 +133,7 @@ class TestPyPIOnlyMethods: @SKIP_IN_CI def test_project_returns_info_and_releases(self): - client = PyPIClient() + client = _make_client(PYPI_HOST) result = client.project(WELL_KNOWN_PACKAGE) assert result["info"]["name"] == WELL_KNOWN_PACKAGE @@ -145,7 +145,7 @@ def test_project_returns_info_and_releases(self): @patch("pypi_tools.pypi.sys") def test_filter_packages_for_compatibility(self, mock_sys): mock_sys.version_info = (2, 7, 0) - client = PyPIClient() + client = _make_client(PYPI_HOST) filtered = client.get_ordered_versions(WELL_KNOWN_PACKAGE, True) unfiltered = client.get_ordered_versions(WELL_KNOWN_PACKAGE, False) assert len(filtered) < len(unfiltered) diff --git a/uv.toml b/uv.toml new file mode 100644 index 000000000000..23e19998c5af --- /dev/null +++ b/uv.toml @@ -0,0 +1,4 @@ +[[index]] +name = "azure-sdk" +url = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" +default = true