From da42a386e284bec5fd40b8b1d1e7486dee8c1a24 Mon Sep 17 00:00:00 2001 From: Federico Kamelhar Date: Wed, 23 Sep 2026 23:44:22 -0400 Subject: [PATCH 1/4] fix(oci): resolve availability domains from the compartment `OCIRegionClient.availability_domains` lists the availability domains of the tenancy in the credentials. The compartment dstack works in does not have to belong to that tenancy: a compartment can be shared with the credentials' tenancy by a cross-tenancy policy, and `compartment_id` in the backend config then points into another tenancy. Availability domain names are tenancy-specific, so in that setup every shape lookup asks for a domain the compartment's tenancy does not have and OCI answers 404 NotAuthorizedOrNotFound. The backend configures, but no offers are ever returned: list_shapes(availability_domain="", compartment_id="") -> ServiceError(status=404, code='NotAuthorizedOrNotFound', operation_name='list_shapes') `ListAvailabilityDomains` accepts any compartment OCID and answers for the tenancy owning it, so pass the compartment dstack was configured with. Lookups that are not compartment-scoped keep using the credentials' tenancy. --- .../_internal/core/backends/oci/region.py | 18 ++++++- .../_internal/core/backends/oci/resources.py | 6 +-- .../core/backends/oci/test_region.py | 54 +++++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 src/tests/_internal/core/backends/oci/test_region.py diff --git a/src/dstack/_internal/core/backends/oci/region.py b/src/dstack/_internal/core/backends/oci/region.py index 806ed76fd1..517b3beb0e 100644 --- a/src/dstack/_internal/core/backends/oci/region.py +++ b/src/dstack/_internal/core/backends/oci/region.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from functools import cached_property +from functools import cached_property, lru_cache from typing import Any, Dict, Iterable, List, Mapping, Set import oci @@ -46,7 +46,21 @@ def work_request_client(self) -> oci.work_requests.WorkRequestClient: @cached_property def availability_domains(self) -> List[oci.identity.models.AvailabilityDomain]: - return self.identity_client.list_availability_domains(self.client_config["tenancy"]).data + return self.availability_domains_in(self.client_config["tenancy"]) + + @lru_cache(maxsize=None) + def availability_domains_in( + self, compartment_id: str + ) -> List[oci.identity.models.AvailabilityDomain]: + """ + Availability domains of the tenancy that owns `compartment_id`. + + The compartment dstack works in need not belong to the tenancy the credentials + authenticate against - a compartment can be shared with another tenancy by a + cross-tenancy policy. Availability domain names are tenancy-specific, so they + have to be resolved from the compartment rather than from the credentials. + """ + return self.identity_client.list_availability_domains(compartment_id).data def make_region_client(region_name: str, creds: AnyOCICreds) -> OCIRegionClient: diff --git a/src/dstack/_internal/core/backends/oci/resources.py b/src/dstack/_internal/core/backends/oci/resources.py index 91e1d890f0..413395a4ca 100644 --- a/src/dstack/_internal/core/backends/oci/resources.py +++ b/src/dstack/_internal/core/backends/oci/resources.py @@ -140,7 +140,7 @@ def list_shapes_in_region(region: OCIRegionClient, compartment_id: str) -> Dict[ """ result = {} - for availability_domain in region.availability_domains: + for availability_domain in region.availability_domains_in(compartment_id): result[availability_domain.name] = list_shapes_in_domain( availability_domain.name, region.compute_client, compartment_id ) @@ -214,7 +214,7 @@ def check_availability_per_domain( all_shapes = set(shape_names) available_shapes_per_domain = {} - for availability_domain in region.availability_domains: + for availability_domain in region.availability_domains_in(compartment_id): shapes_to_check = { shape for shape in all_shapes @@ -277,7 +277,7 @@ def get_available_domains( `shape_name` is available and within `shapes_quota`. """ domains = [] - for domain in region.availability_domains: + for domain in region.availability_domains_in(compartment_id): if shapes_quota.is_within_domain_quota( shape_name, domain.name ) and check_availability_in_domain( diff --git a/src/tests/_internal/core/backends/oci/test_region.py b/src/tests/_internal/core/backends/oci/test_region.py new file mode 100644 index 0000000000..4d901ba0b5 --- /dev/null +++ b/src/tests/_internal/core/backends/oci/test_region.py @@ -0,0 +1,54 @@ +from unittest.mock import MagicMock, patch + +from dstack._internal.core.backends.oci.region import OCIRegionClient + +CREDS_TENANCY = "ocid1.tenancy.oc1..credentials" +OTHER_TENANCY_COMPARTMENT = "ocid1.compartment.oc1..shared-in-from-another-tenancy" + + +def make_region_client() -> OCIRegionClient: + return OCIRegionClient( + { + "tenancy": CREDS_TENANCY, + "region": "us-phoenix-1", + "user": "ocid1.user.oc1..aaaa", + "fingerprint": "00:11:22", + "key_file": "/dev/null", + } + ) + + +class TestAvailabilityDomainsIn: + def test_resolves_domains_from_the_compartment_not_the_credentials(self): + region = make_region_client() + identity_client = MagicMock() + with patch.object( + OCIRegionClient, "identity_client", new_callable=lambda: identity_client + ): + region.availability_domains_in(OTHER_TENANCY_COMPARTMENT) + + identity_client.list_availability_domains.assert_called_once_with( + OTHER_TENANCY_COMPARTMENT + ) + + def test_availability_domains_defaults_to_the_credentials_tenancy(self): + region = make_region_client() + identity_client = MagicMock() + with patch.object( + OCIRegionClient, "identity_client", new_callable=lambda: identity_client + ): + region.availability_domains + + identity_client.list_availability_domains.assert_called_once_with(CREDS_TENANCY) + + def test_result_is_cached_per_compartment(self): + region = make_region_client() + identity_client = MagicMock() + with patch.object( + OCIRegionClient, "identity_client", new_callable=lambda: identity_client + ): + region.availability_domains_in(OTHER_TENANCY_COMPARTMENT) + region.availability_domains_in(OTHER_TENANCY_COMPARTMENT) + region.availability_domains_in(CREDS_TENANCY) + + assert identity_client.list_availability_domains.call_count == 2 From ea1e85fee3a495a62434f0c69de20f4fe692939c Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Mon, 28 Sep 2026 22:12:25 +0200 Subject: [PATCH 2/4] Simplify --- .../_internal/core/backends/oci/region.py | 14 +---- .../core/backends/oci/test_region.py | 54 ------------------- 2 files changed, 1 insertion(+), 67 deletions(-) delete mode 100644 src/tests/_internal/core/backends/oci/test_region.py diff --git a/src/dstack/_internal/core/backends/oci/region.py b/src/dstack/_internal/core/backends/oci/region.py index 517b3beb0e..ab996d32f8 100644 --- a/src/dstack/_internal/core/backends/oci/region.py +++ b/src/dstack/_internal/core/backends/oci/region.py @@ -44,22 +44,10 @@ def virtual_network_client(self) -> oci.core.VirtualNetworkClient: def work_request_client(self) -> oci.work_requests.WorkRequestClient: return oci.work_requests.WorkRequestClient(self.client_config) - @cached_property - def availability_domains(self) -> List[oci.identity.models.AvailabilityDomain]: - return self.availability_domains_in(self.client_config["tenancy"]) - - @lru_cache(maxsize=None) + @lru_cache() def availability_domains_in( self, compartment_id: str ) -> List[oci.identity.models.AvailabilityDomain]: - """ - Availability domains of the tenancy that owns `compartment_id`. - - The compartment dstack works in need not belong to the tenancy the credentials - authenticate against - a compartment can be shared with another tenancy by a - cross-tenancy policy. Availability domain names are tenancy-specific, so they - have to be resolved from the compartment rather than from the credentials. - """ return self.identity_client.list_availability_domains(compartment_id).data diff --git a/src/tests/_internal/core/backends/oci/test_region.py b/src/tests/_internal/core/backends/oci/test_region.py deleted file mode 100644 index 4d901ba0b5..0000000000 --- a/src/tests/_internal/core/backends/oci/test_region.py +++ /dev/null @@ -1,54 +0,0 @@ -from unittest.mock import MagicMock, patch - -from dstack._internal.core.backends.oci.region import OCIRegionClient - -CREDS_TENANCY = "ocid1.tenancy.oc1..credentials" -OTHER_TENANCY_COMPARTMENT = "ocid1.compartment.oc1..shared-in-from-another-tenancy" - - -def make_region_client() -> OCIRegionClient: - return OCIRegionClient( - { - "tenancy": CREDS_TENANCY, - "region": "us-phoenix-1", - "user": "ocid1.user.oc1..aaaa", - "fingerprint": "00:11:22", - "key_file": "/dev/null", - } - ) - - -class TestAvailabilityDomainsIn: - def test_resolves_domains_from_the_compartment_not_the_credentials(self): - region = make_region_client() - identity_client = MagicMock() - with patch.object( - OCIRegionClient, "identity_client", new_callable=lambda: identity_client - ): - region.availability_domains_in(OTHER_TENANCY_COMPARTMENT) - - identity_client.list_availability_domains.assert_called_once_with( - OTHER_TENANCY_COMPARTMENT - ) - - def test_availability_domains_defaults_to_the_credentials_tenancy(self): - region = make_region_client() - identity_client = MagicMock() - with patch.object( - OCIRegionClient, "identity_client", new_callable=lambda: identity_client - ): - region.availability_domains - - identity_client.list_availability_domains.assert_called_once_with(CREDS_TENANCY) - - def test_result_is_cached_per_compartment(self): - region = make_region_client() - identity_client = MagicMock() - with patch.object( - OCIRegionClient, "identity_client", new_callable=lambda: identity_client - ): - region.availability_domains_in(OTHER_TENANCY_COMPARTMENT) - region.availability_domains_in(OTHER_TENANCY_COMPARTMENT) - region.availability_domains_in(CREDS_TENANCY) - - assert identity_client.list_availability_domains.call_count == 2 From 2935a055efe831c6ada48dceece94510768676ed Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Mon, 28 Sep 2026 22:35:24 +0200 Subject: [PATCH 3/4] Fix cache leak --- src/dstack/_internal/core/backends/oci/region.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dstack/_internal/core/backends/oci/region.py b/src/dstack/_internal/core/backends/oci/region.py index ab996d32f8..2d4f59023d 100644 --- a/src/dstack/_internal/core/backends/oci/region.py +++ b/src/dstack/_internal/core/backends/oci/region.py @@ -1,7 +1,8 @@ from dataclasses import dataclass -from functools import cached_property, lru_cache +from functools import cached_property from typing import Any, Dict, Iterable, List, Mapping, Set +import cachetools import oci from dstack._internal.core.backends.oci.auth import get_client_config @@ -15,6 +16,9 @@ class OCIRegionClient: def __init__(self, client_config: Mapping[str, Any]): self.client_config = client_config + self._availability_domains_cache: Dict[ + str, List[oci.identity.models.AvailabilityDomain] + ] = {} @property def name(self) -> str: @@ -44,7 +48,7 @@ def virtual_network_client(self) -> oci.core.VirtualNetworkClient: def work_request_client(self) -> oci.work_requests.WorkRequestClient: return oci.work_requests.WorkRequestClient(self.client_config) - @lru_cache() + @cachetools.cachedmethod(lambda self: self._availability_domains_cache) def availability_domains_in( self, compartment_id: str ) -> List[oci.identity.models.AvailabilityDomain]: From 8f62a8df796487f6a4475944877b17633adf62f8 Mon Sep 17 00:00:00 2001 From: Jvst Me Date: Mon, 28 Sep 2026 23:32:52 +0200 Subject: [PATCH 4/4] Adjust caching --- src/dstack/_internal/core/backends/oci/region.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/dstack/_internal/core/backends/oci/region.py b/src/dstack/_internal/core/backends/oci/region.py index 2d4f59023d..8967e9ce80 100644 --- a/src/dstack/_internal/core/backends/oci/region.py +++ b/src/dstack/_internal/core/backends/oci/region.py @@ -1,9 +1,10 @@ +import threading from dataclasses import dataclass from functools import cached_property from typing import Any, Dict, Iterable, List, Mapping, Set -import cachetools import oci +from cachetools import Cache, cachedmethod from dstack._internal.core.backends.oci.auth import get_client_config from dstack._internal.core.backends.oci.models import AnyOCICreds @@ -16,9 +17,8 @@ class OCIRegionClient: def __init__(self, client_config: Mapping[str, Any]): self.client_config = client_config - self._availability_domains_cache: Dict[ - str, List[oci.identity.models.AvailabilityDomain] - ] = {} + self._availability_domains_cache = Cache(maxsize=10) + self._availability_domains_lock = threading.Lock() @property def name(self) -> str: @@ -48,7 +48,10 @@ def virtual_network_client(self) -> oci.core.VirtualNetworkClient: def work_request_client(self) -> oci.work_requests.WorkRequestClient: return oci.work_requests.WorkRequestClient(self.client_config) - @cachetools.cachedmethod(lambda self: self._availability_domains_cache) + @cachedmethod( + cache=lambda self: self._availability_domains_cache, + lock=lambda self: self._availability_domains_lock, + ) def availability_domains_in( self, compartment_id: str ) -> List[oci.identity.models.AvailabilityDomain]: