diff --git a/.github/workflows/ci-test.yaml b/.github/workflows/ci-test.yaml index 13de1475f4..87bf2f658f 100644 --- a/.github/workflows/ci-test.yaml +++ b/.github/workflows/ci-test.yaml @@ -4,22 +4,47 @@ on: push: branches: - main - paths-ignore: - - "docker/**" - - "frontend/**" - - "docs/**" pull_request: types: [opened, synchronize, reopened, ready_for_review] - branches: [main] - paths-ignore: - - "docker/**" - - "frontend/**" - - "docs/**" jobs: + # Path filtering at job level, not trigger paths-ignore: an untriggered + # workflow reports no check run, so a required check waits forever; a job + # skipped via `if:` reports conclusion `skipped`, which passes. + changes: + runs-on: ubuntu-latest + outputs: + relevant: ${{ steps.filter.outputs.relevant }} + steps: + - name: Checkout repository + # paths-filter needs the local git history for push events. + uses: actions/checkout@v6 + + - name: Check for changes outside ignored paths + uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + id: filter + with: + # `every`: a file counts as relevant only if it matches ALL the + # negated globs, i.e. lies outside every ignored dir. + predicate-quantifier: every + filters: | + relevant: + - "!docker/**" + - "!frontend/**" + - "!docs/**" + + # Tiers share no state (separate runners, disjoint groups), so they run as + # parallel matrix legs; whatever needs both tiers' results happens in `report`. test: - if: github.event.pull_request.draft == false + needs: changes + if: needs.changes.outputs.relevant == 'true' && github.event.pull_request.draft == false runs-on: ubuntu-latest + strategy: + # One tier failing must not cancel the other; the report job needs + # whatever results exist from both. + fail-fast: false + matrix: + tier: [unit, integration] steps: - name: Checkout repository @@ -37,21 +62,20 @@ jobs: uses: actions/cache@v5 with: path: .tox/ - key: ${{ runner.os }}-tox-uv-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }} + # Tier in the key: legs build different tox envs, and parallel + # saves to one key race (first save wins). + key: ${{ runner.os }}-tox-uv-${{ matrix.tier }}-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }} restore-keys: | + ${{ runner.os }}-tox-uv-${{ matrix.tier }}- ${{ runner.os }}-tox-uv- - name: Restore main-branch test baseline (for regression detection) - # actions/cache only saves on a cache miss. Include the run id in the - # key so each main build writes a fresh cache entry; the prefix in - # restore-keys pulls the most recent baseline. - # - # The unit/integration lane keeps a SEPARATE baseline from the e2e - # workflow because their `scope_groups` don't overlap — restoring an - # e2e-tier baseline here would flag every e2e-covered path as a - # regression in this lane (and vice versa). Each workflow is the - # source of truth for the paths covered by its own tiers. - uses: actions/cache@v5 + # Restore-only: the report job is the sole baseline writer (a save + # here would claim this run's key with an unmerged file). Baseline is + # separate from the e2e workflow's — their scope_groups don't overlap, + # and a cross-lane restore would flag the other lane's paths as + # regressions. + uses: actions/cache/restore@v5 with: path: reports/previous-summary.json key: unstract-test-baseline-ut-main-${{ github.run_id }} @@ -64,35 +88,105 @@ jobs: - name: Validate test manifests # Cheap pre-flight: catches groups.yaml / critical_paths.yaml schema - # errors before we spend minutes on tier runs. Also catches malformed - # manifests on PRs that only touch paths-ignored files (because this - # step always runs). + # errors before we spend minutes on tier runs. run: tox -e rig -- validate - - name: Run unit tier - # Each tier runs as a separate rig invocation so its results land in - # reports// before the next tier starts. --update-baseline (on - # main only) merges this tier's covered paths into the cached - # previous-summary.json; later tiers union on top. + - name: Run ${{ matrix.tier }} tier + # Gap check on PRs too: main is gap-free and PRs test the merge ref, + # so a gap here can only be PR-introduced. No --update-baseline: + # parallel writes would race; the report job merges once (main only). + run: tox -e ${{ matrix.tier }} -- --fail-on-critical-gap + + - name: Suffix coverage data for cross-tier merge + # Both tiers emit a bare reports/.coverage that would clobber on + # download; suffixed names match the `.coverage.*` glob `report + # combine` unions. + if: always() run: | - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - tox -e unit -- --fail-on-critical-gap --update-baseline - else - tox -e unit + if [ -f reports/.coverage ]; then + mv reports/.coverage "reports/.coverage.tier-${{ matrix.tier }}" fi - - name: Run integration tier + - name: Upload tier reports artifact if: always() + uses: actions/upload-artifact@v4 + with: + name: reports-${{ matrix.tier }} + # Baseline excluded: it's cache-managed, and a stale copy inside the + # artifact would overwrite the report job's freshly restored one. + path: | + reports/ + !reports/previous-summary.json + if-no-files-found: ignore + retention-days: 14 + + report: + needs: [changes, test] + # `always()` so a red tier still gets a PR report. Skip only a legitimate + # skip — irrelevant paths / draft, decided by a *successful* changes job. + # A failed changes job skips the tiers, so this must run and go red + # (final step) instead of green-skipping the required check. + if: >- + always() && (needs.changes.result != 'success' + || (needs.changes.outputs.relevant == 'true' && github.event.pull_request.draft == false)) + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + version: "0.6.14" + python-version: 3.12.9 + + - name: Cache tox environments + uses: actions/cache@v5 + with: + path: .tox/ + key: ${{ runner.os }}-tox-uv-rig-${{ hashFiles('**/pyproject.toml', '**/tox.ini') }} + restore-keys: | + ${{ runner.os }}-tox-uv-rig- + ${{ runner.os }}-tox-uv- + + - name: Restore main-branch test baseline + uses: actions/cache/restore@v5 + with: + path: reports/previous-summary.json + key: unstract-test-baseline-ut-main-${{ github.run_id }} + restore-keys: | + unstract-test-baseline-ut-main- + unstract-test-baseline-ut- + + - name: Install tox with UV + run: uv tool install tox --with tox-uv + + - name: Download tier reports + uses: actions/download-artifact@v4 + with: + pattern: reports-* + path: reports/ + merge-multiple: true + + - name: Combine reports from both tiers + # On green main builds also merge covered paths into the baseline + # (single writer; the rig refuses the merge if any group is red). run: | - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - tox -e integration -- --fail-on-critical-gap --update-baseline - else - tox -e integration + flags="" + if [ "${{ github.ref }}" = "refs/heads/main" ] && [ "${{ needs.test.result }}" = "success" ]; then + flags="--update-baseline" fi + tox -e rig -- report combine $flags - - name: Re-aggregate reports from both tiers - if: always() - run: tox -e rig -- report combine + - name: Save updated baseline + # actions/cache saves only on a miss; the run id keys each main build + # a fresh entry, and restore-keys' prefix pulls the latest next run. + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@v5 + with: + path: reports/previous-summary.json + key: unstract-test-baseline-ut-main-${{ github.run_id }} - name: Render combined test report to PR uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3.0.2 @@ -116,3 +210,10 @@ jobs: path: reports/ if-no-files-found: ignore retention-days: 14 + + - name: Fail if any tier failed + # Lets this job serve as the single branch-protection check: a matrix + # `result` is success only when every leg passed. + if: always() + run: | + [ "${{ needs.test.result }}" = "success" ] || exit 1 diff --git a/backend/adapter_processor_v2/tests/test_adapter_api.py b/backend/adapter_processor_v2/tests/test_adapter_api.py new file mode 100644 index 0000000000..5df0d748f6 --- /dev/null +++ b/backend/adapter_processor_v2/tests/test_adapter_api.py @@ -0,0 +1,61 @@ +"""Critical path ``adapter-register-llm``: POST /api/v1/adapter/ registers an +LLM adapter. Exercises the real endpoint wiring — auth, serializer, metadata +encryption, org-scoped persistence — with only the SDK context-window lookup +(a provider-shaped call) mocked. Needs a live DB (integration tier). +""" + +from __future__ import annotations + +import secrets +from unittest.mock import patch + +import pytest +from account_v2.models import Organization, User +from django.test import TestCase +from rest_framework import status +from rest_framework.test import APIRequestFactory, force_authenticate +from tenant_account_v2.models import OrganizationMember +from utils.user_context import UserContext + +from adapter_processor_v2.models import AdapterInstance +from adapter_processor_v2.views import AdapterInstanceViewSet + + +class AdapterRegisterLLMAPITest(TestCase): + def setUp(self) -> None: + self.org = Organization.objects.create( + name="org-a", display_name="Org A", organization_id="org-a" + ) + UserContext.set_organization_identifier(self.org.organization_id) + self.user = User.objects.create_user( + username="owner@example.com", + email="owner@example.com", + password=secrets.token_urlsafe(), + ) + OrganizationMember.objects.create( + organization=self.org, user=self.user, role="user" + ) + self.create_view = AdapterInstanceViewSet.as_view({"post": "create"}) + + @pytest.mark.critical_path("adapter-register-llm") + @patch.object(AdapterInstance, "get_context_window_size", return_value=4096) + def test_register_llm_adapter_persists_encrypted(self, _ctx_window) -> None: + payload = { + "adapter_id": "openai|test-llm", + "adapter_name": "my-openai", + "adapter_type": "LLM", + "adapter_metadata": {"api_key": "sk-test", "model": "gpt-4o-mini"}, + } + request = APIRequestFactory().post("/api/v1/adapter/", payload, format="json") + force_authenticate(request, user=self.user) + + response = self.create_view(request) + + assert response.status_code == status.HTTP_201_CREATED, response.data + instance = AdapterInstance.objects.get(adapter_name="my-openai") + # persisted under the request user's org, created_by the request user + assert instance.organization_id == self.org.id + assert instance.created_by == self.user + # metadata stored encrypted (binary), decrypts back via .metadata + assert instance.adapter_metadata_b is not None + assert instance.metadata["model"] == "gpt-4o-mini" diff --git a/backend/api_v2/tests/test_deployment_helper.py b/backend/api_v2/tests/test_deployment_helper.py index 4a9e40a045..39b23e5b16 100644 --- a/backend/api_v2/tests/test_deployment_helper.py +++ b/backend/api_v2/tests/test_deployment_helper.py @@ -3,199 +3,89 @@ When an API-deployment run fails synchronously at the "Staging files in API storage" step (``SourceConnector.add_input_file_to_api_storage``, before async dispatch), the PENDING ``WorkflowExecution`` row created earlier must be marked -ERROR — otherwise the UI shows the run as stuck/running forever. - -Before the fix, staging sat *outside* the try/except in -``DeploymentHelper.execute_workflow``, so a staging exception propagated out of -the method and the row stayed PENDING. The fix gives staging its own -try/except that marks the execution ERROR (with the error-marking isolated so -cleanup still runs if that DB write fails) and then releases the rate-limit -slot and cleans up storage. - -Like ``usage_v2/tests/test_helper.py``, this test does not require a live Django -database (the backend test env has no ``pytest-django`` / no DB). It stubs the -module's cross-app imports in ``sys.modules`` *before* importing the helper, so -the real ``execute_workflow`` control flow is exercised with the collaborators -mocked. Runnable under pytest or directly: ``python3 test_deployment_helper.py``. -""" +ERROR — otherwise the UI shows the run as stuck/running forever. The +error-marking is isolated so the rate-limit slot release and storage cleanup +still run even if that DB write fails. -from __future__ import annotations +Unit tests: the real ``execute_workflow`` control flow runs with every +DB/storage-touching collaborator patched on the imported module, so no +database is needed. +""" -import sys -import types -from pathlib import Path +from unittest import mock from unittest.mock import MagicMock -# Ensure the backend dir (which holds the ``api_v2`` package) is importable when -# this file is run directly, not just under pytest's rootdir. -_BACKEND_DIR = Path(__file__).resolve().parents[2] -if str(_BACKEND_DIR) not in sys.path: - sys.path.insert(0, str(_BACKEND_DIR)) - - -class _AutoModule(types.ModuleType): - """Module whose attribute access lazily returns (and caches) a MagicMock.""" - - def __getattr__(self, name: str) -> MagicMock: - mock = MagicMock(name=f"{self.__name__}.{name}") - setattr(self, name, mock) - return mock - - -def _install_stub(dotted: str) -> None: - """Register an ``_AutoModule`` for ``dotted`` and every parent prefix. - - Existing entries (e.g. the real, empty ``api_v2`` package) are preserved so - the real ``api_v2.deployment_helper`` submodule can still be imported. - """ - parts = dotted.split(".") - for i in range(1, len(parts) + 1): - prefix = ".".join(parts[:i]) - if prefix not in sys.modules: - sys.modules[prefix] = _AutoModule(prefix) - - -def _load_deployment_helper(): - """Stub cross-app imports, then import and return the real helper module.""" - import api_v2 # real, empty package — keep it so the submodule resolves - - assert not isinstance(api_v2, _AutoModule), "api_v2 must be the real package" - - for dotted in ( - "requests", - "configuration.config_registry", - "configuration.models", - "django.conf", - "django.core.files.uploadedfile", - "plugins.workflow_manager.workflow_v2.api_hub_usage_utils", - "rest_framework.request", - "rest_framework.serializers", - "rest_framework.utils.serializer_helpers", - "tags.models", - "usage_v2.helper", - "utils.constants", - "utils.local_context", - "workflow_manager.endpoint_v2.destination", - "workflow_manager.endpoint_v2.source", - "workflow_manager.workflow_v2.dto", - "workflow_manager.workflow_v2.enums", - "workflow_manager.workflow_v2.execution", - "workflow_manager.workflow_v2.models", - "workflow_manager.workflow_v2.workflow_helper", - "api_v2.api_key_validator", - "api_v2.dto", - "api_v2.exceptions", - "api_v2.key_helper", - "api_v2.models", - "api_v2.rate_limiter", - "api_v2.serializers", - "api_v2.utils", - ): - _install_stub(dotted) - - # ``class DeploymentHelper(BaseAPIKeyValidator)`` needs a real base class, - # not a MagicMock instance. - sys.modules["api_v2.api_key_validator"].BaseAPIKeyValidator = type( - "BaseAPIKeyValidator", (), {} - ) - - import api_v2.deployment_helper as helper - - return helper - - -def _make_helper_and_api(staging_error: Exception): - """Load the helper with a known execution id and a failing staging call.""" - helper = _load_deployment_helper() - - # The module is cached in sys.modules, so its mocked collaborators are shared - # across tests. Reset call counts and side effects so each test is isolated. - for collaborator in ( - helper.WorkflowExecutionServiceHelper, - helper.SourceConnector, - helper.APIDeploymentRateLimiter, - helper.DestinationConnector, - helper.WorkflowHelper, - ): - collaborator.reset_mock(return_value=True, side_effect=True) - - # Known execution id so we can assert it is the one marked ERROR. - execution_row = MagicMock() - execution_row.id = "exec-123" - helper.WorkflowExecutionServiceHelper.create_workflow_execution.return_value = ( - execution_row - ) +import pytest + +import api_v2.deployment_helper as dh + + +@pytest.fixture +def collaborators(): + """Patch execute_workflow's collaborators; staging fails with 'boom'.""" + with mock.patch.multiple( + dh, + WorkflowExecutionServiceHelper=mock.DEFAULT, + SourceConnector=mock.DEFAULT, + DestinationConnector=mock.DEFAULT, + APIDeploymentRateLimiter=mock.DEFAULT, + WorkflowHelper=mock.DEFAULT, + Tag=mock.DEFAULT, + logger=mock.DEFAULT, + ) as mocks: + execution_row = MagicMock() + execution_row.id = "exec-123" + mocks[ + "WorkflowExecutionServiceHelper" + ].create_workflow_execution.return_value = execution_row + mocks["SourceConnector"].add_input_file_to_api_storage.side_effect = ( + RuntimeError("boom") + ) + yield mocks - # Simulate the synchronous staging failure (e.g. an object-store 403). - helper.SourceConnector.add_input_file_to_api_storage.side_effect = staging_error +def _api() -> MagicMock: api = MagicMock() api.workflow.id = "wf-1" api.id = "pipe-1" - return helper, api + return api -def _run_staging_failure() -> None: +def test_staging_failure_marks_execution_error(collaborators) -> None: """A staging failure marks the execution ERROR instead of leaving it PENDING.""" - helper, api = _make_helper_and_api(RuntimeError("boom")) - # Must NOT raise — the failure should be handled, not propagated. - helper.DeploymentHelper.execute_workflow( + dh.DeploymentHelper.execute_workflow( organization_name="org", - api=api, + api=_api(), file_objs=[], timeout=-1, ) # The PENDING row is marked ERROR with the surfaced reason. - helper.WorkflowExecutionServiceHelper.update_execution_err.assert_called_once_with( - "exec-123", "boom" - ) + collaborators[ + "WorkflowExecutionServiceHelper" + ].update_execution_err.assert_called_once_with("exec-123", "boom") # And the slot/storage cleanup still runs. - helper.APIDeploymentRateLimiter.release_slot.assert_called_once() - helper.DestinationConnector.delete_api_storage_dir.assert_called_once() - + collaborators["APIDeploymentRateLimiter"].release_slot.assert_called_once() + collaborators["DestinationConnector"].delete_api_storage_dir.assert_called_once() # Async dispatch is never reached when staging fails. - helper.WorkflowHelper.execute_workflow_async.assert_not_called() + collaborators["WorkflowHelper"].execute_workflow_async.assert_not_called() -def _run_staging_failure_db_marking_raises() -> None: +def test_staging_failure_cleanup_survives_db_marking_error(collaborators) -> None: """If marking the row ERROR itself raises, cleanup must still run (not propagate).""" - helper, api = _make_helper_and_api(RuntimeError("boom")) - - # The DB write to mark ERROR fails (e.g. transient DB error). - helper.WorkflowExecutionServiceHelper.update_execution_err.side_effect = RuntimeError( - "db down" + collaborators["WorkflowExecutionServiceHelper"].update_execution_err.side_effect = ( + RuntimeError("db down") ) # Must NOT raise — a failed error-marking should not break cleanup. - # The helper logs the failure via logger.exception; silence it so the - # expected, handled error doesn't look like a test failure in the output. - helper.logger.disabled = True - try: - helper.DeploymentHelper.execute_workflow( - organization_name="org", - api=api, - file_objs=[], - timeout=-1, - ) - finally: - helper.logger.disabled = False + dh.DeploymentHelper.execute_workflow( + organization_name="org", + api=_api(), + file_objs=[], + timeout=-1, + ) # Cleanup still runs even though error-marking raised. - helper.APIDeploymentRateLimiter.release_slot.assert_called_once() - helper.DestinationConnector.delete_api_storage_dir.assert_called_once() - - -def test_staging_failure_marks_execution_error() -> None: - _run_staging_failure() - - -def test_staging_failure_cleanup_survives_db_marking_error() -> None: - _run_staging_failure_db_marking_raises() - - -if __name__ == "__main__": - _run_staging_failure() - _run_staging_failure_db_marking_raises() - print("OK: staging failure marks execution ERROR + cleanup survives DB error") + collaborators["APIDeploymentRateLimiter"].release_slot.assert_called_once() + collaborators["DestinationConnector"].delete_api_storage_dir.assert_called_once() diff --git a/backend/conftest.py b/backend/conftest.py index 16452bb9ef..bbf1910425 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -22,3 +22,23 @@ # to make a mis-located file debuggable instead of silently empty. if not load_dotenv(Path(__file__).parent / "test.env", override=False): print("[conftest] backend/test.env not found; using ambient environment", flush=True) + + +def pytest_collection_modifyitems(items): + """Auto-mark every DB-bound test as ``integration`` so the rig's unit tier + (``-m 'not integration'``) skips it while the integration tier (live + Postgres) runs it. Detects Django ``TestCase``/``TransactionTestCase`` + subclasses and any item using the ``django_db`` marker — the two ways a + backend test needs a database. Kept central so tests declare their DB need + by how they're written, not by a hand-maintained marker on each file. + """ + import pytest + from django.test import TestCase, TransactionTestCase + + for item in items: + cls = getattr(item, "cls", None) + needs_db = item.get_closest_marker("django_db") is not None or ( + cls is not None and issubclass(cls, (TestCase, TransactionTestCase)) + ) + if needs_db: + item.add_marker(pytest.mark.integration) diff --git a/backend/connector_v2/tests/conftest.py b/backend/connector_v2/tests/conftest.py deleted file mode 100644 index 89f1715a16..0000000000 --- a/backend/connector_v2/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest -from django.core.management import call_command - - -@pytest.fixture(scope="session") -def django_db_setup(django_db_blocker): # type: ignore - fixtures = ["./connector/tests/fixtures/fixtures_0001.json"] - with django_db_blocker.unblock(): - call_command("loaddata", *fixtures) diff --git a/backend/connector_v2/tests/connector_tests.py b/backend/connector_v2/tests/connector_tests.py deleted file mode 100644 index 2f7960de6b..0000000000 --- a/backend/connector_v2/tests/connector_tests.py +++ /dev/null @@ -1,314 +0,0 @@ -# mypy: ignore-errors -import pytest -from connector_v2.models import ConnectorInstance -from django.urls import reverse -from rest_framework import status -from rest_framework.test import APITestCase - -pytestmark = pytest.mark.django_db - - -@pytest.mark.connector -class TestConnector(APITestCase): - def test_connector_list(self) -> None: - """Tests to List the connectors.""" - url = reverse("connectors_v1-list") - response = self.client.get(url) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - - def test_connectors_detail(self) -> None: - """Tests to fetch a connector with given pk.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - response = self.client.get(url) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - - def test_connectors_detail_not_found(self) -> None: - """Tests for negative case to fetch non exiting key.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 768}) - response = self.client.get(url) - - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - - def test_connectors_create(self) -> None: - """Tests to create a new ConnectorInstance.""" - url = reverse("connectors_v1-list") - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConnectorInstance.objects.count(), 2) - - def test_connectors_create_with_json_list(self) -> None: - """Tests to create a new connector with list included in the json - field. - """ - url = reverse("connectors_v1-list") - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - "file_name_list": ["a1", "a2"], - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConnectorInstance.objects.count(), 2) - - def test_connectors_create_with_nested_json(self) -> None: - """Tests to create a new connector with json field as nested json.""" - url = reverse("connectors_v1-list") - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConnectorInstance.objects.count(), 2) - - def test_connectors_create_bad_request(self) -> None: - """Tests for negative case to throw error on a wrong access.""" - url = reverse("connectors_v1-list") - data = { - "org": 5, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.post(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - - def test_connectors_update_json_field(self) -> None: - """Tests to update connector with json field update.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - drive_link = response.data["connector_metadata"]["drive_link"] - self.assertEqual(drive_link, "new_sample_url") - - def test_connectors_update(self) -> None: - """Tests to update connector update single field.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 1, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - modified_by = response.data["modified_by"] - self.assertEqual(modified_by, 2) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - def test_connectors_update_pk(self) -> None: - """Tests the PUT method for 400 error.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 2, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - - def test_connectors_update_json_fields(self) -> None: - """Tests to update ConnectorInstance.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - }, - } - response = self.client.put(url, data, format="json") - nested_value = response.data["connector_metadata"]["sample_metadata_json"]["key1"] - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(nested_value, "value1") - - def test_connectors_update_json_list_fields(self) -> None: - """Tests to update connector to the third second level of json.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - "connector_metadata": { - "drive_link": "new_sample_url", - "sharable_link": True, - "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - "file_list": ["a1", "a2", "a3"], - }, - } - response = self.client.put(url, data, format="json") - nested_value = response.data["connector_metadata"]["sample_metadata_json"]["key1"] - nested_list = response.data["connector_metadata"]["file_list"] - last_val = nested_list.pop() - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(nested_value, "value1") - self.assertEqual(last_val, "a3") - - # @pytest.mark.xfail(raises=KeyError) - # def test_connectors_update_json_fields_failed(self) -> None: - # """Tests to update connector to the second level of JSON with a wrong - # key.""" - - # url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - # data = { - # "org": 1, - # "project": 1, - # "created_by": 2, - # "modified_by": 2, - # "modified_at": "2023-06-14T05:28:47.759Z", - # "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - # "connector_metadata": { - # "drive_link": "new_sample_url", - # "sharable_link": True, - # "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - # }, - # } - # response = self.client.put(url, data, format="json") - # nested_value = response.data["connector_metadata"]["sample_metadata_json"][ - # "key00" - # ] - - # @pytest.mark.xfail(raises=KeyError) - # def test_connectors_update_json_nested_failed(self) -> None: - # """Tests to update connector to test a first level of json with a wrong - # key.""" - - # url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - # data = { - # "org": 1, - # "project": 1, - # "created_by": 2, - # "modified_by": 2, - # "modified_at": "2023-06-14T05:28:47.759Z", - # "connector_id": "e3a4512m-efgb-48d5-98a9-3983nd77f", - # "connector_metadata": { - # "drive_link": "new_sample_url", - # "sharable_link": True, - # "sample_metadata_json": {"key1": "value1", "key2": "value2"}, - # }, - # } - # response = self.client.put(url, data, format="json") - # nested_value = response.data["connector_metadata"]["sample_metadata_jsonNew"] - - def test_connectors_update_field(self) -> None: - """Tests the PATCH method.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = {"connector_id": "e3a4512m-efgb-48d5-98a9-3983ntest"} - response = self.client.patch(url, data, format="json") - self.assertEqual(response.status_code, status.HTTP_200_OK) - connector_id = response.data["connector_id"] - - self.assertEqual( - connector_id, - ConnectorInstance.objects.get(connector_id=connector_id).connector_id, - ) - - def test_connectors_update_json_field_patch(self) -> None: - """Tests the PATCH method.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - data = { - "connector_metadata": { - "drive_link": "patch_update_url", - "sharable_link": True, - "sample_metadata_json": { - "key1": "patch_update1", - "key2": "value2", - }, - } - } - - response = self.client.patch(url, data, format="json") - self.assertEqual(response.status_code, status.HTTP_200_OK) - drive_link = response.data["connector_metadata"]["drive_link"] - - self.assertEqual(drive_link, "patch_update_url") - - def test_connectors_delete(self) -> None: - """Tests the DELETE method.""" - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - response = self.client.delete(url, format="json") - self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) - - url = reverse("connectors_v1-detail", kwargs={"pk": 1}) - response = self.client.get(url) - self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) diff --git a/backend/connector_v2/tests/fixtures/fixtures_0001.json b/backend/connector_v2/tests/fixtures/fixtures_0001.json deleted file mode 100644 index 55b39e6d86..0000000000 --- a/backend/connector_v2/tests/fixtures/fixtures_0001.json +++ /dev/null @@ -1,67 +0,0 @@ -[ - { - "model": "account.org", - "pk": 1, - "fields": { - "org_name": "Zipstack", - "created_by": 1, - "modified_by": 1, - "modified_at": "2023-06-14T05:28:47.739Z" - } - }, - { - "model": "account.user", - "pk": 1, - "fields": { - "org": 1, - "email": "johndoe@gmail.com", - "first_name": "John", - "last_name": "Doe", - "is_admin": true, - "created_by": null, - "modified_by": null, - "modified_at": "2023-06-14T05:28:47.744Z" - } - }, - { - "model": "account.user", - "pk": 2, - "fields": { - "org": 1, - "email": "user1@gmail.com", - "first_name": "Ron", - "last_name": "Stone", - "is_admin": false, - "created_by": 1, - "modified_by": 1, - "modified_at": "2023-06-14T05:28:47.750Z" - } - }, - { - "model": "project.project", - "pk": 1, - "fields": { - "org": 1, - "project_name": "Unstract Test", - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z" - } - }, - { - "model": "connector.connector", - "pk": 1, - "fields": { - "org": 1, - "project": 1, - "created_by": 2, - "modified_by": 2, - "modified_at": "2023-06-14T05:28:47.759Z", - "connector_id": "e38a59b7-efbb-48d5-9da6-3a0cf2d882a0", - "connector_metadata": { - "connector_type": "gdrive", - "auth_type": "oauth" - } - } - } -] diff --git a/backend/dashboard_metrics/tests/test_tasks.py b/backend/dashboard_metrics/tests/test_tasks.py index 1e38e2ca89..ac45887d4c 100644 --- a/backend/dashboard_metrics/tests/test_tasks.py +++ b/backend/dashboard_metrics/tests/test_tasks.py @@ -1,11 +1,11 @@ """Unit tests for Dashboard Metrics Celery tasks.""" -import uuid from datetime import datetime, timedelta from django.test import TestCase, TransactionTestCase from django.utils import timezone +from account_v2.models import Organization from dashboard_metrics.models import ( EventMetricsDaily, EventMetricsHourly, @@ -86,7 +86,10 @@ class TestCleanupTasks(TransactionTestCase): def setUp(self): """Set up test fixtures.""" - self.org_id = str(uuid.uuid4()) + # organization FK targets Organization's int PK, not a UUID. + self.org = Organization.objects.create( + organization_id="test-org", name="test-org", display_name="Test Org" + ) def test_cleanup_hourly_metrics_deletes_old_records(self): """Test that cleanup deletes hourly records older than retention.""" @@ -96,7 +99,7 @@ def test_cleanup_hourly_metrics_deletes_old_records(self): # Create old record EventMetricsHourly.objects.create( - organization_id=self.org_id, + organization=self.org, timestamp=old_timestamp, metric_name="old_metric", metric_type=MetricType.COUNTER, @@ -107,7 +110,7 @@ def test_cleanup_hourly_metrics_deletes_old_records(self): # Create recent record EventMetricsHourly.objects.create( - organization_id=self.org_id, + organization=self.org, timestamp=recent_timestamp, metric_name="recent_metric", metric_type=MetricType.COUNTER, @@ -122,9 +125,10 @@ def test_cleanup_hourly_metrics_deletes_old_records(self): assert result["deleted"] == 1 assert result["retention_days"] == 30 - # Verify old is deleted, recent remains - assert not EventMetricsHourly.objects.filter(metric_name="old_metric").exists() - assert EventMetricsHourly.objects.filter(metric_name="recent_metric").exists() + # _base_manager bypasses the org-scoped default manager, which filters + # by UserContext.get_organization() — None here, so .objects sees nothing. + assert not EventMetricsHourly._base_manager.filter(metric_name="old_metric").exists() + assert EventMetricsHourly._base_manager.filter(metric_name="recent_metric").exists() def test_cleanup_daily_metrics_deletes_old_records(self): """Test that cleanup deletes daily records older than retention.""" @@ -134,7 +138,7 @@ def test_cleanup_daily_metrics_deletes_old_records(self): # Create old record EventMetricsDaily.objects.create( - organization_id=self.org_id, + organization=self.org, date=old_date, metric_name="old_daily_metric", metric_type=MetricType.COUNTER, @@ -145,7 +149,7 @@ def test_cleanup_daily_metrics_deletes_old_records(self): # Create recent record EventMetricsDaily.objects.create( - organization_id=self.org_id, + organization=self.org, date=recent_date, metric_name="recent_daily_metric", metric_type=MetricType.COUNTER, @@ -160,10 +164,10 @@ def test_cleanup_daily_metrics_deletes_old_records(self): assert result["deleted"] == 1 # Verify old is deleted, recent remains - assert not EventMetricsDaily.objects.filter( + assert not EventMetricsDaily._base_manager.filter( metric_name="old_daily_metric" ).exists() - assert EventMetricsDaily.objects.filter( + assert EventMetricsDaily._base_manager.filter( metric_name="recent_daily_metric" ).exists() @@ -173,7 +177,7 @@ def test_cleanup_hourly_with_custom_retention(self): old_timestamp = now - timedelta(days=10) EventMetricsHourly.objects.create( - organization_id=self.org_id, + organization=self.org, timestamp=old_timestamp, metric_name="custom_retention_metric", metric_type=MetricType.COUNTER, diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py index adb5713243..d82156b140 100644 --- a/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py +++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py @@ -13,293 +13,21 @@ 4. On an error inside ``check_extraction_status``, swallow the error and fall back to full extraction — the dispatch must not fail. -The backend test environment has no ``pytest-django``, no SQLite -fallback, and the helper has a heavy Django-coupled import surface. -Rather than spin up Django, we stub every collaborator as a -``MagicMock`` on ``sys.modules`` *before* importing the helper, and -then patch ``PromptStudioHelper`` class methods per-test. This mirrors -the ``usage_v2/tests/test_helper.py`` approach. - -If the helper module cannot be imported in a given environment (for -example because the stub surface has drifted), all tests in the module -are skipped with a clear reason. +Unit tests: the real helper module is imported (Django is loaded by the +rig's test env) and every collaborator is patched on it per-test, so no +database is touched. """ from __future__ import annotations -import sys -import types +from contextlib import ExitStack from typing import Any from unittest.mock import MagicMock, patch -import pytest - - -# --------------------------------------------------------------------------- -# Stub every collaborator module on sys.modules before importing the helper. -# These stubs are intentionally broad MagicMocks — the tests patch the -# specific attributes they care about via ``unittest.mock.patch``. -# --------------------------------------------------------------------------- - - -def _install(name: str, attrs: dict[str, Any] | None = None) -> types.ModuleType: - """Install (or replace) a fake module into ``sys.modules``. - - Always creates a fresh ``ModuleType``; this is important because the - real module may already have been imported before these stubs run - (via pytest collection, conftest, etc.), and we need our fake to - actually take effect. - """ - mod = types.ModuleType(name) - if attrs: - for key, value in attrs.items(): - setattr(mod, key, value) - sys.modules[name] = mod - return mod - - -def _install_package(name: str) -> types.ModuleType: - """Install a fake package (marked with ``__path__``). - - Only stubs the package if it is not already in ``sys.modules``. - This prevents clobbering packages like ``unstract.core`` that must - retain their real ``__path__`` for submodule resolution. The child - modules we care about are always replaced explicitly via - ``_install``. - """ - if name in sys.modules: - return sys.modules[name] - mod = types.ModuleType(name) - mod.__path__ = [] # type: ignore[attr-defined] - sys.modules[name] = mod - return mod - - -try: - # Account / adapter stubs - _install_package("account_v2") - _install( - "account_v2.constants", - {"Common": type("Common", (), {"LOG_EVENTS_ID": "log_events_id", - "REQUEST_ID": "request_id"})}, - ) - _install("account_v2.models", {"User": MagicMock(name="User")}) - _install_package("adapter_processor_v2") - _install( - "adapter_processor_v2.constants", - {"AdapterKeys": type("AdapterKeys", (), {})}, - ) - _install( - "adapter_processor_v2.models", - {"AdapterInstance": MagicMock(name="AdapterInstance")}, - ) - - # Plugins stub - _install("plugins", {"get_plugin": MagicMock(return_value=None)}) - - # utils stubs - _install_package("utils") - _install_package("utils.file_storage") - _install( - "utils.file_storage.constants", - { - "FileStorageKeys": type( - "FileStorageKeys", - (), - {"PERMANENT_REMOTE_STORAGE": "permanent"}, - ) - }, - ) - _install_package("utils.file_storage.helpers") - _install( - "utils.file_storage.helpers.prompt_studio_file_helper", - {"PromptStudioFileHelper": MagicMock(name="PromptStudioFileHelper")}, - ) - _install( - "utils.local_context", - {"StateStore": MagicMock(name="StateStore")}, - ) - - # backend.celery_service stub - _install_package("backend") - _install( - "backend.celery_service", - {"app": MagicMock(name="celery_app")}, - ) - - # prompt_studio stubs - _install_package("prompt_studio") - _install_package("prompt_studio.prompt_profile_manager_v2") - _install( - "prompt_studio.prompt_profile_manager_v2.models", - {"ProfileManager": MagicMock(name="ProfileManager")}, - ) - _install( - "prompt_studio.prompt_profile_manager_v2.profile_manager_helper", - {"ProfileManagerHelper": MagicMock(name="ProfileManagerHelper")}, - ) - - _install_package("prompt_studio.prompt_studio_document_manager_v2") - _install( - "prompt_studio.prompt_studio_document_manager_v2.models", - {"DocumentManager": MagicMock(name="DocumentManager")}, - ) - - _install_package("prompt_studio.prompt_studio_index_manager_v2") - _install( - "prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper", - {"PromptStudioIndexHelper": MagicMock(name="PromptStudioIndexHelper")}, - ) +from prompt_studio.prompt_studio_core_v2 import prompt_studio_helper as _psh_mod - _install_package("prompt_studio.prompt_studio_output_manager_v2") - _install( - "prompt_studio.prompt_studio_output_manager_v2.output_manager_helper", - {"OutputManagerHelper": MagicMock(name="OutputManagerHelper")}, - ) - - _install_package("prompt_studio.prompt_studio_v2") - _install( - "prompt_studio.prompt_studio_v2.models", - {"ToolStudioPrompt": MagicMock(name="ToolStudioPrompt")}, - ) - - # Stub the prompt_studio_core_v2 sibling modules too — several of them - # transitively import modules (like ``utils.cache_service``) that we - # don't want to pull in for these unit tests. - _install_package("prompt_studio.prompt_studio_core_v2") - _install( - "prompt_studio.prompt_studio_core_v2.document_indexing_service", - {"DocumentIndexingService": MagicMock(name="DocumentIndexingService")}, - ) - - # Real exception classes — build_index_payload uses ``raise``. - class _FakeExc(Exception): - pass - - _install( - "prompt_studio.prompt_studio_core_v2.exceptions", - { - "AnswerFetchError": type("AnswerFetchError", (_FakeExc,), {}), - "DefaultProfileError": type("DefaultProfileError", (_FakeExc,), {}), - "EmptyPromptError": type("EmptyPromptError", (_FakeExc,), {}), - "ExtractionAPIError": type("ExtractionAPIError", (_FakeExc,), {}), - "IndexingAPIError": type("IndexingAPIError", (_FakeExc,), {}), - "NoPromptsFound": type("NoPromptsFound", (_FakeExc,), {}), - "OperationNotSupported": type("OperationNotSupported", (_FakeExc,), {}), - "PermissionError": type("PermissionError", (_FakeExc,), {}), - }, - ) - _install( - "prompt_studio.prompt_studio_core_v2.migration_utils", - {"SummarizeMigrationUtils": MagicMock(name="SummarizeMigrationUtils")}, - ) - _install( - "prompt_studio.prompt_studio_core_v2.models", - {"CustomTool": MagicMock(name="CustomTool")}, - ) - _install( - "prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool", - {"PromptIdeBaseTool": MagicMock(name="PromptIdeBaseTool")}, - ) - _install( - "prompt_studio.prompt_studio_core_v2.prompt_variable_service", - {"PromptStudioVariableService": MagicMock(name="PromptStudioVariableService")}, - ) - - # unstract.core.pubsub_helper stub (LogPublisher isn't used by - # build_index_payload but the module-level import must succeed). - _install_package("unstract.core") - _install( - "unstract.core.pubsub_helper", - {"LogPublisher": MagicMock(name="LogPublisher")}, - ) - - # unstract.sdk1 stubs — these heavy modules transitively pull in - # ``unstract.core.cache.redis_client`` which isn't on the python - # path for the backend tests. We only need the leaf classes. - _install_package("unstract.sdk1") - _install( - "unstract.sdk1.constants", - { - "LogLevel": type( - "LogLevel", (), {"INFO": "INFO", "WARN": "WARN", "ERROR": "ERROR"} - ) - }, - ) - _install( - "unstract.sdk1.exceptions", - { - "IndexingError": type("IndexingError", (Exception,), {}), - "SdkError": type("SdkError", (Exception,), {}), - }, - ) - _install_package("unstract.sdk1.execution") - - class _FakeExecutionContext: - """Minimal ExecutionContext that keeps ``executor_params`` as - the real dict we pass in (the tests inspect it).""" - - def __init__(self, **kwargs: Any) -> None: - self.executor_name = kwargs.get("executor_name") - self.operation = kwargs.get("operation") - self.run_id = kwargs.get("run_id") - self.execution_source = kwargs.get("execution_source") - self.organization_id = kwargs.get("organization_id") - self.executor_params = kwargs.get("executor_params") or {} - self.request_id = kwargs.get("request_id") - self.log_events_id = kwargs.get("log_events_id") - - _install( - "unstract.sdk1.execution.context", - {"ExecutionContext": _FakeExecutionContext}, - ) - _install( - "unstract.sdk1.execution.dispatcher", - {"ExecutionDispatcher": MagicMock(name="ExecutionDispatcher")}, - ) - _install_package("unstract.sdk1.file_storage") - _install( - "unstract.sdk1.file_storage.constants", - {"StorageType": type("StorageType", (), {"PERMANENT": "permanent"})}, - ) - _install( - "unstract.sdk1.file_storage.env_helper", - {"EnvHelper": MagicMock(name="EnvHelper")}, - ) - _install_package("unstract.sdk1.utils") - _install( - "unstract.sdk1.utils.indexing", - {"IndexingUtils": MagicMock(name="IndexingUtils")}, - ) - _install( - "unstract.sdk1.utils.tool", - {"ToolUtils": MagicMock(name="ToolUtils")}, - ) - - # Now import the helper module. If this fails, all tests below will - # be skipped via the ``_IMPORT_ERROR`` sentinel. - from prompt_studio.prompt_studio_core_v2 import prompt_studio_helper as _psh_mod # noqa: E402 - - PromptStudioHelper = _psh_mod.PromptStudioHelper - IKeys = _psh_mod.IKeys - _IMPORT_ERROR: str | None = None -except Exception as exc: # pragma: no cover — environment guard - _IMPORT_ERROR = ( - f"prompt_studio_helper could not be imported in this environment: " - f"{type(exc).__name__}: {exc}" - ) - PromptStudioHelper = None # type: ignore[assignment] - IKeys = None # type: ignore[assignment] - - -pytestmark = pytest.mark.skipif( - _IMPORT_ERROR is not None, reason=_IMPORT_ERROR or "" -) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- +PromptStudioHelper = _psh_mod.PromptStudioHelper +IKeys = _psh_mod.IKeys def _make_tool(enable_highlight: bool = False, summarize_context: bool = False): @@ -354,72 +82,35 @@ def _dispatch_build( else: check_mock.return_value = check_return - # Patch everything via context managers so each test starts clean. - patches = [ - patch.object( - _psh_mod.CustomTool, - "objects", - MagicMock(get=MagicMock(return_value=tool)), - ), - patch.object( - _psh_mod.PromptStudioFileHelper, - "get_or_create_prompt_studio_subdirectory", - return_value="/prompt-studio/org/user/tool", - ), - patch.object( - _psh_mod.ProfileManager, - "get_default_llm_profile", - return_value=profile, - ), - patch.object( - PromptStudioHelper, - "validate_adapter_status", - return_value=None, - ), - patch.object( - PromptStudioHelper, - "validate_profile_manager_owner_access", - return_value=None, - ), - patch.object( - PromptStudioHelper, - "_get_platform_api_key", - return_value="pk-test", - ), - patch.object( - PromptStudioHelper, - "_build_summarize_params", - return_value=(None, "", MagicMock()), - ), - patch.object( - _psh_mod.EnvHelper, - "get_storage", - return_value=fs_instance, - ), - patch.object( - _psh_mod.PromptStudioIndexHelper, - "check_extraction_status", - check_mock, - ), - patch.object( - _psh_mod.IndexingUtils, - "generate_index_key", - return_value="doc-key-1", - ), - patch.object( - _psh_mod, - "PromptIdeBaseTool", - MagicMock(return_value=MagicMock()), - ), - patch.object( - _psh_mod.StateStore, - "get", - return_value="", - ), - ] - for p in patches: - p.start() - try: + with ExitStack() as stack: + for target, attr, value in ( + (_psh_mod.CustomTool, "objects", MagicMock(get=MagicMock(return_value=tool))), + ( + _psh_mod.PromptStudioFileHelper, + "get_or_create_prompt_studio_subdirectory", + MagicMock(return_value="/prompt-studio/org/user/tool"), + ), + (_psh_mod.ProfileManager, "get_default_llm_profile", MagicMock(return_value=profile)), + (PromptStudioHelper, "validate_adapter_status", MagicMock(return_value=None)), + ( + PromptStudioHelper, + "validate_profile_manager_owner_access", + MagicMock(return_value=None), + ), + (PromptStudioHelper, "_get_platform_api_key", MagicMock(return_value="pk-test")), + ( + PromptStudioHelper, + "_build_summarize_params", + MagicMock(return_value=(None, "", MagicMock())), + ), + (_psh_mod.EnvHelper, "get_storage", MagicMock(return_value=fs_instance)), + (_psh_mod.PromptStudioIndexHelper, "check_extraction_status", check_mock), + (_psh_mod.IndexingUtils, "generate_index_key", MagicMock(return_value="doc-key-1")), + (_psh_mod, "PromptIdeBaseTool", MagicMock(return_value=MagicMock())), + (_psh_mod.StateStore, "get", MagicMock(return_value="")), + ): + stack.enter_context(patch.object(target, attr, value)) + context, cb_kwargs = PromptStudioHelper.build_index_payload( tool_id="tool-1", file_name="doc.pdf", @@ -429,14 +120,6 @@ def _dispatch_build( run_id="run-1", ) return context, cb_kwargs, fs_instance, check_mock - finally: - for p in patches: - p.stop() - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- class TestBuildIndexPayloadMarker: diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 3ef2c187a4..46f0821131 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -71,7 +71,10 @@ dev = [ "responses>=0.25.7", "psutil>=7.0.0", ] -test = ["pytest>=8.0.1"] +test = [ + "pytest>=8.0.1", + "pytest-django>=4.12.0", +] deploy = [ "gunicorn~=23.0", # For serving the application # Keep versions empty and let uv decide version @@ -101,6 +104,11 @@ constraint-dependencies = [ # Note: test.env is loaded by backend/conftest.py via python-dotenv directly # (replaces the unmaintained pytest-dotenv plugin). addopts = "-s" +python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"] +markers = [ + "integration: needs live infra (Postgres/Redis); runs in the rig's integration tier, not unit (select with -m integration / exclude with -m 'not integration')", + "critical_path(path_id): attests coverage of declared critical path(s) from tests/critical_paths.yaml when this test passes", +] [tool.poe] envfile = ".env" diff --git a/backend/usage_v2/tests/test_helper.py b/backend/usage_v2/tests/test_helper.py index 4f6ffc9c40..f0311c22aa 100644 --- a/backend/usage_v2/tests/test_helper.py +++ b/backend/usage_v2/tests/test_helper.py @@ -5,69 +5,34 @@ bare ``"llm"`` bucket from leaking into API deployment responses when a producer-side LLM call site forgets to set ``llm_usage_reason``. -The tests deliberately do not require a live Django database — the -backend test environment has no ``pytest-django``, no SQLite fallback, -and uses ``django-tenants`` against Postgres in production. Instead -the tests stub ``account_usage.models`` and ``usage_v2.models`` in -``sys.modules`` *before* importing the helper, so the helper module -loads cleanly without triggering Django's app registry checks. The -fake ``Usage.objects.filter`` chain returns a deterministic list of -row dicts shaped exactly like the real ``.values(...).annotate(...)`` -queryset rows the helper iterates over. +The tests exercise only the helper's in-memory aggregation logic, not +the ORM. We rebind the ``Usage`` symbol the helper resolved at import +to a fake whose ``objects.filter`` chain returns a deterministic list +of row dicts shaped exactly like the real +``.values(...).annotate(...)`` queryset rows the helper iterates over. """ from __future__ import annotations -import sys -import types from typing import Any from unittest.mock import MagicMock +import pytest +import usage_v2.helper as helper_mod +from usage_v2.helper import UsageHelper -# --------------------------------------------------------------------------- -# Module-level stubs. Must run BEFORE ``usage_v2.helper`` is imported, so we -# do it at import time and capture the helper reference for the tests below. -# --------------------------------------------------------------------------- - - -def _install_stubs() -> tuple[Any, Any]: - """Install fake ``account_usage.models`` and ``usage_v2.models`` modules - so that ``usage_v2.helper`` can be imported without Django being set up. - - Returns ``(UsageHelper, FakeUsage)`` — the helper class to test and the - fake Usage class whose ``objects.filter`` we will swap per-test. - """ - # Fake account_usage package + models module - if "account_usage" not in sys.modules: - account_usage_pkg = types.ModuleType("account_usage") - account_usage_pkg.__path__ = [] # mark as package - sys.modules["account_usage"] = account_usage_pkg - if "account_usage.models" not in sys.modules: - account_usage_models = types.ModuleType("account_usage.models") - account_usage_models.PageUsage = MagicMock(name="PageUsage") - sys.modules["account_usage.models"] = account_usage_models - - # Fake usage_v2.models with a Usage class whose ``objects`` is a - # MagicMock (so each test can rebind ``filter.return_value``). - if "usage_v2.models" not in sys.modules or not hasattr( - sys.modules["usage_v2.models"], "_is_test_stub" - ): - usage_v2_models = types.ModuleType("usage_v2.models") - usage_v2_models._is_test_stub = True - - class _FakeUsage: - objects = MagicMock(name="Usage.objects") - - usage_v2_models.Usage = _FakeUsage - sys.modules["usage_v2.models"] = usage_v2_models - - # Now import the helper — this picks up our stubs. - from usage_v2.helper import UsageHelper - return UsageHelper, sys.modules["usage_v2.models"].Usage +class FakeUsage: + # objects is a MagicMock so each test can rebind filter.return_value. + objects = MagicMock(name="Usage.objects") -UsageHelper, FakeUsage = _install_stubs() +@pytest.fixture(autouse=True) +def _swap_usage(monkeypatch: pytest.MonkeyPatch) -> None: + # Swap the symbol get_usage_by_model resolves, per-test, so monkeypatch + # restores the real model afterwards — a module-level rebind would leak + # FakeUsage into every later test in the same process. + monkeypatch.setattr(helper_mod, "Usage", FakeUsage) # --------------------------------------------------------------------------- diff --git a/backend/utils/file_storage/__init__.py b/backend/utils/file_storage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/utils/file_storage/helpers/__init__.py b/backend/utils/file_storage/helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py index df09776651..c8b75612fe 100644 --- a/backend/utils/file_storage/helpers/prompt_studio_file_helper.py +++ b/backend/utils/file_storage/helpers/prompt_studio_file_helper.py @@ -6,13 +6,13 @@ from file_management.exceptions import InvalidFileType from file_management.file_management_helper import FileManagerHelper -from utils.file_storage.constants import FileStorageConstants, FileStorageKeys -from utils.file_storage.helpers.streaming_writer import write_streaming from unstract.core.utilities import UnstractUtils from unstract.sdk1.file_storage import FileStorage from unstract.sdk1.file_storage.constants import StorageType from unstract.sdk1.file_storage.env_helper import EnvHelper +from utils.file_storage.constants import FileStorageConstants, FileStorageKeys +from utils.file_storage.helpers.streaming_writer import write_streaming logger = logging.getLogger(__name__) diff --git a/backend/uv.lock b/backend/uv.lock index bad2b30614..20e5fc22da 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -2970,6 +2970,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] +[[package]] +name = "pytest-django" +version = "4.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/2b/db9a193df89e5660137f5428063bcc2ced7ad790003b26974adf5c5ceb3b/pytest_django-4.12.0.tar.gz", hash = "sha256:df94ec819a83c8979c8f6de13d9cdfbe76e8c21d39473cfe2b40c9fc9be3c758", size = 91156, upload-time = "2026-02-14T18:40:49.235Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/a5/41d091f697c09609e7ef1d5d61925494e0454ebf51de7de05f0f0a728f1d/pytest_django-4.12.0-py3-none-any.whl", hash = "sha256:3ff300c49f8350ba2953b90297d23bf5f589db69545f56f1ec5f8cff5da83e85", size = 26123, upload-time = "2026-02-14T18:40:47.381Z" }, +] + [[package]] name = "python-crontab" version = "3.3.0" @@ -3721,6 +3733,7 @@ dev = [ ] test = [ { name = "pytest" }, + { name = "pytest-django" }, ] [package.metadata] @@ -3784,7 +3797,10 @@ dev = [ { name = "unstract-tool-sandbox", editable = "../unstract/tool-sandbox" }, { name = "unstract-workflow-execution", editable = "../unstract/workflow-execution" }, ] -test = [{ name = "pytest", specifier = ">=8.0.1" }] +test = [ + { name = "pytest", specifier = ">=8.0.1" }, + { name = "pytest-django", specifier = ">=4.12.0" }, +] [[package]] name = "unstract-connectors" diff --git a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py index 33f31f055a..445affba1d 100644 --- a/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py +++ b/backend/workflow_manager/endpoint_v2/tests/destination-connectors/test_destination_connector_postgres.py @@ -1,19 +1,18 @@ import os from unittest.mock import Mock, patch +import pytest from django.test import TestCase +from unstract.connectors.databases.postgresql import PostgreSQL from workflow_manager.endpoint_v2.constants import DestinationKey from workflow_manager.endpoint_v2.destination import DestinationConnector -from unstract.connectors.databases.postgresql import PostgreSQL - class TestDestinationConnectorPostgreSQL(TestCase): """Integration test for insert_into_db method with real PostgreSQL connector.""" def setUp(self) -> None: """Set up test data and real PostgreSQL configuration.""" - # Real PostgreSQL connection settings for testing self.postgres_config = { "host": os.getenv("DB_HOST", "localhost"), @@ -21,7 +20,7 @@ def setUp(self) -> None: "database": os.getenv("DB_NAME", "test_unstract"), "user": os.getenv("DB_USER", "postgres"), "password": os.getenv("DB_PASSWORD", "password"), - "schema": "test", # Add schema to fix PostgreSQL issue + "schema": os.getenv("DB_SCHEMA", "public"), } # Test data that will be inserted into the database @@ -32,7 +31,9 @@ def setUp(self) -> None: "processing_time": 1.5, } self.input_file_path = "/path/to/test/file.pdf" - self.test_table_name = "OUTPUT_3" + # Lowercase: the connector quotes the name on CREATE (case-preserved) but + # lowercases it when reading information_schema back. + self.test_table_name = "output_3" # Create real PostgreSQL connector instance self.postgres_connector = PostgreSQL(settings=self.postgres_config) @@ -192,22 +193,27 @@ def test_insert_into_db_happy_path_postgresql(self) -> None: f"✅ Successfully inserted test data into PostgreSQL table: {self.test_table_name}" ) + @pytest.mark.xfail( + reason=( + "get_sql_values_for_query serializes data=None as the literal " + "string 'None', which is invalid JSON for the jsonb data column " + "(PR #2115 review). Remove this marker once None maps to SQL NULL." + ), + strict=True, + ) def test_insert_into_db_with_error_postgresql(self) -> None: """Test insertion with error parameter into real PostgreSQL database.""" - # Create mock objects mock_workflow = self.create_mock_workflow() mock_workflow_log = self.create_mock_workflow_log() mock_connector_instance = self.create_real_connector_instance() mock_endpoint = self.create_mock_endpoint(mock_connector_instance) - # Create destination connector destination_connector = self.create_destination_connector( mock_workflow, mock_workflow_log, mock_endpoint ) error_message = "Test processing error occurred" - # Mock the methods that get data with patch.object( destination_connector, "get_tool_execution_result", @@ -218,7 +224,6 @@ def test_insert_into_db_with_error_postgresql(self) -> None: "get_combined_metadata", return_value=self.test_metadata, ): - # Execute with error parameter destination_connector.insert_into_db( input_file_path=self.input_file_path, error=error_message ) @@ -226,10 +231,6 @@ def test_insert_into_db_with_error_postgresql(self) -> None: # Verify that all expected columns were created self.verify_table_columns(self.test_table_name) - print( - f"✅ Successfully inserted error data into PostgreSQL table: {self.test_table_name}" - ) - def test_postgresql_connector_connection(self) -> None: """Test that the PostgreSQL connector can establish a connection.""" # Test the real PostgreSQL connector directly diff --git a/platform-service/tests/test_auth_middleware.py b/platform-service/tests/test_auth_middleware.py deleted file mode 100644 index decf9aaa3b..0000000000 --- a/platform-service/tests/test_auth_middleware.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from unstract.platform_service.main import ( - get_account_from_bearer_token, - validate_bearer_token, -) - - -class TestAuthMiddleware(unittest.TestCase): - def test_auth_middleware(self) -> None: - try: - self.assertTrue(validate_bearer_token("test")) - self.assertEqual(get_account_from_bearer_token("test"), "mock_org") - except Exception as e: - self.fail(f"Authentication Test failed: {e}") - - -if __name__ == "__main__": - unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 17bc80034f..b9d3738924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -178,7 +178,6 @@ keep-dict-typing = true [tool.pytest.ini_options] python_files = ["tests.py", "test_*.py", "*_tests.py"] -DJANGO_SETTINGS_MODULE = "backend.settings.test_cases" testpaths = ["tests"] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", diff --git a/tests/README.md b/tests/README.md index e5677bffc5..40abddd1bf 100644 --- a/tests/README.md +++ b/tests/README.md @@ -216,11 +216,48 @@ Developers can scope local runs however they like via positional args, `--from-f | `tests/e2e//` | HTTP-level tests against a running platform. | | `tests/e2e/hurl/` | Hurl-based HTTP suites. | -After adding tests, either: -1. Reuse an existing group whose `paths` already cover your file, **or** -2. Add a new group to `groups.yaml` (and, if relevant, a `critical_paths.yaml` entry that lists it in `covered_by`). +### Backend: no registration needed + +1. Write a normal test in `backend//tests/test_.py`. The filename + **must** match `test_*.py`, `*_test.py`, `*_tests.py`, or Django's per-app + `tests.py` — anything else is silently never collected. Prefer `test_*.py` + for consistency. +2. Tier is inferred, not declared. A test that touches the database (subclasses + Django `TestCase`/`APITestCase`, or uses `@pytest.mark.django_db`) is + auto-marked `integration` by `backend/conftest.py` and runs in + `integration-backend` against a rig-provisioned Postgres/Redis. Everything + else runs in `unit-backend`. No marker, no `groups.yaml` edit. +3. Don't stub the environment. The rig runs the whole backend tree in one + pytest session with Django fully loaded — tests that patch `sys.modules`, + assume import order, or assume they run alone will break under collection. + Use `unittest.mock.patch` on real modules; mock only true externals + (LLM SDKs, third-party APIs) — never the ORM or serializers. +4. Seed data per test in `setUp` via the ORM. Schema comes from migrations + (run once per session); each test's writes roll back automatically. +5. Needs credentials CI doesn't have (external DB/SaaS)? Guard with + `self.skipTest(" not set")` in `setUp` — it skips in CI and runs + locally when the env vars are exported. + +Run it locally: -Validate with `python -m tests.rig validate` before pushing. +```bash +tox -e groups -- unit-backend --no-coverage # pure tests +tox -e groups -- integration-backend --no-coverage # DB tests (needs Docker) +``` + +List what a group would run (no DB required): + +```bash +cd backend && uv run --group test pytest . -m integration --collect-only -q +``` + +### Other services / new areas + +Outside backend, markers are manual (`workers` enforces `--strict-markers`). +Either reuse an existing group whose `paths` already cover your file, or add a +group to `groups.yaml` (and, if relevant, list it in a `critical_paths.yaml` +entry's `covered_by`). Validate with `python -m tests.rig validate` before +pushing. --- diff --git a/tests/compose/docker-compose.test.yaml b/tests/compose/docker-compose.test.yaml index a513633aaa..ee10d9416d 100644 --- a/tests/compose/docker-compose.test.yaml +++ b/tests/compose/docker-compose.test.yaml @@ -17,11 +17,6 @@ services: environment: - ENVIRONMENT=test - prompt-service: - image: unstract/prompt-service:${UNSTRACT_TEST_VERSION:-latest} - environment: - - ENVIRONMENT=test - platform-service: image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest} environment: diff --git a/tests/critical_paths.yaml b/tests/critical_paths.yaml index 3a062c94d2..ed06895e34 100644 --- a/tests/critical_paths.yaml +++ b/tests/critical_paths.yaml @@ -3,9 +3,24 @@ # A "critical path" is an end-to-end user or system flow whose failure would # constitute a production incident. The rig reports: # ✅ covered — at least one group in `covered_by` ran green this build -# ⚠️ gap — `covered_by` is empty OR no group covering it ran +# ⚠️ gap — no covering group ran green this build # ❌ regression — a path that was ✅ on the cached main baseline is now not ✅ # +# Only one kind of gap gates --fail-on-critical-gap: +# • in-scope gap — a covering group ran in this tier but not green; fails. +# • out-of-scope gap — covered only by an unrun tier, or no group declared; +# warn-only (a tier can't fail for coverage it can't run). +# +# Only wire `covered_by` to a group that really exercises the path — a bogus +# mapping fails the build when that group breaks, for the wrong reason. +# +# `proof` (default: group): +# • group — covered when any covered_by group runs green. Coarse: survives +# the covering test being deleted. +# • marker — additionally requires ≥1 passing test carrying +# @pytest.mark.critical_path("") in this build. Flip each +# path to marker once its tests are marked. +# # We intentionally do NOT chase 100% coverage. Focus on filling these gaps first. version: 1 @@ -21,10 +36,8 @@ paths: - id: adapter-register-llm description: "Register and validate an LLM adapter." entry: "POST /api/v1/adapter/" - # Honest declaration: unit-backend is currently optional/gated and - # e2e-smoke only hits /health/. Track as a gap until a real adapter test - # exists (likely under tests/e2e/smoke/ or a new tests/e2e/adapters/ group). - covered_by: [] + covered_by: [integration-backend] + proof: marker - id: workflow-create-execute description: "Create a workflow, configure source+destination, execute, poll, fetch result." @@ -46,11 +59,6 @@ paths: entry: "POST /api/v1/pipeline/{id}/execute/" covered_by: [] # gap - - id: tool-sandbox-exec - description: "Tool image runs in sandbox container and emits structured output." - entry: "internal: tool-registry → runner → docker run" - covered_by: [unit-runner] - - id: usage-token-tracking description: "Per-execution token usage is recorded and retrievable." entry: "GET /api/v1/usage/get_token_usage/" diff --git a/tests/groups.yaml b/tests/groups.yaml index d85ada8cb3..90435deac8 100644 --- a/tests/groups.yaml +++ b/tests/groups.yaml @@ -32,21 +32,6 @@ groups: uv_sync_group: test coverage_source: src/unstract/sdk1 - unit-runner: - tier: unit - workdir: runner - # Runner co-locates its tests under src/. Pytest recurses from here. - # The project has no `test` uv group, so deps are pip-installed inline. - paths: [src] - pip_install: - - "flask~=3.1.0" - - "docker==6.1.3" - - "redis~=5.2.1" - - "python-dotenv>=1.0.0" - - "kubernetes" - install_editable: true - coverage_source: src/unstract/runner - unit-platform-service: tier: unit workdir: platform-service @@ -66,38 +51,48 @@ groups: unit-backend: tier: unit workdir: backend - # List paths explicitly. `[.]` recurses into every test_*.py in backend/, - # including vendored fixtures and pluggable-app tests that don't belong - # in the OSS rig — keep this list scoped to the apps actually under test. - paths: - - account_v2/tests - - adapter_processor_v2/tests - - api_deployment_v2/tests - - connector_v2/tests - - dashboard_metrics/tests - - file_management/tests - - project/tests - - prompt_studio/prompt_studio_registry_v2/tests - - tenant_account_v2/tests - - usage_v2/tests - - utils/tests - - workflow_manager/endpoint_v2/tests + # Pure backend tests — no DB. Collect the whole tree and let markers, not a + # hand-kept file list, decide membership: tests needing live infra carry + # `@pytest.mark.integration` (see integration-backend) and are excluded here. + paths: ["."] + markers: "not integration" uv_sync_group: test - env: - DJANGO_SETTINGS_MODULE: backend.settings.test_cases - # Backend ORM imports require a real Postgres; rig provisions it via - # testcontainers or compose when this group is selected. - requires_services: [postgres, redis] + # Anchored: integration-backend reuses the identical Django settings env so + # the two halves of the backend suite can't drift apart. + env: &backend_test_env + DJANGO_SETTINGS_MODULE: backend.settings.test + # Tenancy is row-level, not schema-per-tenant; tests run in public. + DB_SCHEMA: public + # base.py resolves these at import time with no default; supply test-safe + # values here. + DJANGO_SECRET_KEY: test-secret-key-not-for-production + # All-zero Fernet key: valid format, obviously a placeholder. + ENCRYPTION_KEY: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + CELERY_BROKER_BASE_URL: redis://localhost:6379 + CELERY_BROKER_USER: guest + CELERY_BROKER_PASS: guest + INDEXING_FLAG_TTL: "3600" + ENABLE_LOG_HISTORY: "False" + STRUCTURE_TOOL_IMAGE_URL: docker:test + STRUCTURE_TOOL_IMAGE_NAME: test-structure-tool + STRUCTURE_TOOL_IMAGE_TAG: test + SYSTEM_ADMIN_USERNAME: admin + SYSTEM_ADMIN_PASSWORD: admin + SYSTEM_ADMIN_EMAIL: admin@example.com + # ExecutionFileHandler builds execution-dir paths from this at init; only + # constructed, never written to in these tests. + WORKFLOW_EXECUTION_DIR_PREFIX: /tmp/unstract-workflow-exec coverage_source: . - optional: true # gated until backend test_cases settings are complete unit-connectors: tier: unit workdir: unstract/connectors paths: [tests] + # Pure connector tests only. Credential / live-infra tests are marked + # `@pytest.mark.integration` and run in `integration-connectors`. + markers: "not integration" uv_sync_group: test coverage_source: src - optional: true unit-core: tier: unit @@ -106,17 +101,38 @@ groups: # No `test` uv group in unstract/core today; rig still injects pytest plugins. install_editable: true coverage_source: src - optional: true - unit-tool-registry: - tier: unit - workdir: unstract/tool-registry + # ── Integration tier: needs infra but not full platform ──────────────────── + integration-backend: + tier: integration + workdir: backend + # Backend ORM tests — need live infra. The rig provisions the declared + # requires_services via testcontainers and injects their connection env into + # pytest (tests/rig/cli.py:_inject_infra_env). Not optional: these gate the + # integration tier. + # Marker-only selection — exact complement of unit-backend: conftest.py + # auto-marks DB-bound tests `integration`, so new DB tests join this + # group with no edit here. Tests needing external creds skip without them. + paths: ["."] + markers: "integration" + uv_sync_group: test + env: *backend_test_env + requires_services: [postgres, redis] + coverage_source: . + + integration-connectors: + tier: integration + workdir: unstract/connectors paths: [tests] + markers: "integration" + # Most connector integration tests need real third-party credentials + # (Snowflake, GDrive, Box, Dropbox, …) and skip when those are absent. The + # MinIO test actually runs: the rig provisions MinIO via testcontainers and + # injects MINIO_* creds (tests/rig/cli.py). + requires_services: [minio] uv_sync_group: test coverage_source: src - optional: true - # ── Integration tier: needs infra but not full platform ──────────────────── integration-workflow-execution: tier: integration paths: [tests/integration/workflow_execution] diff --git a/tests/rig/cli.py b/tests/rig/cli.py index 2876c42a97..e376498523 100644 --- a/tests/rig/cli.py +++ b/tests/rig/cli.py @@ -21,6 +21,7 @@ import uuid from functools import lru_cache from pathlib import Path +from urllib.parse import urlsplit from xml.sax import saxutils from tests.rig import critical_paths as cp @@ -31,8 +32,18 @@ GroupDefinition, load_groups, ) -from tests.rig.reporting import GroupResult, parse_junit, write_summary -from tests.rig.runtime import PlatformEndpoints, PlatformRuntime, pick_runtime +from tests.rig.reporting import ( + GroupResult, + parse_junit, + passed_critical_path_ids, + write_summary, +) +from tests.rig.runtime import ( + PlatformEndpoints, + PlatformRuntime, + TestcontainersRuntime, + pick_runtime, +) from tests.rig.selection import resolve # Pytest exit codes that the rig treats as non-failure for aggregation: @@ -40,6 +51,10 @@ # 5 — no tests collected (optional placeholders, empty hurl group, etc.) _NON_FAILING_PYTEST_EXIT_CODES = (0, 5) +# Injected into every group's pytest run (PYTHONPATH + -p) so +# @pytest.mark.critical_path marker args land in junit properties. +_PYTEST_PLUGIN_DIR = REPO_ROOT / "tests" / "rig" / "pytest_plugin" + @lru_cache(maxsize=1) def _rig_session_id() -> str: @@ -159,6 +174,21 @@ def _build_parser() -> argparse.ArgumentParser: pre = sub.add_parser("report", help="Re-aggregate existing reports/.") pre.add_argument("action", choices=["combine"]) pre.add_argument("--reports-dir", type=Path, default=REPO_ROOT / "reports") + pre.add_argument( + "--baseline", + type=Path, + default=None, + help="Baseline path (default: /previous-summary.json).", + ) + pre.add_argument( + "--update-baseline", + action="store_true", + help=( + "Merge covered paths from all tiers' reports into the baseline in " + "one write (parallel per-tier writes would race). Refused (exit 1) " + "if any non-optional group is red or the baseline is corrupt." + ), + ) pre.set_defaults(func=cmd_report) return p @@ -289,15 +319,21 @@ def cmd_report(args: argparse.Namespace) -> int: result = parse_junit(name, tier, reports_dir) if result is not None: group_results.append(result) - green = _green_group_names(group_results) + green = _coverage_attesting_groups(group_results) + proven, unknown_marker_ids = _marker_proven_paths( + registry, group_results, reports_dir + ) + baseline_path = args.baseline or reports_dir / "previous-summary.json" baseline_corrupt = False try: - baseline = cp.load_baseline(reports_dir / "previous-summary.json") + baseline = cp.load_baseline(baseline_path) except cp.BaselineCorruptError as exc: print(f"[rig] {exc}", file=sys.stderr) baseline = None baseline_corrupt = True - statuses = cp.evaluate(registry, groups_run_green=green, baseline=baseline) + statuses = cp.evaluate( + registry, groups_run_green=green, baseline=baseline, marker_proven=proven + ) write_summary( reports_dir=reports_dir, group_results=group_results, @@ -305,7 +341,30 @@ def cmd_report(args: argparse.Namespace) -> int: baseline_corrupt=baseline_corrupt, ) print(f"Wrote {reports_dir / 'summary.md'}") - return 0 + if unknown_marker_ids: + print( + f"[rig] ❌ @pytest.mark.critical_path references unknown path " + f"id(s): {', '.join(unknown_marker_ids)} " + f"(not in tests/critical_paths.yaml)", + file=sys.stderr, + ) + if args.update_baseline: + red = [ + r.name + for r in group_results + if r.status not in ("pass", "empty") and not manifest.get(r.name).optional + ] + if baseline_corrupt or red: + reason = ( + "corrupt baseline" + if baseline_corrupt + else f"red groups: {', '.join(red)}" + ) + print(f"[rig] ❌ baseline update refused ({reason})", file=sys.stderr) + return 1 + cp.merge_into_baseline(statuses, baseline_path) + print(f"[rig] merged into baseline: {baseline_path}") + return 1 if unknown_marker_ids else 0 def cmd_run(args: argparse.Namespace) -> int: @@ -349,6 +408,11 @@ def cmd_run(args: argparse.Namespace) -> int: reports_dir.mkdir(parents=True, exist_ok=True) needs_platform = any(manifest.get(n).requires_platform for n in runnable) + # A group can declare `requires_services` (stateful infra like Postgres/ + # Redis) without needing the whole platform — provision just that infra via + # testcontainers instead of standing up every compose service. Platform wins + # when both are set. + needs_services = any(manifest.get(n).requires_services for n in runnable) runtime: PlatformRuntime | None = None endpoints: PlatformEndpoints | None = None group_results: list[GroupResult] = [] @@ -361,6 +425,16 @@ def cmd_run(args: argparse.Namespace) -> int: # `up()` is inside the try so a failure here still triggers `down()` # in the finally, cleaning up any partial stack. endpoints = runtime.up() + elif needs_services and not args.dry_run: + # Infra-only: testcontainers Postgres/Redis/etc., no platform + # services. ponytail: up() starts the full infra set even if a run + # only needs Postgres; trim to the requested services if startup + # cost ever matters. + runtime = TestcontainersRuntime() + print( + f"[rig] bringing infra up via runtime={runtime.name} (requires_services)" + ) + endpoints = runtime.up() # TODO(runtime-gate-skip): groups run unconditionally in topo order; # there is no skip-if-a-dependency-failed logic yet. The dep edge to @@ -437,7 +511,19 @@ def cmd_run(args: argparse.Namespace) -> int: if args.coverage and not args.dry_run: combine_and_report(reports_dir) - green = _green_group_names(group_results) + green = _coverage_attesting_groups(group_results) + proven, unknown_marker_ids = _marker_proven_paths( + registry, group_results, reports_dir + ) + if unknown_marker_ids: + print( + f"[rig] ❌ @pytest.mark.critical_path references unknown path " + f"id(s): {', '.join(unknown_marker_ids)} " + f"(not in tests/critical_paths.yaml)", + file=sys.stderr, + ) + if overall_exit == 0: + overall_exit = 1 # A corrupt baseline can't be silently treated as empty (that would turn # the next build into a regression festival once a one-tier baseline gets # written back). But we must still write the per-group summary so the @@ -456,6 +542,7 @@ def cmd_run(args: argparse.Namespace) -> int: groups_run_green=green, baseline=baseline, scope_groups=scope_groups, + marker_proven=proven, ) write_summary( reports_dir=reports_dir, @@ -486,11 +573,25 @@ def cmd_run(args: argparse.Namespace) -> int: if overall_exit == 0: overall_exit = 1 + # Only in-scope gaps gate: a declared covering group ran in this tier but + # not green. Out-of-scope gaps (covered only by other tiers, or undeclared) + # are reported but must not fail a tier for coverage it can't produce. gaps = [s for s in statuses if s.state == "gap"] - if gaps and args.fail_on_critical_gap: + in_scope_gaps = [s for s in gaps if s.in_scope] + out_of_scope_gaps = [s for s in gaps if not s.in_scope] + if out_of_scope_gaps: + ids = ", ".join(s.path.id for s in out_of_scope_gaps) print( - f"\n[rig] ⚠️ {len(gaps)} critical-path gap(s) detected " - f"(fail-on-critical-gap)", + f"[rig] ℹ️ {len(out_of_scope_gaps)} critical-path gap(s) out of scope " + f"for this run (warn-only, not covered by any group in this tier): " + f"{ids}", + file=sys.stderr, + ) + if in_scope_gaps and args.fail_on_critical_gap: + ids = ", ".join(s.path.id for s in in_scope_gaps) + print( + f"\n[rig] ⚠️ {len(in_scope_gaps)} critical-path gap(s) detected " + f"(fail-on-critical-gap): {ids}", file=sys.stderr, ) if overall_exit == 0: @@ -510,8 +611,78 @@ def cmd_run(args: argparse.Namespace) -> int: # ── execution helpers ───────────────────────────────────────────────────────── -def _green_group_names(results: list[GroupResult]) -> set[str]: - return {r.name for r in results if r.status in ("pass", "empty")} +def _db_env_from_postgres_url(url: str) -> dict[str, str]: + """Translate a provisioned Postgres URL into the discrete ``DB_*`` vars + Django reads (``backend/settings/base.py``). + + The rig provisions a throwaway Postgres via testcontainers for groups + declaring ``requires_services: [postgres]``. Without this translation the + backend falls back to the compose hostname ``backend-db-1``, unreachable + from the host-side pytest, and every ``django_db`` test errors on connect. + """ + # e.g. postgresql+psycopg2://user:pass@host:49153/dbname + parts = urlsplit(url) + return { + "DB_HOST": parts.hostname or "localhost", + "DB_PORT": str(parts.port or 5432), + "DB_USER": parts.username or "test", + "DB_PASSWORD": parts.password or "test", + "DB_NAME": parts.path.lstrip("/") or "test", + } + + +def _inject_infra_env( + env: dict[str, str], + group: GroupDefinition, + endpoints: PlatformEndpoints | None, +) -> None: + # Postgres/Redis override so a stale shell value can't shadow the + # provisioned testcontainer; MinIO uses setdefault so a developer's own + # endpoint wins. + if endpoints is None: + return + infra = endpoints.infra + if "postgres" in group.requires_services and infra.postgres_url: + env.update(_db_env_from_postgres_url(infra.postgres_url)) + if "minio" in group.requires_services and infra.minio_endpoint: + # http: this is a local, throwaway testcontainers MinIO with no TLS. + env.setdefault( + "MINIO_ENDPOINT_URL", + f"http://{infra.minio_endpoint}", # NOSONAR + ) + if infra.minio_access_key: + env.setdefault("MINIO_ACCESS_KEY_ID", infra.minio_access_key) + if infra.minio_secret_key: + env.setdefault("MINIO_SECRET_ACCESS_KEY", infra.minio_secret_key) + if "redis" in group.requires_services and infra.redis_host: + redis_port = str(infra.redis_port or 6379) + env["REDIS_HOST"] = infra.redis_host + env["REDIS_PORT"] = redis_port + env["CELERY_BROKER_BASE_URL"] = f"redis://{infra.redis_host}:{redis_port}" + + +def _coverage_attesting_groups(results: list[GroupResult]) -> set[str]: + # "empty" (exit 5) stays non-failing for the build, but a covering group + # that collected zero tests must not attest critical-path coverage — a + # broken marker expression would otherwise report ✅ with zero tests run. + return {r.name for r in results if r.status == "pass"} + + +def _marker_proven_paths( + registry: cp.CriticalPathRegistry, + group_results: list[GroupResult], + reports_dir: Path, +) -> tuple[set[str], list[str]]: + """Union critical-path ids attested by passing marked tests across groups. + + Returns ``(known_ids, unknown_ids)`` — unknown ids (marker typos) must fail + the build or the marked test silently attests nothing. + """ + registry_ids = {p.id for p in registry.paths} + proven: set[str] = set() + for r in group_results: + proven |= passed_critical_path_ids(r.name, reports_dir) + return proven & registry_ids, sorted(proven - registry_ids) def _is_missing_placeholder(group: GroupDefinition) -> bool: @@ -541,6 +712,9 @@ def _execute_group( md_report = group_reports / "report.md" env = _subprocess_env() + env["PYTHONPATH"] = os.pathsep.join( + p for p in (str(_PYTEST_PLUGIN_DIR), env.get("PYTHONPATH")) if p + ) env.update(group.env) if endpoints is not None: env.setdefault("UNSTRACT_BACKEND_URL", endpoints.backend_url) @@ -553,6 +727,7 @@ def _execute_group( # leaked in". `setdefault` would let a leaked sentinel win, which # defeats the purpose — set unconditionally. env["UNSTRACT_RIG_SESSION_ID"] = _rig_session_id() + _inject_infra_env(env, group, endpoints) if coverage and group.coverage_source: env.update(coverage_env(group.name, reports_dir)) @@ -628,13 +803,8 @@ def _prepare_group_env(group: GroupDefinition, *, env: dict[str, str]) -> None: env=env, check=False, ) - if group.install_editable: - subprocess.run( - ["uv", "pip", "install", "-e", "."], - cwd=workdir, - env=env, - check=False, - ) + # install_editable is handled in _pytest_command via `--with-editable`; + # installing it here would be wiped by `uv run`'s venv re-sync. if group.pip_install: subprocess.run( ["uv", "pip", "install", *group.pip_install], @@ -647,6 +817,20 @@ def _prepare_group_env(group: GroupDefinition, *, env: dict[str, str]) -> None: # That avoids losing them on the next `uv run` (which re-syncs the venv). +def _pytest_base_cmd(group: GroupDefinition, workdir: Path) -> list[str]: + if not shutil.which("uv"): + return [sys.executable, "-m", "pytest"] + # `uv run` re-syncs the venv each call, wiping anything from `uv pip + # install`. `--with`/`--with-editable` inject plugins + the project into the + # ephemeral run env instead, surviving the sync. + with_args: list[str] = [] + for spec in RIG_PYTEST_PLUGINS: + with_args += ["--with", spec] + if group.install_editable: + with_args += ["--with-editable", str(workdir)] + return ["uv", "run", *with_args, "pytest"] + + def _pytest_command( group: GroupDefinition, *, @@ -660,23 +844,21 @@ def _pytest_command( workers: str, timeout: int, ) -> list[str]: - use_uv = shutil.which("uv") is not None - if use_uv: - # `uv run` re-syncs the project's venv each call, which would wipe any - # plugins added via `uv pip install`. `--with` injects them into the - # ephemeral run environment, surviving the sync. - with_args: list[str] = [] - for spec in RIG_PYTEST_PLUGINS: - with_args += ["--with", spec] - base: list[str] = ["uv", "run", *with_args, "pytest"] - else: - base = [sys.executable, "-m", "pytest"] + base = _pytest_base_cmd(group, workdir) cmd = [ *base, "-v", f"--junitxml={junit}", f"--timeout={timeout}", + # Marker→junit-property plugin (dir added to PYTHONPATH in + # _execute_group). junit_family=legacy: the default xunit2 schema + # rejects per-testcase , which the coverage attestation + # reads. + "-p", + "rig_critical_path", + "-o", + "junit_family=legacy", ] # pytest-md-report does not aggregate worker output reliably under xdist. # Emit markdown only on serial runs; junit + reporting.py's _render_markdown diff --git a/tests/rig/critical_paths.py b/tests/rig/critical_paths.py index b0cbbd0596..f0b8fb9147 100644 --- a/tests/rig/critical_paths.py +++ b/tests/rig/critical_paths.py @@ -36,6 +36,11 @@ class CriticalPath: description: str entry: str covered_by: tuple[str, ...] + # "group": covered when any covered_by group runs green (coarse — survives + # the covering test being deleted). "marker": additionally requires ≥1 + # passing test marked @pytest.mark.critical_path("") in this run. + # Ratchet each path to "marker" as its tests get marked. + proof: Literal["group", "marker"] = "group" @dataclass(frozen=True) @@ -65,6 +70,12 @@ class CriticalPathStatus: state: CriticalPathState covering_groups_run: tuple[str, ...] notes: str = "" + # True when a declared covering group is in this run's scope (or scoping is + # off). An out-of-scope gap (coverage only in an unrun tier, or none + # declared) must not gate under --fail-on-critical-gap. Defaults False so a + # regression that forgets to pass it can only under-gate (spurious warning), + # never over-gate (spurious build block). + in_scope: bool = False def __post_init__(self) -> None: # Make the contradictory states unrepresentable rather than relying on @@ -80,17 +91,24 @@ def load_critical_paths(path: Path | None = None) -> CriticalPathRegistry: raw = yaml.safe_load((path or DEFAULT_REGISTRY).read_text()) if not isinstance(raw, dict) or "paths" not in raw: raise ValueError(f"{path or DEFAULT_REGISTRY}: expected top-level `paths:` list") - return CriticalPathRegistry( - paths=tuple( + paths = [] + for p in raw["paths"]: + proof = str(p.get("proof", "group")) + if proof not in ("group", "marker"): + raise ValueError( + f"critical path {p.get('id')!r}: proof must be 'group' or " + f"'marker', got {proof!r}" + ) + paths.append( CriticalPath( id=str(p["id"]), description=str(p.get("description", "")), entry=str(p.get("entry", "")), covered_by=tuple(p.get("covered_by") or ()), + proof=proof, ) - for p in raw["paths"] ) - ) + return CriticalPathRegistry(paths=tuple(paths)) def validate_registry_against_manifest( @@ -106,6 +124,11 @@ def validate_registry_against_manifest( f"critical path {path.id!r}: " f"covered_by references unknown group {g!r}" ) + if path.proof == "marker" and not path.covered_by: + errors.append( + f"critical path {path.id!r}: proof 'marker' requires a " + f"non-empty covered_by (marked tests must live in a group)" + ) return errors @@ -115,21 +138,27 @@ def evaluate( groups_run_green: Collection[str], baseline: dict[str, Any] | None, scope_groups: Collection[str] | None = None, + marker_proven: Collection[str] = (), ) -> list[CriticalPathStatus]: """Compute the status for each critical path against this build's results. Args: registry: parsed critical-paths registry. groups_run_green: names of groups that ran AND passed in this build. + Pass only ``status == "pass"`` groups — a group that collected + zero tests ("empty") must not attest coverage. baseline: parsed previous-summary.json from the main-branch cache, or None. Expected shape: ``{"covered_paths": ["auth-login", ...]}``. - scope_groups: collection of every group the caller considered running - this invocation (including dep-expanded deps and skipped - optional placeholders). When a critical path's ``covered_by`` - is fully outside ``scope_groups``, the path is classified as - ``gap`` rather than ``regression`` — running only the unit - tier shouldn't flag e2e-tier paths as regressed. If ``None``, - no scoping is applied (back-compat). + scope_groups: the groups this invocation actually runs (dep-expanded). + When a critical path's ``covered_by`` is fully outside + ``scope_groups``, the path is classified as ``gap`` rather than + ``regression`` — running only the unit tier shouldn't flag + e2e-tier paths as regressed. If ``None``, no scoping is applied + (back-compat). + marker_proven: path ids with at least one passing + ``@pytest.mark.critical_path`` test in this build. Paths with + ``proof: marker`` require membership here on top of a green + covering group. Returns: Statuses in the original registry order. @@ -139,34 +168,52 @@ def evaluate( # when callers pass lists/tuples; the public signature accepts Collection # to leave that choice to them. green = set(groups_run_green) + proven = set(marker_proven) scope = None if scope_groups is None else set(scope_groups) - statuses: list[CriticalPathStatus] = [] - for path in registry.paths: - covering = tuple(g for g in path.covered_by if g in green) - in_scope = scope is None or any(g in scope for g in path.covered_by) - state: CriticalPathState - if covering: - state = "covered" - note = "" - elif path.id in previously_covered and in_scope: - state = "regression" - note = "Was covered on the cached baseline; not covered in this build." - else: - state = "gap" - note = ( - "Out of scope for this invocation." - if not in_scope - else "No group covering this path ran green in this build." - ) - statuses.append( - CriticalPathStatus( - path=path, - state=state, - covering_groups_run=covering, - notes=note, - ) + return [ + _status_for(path, green, proven, previously_covered, scope) + for path in registry.paths + ] + + +def _status_for( + path: CriticalPath, + green: set[str], + proven: set[str], + previously_covered: set[str], + scope: set[str] | None, +) -> CriticalPathStatus: + covering = tuple(g for g in path.covered_by if g in green) + if path.proof == "marker" and path.id not in proven: + covering = () + in_scope = scope is None or any(g in scope for g in path.covered_by) + if covering: + state: CriticalPathState = "covered" + note = "" + elif path.id in previously_covered and in_scope: + state = "regression" + note = "Was covered on the cached baseline; not covered in this build." + else: + state = "gap" + note = _gap_note(path, green, in_scope) + return CriticalPathStatus( + path=path, + state=state, + covering_groups_run=covering, + notes=note, + in_scope=in_scope, + ) + + +def _gap_note(path: CriticalPath, green: set[str], in_scope: bool) -> str: + if not in_scope: + return "Out of scope for this invocation." + if path.proof == "marker" and any(g in green for g in path.covered_by): + return ( + "Covering group ran green but no passing " + "@pytest.mark.critical_path test attested this path." ) - return statuses + return "No group covering this path ran green in this build." def merge_into_baseline(statuses: list[CriticalPathStatus], destination: Path) -> None: diff --git a/tests/rig/pytest_plugin/rig_critical_path.py b/tests/rig/pytest_plugin/rig_critical_path.py new file mode 100644 index 0000000000..2704465b56 --- /dev/null +++ b/tests/rig/pytest_plugin/rig_critical_path.py @@ -0,0 +1,29 @@ +"""Pytest plugin injected by the rig into every group's pytest run. + +Copies ``@pytest.mark.critical_path("")`` marker args into junit +testcase ```` so the rig can attest critical-path coverage from +tests that actually passed, not just from a group's overall exit code. A test +may carry the marker multiple times (or pass several ids) to attest several +paths. + +Lives in its own directory (not ``tests/rig/``) because the rig injects it via +``PYTHONPATH`` + ``-p`` into each group's own venv — putting ``tests/rig`` on +``PYTHONPATH`` would shadow real packages (e.g. ``coverage``). Stdlib-only for +the same reason. +""" + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "critical_path(path_id, ...): attests coverage of declared critical " + "path(s) from tests/critical_paths.yaml when this test passes", + ) + + +# pytest matches hook args by name, so unused params can be omitted. +def pytest_collection_modifyitems(items): + for item in items: + for marker in item.iter_markers("critical_path"): + for path_id in marker.args: + item.user_properties.append(("critical_path", str(path_id))) diff --git a/tests/rig/reporting.py b/tests/rig/reporting.py index 8bff0a00cb..51b5bea52c 100644 --- a/tests/rig/reporting.py +++ b/tests/rig/reporting.py @@ -139,6 +139,32 @@ def parse_junit(group_name: str, tier: str, reports_dir: Path) -> GroupResult | ) +def passed_critical_path_ids(group_name: str, reports_dir: Path) -> set[str]: + """Extract critical-path ids attested by *passing* tests in a group's junit. + + The rig's injected pytest plugin copies ``@pytest.mark.critical_path`` args + into per-testcase ````. Only testcases with no failure/error/ + skipped child count — a skipped or failing marked test proves nothing. + Missing or malformed junit yields an empty set (the group-level result + already surfaces that as an error). + """ + junit_path = reports_dir / group_name / "junit.xml" + if not junit_path.exists(): + return set() + try: + root = ET.parse(junit_path).getroot() + except ET.ParseError: + return set() + ids: set[str] = set() + for case in root.iter("testcase"): + if any(case.find(tag) is not None for tag in ("failure", "error", "skipped")): + continue + for prop in case.iter("property"): + if prop.get("name") == "critical_path" and prop.get("value"): + ids.add(prop.get("value")) + return ids + + def _read_exit_code(exit_path: Path) -> int: if not exit_path.exists(): return -1 diff --git a/tests/rig/runtime.py b/tests/rig/runtime.py index 97be22d748..70de5db40c 100644 --- a/tests/rig/runtime.py +++ b/tests/rig/runtime.py @@ -51,6 +51,8 @@ class InfraEndpoints: rabbitmq_host: str | None = None rabbitmq_port: int | None = None minio_endpoint: str | None = None + minio_access_key: str | None = None + minio_secret_key: str | None = None def __post_init__(self) -> None: for host, port, label in ( @@ -205,6 +207,10 @@ def up(self) -> PlatformEndpoints: minio_endpoint=( f"{minio.get_container_host_ip()}:{minio.get_exposed_port(9000)}" ), + # Default testcontainers MinIO root creds; surfaced so the + # rig can inject them into connector integration tests. + minio_access_key=getattr(minio, "access_key", "minioadmin"), + minio_secret_key=getattr(minio, "secret_key", "minioadmin"), ), ) except Exception: diff --git a/tests/rig/tests/test_cli.py b/tests/rig/tests/test_cli.py index a975f43d62..3e3ac4dfd7 100644 --- a/tests/rig/tests/test_cli.py +++ b/tests/rig/tests/test_cli.py @@ -216,6 +216,109 @@ def fake_execute_group(group, **kwargs): ) +def _run_gap_scenario( + tmp_path: Path, monkeypatch, *, covered_by: str, fail_on_gap: bool +) -> int: + """Drive cmd_run with a single optional group ``unit-cov`` that runs RED and + one critical path covered by ``covered_by`` (a YAML list literal like + ``[unit-cov]`` or ``[]``). The group is optional so its own red exit never + gates — isolating the critical-gap logic. Returns the overall exit code. + """ + from tests.rig.reporting import GroupResult + + test_dir = Path(__file__).parent + manifest_yaml = ( + "version: 1\n" + "groups:\n" + " unit-cov:\n" + " tier: unit\n" + f" workdir: {test_dir}\n" + " paths: [.]\n" + " optional: true\n" + ) + cp_yaml = ( + "version: 1\n" + "paths:\n" + " - id: p1\n" + " description: ''\n" + " entry: ''\n" + f" covered_by: {covered_by}\n" + ) + (tmp_path / "groups.yaml").write_text(manifest_yaml) + (tmp_path / "critical_paths.yaml").write_text(cp_yaml) + + import tests.rig.cli as cli_mod + import tests.rig.critical_paths as cp_mod + import tests.rig.groups as groups_mod + + monkeypatch.setattr(groups_mod, "DEFAULT_MANIFEST", tmp_path / "groups.yaml") + monkeypatch.setattr(cp_mod, "DEFAULT_REGISTRY", tmp_path / "critical_paths.yaml") + + def fake_execute_group(group, **kwargs): + # The covering group runs red, so it never counts as green coverage. + result = GroupResult( + name=group.name, + tier=group.tier, + exit_code=1, + passed=0, + failed=1, + errors=0, + skipped=0, + duration_seconds=0.01, + ) + return result, 1 + + monkeypatch.setattr(cli_mod, "_execute_group", fake_execute_group) + + argv = [ + "run", + "unit-cov", + "--no-coverage", + "--no-parallel", + "--reports-dir", + str(tmp_path / "reports"), + "--baseline", + str(tmp_path / "reports" / "previous-summary.json"), + ] + if fail_on_gap: + argv.append("--fail-on-critical-gap") + args = cli_mod._build_parser().parse_args(argv) + return cli_mod.cmd_run(args) + + +def test_fail_on_critical_gap_gates_on_in_scope_gap(tmp_path: Path, monkeypatch) -> None: + """A critical path covered by an in-tier group that ran red is an IN-SCOPE + gap: --fail-on-critical-gap must fail the build on it (real coverage is + gone). Without the flag, it's reported but doesn't gate. + """ + assert ( + _run_gap_scenario( + tmp_path, monkeypatch, covered_by="[unit-cov]", fail_on_gap=True + ) + == 1 + ) + assert ( + _run_gap_scenario( + tmp_path, monkeypatch, covered_by="[unit-cov]", fail_on_gap=False + ) + == 0 + ) + + +def test_fail_on_critical_gap_ignores_out_of_scope_gap( + tmp_path: Path, monkeypatch +) -> None: + """A path with no declared coverage (or coverage only in another tier) is an + OUT-OF-SCOPE gap: --fail-on-critical-gap must NOT fail this tier on it. + This is the fix for the perma-red `main`: e2e-only and not-yet-covered paths + can't fail the unit/integration tiers. + """ + assert ( + _run_gap_scenario(tmp_path, monkeypatch, covered_by="[]", fail_on_gap=True) + == 0 + ) + + def test_cmd_run_teardown_failure_does_not_mask_up_failure( tmp_path: Path, monkeypatch ) -> None: @@ -465,3 +568,46 @@ def test_cmd_report_re_aggregates_existing_junit(tmp_path: Path, monkeypatch) -> for artifact in ("summary.md", "summary.json", "combined-test-report.md"): assert (reports_dir / artifact).exists(), f"missing {artifact}" assert "unit-x" in (reports_dir / "summary.md").read_text() + + +def test_db_env_from_postgres_url_maps_discrete_vars() -> None: + """The provisioned-Postgres URL (testcontainers, with a `+driver` scheme + and a random host port) must translate into the discrete DB_* vars Django + reads — otherwise integration-backend falls back to `backend-db-1` and + every django_db test errors on connect. + """ + import tests.rig.cli as cli_mod + + env = cli_mod._db_env_from_postgres_url( + "postgresql+psycopg2://tcuser:tcpass@127.0.0.1:49231/testdb" + ) + assert env == { + "DB_HOST": "127.0.0.1", + "DB_PORT": "49231", + "DB_USER": "tcuser", + "DB_PASSWORD": "tcpass", # NOSONAR - test placeholder, not a real credential + "DB_NAME": "testdb", + } + + +def test_inject_infra_env_wires_provisioned_redis() -> None: + """A group declaring `requires_services: [redis]` must get REDIS_HOST/PORT + + the Celery broker URL rewritten to the provisioned endpoint — otherwise + Redis-backed tests silently hit the localhost default and bypass the + testcontainer. + """ + import tests.rig.cli as cli_mod + from tests.rig.groups import GroupDefinition + from tests.rig.runtime import InfraEndpoints, PlatformEndpoints + + endpoints = PlatformEndpoints.from_env( + infra=InfraEndpoints(redis_host="redis.internal", redis_port=49999) + ) + group = GroupDefinition( + name="g", tier="integration", paths=("tests",), requires_services=("redis",) + ) + env: dict[str, str] = {} + cli_mod._inject_infra_env(env, group, endpoints) + assert env["REDIS_HOST"] == "redis.internal" + assert env["REDIS_PORT"] == "49999" + assert env["CELERY_BROKER_BASE_URL"] == "redis://redis.internal:49999" diff --git a/tests/rig/tests/test_critical_paths.py b/tests/rig/tests/test_critical_paths.py index b1798d3c1b..8ee4251798 100644 --- a/tests/rig/tests/test_critical_paths.py +++ b/tests/rig/tests/test_critical_paths.py @@ -158,6 +158,43 @@ def test_scope_demotes_out_of_scope_regressions_to_gaps() -> None: assert by_id["straddle-path"].state == "regression" # partially in scope +def test_in_scope_flag_distinguishes_gap_flavours() -> None: + """The ``in_scope`` flag on a status is what lets --fail-on-critical-gap + gate only on coverage that this tier was actually responsible for. An + out-of-scope gap (e2e path during the unit tier, or a path with no declared + coverage) must report ``in_scope=False``; an in-scope gap (a declared + in-tier group that didn't run green) must report ``in_scope=True``. + """ + registry = _registry( + ("in-scope", ("unit-g",)), # declared group is in scope, but not green + ("e2e-only", ("e2e-g",)), # declared group is out of scope this run + ("undeclared", ()), # no declared coverage anywhere + ) + statuses = evaluate( + registry, + groups_run_green=set(), # nothing passed → all three are gaps + baseline=None, + scope_groups={"unit-g"}, + ) + by_id = {s.path.id: s for s in statuses} + assert all(s.state == "gap" for s in statuses) + assert by_id["in-scope"].in_scope is True + assert by_id["e2e-only"].in_scope is False + assert by_id["undeclared"].in_scope is False + + +def test_covered_path_is_in_scope() -> None: + registry = _registry(("p1", ("g1",))) + statuses = evaluate( + registry, + groups_run_green={"g1"}, + baseline=None, + scope_groups={"g1"}, + ) + assert statuses[0].state == "covered" + assert statuses[0].in_scope is True + + def test_scope_none_preserves_legacy_behavior() -> None: """scope_groups=None disables scope-filtering so callers that don't pass it keep the old "everything in baseline counts" semantics. @@ -170,3 +207,98 @@ def test_scope_none_preserves_legacy_behavior() -> None: scope_groups=None, ) assert statuses[0].state == "regression" + + +def _marker_path(path_id: str, covers: tuple[str, ...]) -> CriticalPath: + return CriticalPath( + id=path_id, description="", entry="", covered_by=covers, proof="marker" + ) + + +def test_marker_proof_covered_when_group_green_and_proven() -> None: + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, groups_run_green={"g1"}, baseline=None, marker_proven={"p1"} + ) + assert statuses[0].state == "covered" + assert statuses[0].covering_groups_run == ("g1",) + + +def test_marker_proof_gap_when_group_green_but_unproven() -> None: + """The whole point of proof=marker: a green covering group alone (e.g. the + marked test was deleted and the rest of the group still passes) must not + attest coverage. + """ + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, groups_run_green={"g1"}, baseline=None, marker_proven=set() + ) + assert statuses[0].state == "gap" + assert "critical_path" in statuses[0].notes + + +def test_marker_proof_regression_when_baseline_covered_but_unproven() -> None: + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, + groups_run_green={"g1"}, + baseline={"covered_paths": ["p1"]}, + marker_proven=set(), + ) + assert statuses[0].state == "regression" + + +def test_marker_proof_requires_green_group_too() -> None: + """A passing marked test inside an overall-red group proves nothing — + coverage needs both the proof and a green covering group. + """ + registry = CriticalPathRegistry(paths=(_marker_path("p1", ("g1",)),)) + statuses = evaluate( + registry, groups_run_green=set(), baseline=None, marker_proven={"p1"} + ) + assert statuses[0].state == "gap" + + +def test_marker_proof_does_not_affect_group_proof_paths() -> None: + registry = _registry(("p1", ("g1",))) + statuses = evaluate( + registry, groups_run_green={"g1"}, baseline=None, marker_proven=set() + ) + assert statuses[0].state == "covered" + + +def test_load_rejects_unknown_proof(tmp_path: Path) -> None: + from tests.rig.critical_paths import load_critical_paths + + reg = tmp_path / "critical_paths.yaml" + reg.write_text( + "paths:\n - id: p1\n covered_by: [g1]\n proof: junit\n" + ) + with pytest.raises(ValueError, match="proof"): + load_critical_paths(reg) + + +def test_load_parses_proof_field(tmp_path: Path) -> None: + from tests.rig.critical_paths import load_critical_paths + + reg = tmp_path / "critical_paths.yaml" + reg.write_text( + "paths:\n" + " - id: p1\n covered_by: [g1]\n proof: marker\n" + " - id: p2\n covered_by: [g1]\n" + ) + loaded = load_critical_paths(reg) + assert loaded.by_id("p1").proof == "marker" + assert loaded.by_id("p2").proof == "group" + + +def test_validate_rejects_marker_proof_without_covered_by() -> None: + from tests.rig.critical_paths import validate_registry_against_manifest + from tests.rig.groups import GroupDefinition, GroupManifest + + manifest = GroupManifest( + groups={"g1": GroupDefinition(name="g1", tier="unit", paths=())} + ) + registry = CriticalPathRegistry(paths=(_marker_path("p1", ()),)) + errors = validate_registry_against_manifest(registry, manifest) + assert errors and "marker" in errors[0] diff --git a/tests/rig/tests/test_reporting.py b/tests/rig/tests/test_reporting.py index dca890267b..4913100dc0 100644 --- a/tests/rig/tests/test_reporting.py +++ b/tests/rig/tests/test_reporting.py @@ -104,3 +104,43 @@ def test_status_icon_round_trips() -> None: assert pass_result.status_icon == "✅" assert fail_result.status_icon == "❌" assert empty_result.status_icon == "⚪" + + +def test_passed_critical_path_ids_collects_only_passing_marked_tests( + tmp_path: Path, +) -> None: + from tests.rig.reporting import passed_critical_path_ids + + _write_junit( + tmp_path / "g1", + """ + + + + + + + + + + + + + + + + + +""", + exit_code=1, + ) + ids = passed_critical_path_ids("g1", tmp_path) + assert ids == {"p-pass", "p-second"} + + +def test_passed_critical_path_ids_missing_or_malformed_junit(tmp_path: Path) -> None: + from tests.rig.reporting import passed_critical_path_ids + + assert passed_critical_path_ids("absent", tmp_path) == set() + _write_junit(tmp_path / "broken", " None: """Set up test configuration from environment variables""" - # SSL enabled config for testing SSL scenarios self.mariadb_config_ssl_enabled = { "host": os.getenv("MARIADB_HOST", "localhost"), @@ -86,7 +84,7 @@ def test_authentication_error_handling(self, mock_connect: Any) -> None: error_message = str(context.exception) self.assertIn("Authentication failed", error_message) - self.assertIn("username, password and ssl-settings", error_message) + self.assertIn("username, password and SSL SETTINGS", error_message) self.assertIn("localhost:3306", error_message) self.assertIn("SSL enabled", error_message) diff --git a/unstract/connectors/tests/databases/test_mssql_db.py b/unstract/connectors/tests/databases/test_mssql_db.py deleted file mode 100644 index c78340af91..0000000000 --- a/unstract/connectors/tests/databases/test_mssql_db.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from unstract.connectors.databases.mssql.mssql import MSSQL - - -class TestMSSQL(unittest.TestCase): - def test_user_name_and_password(self): - mssql = MSSQL( - { - "user": "sa", - "password": "Ascon@123", - "server": "localhost", - "port": "1433", - "database": "testdb", - } - ) - query = "SELECT * FROM Employees" - cursor = mssql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_mysql_db.py b/unstract/connectors/tests/databases/test_mysql_db.py deleted file mode 100644 index aa47cd90cc..0000000000 --- a/unstract/connectors/tests/databases/test_mysql_db.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from unstract.connectors.databases.mysql.mysql import MySQL - - -class TestMySQLDB(unittest.TestCase): - def test_user_name_and_password(self): - mysql = MySQL( - { - "user": "visitran", - "password": "mysqlpass", - "host": "localhost", - "port": "3307", - "database": "sakila", - } - ) - query = "SELECT * FROM category" - cursor = mysql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_postgresql_db.py b/unstract/connectors/tests/databases/test_postgresql_db.py deleted file mode 100644 index 96fceddd59..0000000000 --- a/unstract/connectors/tests/databases/test_postgresql_db.py +++ /dev/null @@ -1,50 +0,0 @@ -import unittest - -from unstract.connectors.databases.postgresql.postgresql import PostgreSQL - - -class TestPostgreSqlDB(unittest.TestCase): - def test_user_name_and_password(self): - psql = PostgreSQL( - { - "user": "test", - "password": "ascon", - "host": "localhost", - "port": "5432", - "database": "test7", - "schema": "public", - } - ) - query = "SELECT * FROM account_user LIMIT 3" - cursor = psql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - def test_connection_url(self): - connection_url = ( - "postgres://iamali003:FeQhupi41INg@ep-crimson-wind-434055" - ".us-east-2.aws.neon.tech/neondb" - ) - psql = PostgreSQL( - { - "connection_url": connection_url, - } - ) - query = "SELECT * FROM users LIMIT 3" - cursor = psql.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_redshift_db.py b/unstract/connectors/tests/databases/test_redshift_db.py deleted file mode 100644 index 0b1300bfab..0000000000 --- a/unstract/connectors/tests/databases/test_redshift_db.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest - -from unstract.connectors.databases.redshift.redshift import Redshift - - -class TestRedshift(unittest.TestCase): - def test_user_name_and_password(self): - redshift = Redshift( - { - "user": "awsuser", - "password": "PASSWORD", - "host": "redshift-cluster-1.redshift.amazonaws.com", - "port": "5439", - "database": "dev", - } - ) - query = ( - "SELECT userid, username, firstname, lastname, city, state, email," - "phone, likesports, liketheatre, likeconcerts, likejazz," - "likeclassical, likeopera, likerock, likevegas, likebroadway," - "likemusicals FROM users limit 10" - ) - cursor = redshift.get_engine().cursor() - cursor.execute(query) - results = cursor.fetchall() - - for c in results: - print(c) - - self.assertTrue(len(results) > 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/databases/test_snowflake_db.py b/unstract/connectors/tests/databases/test_snowflake_db.py deleted file mode 100644 index a87bb32733..0000000000 --- a/unstract/connectors/tests/databases/test_snowflake_db.py +++ /dev/null @@ -1,43 +0,0 @@ -import unittest - -from unstract.connectors.databases.snowflake.snowflake import SnowflakeDB - - -class TestSnowflakeDB(unittest.TestCase): - def test_something(self): - sf = SnowflakeDB( - { - "user": "arun", - "password": "PASSWORD", - "account": "JX91721.ap-south-1", - "database": "RESUME_COLLECTION", - "schema": "PUBLIC", - "warehouse": "COMPUTE_WH", - "role": "", - } - ) - # engine = sf.get_engine() - # try: - # with engine.connect() as connection: - # md = sqlalchemy.MetaData() - # table = sqlalchemy.Table( - # 'RESUME', md, autoload=True, autoload_with=engine) - # columns = table.c - # for c in columns: - # print(c.name, c.type) - # # connection.execute("select current_version()") - # except Exception as e: - # print(e) - # - # engine.dispose() - - cursor = sf.get_engine().cursor() - results = cursor.execute("describe table RESUME") - for c in results: - print(c) - - self.assertIsNotNone(results) # add assertion here - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/connectors/tests/filesystems/test_box_fs.py b/unstract/connectors/tests/filesystems/test_box_fs.py index cc062824bc..1aabdd8a81 100644 --- a/unstract/connectors/tests/filesystems/test_box_fs.py +++ b/unstract/connectors/tests/filesystems/test_box_fs.py @@ -1,17 +1,24 @@ import os import unittest +import pytest from unstract.connectors.filesystems.box import BoxFS +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestBoxFS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("TEST_BOX_APP_SETTINGS"), + "Integration test requires TEST_BOX_APP_SETTINGS", + ) def test_basic(self): box_app_settings = os.environ.get("TEST_BOX_APP_SETTINGS") box_fs = BoxFS(settings={"box_app_settings": box_app_settings}) file_path = "/" try: files = box_fs.get_fsspec_fs().ls(file_path) - print(files) self.assertIsNotNone(files) except Exception as e: self.fail(f"TestBoxFS.test_basic failed: {e}") diff --git a/unstract/connectors/tests/filesystems/test_google_drive_fs.py b/unstract/connectors/tests/filesystems/test_google_drive_fs.py index 26b1ac59a8..b3e1b5a3fa 100644 --- a/unstract/connectors/tests/filesystems/test_google_drive_fs.py +++ b/unstract/connectors/tests/filesystems/test_google_drive_fs.py @@ -1,9 +1,19 @@ +import os import unittest +import pytest from unstract.connectors.filesystems.google_drive.google_drive import GoogleDriveFS +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestGoogleDriveFS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("GDRIVE_GOOGLE_SERVICE_ACCOUNT") + and os.environ.get("GDRIVE_GOOGLE_PROJECT_ID"), + "Integration test requires GDRIVE_GOOGLE_SERVICE_ACCOUNT and GDRIVE_GOOGLE_PROJECT_ID", + ) def test_basic(self): self.assertEqual(GoogleDriveFS.requires_oauth(), True) drive = GoogleDriveFS( @@ -14,7 +24,7 @@ def test_basic(self): } ) - print(drive.get_fsspec_fs().ls("")) + self.assertIsNotNone(drive.get_fsspec_fs().ls("")) if __name__ == "__main__": diff --git a/unstract/connectors/tests/filesystems/test_http_fs.py b/unstract/connectors/tests/filesystems/test_http_fs.py index 4548d99543..0fb0f27940 100644 --- a/unstract/connectors/tests/filesystems/test_http_fs.py +++ b/unstract/connectors/tests/filesystems/test_http_fs.py @@ -1,24 +1,28 @@ +import os import unittest +import pytest from unstract.connectors.filesystems.http.http import HttpFS +# Live-HTTP test — integration tier only, so `unit-connectors` (-m "not +# integration") never selects it even when HTTP_FS_TEST_URL is set. +pytestmark = pytest.mark.integration + class TestHttpFS(unittest.TestCase): - # Run a local HTTP server with - # `python -m http.server -b localhost 8080` + # Needs a reachable HTTP server. Start one locally, e.g. + # python -m http.server -b localhost 8080 + # then run with HTTP_FS_TEST_URL=http://localhost:8080/. Skip-guarded so it + # never hits a hard-coded live URL during a plain unit run. + @unittest.skipUnless( + os.environ.get("HTTP_FS_TEST_URL"), + "Integration test requires a reachable HTTP server via HTTP_FS_TEST_URL", + ) def test_basic(self): self.assertEqual(HttpFS.can_write(), False) - # Assuming that the server is run locally - # url = "http://localhost:8080/" - url = "https://filesystem-spec.readthedocs.io/" - http_fs = HttpFS(settings={"base_url": url}) - file_path = "/" - try: - # print(http_fs.get_fsspec_fs().ls(file_path)) - files = http_fs.get_fsspec_fs().ls(file_path) - self.assertIsNotNone(files) - except Exception as e: - self.fail(f"TestHttpFS.test_basic failed: {e}") + http_fs = HttpFS(settings={"base_url": os.environ["HTTP_FS_TEST_URL"]}) + files = http_fs.get_fsspec_fs().ls("/") + self.assertIsNotNone(files) if __name__ == "__main__": diff --git a/unstract/connectors/tests/filesystems/test_miniofs.py b/unstract/connectors/tests/filesystems/test_miniofs.py index 9da9a1dcf0..9c8c4b6e25 100644 --- a/unstract/connectors/tests/filesystems/test_miniofs.py +++ b/unstract/connectors/tests/filesystems/test_miniofs.py @@ -3,10 +3,10 @@ import unittest from unittest.mock import AsyncMock, patch +import pytest from botocore.exceptions import ClientError from s3fs.core import S3FileSystem from s3fs.errors import translate_boto_error - from unstract.connectors.filesystems.minio.exceptions import s3_error_code from unstract.connectors.filesystems.minio.minio import ( MinioFS, @@ -16,38 +16,31 @@ class TestMinoFS(unittest.TestCase): - @unittest.skip("") - def test_s3(self) -> None: - self.assertEqual(MinioFS.requires_oauth(), False) - access_key = os.environ.get("AWS_ACCESS_KEY_ID") - secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") - s3 = MinioFS( - { - "key": access_key, - "secret": secret_key, - "path": "/", - "endpoint_url": "https://s3.amazonaws.com", - } - ) - - print(s3.get_fsspec_fs().ls("unstract-user-storage")) - - # @unittest.skip("Minio is not running") + @pytest.mark.integration + @unittest.skipUnless( + os.environ.get("MINIO_ACCESS_KEY_ID") + and os.environ.get("MINIO_SECRET_ACCESS_KEY"), + "Integration test requires a live MinIO and MINIO_ACCESS_KEY_ID + MINIO_SECRET_ACCESS_KEY", + ) def test_minio(self) -> None: + # Endpoint from the rig's testcontainers MinIO via MINIO_ENDPOINT_URL; + # falls back to the local platform MinIO for manual runs. self.assertEqual(MinioFS.requires_oauth(), False) - access_key = os.environ.get("MINIO_ACCESS_KEY_ID") - secret_key = os.environ.get("MINIO_SECRET_ACCESS_KEY") - print(access_key, secret_key) - s3 = MinioFS( + fs = MinioFS( { - "key": access_key, - "secret": secret_key, - "endpoint_url": "http://localhost:9000", - "path": "/minio-test", + "key": os.environ["MINIO_ACCESS_KEY_ID"], + "secret": os.environ["MINIO_SECRET_ACCESS_KEY"], + "endpoint_url": os.environ.get( + "MINIO_ENDPOINT_URL", "http://localhost:9000" + ), + "path": "/", } - ) - - print(s3.get_fsspec_fs().ls("/minio-test")) + ).get_fsspec_fs() + bucket = "rig-minio-test" + if not fs.exists(bucket): + fs.mkdir(bucket) + listed = [b.rstrip("/").split("/")[-1] for b in fs.ls("")] + self.assertIn(bucket, listed) def _translated_error(code: str) -> BaseException: diff --git a/unstract/connectors/tests/filesystems/test_pcs.py b/unstract/connectors/tests/filesystems/test_pcs.py index fa3f49c432..580f08a55d 100644 --- a/unstract/connectors/tests/filesystems/test_pcs.py +++ b/unstract/connectors/tests/filesystems/test_pcs.py @@ -1,10 +1,19 @@ import os import unittest +import pytest from unstract.connectors.filesystems.ucs import UnstractCloudStorage +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestPCS_FS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID") + and os.environ.get("GOOGLE_STORAGE_SECRET_ACCESS_KEY"), + "Integration test requires GOOGLE_STORAGE_ACCESS_KEY_ID and GOOGLE_STORAGE_SECRET_ACCESS_KEY", + ) def test_pcs(self) -> None: self.assertEqual(UnstractCloudStorage.requires_oauth(), False) access_key = os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID") @@ -18,7 +27,7 @@ def test_pcs(self) -> None: } ) - print(gcs.get_fsspec_fs().ls("unstract-user-storage")) # type:ignore + self.assertIsNotNone(gcs.get_fsspec_fs().ls("unstract-user-storage")) # type:ignore if __name__ == "__main__": diff --git a/unstract/connectors/tests/filesystems/test_sharepoint_fs.py b/unstract/connectors/tests/filesystems/test_sharepoint_fs.py index d976e73a99..6827ce959a 100644 --- a/unstract/connectors/tests/filesystems/test_sharepoint_fs.py +++ b/unstract/connectors/tests/filesystems/test_sharepoint_fs.py @@ -5,6 +5,8 @@ import unittest from datetime import datetime, timezone +import pytest + logger = logging.getLogger(__name__) @@ -115,14 +117,6 @@ def test_connector_initialization_missing_auth(self): SharePointFS(settings=invalid_settings) self.assertIn("requires authentication", str(context.exception)) - def test_json_schema_has_is_personal(self): - """Test that JSON schema includes is_personal field.""" - from unstract.connectors.filesystems.sharepoint import SharePointFS - - schema = SharePointFS.get_json_schema() - self.assertIn("is_personal", schema) - self.assertIn("Personal Account", schema) - def test_json_schema_has_oneof_pattern(self): """Test that JSON schema uses dependencies/oneOf pattern for dual auth methods.""" import json @@ -243,6 +237,7 @@ def test_get_connector_root_dir(self): self.assertEqual(result, "") +@pytest.mark.integration class TestSharePointFSIntegration(unittest.TestCase): """Integration tests for SharePointFS (require real credentials).""" diff --git a/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py b/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py index df35208d7e..a72bc915e3 100644 --- a/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py +++ b/unstract/connectors/tests/filesystems/test_zs_dropbox_fs.py @@ -1,10 +1,18 @@ import os import unittest +import pytest from unstract.connectors.filesystems.zs_dropbox import DropboxFS +# Whole module needs live infra/credentials — integration tier only. +pytestmark = pytest.mark.integration + class TestDropboxFS(unittest.TestCase): + @unittest.skipUnless( + os.environ.get("TEST_DROPBOX_ACCESS_TOKEN"), + "Integration test requires TEST_DROPBOX_ACCESS_TOKEN", + ) def test_access_token(self): access_token = os.environ.get("TEST_DROPBOX_ACCESS_TOKEN") settings = {"token": access_token} @@ -12,9 +20,7 @@ def test_access_token(self): # Leave empty for root file_path = "" try: - # print(dropbox_fs.get_fsspec_fs().ls(file_path)) files = dropbox_fs.get_fsspec_fs().ls(file_path) - print(files) self.assertIsNotNone(files) except Exception as e: self.fail(f"TestDropboxFS.test_access_token failed: {e}") diff --git a/unstract/core/tests/account_services/test_pandora_account.py b/unstract/core/tests/account_services/test_pandora_account.py deleted file mode 100644 index 8e74fe3bc4..0000000000 --- a/unstract/core/tests/account_services/test_pandora_account.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from unstract.core.account_services.unstract_account import UnstractAccount - - -class TestUnstractAccount(unittest.TestCase): - def test_provision_blob(self): - account = UnstractAccount("acme", "johndoe") - account.provision_s3_storage() - account.upload_sample_files() - self.assertEqual(True, True) # add assertion here - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/core/tests/test_pubsub_helper.py b/unstract/core/tests/test_pubsub_helper.py deleted file mode 100644 index ae3e102690..0000000000 --- a/unstract/core/tests/test_pubsub_helper.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -from unstract.core.pubsub_helper import LogHelper as Log - - -class PubSubHelperTestCase(unittest.TestCase): - def test_pubsub(self): - ps1 = Log.publish( - project_guid="test", - message=Log.log(stage="COMPILE", message="Compile process started"), - ) - ps2 = Log.publish( - project_guid="test", - message=Log.log(level="ERROR", stage="COMPILE", message="Compile failed"), - ) - self.assertEqual(ps1, True) - self.assertEqual(ps2, True) - - -if __name__ == "__main__": - unittest.main() diff --git a/unstract/sdk1/pyproject.toml b/unstract/sdk1/pyproject.toml index 950294ea81..b11e153946 100644 --- a/unstract/sdk1/pyproject.toml +++ b/unstract/sdk1/pyproject.toml @@ -172,7 +172,7 @@ keep-dict-typing = true [tool.pytest.ini_options] -python_files = ["tests.py", "test_*.py", "*_tests.py"] +python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"] testpaths = ["tests"] asyncio_mode = "strict" markers = [ diff --git a/workers/pyproject.toml b/workers/pyproject.toml index 6344f8798d..8ddd63cf70 100644 --- a/workers/pyproject.toml +++ b/workers/pyproject.toml @@ -146,7 +146,7 @@ ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests"] -python_files = ["test_*.py", "*_test.py"] +python_files = ["test_*.py", "*_test.py", "*_tests.py", "tests.py"] python_classes = ["Test*"] python_functions = ["test_*"] asyncio_mode = "strict" diff --git a/workers/tests/test_legacy_executor_scaffold.py b/workers/tests/test_legacy_executor_scaffold.py index 48789c218d..335497dab1 100644 --- a/workers/tests/test_legacy_executor_scaffold.py +++ b/workers/tests/test_legacy_executor_scaffold.py @@ -249,24 +249,6 @@ def test_extraction_error_has_code_and_message(self): assert err.message == "extraction failed" assert err.code == 500 - def test_no_flask_import(self): - """Verify exceptions module does NOT import Flask.""" - import importlib - import sys - - # Ensure fresh import - mod_name = "executor.executors.exceptions" - if mod_name in sys.modules: - importlib.reload(sys.modules[mod_name]) - else: - importlib.import_module(mod_name) - - # Check that no flask modules were pulled in - flask_modules = [m for m in sys.modules if m.startswith("flask")] - assert flask_modules == [], ( - f"Flask modules imported: {flask_modules}" - ) - def test_custom_data_error_signature(self): from executor.executors.exceptions import CustomDataError