From bd782708502b0f29260e812563e88a928f4b3d3e Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Tue, 5 May 2026 07:51:10 -0700 Subject: [PATCH] chore(pyamber): move tests to src/test/python to match Maven layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns Python with the Scala source-set layout already used by amber: production code in `src/main/python`, tests in `src/test/python`. The forward-looking comment in `amber/build.sbt` already anticipated this move. - `git mv` 48 `test_*.py` files preserving package structure. - Convert 7 test files from relative imports (`from .x import Y`) to absolute paths now that the test tree is no longer a sibling of the production module. - Drop the hardcoded `core.util.expression_evaluator.test_expression_evaluator` prefix in test_evaluate_object_expression — use `A.__module__` so the assertion tracks whatever the test file's import path resolves to. - Move `pyproject.toml` from `src/main/python/` to `amber/` so a single config sits at the module root (mirroring `amber/build.sbt` for Scala). - Add `[tool.pytest.ini_options]` with `pythonpath = ["src/main/python"]`, `testpaths = ["src/test/python"]`, and `--import-mode=importlib` so test modules don't need `__init__.py` mirrors and don't collide with the production package namespace. - CI: invoke ruff + pytest from `amber/` with explicit `src/{main,test}/python` paths; codecov uploads from `./amber/coverage.xml`. - `bin/fix-format.sh` collapses to a single `cd amber && ruff format src/main/python src/test/python`. Closes #4943 --- .github/labeler.yml | 8 +++++--- .github/workflows/build.yml | 6 +++--- .github/workflows/required-checks.yml | 4 ++-- .gitignore | 3 +++ amber/build.sbt | 8 ++++---- amber/{src/main/python => }/pyproject.toml | 17 ++++++++++++++++- .../control/test_debug_command_handler.py | 0 .../control/test_evaluate_expression_handler.py | 0 .../test_replay_current_tuple_handler.py | 0 .../control/test_update_executor_handler.py | 0 .../managers/test_console_message_manager.py | 0 .../architecture/managers/test_debug_manager.py | 0 .../test_embedded_control_message_manager.py | 0 .../managers/test_exception_manager.py | 0 .../managers/test_executor_manager.py | 0 .../architecture/managers/test_pause_manager.py | 0 .../architecture/managers/test_state_manager.py | 0 .../managers/test_state_processing_manager.py | 0 .../managers/test_statistics_manager.py | 0 .../managers/test_tuple_processing_manager.py | 0 .../architecture/rpc/test_async_rpc_client.py | 0 .../sendsemantics/test_partitioners.py | 0 .../python/core/models/schema/test_schema.py | 0 .../python/core/models/test_operator.py | 0 .../python/core/models/test_state.py | 0 .../python/core/models/test_table.py | 0 .../python/core/models/test_tuple.py | 0 .../core/models/type/test_large_binary.py | 0 .../python/core/proxy/test_proxy_client.py | 4 ++-- .../python/core/proxy/test_proxy_server.py | 2 +- .../core/runnables/test_console_message.py | 0 .../core/runnables/test_data_processor.py | 0 .../python/core/runnables/test_heartbeat.py | 0 .../python/core/runnables/test_main_loop.py | 0 .../core/runnables/test_network_receiver.py | 0 .../core/runnables/test_network_sender.py | 0 .../storage/iceberg/test_iceberg_document.py | 9 +++++++-- .../iceberg/test_iceberg_utils_catalog.py | 0 .../iceberg/test_iceberg_utils_large_binary.py | 0 .../util/console_message/test_replace_print.py | 0 .../core/util/customized_queue/test_inner.py | 0 .../test_linked_blocking_multi_queue.py | 0 .../test_expression_evaluator.py | 2 +- .../python/core/util/thread/test_atomic.py | 0 .../virtual_identity/test_virtual_identity.py | 0 .../storage/test_dataset_file_document.py | 0 .../storage/test_large_binary_input_stream.py | 0 .../storage/test_large_binary_manager.py | 0 .../storage/test_large_binary_output_stream.py | 0 .../udf/examples/test_count_batch_operator.py | 2 +- .../pytexera/udf/examples/test_echo_operator.py | 2 +- .../udf/examples/test_echo_table_operator.py | 2 +- .../examples/test_generator_operator_binary.py | 2 +- .../examples/test_generator_operator_integer.py | 2 +- bin/fix-format.sh | 11 ++++++----- 55 files changed, 55 insertions(+), 29 deletions(-) rename amber/{src/main/python => }/pyproject.toml (53%) rename amber/src/{main => test}/python/core/architecture/handlers/control/test_debug_command_handler.py (100%) rename amber/src/{main => test}/python/core/architecture/handlers/control/test_evaluate_expression_handler.py (100%) rename amber/src/{main => test}/python/core/architecture/handlers/control/test_replay_current_tuple_handler.py (100%) rename amber/src/{main => test}/python/core/architecture/handlers/control/test_update_executor_handler.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_console_message_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_debug_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_embedded_control_message_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_exception_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_executor_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_pause_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_state_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_state_processing_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_statistics_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/managers/test_tuple_processing_manager.py (100%) rename amber/src/{main => test}/python/core/architecture/rpc/test_async_rpc_client.py (100%) rename amber/src/{main => test}/python/core/architecture/sendsemantics/test_partitioners.py (100%) rename amber/src/{main => test}/python/core/models/schema/test_schema.py (100%) rename amber/src/{main => test}/python/core/models/test_operator.py (100%) rename amber/src/{main => test}/python/core/models/test_state.py (100%) rename amber/src/{main => test}/python/core/models/test_table.py (100%) rename amber/src/{main => test}/python/core/models/test_tuple.py (100%) rename amber/src/{main => test}/python/core/models/type/test_large_binary.py (100%) rename amber/src/{main => test}/python/core/proxy/test_proxy_client.py (98%) rename amber/src/{main => test}/python/core/proxy/test_proxy_server.py (98%) rename amber/src/{main => test}/python/core/runnables/test_console_message.py (100%) rename amber/src/{main => test}/python/core/runnables/test_data_processor.py (100%) rename amber/src/{main => test}/python/core/runnables/test_heartbeat.py (100%) rename amber/src/{main => test}/python/core/runnables/test_main_loop.py (100%) rename amber/src/{main => test}/python/core/runnables/test_network_receiver.py (100%) rename amber/src/{main => test}/python/core/runnables/test_network_sender.py (100%) rename amber/src/{main => test}/python/core/storage/iceberg/test_iceberg_document.py (96%) rename amber/src/{main => test}/python/core/storage/iceberg/test_iceberg_utils_catalog.py (100%) rename amber/src/{main => test}/python/core/storage/iceberg/test_iceberg_utils_large_binary.py (100%) rename amber/src/{main => test}/python/core/util/console_message/test_replace_print.py (100%) rename amber/src/{main => test}/python/core/util/customized_queue/test_inner.py (100%) rename amber/src/{main => test}/python/core/util/customized_queue/test_linked_blocking_multi_queue.py (100%) rename amber/src/{main => test}/python/core/util/expression_evaluator/test_expression_evaluator.py (99%) rename amber/src/{main => test}/python/core/util/thread/test_atomic.py (100%) rename amber/src/{main => test}/python/core/util/virtual_identity/test_virtual_identity.py (100%) rename amber/src/{main => test}/python/pytexera/storage/test_dataset_file_document.py (100%) rename amber/src/{main => test}/python/pytexera/storage/test_large_binary_input_stream.py (100%) rename amber/src/{main => test}/python/pytexera/storage/test_large_binary_manager.py (100%) rename amber/src/{main => test}/python/pytexera/storage/test_large_binary_output_stream.py (100%) rename amber/src/{main => test}/python/pytexera/udf/examples/test_count_batch_operator.py (98%) rename amber/src/{main => test}/python/pytexera/udf/examples/test_echo_operator.py (95%) rename amber/src/{main => test}/python/pytexera/udf/examples/test_echo_table_operator.py (95%) rename amber/src/{main => test}/python/pytexera/udf/examples/test_generator_operator_binary.py (93%) rename amber/src/{main => test}/python/pytexera/udf/examples/test_generator_operator_integer.py (93%) diff --git a/.github/labeler.yml b/.github/labeler.yml index 5a1b2db0d80..d0973255b15 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -46,9 +46,9 @@ agent-service: engine: # Non-Python, non-integration parts of amber/. Pure Python changes - # under amber/src/main/python/** intentionally fall through to the - # `python` label (which the labeler also matches via **/*.py), so - # they only trigger the python + amber-integration stacks rather + # under amber/src/{main,test}/python/** intentionally fall through to + # the `python` label (which the labeler also matches via **/*.py), + # so they only trigger the python + amber-integration stacks rather # than the full Scala-only `amber` stack. Integration specs live # under amber/src/test/integration/** (added to sbt's Test sources # via amber/build.sbt) and are caught by the `amber-integration` @@ -88,6 +88,8 @@ python: - changed-files: - any-glob-to-any-file: - 'amber/src/main/python/**' + - 'amber/src/test/python/**' + - 'amber/pyproject.toml' - '**/*.py' - '**/requirements.txt' - '**/*-requirements.txt' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06be31230e8..eede7eba6a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -505,7 +505,7 @@ jobs: PGPASSWORD: postgres - name: Lint with Ruff run: | - cd amber/src/main/python && ruff check . && ruff format --check . + cd amber && ruff check src/main/python src/test/python && ruff format --check src/main/python src/test/python - name: Install dev dependencies # Test-only deps live in amber/dev-requirements.txt and are # installed after the LICENSE-binary snapshot above so they never @@ -516,13 +516,13 @@ jobs: if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi - name: Test with pytest run: | - cd amber/src/main/python && pytest --cov=. --cov-report=xml -sv + cd amber && pytest --cov=src/main/python --cov-report=xml -sv - name: Upload python coverage to Codecov if: matrix.python-version == '3.12' && always() uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./amber/src/main/python/coverage.xml + files: ./amber/coverage.xml flags: python fail_ci_if_error: false diff --git a/.github/workflows/required-checks.yml b/.github/workflows/required-checks.yml index e7b8968ca46..54c86006381 100644 --- a/.github/workflows/required-checks.yml +++ b/.github/workflows/required-checks.yml @@ -137,8 +137,8 @@ jobs: // amber-integration runs the Scala tests tagged // @org.apache.texera.amber.tags.IntegrationTest (e2e specs that // spawn Python UDF workers). The labeler attaches `python` to - // any *.py change (including amber/src/main/python/**), so - // `engine` does not need to fire the python stack itself — + // any *.py change (including amber/src/{main,test}/python/**), + // so `engine` does not need to fire the python stack itself — // pure-Python amber changes pick up `python` directly. The // `amber-integration` label catches *IntegrationSpec.scala // edits so a test-only change does not trigger the full diff --git a/.gitignore b/.gitignore index d283fc70137..8da9edd67b8 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,9 @@ __pycache__/ *$py.class .ipynb_checkpoints .pytype/ +.coverage +coverage.xml +.pytest_cache/ # Ignoring python generated files *.model diff --git a/amber/build.sbt b/amber/build.sbt index 09e4628ab83..141f60d93d5 100644 --- a/amber/build.sbt +++ b/amber/build.sbt @@ -56,10 +56,10 @@ Compile / unmanagedSourceDirectories += baseDirectory.value / "src" / "main" / " // `amber/src/test/integration` holds Scala specs that exercise both // Scala and Python end-to-end (tagged @org.apache.texera.amber.tags.IntegrationTest). -// Sits next to `src/test/scala` and `src/test/java`; a future `src/test/python` -// can drop in the same way. Adding it to Test/unmanagedSourceDirectories means -// scalafmtCheckAll / scalafixAll --check naturally cover these sources, and -// the AMBER_TEST_FILTER env var below routes which tagged subset runs. +// Sits next to `src/test/scala`, `src/test/java`, and `src/test/python`. +// Adding it to Test/unmanagedSourceDirectories means scalafmtCheckAll / +// scalafixAll --check naturally cover these sources, and the +// AMBER_TEST_FILTER env var below routes which tagged subset runs. Test / unmanagedSourceDirectories += baseDirectory.value / "src" / "test" / "integration" // Test-filter switch driven by the AMBER_TEST_FILTER env var so the diff --git a/amber/src/main/python/pyproject.toml b/amber/pyproject.toml similarity index 53% rename from amber/src/main/python/pyproject.toml rename to amber/pyproject.toml index 72bfeb57245..46f8c55db67 100644 --- a/amber/src/main/python/pyproject.toml +++ b/amber/pyproject.toml @@ -24,4 +24,19 @@ extend-exclude = ["proto"] ignore = ["F403", "F405", "E203"] [tool.ruff.lint.mccabe] -max-complexity = 10 \ No newline at end of file +max-complexity = 10 + +# Layout follows the Maven source-set convention used by Scala +# (`src/main/scala` + `src/test/scala`): production code lives in +# `src/main/python` and tests in `src/test/python`. Pytest is invoked +# from `amber/`, so `pythonpath = ["src/main/python"]` keeps +# `from core.x import y` and `from pytexera.x import y` working. +[tool.pytest.ini_options] +pythonpath = ["src/main/python"] +testpaths = ["src/test/python"] +# `importlib` import mode loads each test module by file path without +# inserting it into a parent package, so `src/test/python` doesn't need +# to mirror `src/main/python`'s __init__.py layout to avoid duplicate +# package names. Required for src-style test layouts per the pytest +# docs (https://docs.pytest.org/en/stable/explanation/goodpractices.html). +addopts = "--import-mode=importlib" \ No newline at end of file diff --git a/amber/src/main/python/core/architecture/handlers/control/test_debug_command_handler.py b/amber/src/test/python/core/architecture/handlers/control/test_debug_command_handler.py similarity index 100% rename from amber/src/main/python/core/architecture/handlers/control/test_debug_command_handler.py rename to amber/src/test/python/core/architecture/handlers/control/test_debug_command_handler.py diff --git a/amber/src/main/python/core/architecture/handlers/control/test_evaluate_expression_handler.py b/amber/src/test/python/core/architecture/handlers/control/test_evaluate_expression_handler.py similarity index 100% rename from amber/src/main/python/core/architecture/handlers/control/test_evaluate_expression_handler.py rename to amber/src/test/python/core/architecture/handlers/control/test_evaluate_expression_handler.py diff --git a/amber/src/main/python/core/architecture/handlers/control/test_replay_current_tuple_handler.py b/amber/src/test/python/core/architecture/handlers/control/test_replay_current_tuple_handler.py similarity index 100% rename from amber/src/main/python/core/architecture/handlers/control/test_replay_current_tuple_handler.py rename to amber/src/test/python/core/architecture/handlers/control/test_replay_current_tuple_handler.py diff --git a/amber/src/main/python/core/architecture/handlers/control/test_update_executor_handler.py b/amber/src/test/python/core/architecture/handlers/control/test_update_executor_handler.py similarity index 100% rename from amber/src/main/python/core/architecture/handlers/control/test_update_executor_handler.py rename to amber/src/test/python/core/architecture/handlers/control/test_update_executor_handler.py diff --git a/amber/src/main/python/core/architecture/managers/test_console_message_manager.py b/amber/src/test/python/core/architecture/managers/test_console_message_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_console_message_manager.py rename to amber/src/test/python/core/architecture/managers/test_console_message_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_debug_manager.py b/amber/src/test/python/core/architecture/managers/test_debug_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_debug_manager.py rename to amber/src/test/python/core/architecture/managers/test_debug_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_embedded_control_message_manager.py b/amber/src/test/python/core/architecture/managers/test_embedded_control_message_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_embedded_control_message_manager.py rename to amber/src/test/python/core/architecture/managers/test_embedded_control_message_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_exception_manager.py b/amber/src/test/python/core/architecture/managers/test_exception_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_exception_manager.py rename to amber/src/test/python/core/architecture/managers/test_exception_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_executor_manager.py b/amber/src/test/python/core/architecture/managers/test_executor_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_executor_manager.py rename to amber/src/test/python/core/architecture/managers/test_executor_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_pause_manager.py b/amber/src/test/python/core/architecture/managers/test_pause_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_pause_manager.py rename to amber/src/test/python/core/architecture/managers/test_pause_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_state_manager.py b/amber/src/test/python/core/architecture/managers/test_state_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_state_manager.py rename to amber/src/test/python/core/architecture/managers/test_state_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_state_processing_manager.py b/amber/src/test/python/core/architecture/managers/test_state_processing_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_state_processing_manager.py rename to amber/src/test/python/core/architecture/managers/test_state_processing_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_statistics_manager.py b/amber/src/test/python/core/architecture/managers/test_statistics_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_statistics_manager.py rename to amber/src/test/python/core/architecture/managers/test_statistics_manager.py diff --git a/amber/src/main/python/core/architecture/managers/test_tuple_processing_manager.py b/amber/src/test/python/core/architecture/managers/test_tuple_processing_manager.py similarity index 100% rename from amber/src/main/python/core/architecture/managers/test_tuple_processing_manager.py rename to amber/src/test/python/core/architecture/managers/test_tuple_processing_manager.py diff --git a/amber/src/main/python/core/architecture/rpc/test_async_rpc_client.py b/amber/src/test/python/core/architecture/rpc/test_async_rpc_client.py similarity index 100% rename from amber/src/main/python/core/architecture/rpc/test_async_rpc_client.py rename to amber/src/test/python/core/architecture/rpc/test_async_rpc_client.py diff --git a/amber/src/main/python/core/architecture/sendsemantics/test_partitioners.py b/amber/src/test/python/core/architecture/sendsemantics/test_partitioners.py similarity index 100% rename from amber/src/main/python/core/architecture/sendsemantics/test_partitioners.py rename to amber/src/test/python/core/architecture/sendsemantics/test_partitioners.py diff --git a/amber/src/main/python/core/models/schema/test_schema.py b/amber/src/test/python/core/models/schema/test_schema.py similarity index 100% rename from amber/src/main/python/core/models/schema/test_schema.py rename to amber/src/test/python/core/models/schema/test_schema.py diff --git a/amber/src/main/python/core/models/test_operator.py b/amber/src/test/python/core/models/test_operator.py similarity index 100% rename from amber/src/main/python/core/models/test_operator.py rename to amber/src/test/python/core/models/test_operator.py diff --git a/amber/src/main/python/core/models/test_state.py b/amber/src/test/python/core/models/test_state.py similarity index 100% rename from amber/src/main/python/core/models/test_state.py rename to amber/src/test/python/core/models/test_state.py diff --git a/amber/src/main/python/core/models/test_table.py b/amber/src/test/python/core/models/test_table.py similarity index 100% rename from amber/src/main/python/core/models/test_table.py rename to amber/src/test/python/core/models/test_table.py diff --git a/amber/src/main/python/core/models/test_tuple.py b/amber/src/test/python/core/models/test_tuple.py similarity index 100% rename from amber/src/main/python/core/models/test_tuple.py rename to amber/src/test/python/core/models/test_tuple.py diff --git a/amber/src/main/python/core/models/type/test_large_binary.py b/amber/src/test/python/core/models/type/test_large_binary.py similarity index 100% rename from amber/src/main/python/core/models/type/test_large_binary.py rename to amber/src/test/python/core/models/type/test_large_binary.py diff --git a/amber/src/main/python/core/proxy/test_proxy_client.py b/amber/src/test/python/core/proxy/test_proxy_client.py similarity index 98% rename from amber/src/main/python/core/proxy/test_proxy_client.py rename to amber/src/test/python/core/proxy/test_proxy_client.py index b28b2cfe999..891c2fda751 100644 --- a/amber/src/main/python/core/proxy/test_proxy_client.py +++ b/amber/src/test/python/core/proxy/test_proxy_client.py @@ -20,8 +20,8 @@ from pyarrow import ArrowNotImplementedError, Table from queue import Queue -from .proxy_client import ProxyClient -from .proxy_server import ProxyServer +from core.proxy.proxy_client import ProxyClient +from core.proxy.proxy_server import ProxyServer class TestProxyClient: diff --git a/amber/src/main/python/core/proxy/test_proxy_server.py b/amber/src/test/python/core/proxy/test_proxy_server.py similarity index 98% rename from amber/src/main/python/core/proxy/test_proxy_server.py rename to amber/src/test/python/core/proxy/test_proxy_server.py index a4a422e16f8..22aec0cc69a 100644 --- a/amber/src/main/python/core/proxy/test_proxy_server.py +++ b/amber/src/test/python/core/proxy/test_proxy_server.py @@ -18,7 +18,7 @@ import pytest from pyarrow.flight import Action -from .proxy_server import ProxyServer +from core.proxy.proxy_server import ProxyServer class TestProxyServer: diff --git a/amber/src/main/python/core/runnables/test_console_message.py b/amber/src/test/python/core/runnables/test_console_message.py similarity index 100% rename from amber/src/main/python/core/runnables/test_console_message.py rename to amber/src/test/python/core/runnables/test_console_message.py diff --git a/amber/src/main/python/core/runnables/test_data_processor.py b/amber/src/test/python/core/runnables/test_data_processor.py similarity index 100% rename from amber/src/main/python/core/runnables/test_data_processor.py rename to amber/src/test/python/core/runnables/test_data_processor.py diff --git a/amber/src/main/python/core/runnables/test_heartbeat.py b/amber/src/test/python/core/runnables/test_heartbeat.py similarity index 100% rename from amber/src/main/python/core/runnables/test_heartbeat.py rename to amber/src/test/python/core/runnables/test_heartbeat.py diff --git a/amber/src/main/python/core/runnables/test_main_loop.py b/amber/src/test/python/core/runnables/test_main_loop.py similarity index 100% rename from amber/src/main/python/core/runnables/test_main_loop.py rename to amber/src/test/python/core/runnables/test_main_loop.py diff --git a/amber/src/main/python/core/runnables/test_network_receiver.py b/amber/src/test/python/core/runnables/test_network_receiver.py similarity index 100% rename from amber/src/main/python/core/runnables/test_network_receiver.py rename to amber/src/test/python/core/runnables/test_network_receiver.py diff --git a/amber/src/main/python/core/runnables/test_network_sender.py b/amber/src/test/python/core/runnables/test_network_sender.py similarity index 100% rename from amber/src/main/python/core/runnables/test_network_sender.py rename to amber/src/test/python/core/runnables/test_network_sender.py diff --git a/amber/src/main/python/core/storage/iceberg/test_iceberg_document.py b/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py similarity index 96% rename from amber/src/main/python/core/storage/iceberg/test_iceberg_document.py rename to amber/src/test/python/core/storage/iceberg/test_iceberg_document.py index 9b374f7d5c7..a218c64a2d8 100644 --- a/amber/src/main/python/core/storage/iceberg/test_iceberg_document.py +++ b/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py @@ -18,6 +18,7 @@ import datetime import pytest import random +import tempfile import uuid from concurrent.futures import as_completed from concurrent.futures.thread import ThreadPoolExecutor @@ -35,7 +36,11 @@ PhysicalOpIdentity, ) -# Hardcoded storage config only for test purposes. +# Hardcoded storage config only for test purposes. The iceberg warehouse +# directory must be a writable absolute path; using `tempfile.mkdtemp()` +# avoids depending on pytest's cwd (an earlier `"../../../../../../amber/ +# user-resources/..."` value silently relied on CWD = amber/src/main/python +# and broke when the cwd moved up to amber/). StorageConfig.initialize( catalog_type="postgres", postgres_uri_without_scheme="localhost:5432/texera_iceberg_catalog", @@ -44,7 +49,7 @@ rest_catalog_uri="http://localhost:8181/catalog/", rest_catalog_warehouse_name="texera", table_result_namespace="operator-port-result", - directory_path="../../../../../../amber/user-resources/workflow-results", + directory_path=tempfile.mkdtemp(prefix="texera-iceberg-warehouse-"), commit_batch_size=4096, s3_endpoint="http://localhost:9000", s3_region="us-east-1", diff --git a/amber/src/main/python/core/storage/iceberg/test_iceberg_utils_catalog.py b/amber/src/test/python/core/storage/iceberg/test_iceberg_utils_catalog.py similarity index 100% rename from amber/src/main/python/core/storage/iceberg/test_iceberg_utils_catalog.py rename to amber/src/test/python/core/storage/iceberg/test_iceberg_utils_catalog.py diff --git a/amber/src/main/python/core/storage/iceberg/test_iceberg_utils_large_binary.py b/amber/src/test/python/core/storage/iceberg/test_iceberg_utils_large_binary.py similarity index 100% rename from amber/src/main/python/core/storage/iceberg/test_iceberg_utils_large_binary.py rename to amber/src/test/python/core/storage/iceberg/test_iceberg_utils_large_binary.py diff --git a/amber/src/main/python/core/util/console_message/test_replace_print.py b/amber/src/test/python/core/util/console_message/test_replace_print.py similarity index 100% rename from amber/src/main/python/core/util/console_message/test_replace_print.py rename to amber/src/test/python/core/util/console_message/test_replace_print.py diff --git a/amber/src/main/python/core/util/customized_queue/test_inner.py b/amber/src/test/python/core/util/customized_queue/test_inner.py similarity index 100% rename from amber/src/main/python/core/util/customized_queue/test_inner.py rename to amber/src/test/python/core/util/customized_queue/test_inner.py diff --git a/amber/src/main/python/core/util/customized_queue/test_linked_blocking_multi_queue.py b/amber/src/test/python/core/util/customized_queue/test_linked_blocking_multi_queue.py similarity index 100% rename from amber/src/main/python/core/util/customized_queue/test_linked_blocking_multi_queue.py rename to amber/src/test/python/core/util/customized_queue/test_linked_blocking_multi_queue.py diff --git a/amber/src/main/python/core/util/expression_evaluator/test_expression_evaluator.py b/amber/src/test/python/core/util/expression_evaluator/test_expression_evaluator.py similarity index 99% rename from amber/src/main/python/core/util/expression_evaluator/test_expression_evaluator.py rename to amber/src/test/python/core/util/expression_evaluator/test_expression_evaluator.py index 19340bdccda..729569f3277 100644 --- a/amber/src/main/python/core/util/expression_evaluator/test_expression_evaluator.py +++ b/amber/src/test/python/core/util/expression_evaluator/test_expression_evaluator.py @@ -201,7 +201,7 @@ def __init__(self): expression="a", value_ref="a", value_str=( - ".A" f" object at {hex(id(a))}>" ), diff --git a/amber/src/main/python/core/util/thread/test_atomic.py b/amber/src/test/python/core/util/thread/test_atomic.py similarity index 100% rename from amber/src/main/python/core/util/thread/test_atomic.py rename to amber/src/test/python/core/util/thread/test_atomic.py diff --git a/amber/src/main/python/core/util/virtual_identity/test_virtual_identity.py b/amber/src/test/python/core/util/virtual_identity/test_virtual_identity.py similarity index 100% rename from amber/src/main/python/core/util/virtual_identity/test_virtual_identity.py rename to amber/src/test/python/core/util/virtual_identity/test_virtual_identity.py diff --git a/amber/src/main/python/pytexera/storage/test_dataset_file_document.py b/amber/src/test/python/pytexera/storage/test_dataset_file_document.py similarity index 100% rename from amber/src/main/python/pytexera/storage/test_dataset_file_document.py rename to amber/src/test/python/pytexera/storage/test_dataset_file_document.py diff --git a/amber/src/main/python/pytexera/storage/test_large_binary_input_stream.py b/amber/src/test/python/pytexera/storage/test_large_binary_input_stream.py similarity index 100% rename from amber/src/main/python/pytexera/storage/test_large_binary_input_stream.py rename to amber/src/test/python/pytexera/storage/test_large_binary_input_stream.py diff --git a/amber/src/main/python/pytexera/storage/test_large_binary_manager.py b/amber/src/test/python/pytexera/storage/test_large_binary_manager.py similarity index 100% rename from amber/src/main/python/pytexera/storage/test_large_binary_manager.py rename to amber/src/test/python/pytexera/storage/test_large_binary_manager.py diff --git a/amber/src/main/python/pytexera/storage/test_large_binary_output_stream.py b/amber/src/test/python/pytexera/storage/test_large_binary_output_stream.py similarity index 100% rename from amber/src/main/python/pytexera/storage/test_large_binary_output_stream.py rename to amber/src/test/python/pytexera/storage/test_large_binary_output_stream.py diff --git a/amber/src/main/python/pytexera/udf/examples/test_count_batch_operator.py b/amber/src/test/python/pytexera/udf/examples/test_count_batch_operator.py similarity index 98% rename from amber/src/main/python/pytexera/udf/examples/test_count_batch_operator.py rename to amber/src/test/python/pytexera/udf/examples/test_count_batch_operator.py index 9ab084a4050..4721861682b 100644 --- a/amber/src/main/python/pytexera/udf/examples/test_count_batch_operator.py +++ b/amber/src/test/python/pytexera/udf/examples/test_count_batch_operator.py @@ -20,7 +20,7 @@ from collections import deque from pytexera import * -from .count_batch_operator import CountBatchOperator +from pytexera.udf.examples.count_batch_operator import CountBatchOperator class TestCountBatchOperator: diff --git a/amber/src/main/python/pytexera/udf/examples/test_echo_operator.py b/amber/src/test/python/pytexera/udf/examples/test_echo_operator.py similarity index 95% rename from amber/src/main/python/pytexera/udf/examples/test_echo_operator.py rename to amber/src/test/python/pytexera/udf/examples/test_echo_operator.py index a70b8a0e7c1..77412725948 100644 --- a/amber/src/main/python/pytexera/udf/examples/test_echo_operator.py +++ b/amber/src/test/python/pytexera/udf/examples/test_echo_operator.py @@ -18,7 +18,7 @@ import pytest from pytexera import Tuple -from .echo_operator import EchoOperator +from pytexera.udf.examples.echo_operator import EchoOperator class TestEchoOperator: diff --git a/amber/src/main/python/pytexera/udf/examples/test_echo_table_operator.py b/amber/src/test/python/pytexera/udf/examples/test_echo_table_operator.py similarity index 95% rename from amber/src/main/python/pytexera/udf/examples/test_echo_table_operator.py rename to amber/src/test/python/pytexera/udf/examples/test_echo_table_operator.py index d619f53a66b..105308c2781 100644 --- a/amber/src/main/python/pytexera/udf/examples/test_echo_table_operator.py +++ b/amber/src/test/python/pytexera/udf/examples/test_echo_table_operator.py @@ -20,7 +20,7 @@ from core.models.table import all_output_to_tuple from pytexera import Tuple -from .echo_table_operator import EchoTableOperator +from pytexera.udf.examples.echo_table_operator import EchoTableOperator class TestEchoTableOperator: diff --git a/amber/src/main/python/pytexera/udf/examples/test_generator_operator_binary.py b/amber/src/test/python/pytexera/udf/examples/test_generator_operator_binary.py similarity index 93% rename from amber/src/main/python/pytexera/udf/examples/test_generator_operator_binary.py rename to amber/src/test/python/pytexera/udf/examples/test_generator_operator_binary.py index 4c7e5d8b407..2d19575d2b7 100644 --- a/amber/src/main/python/pytexera/udf/examples/test_generator_operator_binary.py +++ b/amber/src/test/python/pytexera/udf/examples/test_generator_operator_binary.py @@ -18,7 +18,7 @@ import pytest from pytexera import Tuple -from .generator_operator_binary import GeneratorOperatorBinary +from pytexera.udf.examples.generator_operator_binary import GeneratorOperatorBinary class TestEchoOperator: diff --git a/amber/src/main/python/pytexera/udf/examples/test_generator_operator_integer.py b/amber/src/test/python/pytexera/udf/examples/test_generator_operator_integer.py similarity index 93% rename from amber/src/main/python/pytexera/udf/examples/test_generator_operator_integer.py rename to amber/src/test/python/pytexera/udf/examples/test_generator_operator_integer.py index 3ab19064623..dc50ba0329c 100644 --- a/amber/src/main/python/pytexera/udf/examples/test_generator_operator_integer.py +++ b/amber/src/test/python/pytexera/udf/examples/test_generator_operator_integer.py @@ -18,7 +18,7 @@ import pytest from pytexera import Tuple -from .generator_operator_integer import GeneratorOperatorInteger +from pytexera.udf.examples.generator_operator_integer import GeneratorOperatorInteger class TestEchoOperator: diff --git a/bin/fix-format.sh b/bin/fix-format.sh index 23a53ac8da5..e2cb90a1b6c 100755 --- a/bin/fix-format.sh +++ b/bin/fix-format.sh @@ -42,10 +42,11 @@ fi # --- Key directories --- FRONTEND_DIR="$TEXERA_HOME/frontend" -AMBER_PY_DIR="$TEXERA_HOME/amber/src/main/python" +AMBER_DIR="$TEXERA_HOME/amber" [[ -d "$FRONTEND_DIR" ]] || { tx_error "Frontend directory not found: $FRONTEND_DIR"; exit 1; } -[[ -d "$AMBER_PY_DIR" ]] || { tx_error "Amber Python directory not found: $AMBER_PY_DIR"; exit 1; } +[[ -d "$AMBER_DIR/src/main/python" ]] || { tx_error "Amber Python directory not found: $AMBER_DIR/src/main/python"; exit 1; } +[[ -d "$AMBER_DIR/src/test/python" ]] || { tx_error "Amber Python test directory not found: $AMBER_DIR/src/test/python"; exit 1; } # --- Argument parsing --- TARGET="${1:-all}" @@ -95,14 +96,14 @@ fi # --- 3) Python formatting --- if $run_python; then - tx_info "Running ruff in amber/src/main/python..." + tx_info "Running ruff in amber/src/{main,test}/python..." if ! command -v ruff >/dev/null 2>&1; then tx_error "ruff not found. Install with: pip install ruff" exit 1 fi ( - cd "$AMBER_PY_DIR" - ruff format . + cd "$AMBER_DIR" + ruff format src/main/python src/test/python ) tx_success "Python formatting completed." fi