From f5193aac70d513760128e8494b70b8b46dca7b56 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 1 Jul 2026 10:15:00 +0200 Subject: [PATCH 01/14] test(dbsync_config): scaffolding for config-coverage expansion Prepare to extend db-sync config-option coverage: - Add Column members used by upcoming subtests (Redeemer.FEE, Tx.DEPOSIT, TxOut.CONSUMED_BY_TX_ID, StakeRegistration.DEPOSIT, PoolUpdate.DEPOSIT, GovActionProposal.X_EPOCH/EXPIRATION). - Add ColumnCondition.NOT_ZERO and IS_NOT_NULL for positive column checks. - Extract a reusable wait_for_tables_not_empty() retry helper and use it in the governance subtest (behavior preserving). --- .../tests/test_dbsync_config.py | 38 +++++++++++++------ .../utils/dbsync_service_manager.py | 15 ++++++++ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index c0f8c55b9..be4ef6dbd 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -42,7 +42,9 @@ class ColumnCondition(enum.StrEnum): """Enum for column-level db-sync condition checks.""" ZERO = "column_condition:=0" + NOT_ZERO = "column_condition:!= 0" IS_NULL = "column_condition:IS NULL" + IS_NOT_NULL = "column_condition:IS NOT NULL" GOVERNANCE_TABLES = ( @@ -125,6 +127,29 @@ def check_dbsync_state( raise ValueError(error_msg) +def wait_for_tables_not_empty( + tables: tp.Iterable[str | db_sync.Table], + *, + timeout: int = 600, +) -> None: + """Wait until all given db-sync tables have data. + + Off-chain data (pool/vote metadata) is fetched asynchronously and can appear up to + several minutes after db-sync starts (the fetch loop sleeps ~300s between passes), so + such tables must be polled rather than checked once. Raises ``TimeoutError`` (via + ``retry_query``) if any table is still empty after ``timeout`` seconds. + """ + + def _query_func() -> bool: + empty_tables = [table for table in tables if dbsync_utils.table_empty(table=table)] + if empty_tables: + msg = f"Following tables are still empty: {empty_tables}" + raise dbsync_utils.DbSyncNoResponseError(msg) + return True + + dbsync_utils.retry_query(query_func=_query_func, timeout=timeout) + + @pytest.fixture def db_sync_manager( cluster_singleton: clusterlib.ClusterLib, # noqa: ARG001 @@ -211,18 +236,7 @@ def governance( ) # Off-chain data is inserted into the DB a few minutes after the restart of db-sync - def _query_func(): - empty_tables = [ - table for table in GOVERNANCE_TABLES if dbsync_utils.table_empty(table=table) - ] - - if empty_tables: - msg = f"Following tables are still empty: {empty_tables}" - raise dbsync_utils.DbSyncNoResponseError(msg) - - return True - - dbsync_utils.retry_query(query_func=_query_func, timeout=600) + wait_for_tables_not_empty(GOVERNANCE_TABLES, timeout=600) check_dbsync_state( expected_state={t: TableCondition.NOT_EMPTY for t in GOVERNANCE_TABLES} # noqa: C420 diff --git a/cardano_node_tests/utils/dbsync_service_manager.py b/cardano_node_tests/utils/dbsync_service_manager.py index 29f673d01..27e3e11ef 100644 --- a/cardano_node_tests/utils/dbsync_service_manager.py +++ b/cardano_node_tests/utils/dbsync_service_manager.py @@ -114,9 +114,24 @@ class View(enum.StrEnum): class Column: class Tx(enum.StrEnum): FEE = "tx.fee" + DEPOSIT = "tx.deposit" class Redeemer(enum.StrEnum): SCRIPT_HASH = "redeemer.script_hash" + FEE = "redeemer.fee" + + class TxOut(enum.StrEnum): + CONSUMED_BY_TX_ID = "tx_out.consumed_by_tx_id" + + class StakeRegistration(enum.StrEnum): + DEPOSIT = "stake_registration.deposit" + + class PoolUpdate(enum.StrEnum): + DEPOSIT = "pool_update.deposit" + + class GovActionProposal(enum.StrEnum): + X_EPOCH = "gov_action_proposal.x_epoch" + EXPIRATION = "gov_action_proposal.expiration" class SettingState(enum.StrEnum): From d78d238a63fa1dcfdbc22c56d5fdbedef899d97b Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Thu, 2 Jul 2026 14:30:00 +0200 Subject: [PATCH 02/14] test(dbsync_config): cover plutus/metadata disable and tx_out modes Add Phase 1 config-option subtests that need no extra on-chain activity: - plutus disable: redeemer / redeemer_data / datum stay empty. - metadata disable: tx_metadata stays empty. - tx_out consumed: tx_in empty, consumed_by_tx_id column added; with force_tx_in=True tx_in is populated again. - tx_out use_address_table: the address table exists and is populated. Group the new scenarios under a per-phase `_subtests_phase1()` generator so `get_subtests` stays within complexity limits as coverage grows. Add `column_exists` / `column_data_type` helpers (and `query_column_data_type`). --- .../tests/test_dbsync_config.py | 122 ++++++++++++++++++ cardano_node_tests/utils/dbsync_queries.py | 12 ++ cardano_node_tests/utils/dbsync_utils.py | 10 ++ 3 files changed, 144 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index be4ef6dbd..9d25ba393 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -303,6 +303,128 @@ def multi_asset_disable( yield multi_asset_disable + yield from self._subtests_phase1() + + def _subtests_phase1(self) -> tp.Generator[tp.Callable]: + """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" + + def plutus_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `plutus` option. + + With Plutus disabled, db-sync must not insert script-execution data, so the + redeemer / redeemer_data / datum tables stay empty even though the chain + contains a Plutus transaction. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=False)) + check_dbsync_state( + expected_state={ + db_sync.Table.REDEEMER: TableCondition.EMPTY, + db_sync.Table.REDEEMER_DATA: TableCondition.EMPTY, + db_sync.Table.DATUM: TableCondition.EMPTY, + } + ) + + yield plutus_disable + + def metadata_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `metadata` option. + + With metadata disabled, the tx_metadata table stays empty even though the + chain contains a transaction carrying metadata. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=False)) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.EMPTY}) + + yield metadata_disable + + def tx_out_consumed( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out` in `consumed` mode (without `force_tx_in`). + + In `consumed` mode db-sync records consumption via the new + `tx_out.consumed_by_tx_id` column instead of populating `tx_in`, so `tx_in` + stays empty while `tx_out` / `ma_tx_out` are populated. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.CONSUMED, force_tx_in=False + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_IN: TableCondition.EMPTY, + } + ) + assert dbsync_utils.column_exists( + table=db_sync.Table.TX_OUT, column="consumed_by_tx_id" + ), "`consumed` mode should add the `tx_out.consumed_by_tx_id` column" + + yield tx_out_consumed + + def tx_out_consumed_force_tx_in( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out` in `consumed` mode with `force_tx_in=True`. + + `force_tx_in` re-enables population of the `tx_in` table on top of `consumed` + mode, so both `tx_out` and `tx_in` are populated. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.CONSUMED, force_tx_in=True + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_IN: TableCondition.NOT_EMPTY, + } + ) + + yield tx_out_consumed_force_tx_in + + def tx_out_use_address_table( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out` with `use_address_table=True`. + + With the address table enabled, db-sync normalizes addresses into a separate + `address` table (which otherwise does not exist). + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.ENABLE, use_address_table=True + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.ADDRESS: TableCondition.EXISTS, + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + } + ) + assert not dbsync_utils.table_empty(table=db_sync.Table.ADDRESS), ( + "`use_address_table` should populate the `address` table" + ) + + yield tx_out_use_address_table + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, diff --git a/cardano_node_tests/utils/dbsync_queries.py b/cardano_node_tests/utils/dbsync_queries.py index 4e8cac15a..2879714d3 100644 --- a/cardano_node_tests/utils/dbsync_queries.py +++ b/cardano_node_tests/utils/dbsync_queries.py @@ -1211,6 +1211,18 @@ def query_view_names() -> list[str]: return view_names +def query_column_data_type(*, table: str, column: str) -> str | None: + """Query the SQL data type of a column, or `None` if the column does not exist.""" + query = ( + "SELECT data_type FROM information_schema.columns " + "WHERE table_name = %s AND column_name = %s;" + ) + + with execute(query=query, vars=(table, column)) as cur: + result = cur.fetchone() + return result[0] if result is not None else None + + def query_datum(*, datum_hash: str) -> tp.Generator[DatumDBRow]: """Query datum record in db-sync.""" query = "SELECT id, hash, tx_id, value, bytes FROM datum WHERE hash = %s;" diff --git a/cardano_node_tests/utils/dbsync_utils.py b/cardano_node_tests/utils/dbsync_utils.py index 283311ecf..18906b97d 100644 --- a/cardano_node_tests/utils/dbsync_utils.py +++ b/cardano_node_tests/utils/dbsync_utils.py @@ -1666,6 +1666,16 @@ def table_exists(*, table: str) -> bool: return table in table_names +def column_exists(*, table: str, column: str) -> bool: + """Check if a column exists in a database table.""" + return dbsync_queries.query_column_data_type(table=table, column=column) is not None + + +def column_data_type(*, table: str, column: str) -> str | None: + """Return the SQL data type of a column, or `None` if it does not exist.""" + return dbsync_queries.query_column_data_type(table=table, column=column) + + def check_epoch_state(*, epoch_no: int, txid: str, action_type: ActionTypes) -> None: """Check governance stats per epoch in dbsync.""" if not configuration.HAS_DBSYNC: From 736b72afc467e29505b57a464c56fe4cc136bba9 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Sat, 4 Jul 2026 09:45:00 +0200 Subject: [PATCH 03/14] test(dbsync_config): cover plutus/metadata/shelley enable side Add Phase 2 enable-side subtests (depend on on-chain activity): - plutus enable: script / redeemer / redeemer_data / datum populated. - metadata enable: tx_metadata populated; keys filter keeps only the configured metadata key. - shelley enable: stake_registration / pool_update populated. - shelley disable independence: epoch_stake stays populated (ledger-controlled). Document a db-sync discrepancy found via TDD: legacy ShelleyRegCert stake registrations are inserted regardless of shelley=disable (Certificate.hs insertDelegCert is unguarded, unlike the Conway path). The documented expectation (stake_registration empty) is kept and marked xfail until db-sync gates the legacy path. --- .../tests/test_dbsync_config.py | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 9d25ba393..c2e71a7bb 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -304,6 +304,7 @@ def multi_asset_disable( yield multi_asset_disable yield from self._subtests_phase1() + yield from self._subtests_phase2() def _subtests_phase1(self) -> tp.Generator[tp.Callable]: """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" @@ -425,6 +426,148 @@ def tx_out_use_address_table( yield tx_out_use_address_table + def _subtests_phase2(self) -> tp.Generator[tp.Callable]: + """Phase 2 subtests: enable-side population (depends on prior on-chain activity).""" + + def plutus_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `plutus` option. + + With Plutus enabled, script-execution data is inserted: the script, redeemer, + redeemer_data and datum tables are populated (the chain must contain a Plutus + transaction that locks/spends with a datum). + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=True)) + check_dbsync_state( + expected_state={ + db_sync.Table.SCRIPT: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER_DATA: TableCondition.NOT_EMPTY, + db_sync.Table.DATUM: TableCondition.NOT_EMPTY, + } + ) + + yield plutus_enable + + def metadata_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `metadata` option. + + With metadata enabled, the tx_metadata table is populated from transactions + carrying metadata. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=True)) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY}) + + yield metadata_enable + + def metadata_keys_filter( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `metadata.keys` filter. + + When `metadata.keys` lists specific metadata keys, db-sync stores only metadata + with those keys; every tx_metadata row must therefore have the configured key. + """ + keep_key = 2 + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_metadata(enable=True, keys=[keep_key]) + ) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY}) + # Every stored metadata row must have the single kept key. + dbsync_utils.check_column_condition( + table=db_sync.Table.TX_METADATA, column="key", condition=f"= {keep_key}" + ) + + yield metadata_keys_filter + + def shelley_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `shelley` option. + + Shelley-era data (certificates, etc.) is inserted: the stake_registration and + pool_update tables are populated. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_shelley(enable=True)) + check_dbsync_state( + expected_state={ + db_sync.Table.STAKE_REGISTRATION: TableCondition.NOT_EMPTY, + db_sync.Table.POOL_UPDATE: TableCondition.NOT_EMPTY, + } + ) + + yield shelley_enable + + def shelley_disable_independence( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `shelley` option and its independence from `ledger`. + + With Shelley disabled, ledger-derived data (epoch_stake) is still populated + because it is controlled by the `ledger` option, not `shelley`. + + Note: certificate tables are deliberately NOT asserted empty here. Genesis pool + registrations are inserted via an ungated path (Shelley/Genesis.hs), so + `pool_update` stays populated regardless of the `shelley` flag, and legacy + stake-registration certs are likewise ungated (covered separately by + ``shelley_disable_stake_registration``). `epoch_stake` is therefore the clean + signal for shelley/ledger independence. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_shelley(enable=False)) + check_dbsync_state( + expected_state={ + db_sync.Table.EPOCH_STAKE: TableCondition.NOT_EMPTY, + } + ) + + yield shelley_disable_independence + + def shelley_disable_stake_registration( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test that `shelley=disable` suppresses stake_registration (per the docs). + + doc/configuration.md states `shelley.enable` disables "all certificates", which + includes stake registrations. The assertion below expresses that documented + behavior. + + KNOWN db-sync discrepancy: legacy ``ShelleyRegCert`` stake registrations are + inserted regardless of the flag, because + ``Cardano.DbSync.Era.Universal.Insert.Certificate.insertDelegCert`` calls + ``insertStakeRegistration`` without a ``when (ioShelley iopts)`` guard, unlike + the Conway ``insertConwayDelegCert`` path which is guarded. The assertion is kept + as the documented expectation and marked xfail until db-sync gates the legacy + path (pending cardano-db-sync issue; convert to ``issues.dbsync_.finish_test`` + once filed). + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config(custom_config=db_config.with_shelley(enable=False)) + if not dbsync_utils.table_empty(table=db_sync.Table.STAKE_REGISTRATION): + pytest.xfail( + "db-sync inserts legacy ShelleyRegCert stake registrations regardless of " + "shelley=disable (Certificate.insertDelegCert is unguarded); docs say " + "shelley disables all certificates." + ) + check_dbsync_state( + expected_state={db_sync.Table.STAKE_REGISTRATION: TableCondition.EMPTY} + ) + + yield shelley_disable_stake_registration + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, From 709da6c33580cc2b846a0d1c1ae694b0854512d4 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Mon, 6 Jul 2026 16:20:00 +0200 Subject: [PATCH 04/14] test(dbsync_config): cover ledger modes and pool_stat Add Phase 3 subtests for ledger-derived data: - ledger enable: reward / epoch_stake / ada_pots / epoch_param populated, redeemer.fee computed, and a positive (ledger-derived) tx deposit recorded. - ledger disable: those tables empty, redeemer.fee null, and no positive tx deposit remains. tx.deposit is not asserted uniformly NULL because db-sync still records 0 for some txs even without ledger state; the meaningful effect is that ledger-derived (positive) deposits are dropped. - ledger ignore: ledger-derived tables empty (state maintained but unused). - pool_stat enable/disable: pool_stat populated only when enabled. --- .../tests/test_dbsync_config.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index c2e71a7bb..69b34345e 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -13,6 +13,7 @@ from cardano_node_tests.tests import common from cardano_node_tests.utils import cluster_nodes from cardano_node_tests.utils import configuration +from cardano_node_tests.utils import dbsync_queries from cardano_node_tests.utils import dbsync_service_manager as db_sync from cardano_node_tests.utils import dbsync_utils from cardano_node_tests.utils import helpers @@ -305,6 +306,7 @@ def multi_asset_disable( yield from self._subtests_phase1() yield from self._subtests_phase2() + yield from self._subtests_phase3() def _subtests_phase1(self) -> tp.Generator[tp.Callable]: """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" @@ -568,6 +570,122 @@ def shelley_disable_stake_registration( yield shelley_disable_stake_registration + def _subtests_phase3(self) -> tp.Generator[tp.Callable]: + """Phase 3 subtests: `ledger` modes and `pool_stat` (ledger-derived data).""" + # Tables populated only from ledger state; empty unless `ledger` maintains and uses it. + ledger_derived_tables = ( + db_sync.Table.REWARD, + db_sync.Table.EPOCH_STAKE, + db_sync.Table.ADA_POTS, + db_sync.Table.EPOCH_PARAM, + ) + + def ledger_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `ledger` option. + + With ledger enabled, db-sync maintains ledger state and populates the + ledger-derived tables (reward, epoch_stake, ada_pots, epoch_param), computes + redeemer fees, and records ledger-derived deposits. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_ledger(value=db_sync.LedgerMode.ENABLE) + ) + check_dbsync_state( + expected_state={ + **dict.fromkeys(ledger_derived_tables, TableCondition.NOT_EMPTY), + db_sync.Column.Redeemer.FEE: ColumnCondition.IS_NOT_NULL, + } + ) + # Ledger state lets db-sync compute deposits: at least one tx carries a positive + # deposit (e.g. a registration deposit). Contrast with the ledger=disable case. + assert ( + dbsync_queries.query_rows_count(table="tx", column="deposit", condition="> 0") > 0 + ), "ledger=enable should record ledger-derived (positive) tx deposits" + + yield ledger_enable + + def ledger_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `ledger` option. + + With ledger disabled, db-sync does not maintain ledger state: the ledger-derived + tables stay empty and ledger-derived columns are null (redeemer.fee, tx.deposit). + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_ledger(value=db_sync.LedgerMode.DISABLE) + ) + check_dbsync_state( + expected_state={ + **dict.fromkeys(ledger_derived_tables, TableCondition.EMPTY), + db_sync.Column.Redeemer.FEE: ColumnCondition.IS_NULL, + } + ) + # Ledger-derived deposits are dropped without ledger state: no tx has a positive + # deposit. (Note: tx.deposit is not uniformly NULL - db-sync still records 0 for + # some txs - so we assert the meaningful effect: no positive deposit remains.) + assert ( + dbsync_queries.query_rows_count(table="tx", column="deposit", condition="> 0") == 0 + ), "ledger=disable should drop ledger-derived (positive) tx deposits" + + yield ledger_disable + + def ledger_ignore( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `ledger` in `ignore` mode. + + In `ignore` mode db-sync maintains ledger state but does not use any of its data + (except UTxO for bootstrap), so the ledger-derived tables stay empty, like + `disable`. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_ledger(value=db_sync.LedgerMode.IGNORE) + ) + check_dbsync_state( + expected_state=dict.fromkeys(ledger_derived_tables, TableCondition.EMPTY) + ) + + yield ledger_ignore + + def pool_stat_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `pool_stat` option. + + With pool_stat enabled, per-epoch pool statistics are stored in the pool_stat + table once an epoch boundary has been crossed. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_pool_stat(value=db_sync.SettingState.ENABLE) + ) + check_dbsync_state(expected_state={db_sync.Table.POOL_STAT: TableCondition.NOT_EMPTY}) + + yield pool_stat_enable + + def pool_stat_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `pool_stat` option.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_pool_stat(value=db_sync.SettingState.DISABLE) + ) + check_dbsync_state(expected_state={db_sync.Table.POOL_STAT: TableCondition.EMPTY}) + + yield pool_stat_disable + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, From c31fdc787d8122e6001f0af5282530d551272a0e Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 8 Jul 2026 11:05:00 +0200 Subject: [PATCH 05/14] test(dbsync_config): cover offchain_pool_data + enable CI flag Add Phase 4 off-chain pool metadata subtests: - offchain_pool_data enable: db-sync fetches the cluster pools' registered metadata into off_chain_pool_data with no fetch errors (polled, since the fetch loop is asynchronous). Skipped when db-sync was not started with --allow-private-offchain-urls. - offchain_pool_data disable: no fetch happens (off_chain_pool_data and off_chain_pool_fetch_error empty); the on-chain pool_metadata_ref remains. Add dbsync_utils.allow_private_offchain_urls_enabled() to gate the off-chain tests (detected from the running db-sync process args). Enable --allow-private-offchain-urls for the "dbsync config" CI selection so the off-chain tests can fetch localhost metadata. --- .../tests/test_dbsync_config.py | 56 +++++++++++++++++++ cardano_node_tests/utils/dbsync_utils.py | 23 ++++++++ runner/regression.sh | 5 ++ 3 files changed, 84 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 69b34345e..6bceb6da4 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -307,6 +307,7 @@ def multi_asset_disable( yield from self._subtests_phase1() yield from self._subtests_phase2() yield from self._subtests_phase3() + yield from self._subtests_phase4() def _subtests_phase1(self) -> tp.Generator[tp.Callable]: """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" @@ -686,6 +687,61 @@ def pool_stat_disable( yield pool_stat_disable + def _subtests_phase4(self) -> tp.Generator[tp.Callable]: + """Phase 4 subtests: off-chain pool metadata (needs --allow-private-offchain-urls).""" + + def offchain_pool_data_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `offchain_pool_data` option. + + With off-chain pool data enabled (and db-sync allowed to fetch private/localhost + URLs), db-sync fetches the cluster pools' registered metadata and stores it in + off_chain_pool_data, with no fetch errors. The fetch is asynchronous (the fetch + loop sleeps ~300s between passes), so off_chain_pool_data is polled. + """ + if not dbsync_utils.allow_private_offchain_urls_enabled(): + pytest.skip("requires db-sync started with --allow-private-offchain-urls") + + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_offchain_pool_data(value=db_sync.SettingState.ENABLE) + ) + wait_for_tables_not_empty([db_sync.Table.OFF_CHAIN_POOL_DATA], timeout=600) + check_dbsync_state( + expected_state={ + db_sync.Table.OFF_CHAIN_POOL_DATA: TableCondition.NOT_EMPTY, + db_sync.Table.OFF_CHAIN_POOL_FETCH_ERROR: TableCondition.EMPTY, + } + ) + + yield offchain_pool_data_enable + + def offchain_pool_data_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `offchain_pool_data` option. + + With off-chain pool data disabled, db-sync does not fetch pool metadata, so both + off_chain_pool_data and off_chain_pool_fetch_error stay empty. The on-chain + metadata reference (pool_metadata_ref) is still recorded. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_offchain_pool_data(value=db_sync.SettingState.DISABLE) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.OFF_CHAIN_POOL_DATA: TableCondition.EMPTY, + db_sync.Table.OFF_CHAIN_POOL_FETCH_ERROR: TableCondition.EMPTY, + db_sync.Table.POOL_METADATA_REF: TableCondition.NOT_EMPTY, + } + ) + + yield offchain_pool_data_disable + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, diff --git a/cardano_node_tests/utils/dbsync_utils.py b/cardano_node_tests/utils/dbsync_utils.py index 18906b97d..21d229ff1 100644 --- a/cardano_node_tests/utils/dbsync_utils.py +++ b/cardano_node_tests/utils/dbsync_utils.py @@ -4,6 +4,7 @@ import functools import itertools import logging +import pathlib as pl import time import typing as tp @@ -1676,6 +1677,28 @@ def column_data_type(*, table: str, column: str) -> str | None: return dbsync_queries.query_column_data_type(table=table, column=column) +def allow_private_offchain_urls_enabled() -> bool: + """Check whether the running db-sync uses ``--allow-private-offchain-urls``. + + This is a start-time CLI flag (gated by the ``DBSYNC_ALLOW_PRIVATE_OFFCHAIN_URLS`` env + var in the ``run-cardano-dbsync`` script), not an insert option, so it cannot be toggled + per test. Off-chain fetching of private / localhost metadata URLs only works when it is + set. Detected from the running process arguments, because the run script always contains + the conditional flag and so its text is not a reliable signal. + """ + proc_root = pl.Path("/proc") + for proc_dir in proc_root.iterdir(): + if not proc_dir.name.isdigit(): + continue + try: + cmdline = (proc_dir / "cmdline").read_bytes() + except OSError: + continue + if b"cardano-db-sync" in cmdline and b"--allow-private-offchain-urls" in cmdline: + return True + return False + + def check_epoch_state(*, epoch_no: int, txid: str, action_type: ActionTypes) -> None: """Check governance stats per epoch in dbsync.""" if not configuration.HAS_DBSYNC: diff --git a/runner/regression.sh b/runner/regression.sh index 4c2c8e4d5..2cc27ddf1 100755 --- a/runner/regression.sh +++ b/runner/regression.sh @@ -64,6 +64,11 @@ elif [ "$MARKEXPR" = "conway only" ]; then elif [ "$MARKEXPR" = "dbsync config" ]; then export CLUSTERS_COUNT=1 export MARKEXPR="(dbsync and smoke) or dbsync_config" + # Allow db-sync to fetch off-chain metadata from the cluster's private/localhost + # URLs so the off-chain config tests can exercise offchain_pool_data / + # offchain_vote_data (the flag only permits private URLs; the tests still enable + # the corresponding insert options themselves). + export DBSYNC_ALLOW_PRIVATE_OFFCHAIN_URLS=true fi if [ -n "${CLUSTERS_COUNT:-}" ]; then From 8afe59e485563d0ee6e5c3245220c500e2261cbf Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Thu, 9 Jul 2026 15:40:00 +0200 Subject: [PATCH 06/14] test(dbsync_config): cover offchain_vote_data + builder support Add Phase 5 off-chain vote metadata coverage: - Extend DBSyncConfigBuilder with `offchain_vote_data` (default, builder method and build() output); db-sync defaults it to "disable" since 13.7.0.1. - offchain_vote_data disable: with governance enabled, on-chain anchors (voting_anchor) are recorded but all off_chain_vote_* tables stay empty, proving the fetch is gated independently of the governance flag. - offchain_vote_data enable: fetches anchor metadata (recorded as data or a fetch error). Skipped unless db-sync allows private URLs and a fetchable vote anchor exists; the suite's anchors are non-CIP and unpublished, so the CIP sub-tables are not asserted here. Move the off_chain_vote_* tables out of GOVERNANCE_TABLES into a dedicated OFFCHAIN_VOTE_TABLES group so the governance subtest covers only on-chain governance data and off-chain vote coverage is flag-gated. --- .../tests/test_dbsync_config.py | 94 ++++++++++++++++++- .../utils/dbsync_service_manager.py | 7 ++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 6bceb6da4..203731494 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -48,6 +48,9 @@ class ColumnCondition(enum.StrEnum): IS_NOT_NULL = "column_condition:IS NOT NULL" +# On-chain governance tables. Off-chain vote metadata is controlled by `offchain_vote_data` +# (and needs --allow-private-offchain-urls), so it is covered separately by the +# offchain_vote_data subtests rather than bundled here with on-chain governance data. GOVERNANCE_TABLES = ( db_sync.Table.COMMITTEE_DE_REGISTRATION, db_sync.Table.COMMITTEE_MEMBER, @@ -59,15 +62,21 @@ class ColumnCondition(enum.StrEnum): db_sync.Table.DREP_REGISTRATION, db_sync.Table.EPOCH_STATE, db_sync.Table.GOV_ACTION_PROPOSAL, + db_sync.Table.VOTING_ANCHOR, + db_sync.Table.VOTING_PROCEDURE, + db_sync.Table.TREASURY_WITHDRAWAL, +) + +# Off-chain vote metadata tables, populated only when `offchain_vote_data` is enabled and +# db-sync is allowed to fetch the (private/localhost) anchor URLs. +OFFCHAIN_VOTE_TABLES = ( db_sync.Table.OFF_CHAIN_VOTE_DATA, db_sync.Table.OFF_CHAIN_VOTE_DREP_DATA, db_sync.Table.OFF_CHAIN_VOTE_EXTERNAL_UPDATE, db_sync.Table.OFF_CHAIN_VOTE_FETCH_ERROR, db_sync.Table.OFF_CHAIN_VOTE_GOV_ACTION_DATA, db_sync.Table.OFF_CHAIN_VOTE_REFERENCE, - db_sync.Table.VOTING_ANCHOR, - db_sync.Table.VOTING_PROCEDURE, - db_sync.Table.TREASURY_WITHDRAWAL, + db_sync.Table.OFF_CHAIN_VOTE_AUTHOR, ) @@ -308,6 +317,7 @@ def multi_asset_disable( yield from self._subtests_phase2() yield from self._subtests_phase3() yield from self._subtests_phase4() + yield from self._subtests_phase5() def _subtests_phase1(self) -> tp.Generator[tp.Callable]: """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" @@ -742,6 +752,84 @@ def offchain_pool_data_disable( yield offchain_pool_data_disable + def _subtests_phase5(self) -> tp.Generator[tp.Callable]: + """Phase 5 subtests: off-chain vote metadata (needs --allow-private-offchain-urls).""" + + def offchain_vote_data_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `offchain_vote_data`, independent of `governance`. + + With governance enabled but off-chain vote data disabled, db-sync records on-chain + governance anchors (voting_anchor) but does not fetch any anchor metadata, so all + off_chain_vote_* tables stay empty. This proves `offchain_vote_data` gates the + metadata fetch independently of the `governance` flag. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_governance( + value=db_sync.SettingState.ENABLE + ).with_offchain_vote_data(value=db_sync.SettingState.DISABLE) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.VOTING_ANCHOR: TableCondition.NOT_EMPTY, + **dict.fromkeys(OFFCHAIN_VOTE_TABLES, TableCondition.EMPTY), + } + ) + + yield offchain_vote_data_disable + + def offchain_vote_data_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `offchain_vote_data`. + + With off-chain vote data enabled (and db-sync allowed to fetch private/localhost + URLs), db-sync fetches governance anchor metadata; the fetch is recorded in + off_chain_vote_data (or off_chain_vote_fetch_error on failure). + + Skipped unless db-sync allows private URLs and the chain has a fetchable vote + anchor. The framework's governance activity publishes only non-CIP anchors and + unpublishes them, so on a standard cluster there is nothing for db-sync to fetch; + full off-chain vote coverage requires anchored governance activity with reachable + metadata. (The CIP sub-tables - off_chain_vote_gov_action_data / drep_data / + author / reference - additionally need CIP-compliant anchors and are not asserted + here.) + """ + if not dbsync_utils.allow_private_offchain_urls_enabled(): + pytest.skip("requires db-sync started with --allow-private-offchain-urls") + if ( + dbsync_queries.query_rows_count( + table="voting_anchor", column="url", condition="!= ''" + ) + == 0 + ): + pytest.skip("no fetchable governance vote anchors on chain") + + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_governance( + value=db_sync.SettingState.ENABLE + ).with_offchain_vote_data(value=db_sync.SettingState.ENABLE) + ) + + # The fetch is asynchronous; wait until db-sync has recorded the result either as + # fetched data or as a fetch error. + def _query_func() -> bool: + data = not dbsync_utils.table_empty(table=db_sync.Table.OFF_CHAIN_VOTE_DATA) + err = not dbsync_utils.table_empty(table=db_sync.Table.OFF_CHAIN_VOTE_FETCH_ERROR) + if not (data or err): + msg = "off_chain_vote_data / off_chain_vote_fetch_error still empty" + raise dbsync_utils.DbSyncNoResponseError(msg) + return True + + dbsync_utils.retry_query(query_func=_query_func, timeout=600) + + yield offchain_vote_data_enable + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, diff --git a/cardano_node_tests/utils/dbsync_service_manager.py b/cardano_node_tests/utils/dbsync_service_manager.py index 27e3e11ef..d02c5081c 100644 --- a/cardano_node_tests/utils/dbsync_service_manager.py +++ b/cardano_node_tests/utils/dbsync_service_manager.py @@ -200,6 +200,7 @@ def __init__(self) -> None: "plutus": PlutusConfig(), "governance": SettingState.ENABLE, "offchain_pool_data": SettingState.ENABLE, + "offchain_vote_data": SettingState.DISABLE, "pool_stat": SettingState.ENABLE, "remove_jsonb_from_schema": SettingState.DISABLE, } @@ -322,6 +323,11 @@ def with_offchain_pool_data(self, *, value: SettingState) -> tp.Self: self._config["offchain_pool_data"] = value return self + def with_offchain_vote_data(self, *, value: SettingState) -> tp.Self: + if not self._preset_applied: + self._config["offchain_vote_data"] = value + return self + def with_pool_stat(self, *, value: SettingState) -> tp.Self: if not self._preset_applied: self._config["pool_stat"] = value @@ -356,6 +362,7 @@ def build(self) -> dict[str, tp.Any]: "plutus": {"enable": plutus.enable}, "governance": self._enum_to_value(self._config["governance"]), "offchain_pool_data": self._enum_to_value(self._config["offchain_pool_data"]), + "offchain_vote_data": self._enum_to_value(self._config["offchain_vote_data"]), "pool_stat": self._enum_to_value(self._config["pool_stat"]), "remove_jsonb_from_schema": self._enum_to_value( self._config["remove_jsonb_from_schema"] From 649e63d04d8dee8082ad764c03db7948eff7609d Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Sat, 11 Jul 2026 10:25:00 +0200 Subject: [PATCH 07/14] test(dbsync_config): cover remove_jsonb_from_schema Add Phase 6 subtests for the remove_jsonb_from_schema option: - disable: jsonb columns (datum.value, cost_model.costs, gov_action_proposal.description) keep the jsonb type. - enable: those columns no longer use the jsonb type. Assertions use column data types (schema-level, independent of row counts) and are limited to jsonb columns not also governed by json_type, to isolate this option's effect. --- .../tests/test_dbsync_config.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 203731494..722fd33cc 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -318,6 +318,7 @@ def multi_asset_disable( yield from self._subtests_phase3() yield from self._subtests_phase4() yield from self._subtests_phase5() + yield from self._subtests_phase6() def _subtests_phase1(self) -> tp.Generator[tp.Callable]: """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" @@ -830,6 +831,55 @@ def _query_func() -> bool: yield offchain_vote_data_enable + def _subtests_phase6(self) -> tp.Generator[tp.Callable]: + """Phase 6 subtests: `remove_jsonb_from_schema` column-type effects.""" + # jsonb columns controlled by remove_jsonb_from_schema. Limited to columns NOT also + # governed by `json_type` (the *.json columns), to isolate this option's effect. + # Column types are schema-level, so these checks hold regardless of row counts. + jsonb_columns = ( + (db_sync.Table.DATUM, "value"), + (db_sync.Table.COST_MODEL, "costs"), + (db_sync.Table.GOV_ACTION_PROPOSAL, "description"), + ) + + def remove_jsonb_disable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test disabled `remove_jsonb_from_schema`: jsonb columns keep the jsonb type.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_remove_jsonb_from_schema( + value=db_sync.SettingState.DISABLE + ) + ) + for table, column in jsonb_columns: + assert dbsync_utils.column_data_type(table=table, column=column) == "jsonb", ( + f"{table}.{column} should be jsonb when remove_jsonb_from_schema is disabled" + ) + + yield remove_jsonb_disable + + def remove_jsonb_enable( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test enabled `remove_jsonb_from_schema`: jsonb columns drop the jsonb type.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_remove_jsonb_from_schema( + value=db_sync.SettingState.ENABLE + ) + ) + for table, column in jsonb_columns: + dtype = dbsync_utils.column_data_type(table=table, column=column) + assert dtype is not None and dtype != "jsonb", ( + f"{table}.{column} should not be jsonb when remove_jsonb_from_schema is " + f"enabled (got {dtype})" + ) + + yield remove_jsonb_enable + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, From f12f692b6373af6f4bb049de1004ba93a75fc02a Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Mon, 13 Jul 2026 13:50:00 +0200 Subject: [PATCH 08/14] test(dbsync_config): cover insert-option presets Add Phase 7 preset subtests that exercise db-sync's own preset expansion by emitting a raw `preset` key (DBSyncConfigBuilder.build now emits {"preset": ...} when a preset is selected): - full: tx_out / ma_tx_out / tx_metadata / redeemer / drep_registration populated, tx_cbor empty. - only_governance: governance data populated, tx_out / ma_tx_out empty. - disable_all: tx_out / ma_tx_out / redeemer / drep_registration empty. - only_utxo: metadata / plutus off (tx_metadata, redeemer empty). tx_out / ma_tx_out are not asserted (bootstrap bulk UTxO load is environment-sensitive and unreliable on a fast dev cluster). Document a db-sync discrepancy found via TDD: the only_utxo preset enables governance (onlyUTxOInsertOptions sioGovernance=True) although the docs describe it as UTxO-only. The documented expectation (drep_registration empty) is kept and marked xfail until db-sync / the docs are reconciled. --- .../tests/test_dbsync_config.py | 107 ++++++++++++++++++ .../utils/dbsync_service_manager.py | 7 ++ 2 files changed, 114 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 722fd33cc..c93f27081 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -319,6 +319,7 @@ def multi_asset_disable( yield from self._subtests_phase4() yield from self._subtests_phase5() yield from self._subtests_phase6() + yield from self._subtests_phase7() def _subtests_phase1(self) -> tp.Generator[tp.Callable]: """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" @@ -880,6 +881,112 @@ def remove_jsonb_enable( yield remove_jsonb_enable + def _subtests_phase7(self) -> tp.Generator[tp.Callable]: + """Phase 7 subtests: insert-option presets (exercise db-sync's own preset expansion).""" + + def preset_full( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `full` preset: all insert options on except tx_cbor and off-chain.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.FULL) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER: TableCondition.NOT_EMPTY, + db_sync.Table.DREP_REGISTRATION: TableCondition.NOT_EMPTY, + db_sync.Table.TX_CBOR: TableCondition.EMPTY, + } + ) + + yield preset_full + + def preset_only_utxo( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `only_utxo` preset: load block/tx/tx_out/ma_tx_out only. + + shelley / metadata / plutus are disabled, so tx_metadata and redeemer stay empty. + + tx_out / ma_tx_out are not asserted here: only_utxo uses tx_out=bootstrap, which + loads the whole UTxO set in bulk only once the chain tip is reached. That bulk + load is environment-sensitive (designed for mainnet-scale initial sync) and does + not reliably complete on a small/fast dev cluster within a reasonable timeout, so + asserting it would be flaky. + + KNOWN doc-vs-code discrepancy: db-sync's only_utxo preset ENABLES governance + (Config/Types.hs ``onlyUTxOInsertOptions`` -> ``sioGovernance = + GovernanceConfig True``), while doc/configuration.md describes only_utxo as + disabling governance ("Only load block, tx, tx_out and ma_tx_out"). The + governance assertion below expresses the documented behavior and is xfailed until + db-sync / the docs are reconciled. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.ONLY_UTXO) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_METADATA: TableCondition.EMPTY, + db_sync.Table.REDEEMER: TableCondition.EMPTY, + } + ) + if not dbsync_utils.table_empty(table=db_sync.Table.DREP_REGISTRATION): + pytest.xfail( + "only_utxo preset enables governance in db-sync " + "(onlyUTxOInsertOptions sioGovernance=True), contrary to the docs" + ) + check_dbsync_state( + expected_state={db_sync.Table.DREP_REGISTRATION: TableCondition.EMPTY} + ) + + yield preset_only_utxo + + def preset_only_governance( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `only_governance` preset: governance data, no tx_out / multi_asset.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.ONLY_GOVERNANCE) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.EMPTY, + db_sync.Table.DREP_REGISTRATION: TableCondition.NOT_EMPTY, + } + ) + + yield preset_only_governance + + def preset_disable_all( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test the `disable_all` preset: only block/tx and ledger-related data.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.DISABLE_ALL) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.EMPTY, + db_sync.Table.REDEEMER: TableCondition.EMPTY, + db_sync.Table.DREP_REGISTRATION: TableCondition.EMPTY, + } + ) + + yield preset_disable_all + @allure.link(helpers.get_vcs_link()) def test_dbsync_config( self, diff --git a/cardano_node_tests/utils/dbsync_service_manager.py b/cardano_node_tests/utils/dbsync_service_manager.py index d02c5081c..81f5cde2d 100644 --- a/cardano_node_tests/utils/dbsync_service_manager.py +++ b/cardano_node_tests/utils/dbsync_service_manager.py @@ -205,9 +205,11 @@ def __init__(self) -> None: "remove_jsonb_from_schema": SettingState.DISABLE, } self._preset_applied = False + self._preset: Preset | None = None def with_preset(self, *, preset: Preset) -> tp.Self: self._preset_applied = True + self._preset = preset if preset == Preset.FULL: self._config.update( @@ -339,6 +341,11 @@ def with_remove_jsonb_from_schema(self, *, value: SettingState) -> tp.Self: return self def build(self) -> dict[str, tp.Any]: + # When a preset is selected, emit only the `preset` key so db-sync expands it with + # its own preset definitions (individual keys would override the preset base). + if self._preset is not None: + return {"preset": self._preset.value} + tx_out = tp.cast(TxOutConfig, self._config["tx_out"]) shelley = tp.cast(ShelleyConfig, self._config["shelley"]) multi_asset = tp.cast(MultiAssetConfig, self._config["multi_asset"]) From fe52c164b08156d6733332a0a6665f75ef372750 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 15 Jul 2026 09:30:00 +0200 Subject: [PATCH 09/14] test(dbsync_config): note off-chain vote coverage pending PR #3497 Mark in offchain_vote_data_enable that full is_valid (TRUE/FALSE/NULL) and off-chain vote sub-table assertions become possible once the anchor test-data vectors land (cardano-node-tests PR #3497) and a governance test registers them. No new data files needed: governance_action_anchor.json already provides a conformant CIP-100 vector, and #3497 adds the non-conformant and invalid-JSON negatives (reusable across anchor types). --- cardano_node_tests/tests/test_dbsync_config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index c93f27081..c6fc24919 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -799,6 +799,14 @@ def offchain_vote_data_enable( metadata. (The CIP sub-tables - off_chain_vote_gov_action_data / drep_data / author / reference - additionally need CIP-compliant anchors and are not asserted here.) + + TODO (covered once cardano-node-tests PR #3497 merges): with the anchor test-data + vectors in place, a governance test can register conformant + non-conformant + + invalid-JSON anchors so this can assert off_chain_vote_data.is_valid (TRUE / FALSE / + NULL) and the sub-tables instead of skipping. No new data files are needed: + governance_action_anchor.json already provides a conformant CIP-100 vector (authors + / references / externalUpdates), and #3497 adds the non-conformant and invalid-JSON + negatives (reusable across all anchor types). """ if not dbsync_utils.allow_private_offchain_urls_enabled(): pytest.skip("requires db-sync started with --allow-private-offchain-urls") From 71203dd7e4e1e5b1ca9a329625f5e60d146b9aa7 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 15 Jul 2026 13:11:18 +0200 Subject: [PATCH 10/14] test(dbsync_config): refine subtests and presets Condense subtest docstrings/comments and materialize the tables argument in wait_for_tables_not_empty so a generator is not exhausted by the first failing poll. Simplify with_preset to emit only the `preset` key and let db-sync expand it, dropping the now-unused per-preset config maps and the Column members / NOT_ZERO condition they relied on. Wire the only_utxo preset to real cardano-db-sync blockers: #2150 (bootstrap crash near tip) xfails the restart, #2151 (governance enabled contrary to the docs) replaces the inline xfail. Add skip guards for offchain_pool_data when no pool metadata is on chain, and drop the unguarded shelley_disable_stake_registration subtest. --- cardano_node_tests/tests/issues.py | 10 + .../tests/test_dbsync_config.py | 259 +++++------------- .../utils/dbsync_service_manager.py | 77 +----- 3 files changed, 89 insertions(+), 257 deletions(-) diff --git a/cardano_node_tests/tests/issues.py b/cardano_node_tests/tests/issues.py index 1795a5f4c..5121c9cff 100644 --- a/cardano_node_tests/tests/issues.py +++ b/cardano_node_tests/tests/issues.py @@ -145,6 +145,16 @@ fixed_in="13.7.0.3", message="Swapped min_pool_cost / coins_per_utxo_size.", ) +dbsync_2150 = blockers.GH( + issue=2150, + repo="IntersectMBO/cardano-db-sync", + message="only_utxo preset (tx_out bootstrap) inserts 0 tx_out and crashes near tip.", +) +dbsync_2151 = blockers.GH( + issue=2151, + repo="IntersectMBO/cardano-db-sync", + message="only_utxo preset populates governance tables, contrary to the docs.", +) ledger_3731 = blockers.GH( issue=3731, diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index c6fc24919..5ab6c2094 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -11,6 +11,7 @@ from cardano_clusterlib import clusterlib from cardano_node_tests.tests import common +from cardano_node_tests.tests import issues from cardano_node_tests.utils import cluster_nodes from cardano_node_tests.utils import configuration from cardano_node_tests.utils import dbsync_queries @@ -43,7 +44,6 @@ class ColumnCondition(enum.StrEnum): """Enum for column-level db-sync condition checks.""" ZERO = "column_condition:=0" - NOT_ZERO = "column_condition:!= 0" IS_NULL = "column_condition:IS NULL" IS_NOT_NULL = "column_condition:IS NOT NULL" @@ -149,6 +149,9 @@ def wait_for_tables_not_empty( such tables must be polled rather than checked once. Raises ``TimeoutError`` (via ``retry_query``) if any table is still empty after ``timeout`` seconds. """ + # Materialize once so a generator argument is not exhausted by the first (failing) + # poll, which would make every subsequent retry see an empty list and pass falsely. + tables = list(tables) def _query_func() -> bool: empty_tables = [table for table in tables if dbsync_utils.table_empty(table=table)] @@ -327,11 +330,9 @@ def _subtests_phase1(self) -> tp.Generator[tp.Callable]: def plutus_disable( db_sync_manager: db_sync.DBSyncManager, ): - """Test disabled `plutus` option. + """Test disabled `plutus`: no script-execution data. - With Plutus disabled, db-sync must not insert script-execution data, so the - redeemer / redeemer_data / datum tables stay empty even though the chain - contains a Plutus transaction. + redeemer / redeemer_data / datum stay empty despite a Plutus tx on chain. """ db_config = db_sync_manager.get_config_builder() @@ -349,11 +350,7 @@ def plutus_disable( def metadata_disable( db_sync_manager: db_sync.DBSyncManager, ): - """Test disabled `metadata` option. - - With metadata disabled, the tx_metadata table stays empty even though the - chain contains a transaction carrying metadata. - """ + """Test disabled `metadata`: tx_metadata stays empty despite a tx with metadata.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=False)) @@ -364,11 +361,10 @@ def metadata_disable( def tx_out_consumed( db_sync_manager: db_sync.DBSyncManager, ): - """Test `tx_out` in `consumed` mode (without `force_tx_in`). + """Test `tx_out=consumed` (no force_tx_in). - In `consumed` mode db-sync records consumption via the new - `tx_out.consumed_by_tx_id` column instead of populating `tx_in`, so `tx_in` - stays empty while `tx_out` / `ma_tx_out` are populated. + Consumption is tracked via `tx_out.consumed_by_tx_id`, so `tx_in` stays empty + while `tx_out` / `ma_tx_out` are populated. """ db_config = db_sync_manager.get_config_builder() @@ -393,11 +389,7 @@ def tx_out_consumed( def tx_out_consumed_force_tx_in( db_sync_manager: db_sync.DBSyncManager, ): - """Test `tx_out` in `consumed` mode with `force_tx_in=True`. - - `force_tx_in` re-enables population of the `tx_in` table on top of `consumed` - mode, so both `tx_out` and `tx_in` are populated. - """ + """Test `tx_out=consumed` with force_tx_in: `tx_in` is populated alongside `tx_out`.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config( @@ -417,11 +409,7 @@ def tx_out_consumed_force_tx_in( def tx_out_use_address_table( db_sync_manager: db_sync.DBSyncManager, ): - """Test `tx_out` with `use_address_table=True`. - - With the address table enabled, db-sync normalizes addresses into a separate - `address` table (which otherwise does not exist). - """ + """Test `tx_out` with use_address_table: addresses go to a separate `address` table.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config( @@ -447,11 +435,9 @@ def _subtests_phase2(self) -> tp.Generator[tp.Callable]: def plutus_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `plutus` option. + """Test enabled `plutus`: script / redeemer / redeemer_data / datum are populated. - With Plutus enabled, script-execution data is inserted: the script, redeemer, - redeemer_data and datum tables are populated (the chain must contain a Plutus - transaction that locks/spends with a datum). + Needs a Plutus tx that locks/spends with a datum on chain. """ db_config = db_sync_manager.get_config_builder() @@ -470,11 +456,7 @@ def plutus_enable( def metadata_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `metadata` option. - - With metadata enabled, the tx_metadata table is populated from transactions - carrying metadata. - """ + """Test enabled `metadata`: tx_metadata is populated from txs carrying metadata.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=True)) @@ -485,11 +467,7 @@ def metadata_enable( def metadata_keys_filter( db_sync_manager: db_sync.DBSyncManager, ): - """Test the `metadata.keys` filter. - - When `metadata.keys` lists specific metadata keys, db-sync stores only metadata - with those keys; every tx_metadata row must therefore have the configured key. - """ + """Test the `metadata.keys` filter: only metadata with the listed key is stored.""" keep_key = 2 db_config = db_sync_manager.get_config_builder() @@ -507,11 +485,7 @@ def metadata_keys_filter( def shelley_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `shelley` option. - - Shelley-era data (certificates, etc.) is inserted: the stake_registration and - pool_update tables are populated. - """ + """Test enabled `shelley`: certificate data (stake_registration, pool_update) is set.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config(custom_config=db_config.with_shelley(enable=True)) @@ -527,17 +501,12 @@ def shelley_enable( def shelley_disable_independence( db_sync_manager: db_sync.DBSyncManager, ): - """Test disabled `shelley` option and its independence from `ledger`. - - With Shelley disabled, ledger-derived data (epoch_stake) is still populated - because it is controlled by the `ledger` option, not `shelley`. + """Test disabled `shelley` is independent of `ledger`. - Note: certificate tables are deliberately NOT asserted empty here. Genesis pool - registrations are inserted via an ungated path (Shelley/Genesis.hs), so - `pool_update` stays populated regardless of the `shelley` flag, and legacy - stake-registration certs are likewise ungated (covered separately by - ``shelley_disable_stake_registration``). `epoch_stake` is therefore the clean - signal for shelley/ledger independence. + `epoch_stake` is ledger-controlled, so it stays populated with shelley off. + Certificate tables are not asserted empty: tx-era certs are gated by `shelley`, + but genesis pool/stake registrations are inserted unconditionally + (Shelley/Genesis.hs), so `pool_update` / `stake_registration` keep genesis rows. """ db_config = db_sync_manager.get_config_builder() @@ -550,39 +519,6 @@ def shelley_disable_independence( yield shelley_disable_independence - def shelley_disable_stake_registration( - db_sync_manager: db_sync.DBSyncManager, - ): - """Test that `shelley=disable` suppresses stake_registration (per the docs). - - doc/configuration.md states `shelley.enable` disables "all certificates", which - includes stake registrations. The assertion below expresses that documented - behavior. - - KNOWN db-sync discrepancy: legacy ``ShelleyRegCert`` stake registrations are - inserted regardless of the flag, because - ``Cardano.DbSync.Era.Universal.Insert.Certificate.insertDelegCert`` calls - ``insertStakeRegistration`` without a ``when (ioShelley iopts)`` guard, unlike - the Conway ``insertConwayDelegCert`` path which is guarded. The assertion is kept - as the documented expectation and marked xfail until db-sync gates the legacy - path (pending cardano-db-sync issue; convert to ``issues.dbsync_.finish_test`` - once filed). - """ - db_config = db_sync_manager.get_config_builder() - - db_sync_manager.restart_with_config(custom_config=db_config.with_shelley(enable=False)) - if not dbsync_utils.table_empty(table=db_sync.Table.STAKE_REGISTRATION): - pytest.xfail( - "db-sync inserts legacy ShelleyRegCert stake registrations regardless of " - "shelley=disable (Certificate.insertDelegCert is unguarded); docs say " - "shelley disables all certificates." - ) - check_dbsync_state( - expected_state={db_sync.Table.STAKE_REGISTRATION: TableCondition.EMPTY} - ) - - yield shelley_disable_stake_registration - def _subtests_phase3(self) -> tp.Generator[tp.Callable]: """Phase 3 subtests: `ledger` modes and `pool_stat` (ledger-derived data).""" # Tables populated only from ledger state; empty unless `ledger` maintains and uses it. @@ -596,11 +532,9 @@ def _subtests_phase3(self) -> tp.Generator[tp.Callable]: def ledger_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `ledger` option. + """Test enabled `ledger`: ledger-derived tables and columns are populated. - With ledger enabled, db-sync maintains ledger state and populates the - ledger-derived tables (reward, epoch_stake, ada_pots, epoch_param), computes - redeemer fees, and records ledger-derived deposits. + reward / epoch_stake / ada_pots / epoch_param, redeemer.fee and tx deposits. """ db_config = db_sync_manager.get_config_builder() @@ -613,8 +547,7 @@ def ledger_enable( db_sync.Column.Redeemer.FEE: ColumnCondition.IS_NOT_NULL, } ) - # Ledger state lets db-sync compute deposits: at least one tx carries a positive - # deposit (e.g. a registration deposit). Contrast with the ledger=disable case. + # With ledger state, deposits are computed: at least one tx has a positive deposit. assert ( dbsync_queries.query_rows_count(table="tx", column="deposit", condition="> 0") > 0 ), "ledger=enable should record ledger-derived (positive) tx deposits" @@ -624,11 +557,7 @@ def ledger_enable( def ledger_disable( db_sync_manager: db_sync.DBSyncManager, ): - """Test disabled `ledger` option. - - With ledger disabled, db-sync does not maintain ledger state: the ledger-derived - tables stay empty and ledger-derived columns are null (redeemer.fee, tx.deposit). - """ + """Test disabled `ledger`: derived tables empty, redeemer.fee null, no deposits.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config( @@ -640,9 +569,8 @@ def ledger_disable( db_sync.Column.Redeemer.FEE: ColumnCondition.IS_NULL, } ) - # Ledger-derived deposits are dropped without ledger state: no tx has a positive - # deposit. (Note: tx.deposit is not uniformly NULL - db-sync still records 0 for - # some txs - so we assert the meaningful effect: no positive deposit remains.) + # No ledger state -> no positive deposits (tx.deposit isn't uniformly NULL; some + # txs keep 0, so assert the meaningful effect: no positive deposit remains). assert ( dbsync_queries.query_rows_count(table="tx", column="deposit", condition="> 0") == 0 ), "ledger=disable should drop ledger-derived (positive) tx deposits" @@ -652,12 +580,7 @@ def ledger_disable( def ledger_ignore( db_sync_manager: db_sync.DBSyncManager, ): - """Test `ledger` in `ignore` mode. - - In `ignore` mode db-sync maintains ledger state but does not use any of its data - (except UTxO for bootstrap), so the ledger-derived tables stay empty, like - `disable`. - """ + """Test `ledger=ignore`: state is kept but unused, so derived tables stay empty.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config( @@ -672,11 +595,7 @@ def ledger_ignore( def pool_stat_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `pool_stat` option. - - With pool_stat enabled, per-epoch pool statistics are stored in the pool_stat - table once an epoch boundary has been crossed. - """ + """Test enabled `pool_stat`: per-epoch pool stats stored after an epoch boundary.""" db_config = db_sync_manager.get_config_builder() db_sync_manager.restart_with_config( @@ -705,12 +624,10 @@ def _subtests_phase4(self) -> tp.Generator[tp.Callable]: def offchain_pool_data_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `offchain_pool_data` option. + """Test enabled `offchain_pool_data`: pool metadata is fetched into off_chain_pool_data. - With off-chain pool data enabled (and db-sync allowed to fetch private/localhost - URLs), db-sync fetches the cluster pools' registered metadata and stores it in - off_chain_pool_data, with no fetch errors. The fetch is asynchronous (the fetch - loop sleeps ~300s between passes), so off_chain_pool_data is polled. + Fetch is async (~300s loop) so the table is polled. Skipped without private URLs + allowed or when no pool metadata is registered on chain. """ if not dbsync_utils.allow_private_offchain_urls_enabled(): pytest.skip("requires db-sync started with --allow-private-offchain-urls") @@ -720,6 +637,8 @@ def offchain_pool_data_enable( db_sync_manager.restart_with_config( custom_config=db_config.with_offchain_pool_data(value=db_sync.SettingState.ENABLE) ) + if dbsync_utils.table_empty(table=db_sync.Table.POOL_METADATA_REF): + pytest.skip("no pool metadata registered on chain to fetch") wait_for_tables_not_empty([db_sync.Table.OFF_CHAIN_POOL_DATA], timeout=600) check_dbsync_state( expected_state={ @@ -733,11 +652,10 @@ def offchain_pool_data_enable( def offchain_pool_data_disable( db_sync_manager: db_sync.DBSyncManager, ): - """Test disabled `offchain_pool_data` option. + """Test disabled `offchain_pool_data`: no fetch. - With off-chain pool data disabled, db-sync does not fetch pool metadata, so both - off_chain_pool_data and off_chain_pool_fetch_error stay empty. The on-chain - metadata reference (pool_metadata_ref) is still recorded. + off_chain_pool_data / off_chain_pool_fetch_error stay empty; the on-chain + pool_metadata_ref is still recorded. """ db_config = db_sync_manager.get_config_builder() @@ -760,12 +678,10 @@ def _subtests_phase5(self) -> tp.Generator[tp.Callable]: def offchain_vote_data_disable( db_sync_manager: db_sync.DBSyncManager, ): - """Test disabled `offchain_vote_data`, independent of `governance`. + """Test `offchain_vote_data=disable` gates the fetch independently of `governance`. - With governance enabled but off-chain vote data disabled, db-sync records on-chain - governance anchors (voting_anchor) but does not fetch any anchor metadata, so all - off_chain_vote_* tables stay empty. This proves `offchain_vote_data` gates the - metadata fetch independently of the `governance` flag. + With governance on but vote data off, voting_anchor is recorded but no anchor + metadata is fetched, so all off_chain_vote_* tables stay empty. """ db_config = db_sync_manager.get_config_builder() @@ -786,27 +702,12 @@ def offchain_vote_data_disable( def offchain_vote_data_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `offchain_vote_data`. - - With off-chain vote data enabled (and db-sync allowed to fetch private/localhost - URLs), db-sync fetches governance anchor metadata; the fetch is recorded in - off_chain_vote_data (or off_chain_vote_fetch_error on failure). - - Skipped unless db-sync allows private URLs and the chain has a fetchable vote - anchor. The framework's governance activity publishes only non-CIP anchors and - unpublishes them, so on a standard cluster there is nothing for db-sync to fetch; - full off-chain vote coverage requires anchored governance activity with reachable - metadata. (The CIP sub-tables - off_chain_vote_gov_action_data / drep_data / - author / reference - additionally need CIP-compliant anchors and are not asserted - here.) - - TODO (covered once cardano-node-tests PR #3497 merges): with the anchor test-data - vectors in place, a governance test can register conformant + non-conformant + - invalid-JSON anchors so this can assert off_chain_vote_data.is_valid (TRUE / FALSE / - NULL) and the sub-tables instead of skipping. No new data files are needed: - governance_action_anchor.json already provides a conformant CIP-100 vector (authors - / references / externalUpdates), and #3497 adds the non-conformant and invalid-JSON - negatives (reusable across all anchor types). + """Test enabled `offchain_vote_data`: anchor metadata is fetched. + + The fetch result lands in off_chain_vote_data (or off_chain_vote_fetch_error). + Skipped without private URLs allowed or with no fetchable vote anchor on chain. + is_valid and the CIP sub-tables need conformant anchors; asserting those (using + the #3497 anchor vectors) is left to a dedicated off-chain test. """ if not dbsync_utils.allow_private_offchain_urls_enabled(): pytest.skip("requires db-sync started with --allow-private-offchain-urls") @@ -842,9 +743,8 @@ def _query_func() -> bool: def _subtests_phase6(self) -> tp.Generator[tp.Callable]: """Phase 6 subtests: `remove_jsonb_from_schema` column-type effects.""" - # jsonb columns controlled by remove_jsonb_from_schema. Limited to columns NOT also - # governed by `json_type` (the *.json columns), to isolate this option's effect. - # Column types are schema-level, so these checks hold regardless of row counts. + # jsonb columns controlled by remove_jsonb_from_schema (excluding *.json columns, which + # `json_type` also governs). Column types are schema-level, so row counts don't matter. jsonb_columns = ( (db_sync.Table.DATUM, "value"), (db_sync.Table.COST_MODEL, "costs"), @@ -917,28 +817,23 @@ def preset_full( def preset_only_utxo( db_sync_manager: db_sync.DBSyncManager, ): - """Test the `only_utxo` preset: load block/tx/tx_out/ma_tx_out only. - - shelley / metadata / plutus are disabled, so tx_metadata and redeemer stay empty. - - tx_out / ma_tx_out are not asserted here: only_utxo uses tx_out=bootstrap, which - loads the whole UTxO set in bulk only once the chain tip is reached. That bulk - load is environment-sensitive (designed for mainnet-scale initial sync) and does - not reliably complete on a small/fast dev cluster within a reasonable timeout, so - asserting it would be flaky. - - KNOWN doc-vs-code discrepancy: db-sync's only_utxo preset ENABLES governance - (Config/Types.hs ``onlyUTxOInsertOptions`` -> ``sioGovernance = - GovernanceConfig True``), while doc/configuration.md describes only_utxo as - disabling governance ("Only load block, tx, tx_out and ma_tx_out"). The - governance assertion below expresses the documented behavior and is xfailed until - db-sync / the docs are reconciled. + """Test the `only_utxo` preset (docs: block/tx/tx_out/ma_tx_out only). + + tx_out/ma_tx_out are not asserted: bootstrap bulk-loads them only at tip, which + is flaky on a dev cluster. db-sync also crashes syncing under this preset + (dbsync #2150), so the restart is xfailed while that is open. If it does sync, the + preset still enables governance contrary to the docs (dbsync #2151). """ db_config = db_sync_manager.get_config_builder() - db_sync_manager.restart_with_config( - custom_config=db_config.with_preset(preset=db_sync.Preset.ONLY_UTXO) - ) + try: + db_sync_manager.restart_with_config( + custom_config=db_config.with_preset(preset=db_sync.Preset.ONLY_UTXO) + ) + except Exception: + # db-sync bootstrap crash under only_utxo (dbsync #2150). + issues.dbsync_2150.finish_test() + check_dbsync_state( expected_state={ db_sync.Table.TX_METADATA: TableCondition.EMPTY, @@ -946,10 +841,7 @@ def preset_only_utxo( } ) if not dbsync_utils.table_empty(table=db_sync.Table.DREP_REGISTRATION): - pytest.xfail( - "only_utxo preset enables governance in db-sync " - "(onlyUTxOInsertOptions sioGovernance=True), contrary to the docs" - ) + issues.dbsync_2151.finish_test() check_dbsync_state( expected_state={db_sync.Table.DREP_REGISTRATION: TableCondition.EMPTY} ) @@ -1005,17 +897,18 @@ def test_dbsync_config( """Test DB-Sync configuration options using multiple subtests. Verifies that different DB-Sync configuration settings correctly control table population - and data insertion behavior. Each subtest modifies the configuration, restarts DB-Sync, - and validates the expected database state. - - * Test `tx_out` option (enable/disable modes with various settings) - * Verify address, tx_in, tx_out, and ma_tx_out tables respond to tx_out configuration - * Test `governance` option (enable/disable) - * Verify all governance-related tables populate when enabled and clear when disabled - * Test `tx_cbor` option (enable/disable) - * Verify tx_cbor table populates when enabled and clears when disabled - * Test `multi_asset` option (enable/disable) - * Verify multi_asset table populates when enabled and clears when disabled + and data insertion behavior. Each subtest modifies the configuration, restarts DB-Sync + (recreating and re-syncing the database), and validates the expected database state. + + Covers, across the phased subtests: + + * `tx_out` (enable/disable/consumed modes, force_tx_in, use_address_table) + * `governance`, `tx_cbor`, `multi_asset` (enable/disable) + * `plutus`, `metadata` (+ keys filter), `shelley` (enable/disable side effects) + * `ledger` (enable/disable/ignore) and `pool_stat` + * `offchain_pool_data` and `offchain_vote_data` (need --allow-private-offchain-urls) + * `remove_jsonb_from_schema` (column-type effects) + * insert-option presets (`full`, `only_utxo`, `only_governance`, `disable_all`) * Restore original DB-Sync configuration after all subtests complete """ cluster = cluster_singleton diff --git a/cardano_node_tests/utils/dbsync_service_manager.py b/cardano_node_tests/utils/dbsync_service_manager.py index 81f5cde2d..fa0eb4f56 100644 --- a/cardano_node_tests/utils/dbsync_service_manager.py +++ b/cardano_node_tests/utils/dbsync_service_manager.py @@ -114,25 +114,11 @@ class View(enum.StrEnum): class Column: class Tx(enum.StrEnum): FEE = "tx.fee" - DEPOSIT = "tx.deposit" class Redeemer(enum.StrEnum): SCRIPT_HASH = "redeemer.script_hash" FEE = "redeemer.fee" - class TxOut(enum.StrEnum): - CONSUMED_BY_TX_ID = "tx_out.consumed_by_tx_id" - - class StakeRegistration(enum.StrEnum): - DEPOSIT = "stake_registration.deposit" - - class PoolUpdate(enum.StrEnum): - DEPOSIT = "pool_update.deposit" - - class GovActionProposal(enum.StrEnum): - X_EPOCH = "gov_action_proposal.x_epoch" - EXPIRATION = "gov_action_proposal.expiration" - class SettingState(enum.StrEnum): ENABLE = "enable" @@ -208,68 +194,11 @@ def __init__(self) -> None: self._preset: Preset | None = None def with_preset(self, *, preset: Preset) -> tp.Self: + # Emit only the `preset` key and let db-sync expand it with its own preset + # definitions (see ``build``). The per-option `with_*` builders no-op once a + # preset is selected, so the individual config values are not used here. self._preset_applied = True self._preset = preset - - if preset == Preset.FULL: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.ENABLE), - "ledger": LedgerMode.ENABLE, - "shelley": ShelleyConfig(enable=True), - "multi_asset": MultiAssetConfig(enable=True), - "metadata": MetadataConfig(enable=True), - "plutus": PlutusConfig(enable=True), - "governance": SettingState.ENABLE, - "offchain_pool_data": SettingState.ENABLE, - "pool_stat": SettingState.ENABLE, - } - ) - elif preset == Preset.ONLY_UTXO: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.BOOTSTRAP), - "ledger": LedgerMode.IGNORE, - "shelley": ShelleyConfig(enable=False), - "metadata": MetadataConfig(enable=False), - "multi_asset": MultiAssetConfig(enable=True), - "plutus": PlutusConfig(enable=False), - "governance": SettingState.DISABLE, - "offchain_pool_data": SettingState.DISABLE, - "pool_stat": SettingState.DISABLE, - } - ) - elif preset == Preset.ONLY_GOVERNANCE: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.DISABLE), - "ledger": LedgerMode.ENABLE, - "shelley": ShelleyConfig(enable=False), - "multi_asset": MultiAssetConfig(enable=False), - "plutus": PlutusConfig(enable=False), - "governance": SettingState.ENABLE, - "offchain_pool_data": SettingState.DISABLE, - "pool_stat": SettingState.ENABLE, - } - ) - elif preset == Preset.DISABLE_ALL: - self._config.update( - { - "tx_cbor": SettingState.DISABLE, - "tx_out": TxOutConfig(value=TxOutMode.DISABLE), - "ledger": LedgerMode.DISABLE, - "shelley": ShelleyConfig(enable=False), - "multi_asset": MultiAssetConfig(enable=False), - "plutus": PlutusConfig(enable=False), - "governance": SettingState.DISABLE, - "offchain_pool_data": SettingState.DISABLE, - "pool_stat": SettingState.DISABLE, - } - ) - return self def with_tx_cbor(self, *, value: SettingState) -> tp.Self: From e6e9cb57d77f2bd3cf24d055cd28a62e6c517dc8 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 15 Jul 2026 15:03:34 +0200 Subject: [PATCH 11/14] test(dbsync_config): group subtests by option Replace the opaque _subtests_phase1..7 helpers with helpers named after the db-sync config option each set exercises (_subtests_tx_out, _subtests_plutus, _subtests_ledger, ...), and fold the previously inline subtests into them so each option's enable and disable scenarios live together. Pure refactor: all 31 subtest bodies are unchanged and still reported by scenario name; no test logic changes. Verified on a local node + db-sync 13.7.2.1 cluster (30 passed, 1 xfailed). --- .../tests/test_dbsync_config.py | 264 ++++++++++-------- 1 file changed, 142 insertions(+), 122 deletions(-) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 5ab6c2094..e3e3c9cbd 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -197,8 +197,24 @@ class TestDBSyncConfig: def get_subtests(self) -> tp.Generator[tp.Callable]: """Get the DB-Sync Config scenarios. - The scenarios are executed as subtests in the `test_dbsync_config` test. + The scenarios are executed as subtests in the `test_dbsync_config` test, + grouped by the db-sync config option each set exercises. """ + yield from self._subtests_tx_out() + yield from self._subtests_governance() + yield from self._subtests_tx_cbor() + yield from self._subtests_multi_asset() + yield from self._subtests_plutus() + yield from self._subtests_metadata() + yield from self._subtests_shelley() + yield from self._subtests_ledger() + yield from self._subtests_offchain_pool() + yield from self._subtests_offchain_vote() + yield from self._subtests_remove_jsonb() + yield from self._subtests_presets() + + def _subtests_tx_out(self) -> tp.Generator[tp.Callable]: + """Subtests for the `tx_out` option (modes, force_tx_in, use_address_table).""" def basic_tx_out( db_sync_manager: db_sync.DBSyncManager, @@ -238,6 +254,80 @@ def basic_tx_out( yield basic_tx_out + def tx_out_consumed( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out=consumed` (no force_tx_in). + + Consumption is tracked via `tx_out.consumed_by_tx_id`, so `tx_in` stays empty + while `tx_out` / `ma_tx_out` are populated. + """ + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.CONSUMED, force_tx_in=False + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.MA_TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_IN: TableCondition.EMPTY, + } + ) + assert dbsync_utils.column_exists( + table=db_sync.Table.TX_OUT, column="consumed_by_tx_id" + ), "`consumed` mode should add the `tx_out.consumed_by_tx_id` column" + + yield tx_out_consumed + + def tx_out_consumed_force_tx_in( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out=consumed` with force_tx_in: `tx_in` is populated alongside `tx_out`.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.CONSUMED, force_tx_in=True + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.TX_IN: TableCondition.NOT_EMPTY, + } + ) + + yield tx_out_consumed_force_tx_in + + def tx_out_use_address_table( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `tx_out` with use_address_table: addresses go to a separate `address` table.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_tx_out( + value=db_sync.TxOutMode.ENABLE, use_address_table=True + ) + ) + check_dbsync_state( + expected_state={ + db_sync.Table.ADDRESS: TableCondition.EXISTS, + db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + } + ) + assert not dbsync_utils.table_empty(table=db_sync.Table.ADDRESS), ( + "`use_address_table` should populate the `address` table" + ) + + yield tx_out_use_address_table + + def _subtests_governance(self) -> tp.Generator[tp.Callable]: + """Subtests for the `governance` option.""" + def governance( db_sync_manager: db_sync.DBSyncManager, ): @@ -264,6 +354,9 @@ def governance( yield governance + def _subtests_tx_cbor(self) -> tp.Generator[tp.Callable]: + """Subtests for the `tx_cbor` option.""" + def tx_cbor_value_enable( db_sync_manager: db_sync.DBSyncManager, ): @@ -290,6 +383,9 @@ def tx_cbor_value_disable( yield tx_cbor_value_disable + def _subtests_multi_asset(self) -> tp.Generator[tp.Callable]: + """Subtests for the `multi_asset` option.""" + def multi_asset_enable( db_sync_manager: db_sync.DBSyncManager, ): @@ -316,153 +412,74 @@ def multi_asset_disable( yield multi_asset_disable - yield from self._subtests_phase1() - yield from self._subtests_phase2() - yield from self._subtests_phase3() - yield from self._subtests_phase4() - yield from self._subtests_phase5() - yield from self._subtests_phase6() - yield from self._subtests_phase7() - - def _subtests_phase1(self) -> tp.Generator[tp.Callable]: - """Phase 1 subtests: config-presence / empties (no extra on-chain activity needed).""" + def _subtests_plutus(self) -> tp.Generator[tp.Callable]: + """Subtests for the `plutus` option.""" - def plutus_disable( + def plutus_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test disabled `plutus`: no script-execution data. + """Test enabled `plutus`: script / redeemer / redeemer_data / datum are populated. - redeemer / redeemer_data / datum stay empty despite a Plutus tx on chain. + Needs a Plutus tx that locks/spends with a datum on chain. """ db_config = db_sync_manager.get_config_builder() - db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=False)) + db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=True)) check_dbsync_state( expected_state={ - db_sync.Table.REDEEMER: TableCondition.EMPTY, - db_sync.Table.REDEEMER_DATA: TableCondition.EMPTY, - db_sync.Table.DATUM: TableCondition.EMPTY, + db_sync.Table.SCRIPT: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER_DATA: TableCondition.NOT_EMPTY, + db_sync.Table.DATUM: TableCondition.NOT_EMPTY, } ) - yield plutus_disable - - def metadata_disable( - db_sync_manager: db_sync.DBSyncManager, - ): - """Test disabled `metadata`: tx_metadata stays empty despite a tx with metadata.""" - db_config = db_sync_manager.get_config_builder() - - db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=False)) - check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.EMPTY}) - - yield metadata_disable + yield plutus_enable - def tx_out_consumed( + def plutus_disable( db_sync_manager: db_sync.DBSyncManager, ): - """Test `tx_out=consumed` (no force_tx_in). + """Test disabled `plutus`: no script-execution data. - Consumption is tracked via `tx_out.consumed_by_tx_id`, so `tx_in` stays empty - while `tx_out` / `ma_tx_out` are populated. + redeemer / redeemer_data / datum stay empty despite a Plutus tx on chain. """ db_config = db_sync_manager.get_config_builder() - db_sync_manager.restart_with_config( - custom_config=db_config.with_tx_out( - value=db_sync.TxOutMode.CONSUMED, force_tx_in=False - ) - ) - check_dbsync_state( - expected_state={ - db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, - db_sync.Table.MA_TX_OUT: TableCondition.NOT_EMPTY, - db_sync.Table.TX_IN: TableCondition.EMPTY, - } - ) - assert dbsync_utils.column_exists( - table=db_sync.Table.TX_OUT, column="consumed_by_tx_id" - ), "`consumed` mode should add the `tx_out.consumed_by_tx_id` column" - - yield tx_out_consumed - - def tx_out_consumed_force_tx_in( - db_sync_manager: db_sync.DBSyncManager, - ): - """Test `tx_out=consumed` with force_tx_in: `tx_in` is populated alongside `tx_out`.""" - db_config = db_sync_manager.get_config_builder() - - db_sync_manager.restart_with_config( - custom_config=db_config.with_tx_out( - value=db_sync.TxOutMode.CONSUMED, force_tx_in=True - ) - ) - check_dbsync_state( - expected_state={ - db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, - db_sync.Table.TX_IN: TableCondition.NOT_EMPTY, - } - ) - - yield tx_out_consumed_force_tx_in - - def tx_out_use_address_table( - db_sync_manager: db_sync.DBSyncManager, - ): - """Test `tx_out` with use_address_table: addresses go to a separate `address` table.""" - db_config = db_sync_manager.get_config_builder() - - db_sync_manager.restart_with_config( - custom_config=db_config.with_tx_out( - value=db_sync.TxOutMode.ENABLE, use_address_table=True - ) - ) + db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=False)) check_dbsync_state( expected_state={ - db_sync.Table.ADDRESS: TableCondition.EXISTS, - db_sync.Table.TX_OUT: TableCondition.NOT_EMPTY, + db_sync.Table.REDEEMER: TableCondition.EMPTY, + db_sync.Table.REDEEMER_DATA: TableCondition.EMPTY, + db_sync.Table.DATUM: TableCondition.EMPTY, } ) - assert not dbsync_utils.table_empty(table=db_sync.Table.ADDRESS), ( - "`use_address_table` should populate the `address` table" - ) - yield tx_out_use_address_table + yield plutus_disable - def _subtests_phase2(self) -> tp.Generator[tp.Callable]: - """Phase 2 subtests: enable-side population (depends on prior on-chain activity).""" + def _subtests_metadata(self) -> tp.Generator[tp.Callable]: + """Subtests for the `metadata` option (enable/disable and keys filter).""" - def plutus_enable( + def metadata_enable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `plutus`: script / redeemer / redeemer_data / datum are populated. - - Needs a Plutus tx that locks/spends with a datum on chain. - """ + """Test enabled `metadata`: tx_metadata is populated from txs carrying metadata.""" db_config = db_sync_manager.get_config_builder() - db_sync_manager.restart_with_config(custom_config=db_config.with_plutus(enable=True)) - check_dbsync_state( - expected_state={ - db_sync.Table.SCRIPT: TableCondition.NOT_EMPTY, - db_sync.Table.REDEEMER: TableCondition.NOT_EMPTY, - db_sync.Table.REDEEMER_DATA: TableCondition.NOT_EMPTY, - db_sync.Table.DATUM: TableCondition.NOT_EMPTY, - } - ) + db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=True)) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY}) - yield plutus_enable + yield metadata_enable - def metadata_enable( + def metadata_disable( db_sync_manager: db_sync.DBSyncManager, ): - """Test enabled `metadata`: tx_metadata is populated from txs carrying metadata.""" + """Test disabled `metadata`: tx_metadata stays empty despite a tx with metadata.""" db_config = db_sync_manager.get_config_builder() - db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=True)) - check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.NOT_EMPTY}) + db_sync_manager.restart_with_config(custom_config=db_config.with_metadata(enable=False)) + check_dbsync_state(expected_state={db_sync.Table.TX_METADATA: TableCondition.EMPTY}) - yield metadata_enable + yield metadata_disable def metadata_keys_filter( db_sync_manager: db_sync.DBSyncManager, @@ -482,6 +499,9 @@ def metadata_keys_filter( yield metadata_keys_filter + def _subtests_shelley(self) -> tp.Generator[tp.Callable]: + """Subtests for the `shelley` option (enable/disable side effects).""" + def shelley_enable( db_sync_manager: db_sync.DBSyncManager, ): @@ -519,8 +539,8 @@ def shelley_disable_independence( yield shelley_disable_independence - def _subtests_phase3(self) -> tp.Generator[tp.Callable]: - """Phase 3 subtests: `ledger` modes and `pool_stat` (ledger-derived data).""" + def _subtests_ledger(self) -> tp.Generator[tp.Callable]: + """Subtests for the `ledger` modes and `pool_stat` (ledger-derived data).""" # Tables populated only from ledger state; empty unless `ledger` maintains and uses it. ledger_derived_tables = ( db_sync.Table.REWARD, @@ -618,8 +638,8 @@ def pool_stat_disable( yield pool_stat_disable - def _subtests_phase4(self) -> tp.Generator[tp.Callable]: - """Phase 4 subtests: off-chain pool metadata (needs --allow-private-offchain-urls).""" + def _subtests_offchain_pool(self) -> tp.Generator[tp.Callable]: + """Subtests for `offchain_pool_data` (needs --allow-private-offchain-urls).""" def offchain_pool_data_enable( db_sync_manager: db_sync.DBSyncManager, @@ -672,8 +692,8 @@ def offchain_pool_data_disable( yield offchain_pool_data_disable - def _subtests_phase5(self) -> tp.Generator[tp.Callable]: - """Phase 5 subtests: off-chain vote metadata (needs --allow-private-offchain-urls).""" + def _subtests_offchain_vote(self) -> tp.Generator[tp.Callable]: + """Subtests for `offchain_vote_data` (needs --allow-private-offchain-urls).""" def offchain_vote_data_disable( db_sync_manager: db_sync.DBSyncManager, @@ -741,8 +761,8 @@ def _query_func() -> bool: yield offchain_vote_data_enable - def _subtests_phase6(self) -> tp.Generator[tp.Callable]: - """Phase 6 subtests: `remove_jsonb_from_schema` column-type effects.""" + def _subtests_remove_jsonb(self) -> tp.Generator[tp.Callable]: + """Subtests for `remove_jsonb_from_schema` column-type effects.""" # jsonb columns controlled by remove_jsonb_from_schema (excluding *.json columns, which # `json_type` also governs). Column types are schema-level, so row counts don't matter. jsonb_columns = ( @@ -789,8 +809,8 @@ def remove_jsonb_enable( yield remove_jsonb_enable - def _subtests_phase7(self) -> tp.Generator[tp.Callable]: - """Phase 7 subtests: insert-option presets (exercise db-sync's own preset expansion).""" + def _subtests_presets(self) -> tp.Generator[tp.Callable]: + """Subtests for insert-option presets (exercise db-sync's own preset expansion).""" def preset_full( db_sync_manager: db_sync.DBSyncManager, @@ -900,7 +920,7 @@ def test_dbsync_config( and data insertion behavior. Each subtest modifies the configuration, restarts DB-Sync (recreating and re-syncing the database), and validates the expected database state. - Covers, across the phased subtests: + Covers, grouped by config option: * `tx_out` (enable/disable/consumed modes, force_tx_in, use_address_table) * `governance`, `tx_cbor`, `multi_asset` (enable/disable) From cd54fbf1e04fc3437755693c1d9ba752c6f90f06 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 15 Jul 2026 16:30:30 +0200 Subject: [PATCH 12/14] test(dbsync_config): fix full-suite CI failures Two failures surfaced only in the full db-sync regression suite: - offchain_pool_data_enable asserted off_chain_pool_fetch_error is empty. In a full run other pools have unreachable metadata and a transient fetch error can precede a successful retry, so drop that assertion and keep the off_chain_pool_data NOT_EMPTY check. - The only_utxo preset crash (db-sync #2150) and benign pool_stat "pool not found" warnings tripped the teardown log-error check. Register ignore rules for those expected lines via add_ignore_rule. --- .../tests/test_dbsync_config.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index e3e3c9cbd..3e80024a6 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -18,6 +18,7 @@ from cardano_node_tests.utils import dbsync_service_manager as db_sync from cardano_node_tests.utils import dbsync_utils from cardano_node_tests.utils import helpers +from cardano_node_tests.utils import logfiles LOGGER = logging.getLogger(__name__) @@ -660,11 +661,11 @@ def offchain_pool_data_enable( if dbsync_utils.table_empty(table=db_sync.Table.POOL_METADATA_REF): pytest.skip("no pool metadata registered on chain to fetch") wait_for_tables_not_empty([db_sync.Table.OFF_CHAIN_POOL_DATA], timeout=600) + # `off_chain_pool_fetch_error` is intentionally not asserted empty: in a full run + # other pools can have unreachable metadata, and db-sync may record a transient + # fetch error before a later successful retry populates the data. check_dbsync_state( - expected_state={ - db_sync.Table.OFF_CHAIN_POOL_DATA: TableCondition.NOT_EMPTY, - db_sync.Table.OFF_CHAIN_POOL_FETCH_ERROR: TableCondition.EMPTY, - } + expected_state={db_sync.Table.OFF_CHAIN_POOL_DATA: TableCondition.NOT_EMPTY} ) yield offchain_pool_data_enable @@ -913,6 +914,7 @@ def test_dbsync_config( cluster_singleton: clusterlib.ClusterLib, db_sync_manager: db_sync.DBSyncManager, subtests: pytest_subtests.SubTests, + worker_id: str, ): """Test DB-Sync configuration options using multiple subtests. @@ -934,6 +936,23 @@ def test_dbsync_config( cluster = cluster_singleton common.get_test_id(cluster) + # Ignore expected db-sync log noise that would otherwise fail the test in teardown: + # * the `only_utxo` preset deliberately triggers the known bootstrap crash (db-sync + # issue #2150), which logs "TxIn not found in memory" and exits the dbsync service; + # * `pool_stat` logs a benign "assume the pool exists and move on" warning while a + # pool is not yet in the active cache (queryPoolHashId / insertPoolStats). + for _glob in ("dbsync.stdout", "dbsync.stderr"): + logfiles.add_ignore_rule( + files_glob=_glob, + regex="TxIn not found in memory|queryPoolHashId", + ignore_file_id=worker_id, + ) + logfiles.add_ignore_rule( + files_glob="supervisord.log", + regex="exited: dbsync .exit status 1", + ignore_file_id=worker_id, + ) + for subt in self.get_subtests(): with subtests.test(scenario=getattr(subt, "__name__", "")): subt(db_sync_manager) From 4868cef2fbcc086b935c8a97f4fe2acdbdb37871 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 15 Jul 2026 19:25:07 +0200 Subject: [PATCH 13/14] test(dbsync_config): skip only_utxo preset The only_utxo preset triggers the db-sync #2150 bootstrap crash, whose unexpected dbsync exit is logged to supervisord.log. The teardown log check has no ignore path for supervisord.log, so that crash fails the run even though the subtest is otherwise known-broken. Skip it (still visible in the report, tracked by #2150) instead of running and crashing db-sync. Also narrow the db-sync log ignore rule to the benign pool_stat queryPoolHashId warning and drop the ineffective supervisord ignore rule. Fix the GOVERNANCE_TABLES comment: off-chain vote anchors need --allow-private-offchain-urls only for cluster-local URLs; public anchor URLs are fetched regardless. --- .../tests/test_dbsync_config.py | 54 +++++-------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 3e80024a6..0783ff8e5 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -11,7 +11,6 @@ from cardano_clusterlib import clusterlib from cardano_node_tests.tests import common -from cardano_node_tests.tests import issues from cardano_node_tests.utils import cluster_nodes from cardano_node_tests.utils import configuration from cardano_node_tests.utils import dbsync_queries @@ -50,8 +49,9 @@ class ColumnCondition(enum.StrEnum): # On-chain governance tables. Off-chain vote metadata is controlled by `offchain_vote_data` -# (and needs --allow-private-offchain-urls), so it is covered separately by the -# offchain_vote_data subtests rather than bundled here with on-chain governance data. +# and has to be fetched from the anchor URLs (only cluster-local/localhost anchors need +# --allow-private-offchain-urls; public URLs are fetched regardless), so it is covered +# separately by the offchain_vote_data subtests rather than bundled here. GOVERNANCE_TABLES = ( db_sync.Table.COMMITTEE_DE_REGISTRATION, db_sync.Table.COMMITTEE_MEMBER, @@ -836,36 +836,17 @@ def preset_full( yield preset_full def preset_only_utxo( - db_sync_manager: db_sync.DBSyncManager, + db_sync_manager: db_sync.DBSyncManager, # noqa: ARG001 ): """Test the `only_utxo` preset (docs: block/tx/tx_out/ma_tx_out only). - tx_out/ma_tx_out are not asserted: bootstrap bulk-loads them only at tip, which - is flaky on a dev cluster. db-sync also crashes syncing under this preset - (dbsync #2150), so the restart is xfailed while that is open. If it does sync, the - preset still enables governance contrary to the docs (dbsync #2151). + Skipped: the preset uses tx_out bootstrap mode, under which db-sync inserts 0 + tx_out and then crashes near tip (dbsync #2150). The crash exits the dbsync + service and that unexpected exit is logged to supervisord.log, which the teardown + log check cannot ignore. Re-enable this (and assert the #2151 governance-vs-docs + behaviour) once #2150 is fixed. """ - db_config = db_sync_manager.get_config_builder() - - try: - db_sync_manager.restart_with_config( - custom_config=db_config.with_preset(preset=db_sync.Preset.ONLY_UTXO) - ) - except Exception: - # db-sync bootstrap crash under only_utxo (dbsync #2150). - issues.dbsync_2150.finish_test() - - check_dbsync_state( - expected_state={ - db_sync.Table.TX_METADATA: TableCondition.EMPTY, - db_sync.Table.REDEEMER: TableCondition.EMPTY, - } - ) - if not dbsync_utils.table_empty(table=db_sync.Table.DREP_REGISTRATION): - issues.dbsync_2151.finish_test() - check_dbsync_state( - expected_state={db_sync.Table.DREP_REGISTRATION: TableCondition.EMPTY} - ) + pytest.skip("only_utxo preset crashes db-sync, see cardano-db-sync issue #2150") yield preset_only_utxo @@ -936,22 +917,15 @@ def test_dbsync_config( cluster = cluster_singleton common.get_test_id(cluster) - # Ignore expected db-sync log noise that would otherwise fail the test in teardown: - # * the `only_utxo` preset deliberately triggers the known bootstrap crash (db-sync - # issue #2150), which logs "TxIn not found in memory" and exits the dbsync service; - # * `pool_stat` logs a benign "assume the pool exists and move on" warning while a - # pool is not yet in the active cache (queryPoolHashId / insertPoolStats). + # `pool_stat` logs a benign "assume the pool exists and move on" warning + # (queryPoolHashId / insertPoolStats) while a pool is not yet in the active cache. + # Ignore it so it does not fail the test during the teardown log check. for _glob in ("dbsync.stdout", "dbsync.stderr"): logfiles.add_ignore_rule( files_glob=_glob, - regex="TxIn not found in memory|queryPoolHashId", + regex="queryPoolHashId", ignore_file_id=worker_id, ) - logfiles.add_ignore_rule( - files_glob="supervisord.log", - regex="exited: dbsync .exit status 1", - ignore_file_id=worker_id, - ) for subt in self.get_subtests(): with subtests.test(scenario=getattr(subt, "__name__", "")): From d84a731c1a4f46df255905b53f9f56dc53e21238 Mon Sep 17 00:00:00 2001 From: Artur Wieczorek Date: Wed, 15 Jul 2026 20:49:58 +0200 Subject: [PATCH 14/14] test(dbsync_config): cover disable_epoch option Add disable_epoch_true / disable_epoch_false subtests: with disable_epoch=true the `epoch` rollup view returns no rows; with false it is populated. Adds a with_disable_epoch builder method, emitted only when explicitly set so the config for all other subtests is unchanged. Verified on a local db-sync 13.7.2.1 cluster. json_type was evaluated but is not covered: in 13.7.2.1 the option is parsed and validated (text/jsonb/disable) yet never referenced by any schema or insert module, so it has no observable effect (text and jsonb produce identical jsonb columns), leaving nothing to assert. --- .../tests/test_dbsync_config.py | 31 +++++++++++++++++++ .../utils/dbsync_service_manager.py | 9 ++++++ 2 files changed, 40 insertions(+) diff --git a/cardano_node_tests/tests/test_dbsync_config.py b/cardano_node_tests/tests/test_dbsync_config.py index 0783ff8e5..09b18aad1 100644 --- a/cardano_node_tests/tests/test_dbsync_config.py +++ b/cardano_node_tests/tests/test_dbsync_config.py @@ -212,6 +212,7 @@ def get_subtests(self) -> tp.Generator[tp.Callable]: yield from self._subtests_offchain_pool() yield from self._subtests_offchain_vote() yield from self._subtests_remove_jsonb() + yield from self._subtests_disable_epoch() yield from self._subtests_presets() def _subtests_tx_out(self) -> tp.Generator[tp.Callable]: @@ -810,6 +811,35 @@ def remove_jsonb_enable( yield remove_jsonb_enable + def _subtests_disable_epoch(self) -> tp.Generator[tp.Callable]: + """Subtests for the `disable_epoch` option (controls the `epoch` rollup view).""" + + def disable_epoch_true( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `disable_epoch=true`: the `epoch` view returns no rows.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_disable_epoch(value=True) + ) + check_dbsync_state(expected_state={db_sync.View.EPOCH: TableCondition.EMPTY}) + + yield disable_epoch_true + + def disable_epoch_false( + db_sync_manager: db_sync.DBSyncManager, + ): + """Test `disable_epoch=false`: the `epoch` view is populated.""" + db_config = db_sync_manager.get_config_builder() + + db_sync_manager.restart_with_config( + custom_config=db_config.with_disable_epoch(value=False) + ) + check_dbsync_state(expected_state={db_sync.View.EPOCH: TableCondition.NOT_EMPTY}) + + yield disable_epoch_false + def _subtests_presets(self) -> tp.Generator[tp.Callable]: """Subtests for insert-option presets (exercise db-sync's own preset expansion).""" @@ -911,6 +941,7 @@ def test_dbsync_config( * `ledger` (enable/disable/ignore) and `pool_stat` * `offchain_pool_data` and `offchain_vote_data` (need --allow-private-offchain-urls) * `remove_jsonb_from_schema` (column-type effects) + * `disable_epoch` (the `epoch` rollup view) * insert-option presets (`full`, `only_utxo`, `only_governance`, `disable_all`) * Restore original DB-Sync configuration after all subtests complete """ diff --git a/cardano_node_tests/utils/dbsync_service_manager.py b/cardano_node_tests/utils/dbsync_service_manager.py index fa0eb4f56..b57797be3 100644 --- a/cardano_node_tests/utils/dbsync_service_manager.py +++ b/cardano_node_tests/utils/dbsync_service_manager.py @@ -189,6 +189,9 @@ def __init__(self) -> None: "offchain_vote_data": SettingState.DISABLE, "pool_stat": SettingState.ENABLE, "remove_jsonb_from_schema": SettingState.DISABLE, + # Optional key: emitted only when explicitly set, otherwise db-sync's own default + # is used (keeps the config for all other subtests unchanged). + "disable_epoch": None, } self._preset_applied = False self._preset: Preset | None = None @@ -269,6 +272,11 @@ def with_remove_jsonb_from_schema(self, *, value: SettingState) -> tp.Self: self._config["remove_jsonb_from_schema"] = value return self + def with_disable_epoch(self, *, value: bool) -> tp.Self: + if not self._preset_applied: + self._config["disable_epoch"] = value + return self + def build(self) -> dict[str, tp.Any]: # When a preset is selected, emit only the `preset` key so db-sync expands it with # its own preset definitions (individual keys would override the preset base). @@ -303,6 +311,7 @@ def build(self) -> dict[str, tp.Any]: "remove_jsonb_from_schema": self._enum_to_value( self._config["remove_jsonb_from_schema"] ), + **self._optional("disable_epoch", self._config["disable_epoch"]), } return config