diff --git a/.github/workflows/c-tests.yaml b/.github/workflows/c-tests.yaml index 08d7519f9..e22d5be0f 100644 --- a/.github/workflows/c-tests.yaml +++ b/.github/workflows/c-tests.yaml @@ -25,8 +25,8 @@ jobs: if: runner.os == 'Windows' run: | rm core/rs/integration_check/rust-toolchain.toml - rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-pc-windows-gnu - rustup default nightly-2023-10-05-x86_64-pc-windows-gnu + rustup component add rust-src --toolchain nightly-2026-02-01-x86_64-pc-windows-gnu + rustup default nightly-2026-02-01-x86_64-pc-windows-gnu - name: Test run: | diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 1808270ac..037f7356d 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -69,8 +69,8 @@ jobs: - name: Rust Nightly run: | - rustup toolchain install nightly-2023-10-05 - rustup target add aarch64-apple-darwin --toolchain nightly-2023-10-05 + rustup toolchain install nightly-2026-02-01 + rustup target add aarch64-apple-darwin --toolchain nightly-2026-02-01 - name: Build MacOS Arm dylib run: | @@ -121,9 +121,9 @@ jobs: - name: Rust Nightly run: | - rustup toolchain install nightly-2023-10-05 - rustup target add aarch64-unknown-linux-gnu --toolchain nightly-2023-10-05 - rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2026-02-01 + rustup target add aarch64-unknown-linux-gnu --toolchain nightly-2026-02-01 + rustup component add rust-src --toolchain nightly-2026-02-01-x86_64-unknown-linux-gnu - name: Build run: | @@ -157,9 +157,9 @@ jobs: - name: Rust Nightly run: | - rustup toolchain install nightly-2023-10-05 - rustup target add x86_64-pc-windows-gnu --toolchain nightly-2023-10-05 - rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2026-02-01 + rustup target add x86_64-pc-windows-gnu --toolchain nightly-2026-02-01 + rustup component add rust-src --toolchain nightly-2026-02-01-x86_64-unknown-linux-gnu - name: mingw-w64 run: | @@ -197,9 +197,9 @@ jobs: - name: Rust Nightly run: | - rustup toolchain install nightly-2023-10-05 - rustup target add i686-pc-windows-gnu --toolchain nightly-2023-10-05 - rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2026-02-01 + rustup target add i686-pc-windows-gnu --toolchain nightly-2026-02-01 + rustup component add rust-src --toolchain nightly-2026-02-01-x86_64-unknown-linux-gnu - name: mingw-w64 run: | @@ -237,9 +237,9 @@ jobs: - name: Rust Nightly run: | - rustup toolchain install nightly-2023-10-05 - rustup target add aarch64-linux-android --toolchain nightly-2023-10-05 - rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-unknown-linux-gnu + rustup toolchain install nightly-2026-02-01 + rustup target add aarch64-linux-android --toolchain nightly-2026-02-01 + rustup component add rust-src --toolchain nightly-2026-02-01-x86_64-unknown-linux-gnu - uses: nttld/setup-ndk@v1 id: setup-ndk diff --git a/.github/workflows/rs-tests.yml b/.github/workflows/rs-tests.yml index b5fd0bc79..b86313926 100644 --- a/.github/workflows/rs-tests.yml +++ b/.github/workflows/rs-tests.yml @@ -25,8 +25,8 @@ jobs: if: runner.os == 'Windows' run: | find . -name rust-toolchain.toml -exec echo rm {} \; - rustup component add rust-src --toolchain nightly-2023-10-05-x86_64-pc-windows-gnu - rustup default nightly-2023-10-05-x86_64-pc-windows-gnu + rustup component add rust-src --toolchain nightly-2026-02-01-x86_64-pc-windows-gnu + rustup default nightly-2026-02-01-x86_64-pc-windows-gnu - name: Test Fractindex run: | diff --git a/.gitmodules b/.gitmodules index 74ce91bdf..35d831f90 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "core/rs/sqlite-rs-embedded"] path = core/rs/sqlite-rs-embedded - url = git@github.com:vlcn-io/sqlite-rs-embedded.git + url = git@github.com:superfly/sqlite-rs-embedded.git diff --git a/README.md b/README.md index 1202b0d21..b6bf1ee8c 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,13 @@ A [run-time loadable extension](https://www.sqlite.org/loadext.html) for [SQLite ## Relationship to Upstream -This fork diverges from upstream cr-sqlite v0.15.0 and introduces significant, breaking changes to the change bookkeeping model. The current version is **0.17.0**. Databases created with cr-sqlite < 0.17.0 are not supported and must be migrated. +This fork diverges from upstream cr-sqlite v0.15.0 and introduces significant, breaking changes to the change bookkeeping model and underlying data format. The current version is **0.18.0**. Databases created with cr-sqlite < 0.17.0 are not supported and must be migrated. Databases on 0.17.0 can incrementally migrate to the new V2 metadata format (see [Migration Guide](#migration-guide-017--018)). + +**0.19.0 will make V2 metadata + V2 wire format the default and remove V1 support entirely.** The core CRDT approach (history-free, last-write-wins per column, causal length sets) remains the same. The differences are in **how changes are tracked, timestamped, and replicated**. -## Key Changes from Upstream +## Key Changes in 0.17.0 ### Per-Site DB Version Tracking @@ -116,6 +118,131 @@ Local writes (insert/update/delete triggers) have been significantly reworked: - **Config lifetime fix**: `crsql_config_set` now properly manages the statement lifetime to prevent use-after-free of returned values. - **`crsql_changes` schema**: The `crsql_changes` virtual table now includes a `ts` column (column index 9). +## What's New in 0.18.0 + +0.18.0 introduces a new V2 metadata format and V2 wire format for change tracking. These are opt-in in 0.18.0 and will become the default (and only) format in 0.19.0. For full design details, see [`v2_metadata_design.md`](./v2_metadata_design.md). + +### Mandatory `crsql_set_ts()` Before All Write Operations + +Starting in 0.18.0, **`crsql_set_ts()` must be called in the same transaction** before any operation that writes to clock tables or metadata. This includes: + +- **`crsql_as_crr()`** — creating a new CRR +- **`crsql_begin_alter()` / `crsql_commit_alter()`** — altering CRR schema +- **`crsql_automigrate()`** — automated schema migrations +- **`crsql_incremental_maintenance()`** — V1→V2 migration batches +- **`INSERT`/`UPDATE`/`DELETE` on CRR tables** — local writes (via triggers) +- **`INSERT INTO crsql_changes`** — applying remote changes (merges) + +If the timestamp is not set (or was reset by a prior transaction commit), these operations will fail with an error: + +``` +crsql_as_crr: timestamp not set — call crsql_set_ts() first +``` + +The timestamp is **transaction-scoped**: it is set via `crsql_set_ts()` and resets to 0 on transaction commit or rollback. This means `crsql_set_ts()` and the operation that needs it must run in the same transaction. In autocommit mode, `SELECT crsql_set_ts(...)` does not trigger a commit (it's a read-only function call), so a subsequent `SELECT crsql_as_crr(...)` in a separate `exec` call will still see the timestamp — but the `crsql_as_crr` call itself writes to the database, which triggers an auto-commit and resets the timestamp for the next operation. + +**Only plain `SELECT` queries from `crsql_changes`** (reading changes without merging) do not require a timestamp to be set. + +Example: + +```sql +-- Set timestamp before each transaction that writes to CRRs +SELECT crsql_set_ts('1719878400000'); +SELECT crsql_as_crr('foo'); +-- ts is now reset (crsql_as_crr triggered a commit) + +-- Must set again for the next write operation +SELECT crsql_set_ts('1719878400000'); +INSERT INTO foo VALUES (1, 'bar'); +``` + +### V2 Metadata Format + +The V1 metadata format uses two tables per CRR: `__crsql_clock` (with `key INTEGER, col_name TEXT` as composite primary key) and `__crsql_pks` (mapping PK blobs to integer keys). This works but suffers from performance degradation at scale: self-joins are needed to retrieve the causal length (`cl`) value, deletion tombstones in the clock table slow down update operations, and the `(crsql_key, col_name)` composite PK is less compact than V2's packed integer `cell_key`. + +V2 replaces this with a more compact schema: + +- **`{table}__crsql_v2_pks`** — Maps hashed PK values to a `pk_key` and stores the causal length (`cl`) directly. PK values are hashed with `xxh3_128` truncated to `PK_HASH_SIZE` bytes, primarily to limit the size of sentinel and tombstone entries which accumulate over time. This also helps when PKs are larger than `PK_HASH_SIZE` (e.g., large compound primary keys). +- **`{table}__crsql_v2_clock`** — Uses a packed `cell_key = (pk_key << COL_ID_BITS) | col_id` as the primary key. Column names are mapped to small integer `col_id`s via `__crsql_v2_col_map`. Tracks per cell (column value) metadata. +- **`{table}__crsql_v2_tombstones`** — Tracks deleted rows separately, keyed by `hashed_pk`, stores the deletion causal length (`cl`). +- **`{table}__crsql_v2_tombstone_pks`** — Maps tombstone hashed PKs back to original PK column values (for V1 wire format compatibility). Once in V2-only mode entries from this table are no longer needed and can be cleaned up. + +A sentinel column (`col_id = 0`) replaces the old `-1` sentinel for PK-only tables and row existence tracking. If a column is later added to a previously PK-only table, it reuses `col_id = 0` for the new column. + +### V2 Wire Format + +The V1 wire format produces one `crsql_changes` row per column change. If a transaction updates 5 columns of a row in the same db_version, that's 5 change rows. The V2 wire format coalesces all column changes for a row within the same db_version into a single packed row: + +- `cid` — `\0`-separated packed column names (e.g., `"col1\0col2\0col3"`) +- `cval` — packed column values (binary blob) +- `col_vrsn` — per-column version numbers + +This speeds up processing of operations touching multiple columns at the same time, for example when inserting a new row into a table. + +### Hashed Primary Keys + +V2 hashes PK values with `xxh3_128` (truncated to `PK_HASH_SIZE` bytes, currently 10) and stores them as blobs. This is primarily to limit the size of tombstone entries, which accumulate over time. V2 also moves tombstones to a dedicated `v2_tombstones` table (separate from the clock table), reducing clock table bloat from row deletions. + +### Incremental Migration + +V2 migration is designed to be incremental — large tables can be migrated in batches without long lock times. + +**Prerequisites:** +- SQLite 3.44.0 or later (see [Minimum SQLite Version](#minimum-sqlite-version)) +- Load the 0.18.0 extension — it will detect the existing 0.17.0 schema and continue operating in V1 mode. No data is lost or changed. + +```sql +-- Step 1: Enable dual-write mode (new changes go to both V1 and V2 tables) +SELECT crsql_config_set('metadata-write-version', 2); + +-- Step 2: Run incremental maintenance in batches until complete +-- Returns remaining rows to migrate (0 = done, -1 = error) +-- Adjust batch size based on table size and lock tolerance +SELECT crsql_incremental_maintenance(1000); +-- ... repeat until the function returns 0 + +-- Step 3: Switch reads to V2 +-- The V1 tables still exist and are maintained but not read from +-- This way you can check that everything works correctly with V2 before dropping V1 +SELECT crsql_config_set('metadata-use-version', 2); + +-- Optional: Enable V2 wire format for more compact change replication +-- This packs all column changes per row per db_version into a single crsql_changes row and crsqlite_changes will return pk hashes instead of full PK values for tombstones. Requires migration to be complete and metadata-use-version set to 2 on ALL NODES in the cluster. +-- Nodes with metadata-use-version set to 1 will emit an error if they receive V2 wire format changes. +SELECT crsql_config_set('sync-log-version', 2); + +-- Step 4: Once confident, drop V1 tables by switching to V2-only mode +-- This is IRREVERSIBLE. If something went wrong, roll back to 1 while still in dual-write mode (step 2). +SELECT crsql_config_set('metadata-write-version', 3); + +-- Step 5: Run incremental maintenance again to clean up any remaining V1 data and tables +SELECT crsql_incremental_maintenance(1000); +-- ... repeat until the function returns 0 +``` + +The `metadata-write-version` config has three levels: +- **1** — V1 only (default, legacy) +- **2** — Dual write (V1 + V2, migration in progress, creates V2 tables) +- **3** — V2 only (V1 tables dropped, migration complete, irreversible) + +For small databases, you can migrate in one shot with a large batch size. For large production databases, use smaller batches and run periodically (e.g., in a background task) to avoid long lock times. + +### Rollback (from dual-write mode only) + +To roll back to V1 while still in dual-write mode (`metadata-write-version = 2`): + +```sql +SELECT crsql_config_set('metadata-write-version', 1); +``` + +This automatically resets `metadata-use-version` and `sync-log-version` to 1 as well. V2 tables are dropped via `crsql_incremental_maintenance`. V1 data is still intact (it was kept in sync during dual-write). Rollback is not possible once `metadata-write-version = 3` has been set. + +> **Note**: Once 0.19.0 is released, V1 support will be removed and rollback will not be possible. Complete the migration before upgrading to 0.19.0. + +### Minimum SQLite Version + +0.18.0 requires **SQLite 3.44.0 or later** (for `ORDER BY` in aggregate functions). The extension checks at load time. If loading into an external SQLite, verify with `SELECT sqlite_version();`. + ## Usage ```sql @@ -128,10 +255,13 @@ CREATE TABLE foo (a PRIMARY KEY NOT NULL, b); CREATE TABLE baz (a PRIMARY KEY NOT NULL, b, c, d); -- upgrade tables to be CRRs (conflict-free replicated relations) +-- crsql_set_ts must be called before crsql_as_crr in the same transaction +SELECT crsql_set_ts('1719878400000'); SELECT crsql_as_crr('foo'); +SELECT crsql_set_ts('1719878400000'); SELECT crsql_as_crr('baz'); --- optionally set a timestamp for this transaction +-- set a timestamp for this transaction's writes SELECT crsql_set_ts('1719878400000'); -- insert data as normal @@ -156,6 +286,7 @@ SELECT crsql_finalize(); ### Altering CRR Tables ```sql +SELECT crsql_set_ts('1719878400000'); SELECT crsql_begin_alter('table_name'); -- 1 or more alterations ALTER TABLE table_name ...; @@ -265,9 +396,17 @@ cargo run -- ../core/dist/crsqlite ### Internal Tables (per CRR table) +**V1 (0.17.0, deprecated — removed in 0.19.0):** - **`{table}__crsql_clock`** — Per-column clock metadata (col_version, db_version, site_id, seq, ts) - **`{table}__crsql_pks`** — Maps primary key values to integer keys for clock table lookups +**V2 (0.18.0+):** +- **`{table}__crsql_v2_pks`** — Maps hashed PK values to `pk_key` and stores causal length (`cl`). For tables with a rowid or single INTEGER PRIMARY KEY, PK columns are not stored separately — the integer key is the rowid of the `v2_pks` table itself. +- **`{table}__crsql_v2_col_map`** — Maps column names to small integer `col_id`s +- **`{table}__crsql_v2_clock`** — Per-cell clock metadata keyed by packed `cell_key = (pk_key << COL_ID_BITS) | col_id` +- **`{table}__crsql_v2_tombstones`** — Deleted row tracking, keyed by `hashed_pk`, stores the deletion causal length (`cl`) +- **`{table}__crsql_v2_tombstone_pks`** — Maps tombstone hashed PKs back to original PK column values (V1 wire compat). Can be pruned when V2 wire format is enabled. + ### Global Tables - **`crsql_db_versions`** — Per-site db_version tracking @@ -278,9 +417,8 @@ cargo run -- ../core/dist/crsqlite CR-SQLite uses history-free CRDTs based on [causal length sets](https://dl.acm.org/doi/pdf/10.1145/3380787.3393678). Each table upgraded to a CRR gets: -1. **Clock tables** (`__crsql_clock`) that track per-column version metadata (col_version, db_version, site_id, seq, ts) -2. **PK lookaside tables** (`__crsql_pks`) that map composite primary keys to integer keys for efficient clock table joins -3. **Triggers** (insert, update, delete) that automatically record changes to the clock tables +1. **Clock tables** that track per-column version metadata (col_version, db_version, site_id, seq, ts). In V1 these are `__crsql_clock` + `__crsql_pks`; in V2 these are `__crsql_v2_clock` (packed cell keys) + `__crsql_v2_pks` (hashed PKs with causal length) + `__crsql_v2_tombstones` (deleted rows) + `__crsql_v2_col_map` (column name → col_id mapping) + `__crsql_v2_tombstone_pks` (tombstone PK values for V1 wire compat). +2. **Triggers** (insert, update, delete) that automatically record changes into the clock tables. The same triggers work for both V1 and V2 — the extension routes internally based on the configured metadata version. Merging works by comparing column versions and causal lengths. For each incoming change: - If the incoming `col_version` is greater than the local one, the change wins diff --git a/core/rs/bundle/Cargo.lock b/core/rs/bundle/Cargo.lock index 8b964eb60..8bbaf2b72 100644 --- a/core/rs/bundle/Cargo.lock +++ b/core/rs/bundle/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "autocfg" @@ -10,17 +10,15 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", @@ -28,7 +26,6 @@ dependencies = [ "rustc-hash", "shlex", "syn", - "which", ] [[package]] @@ -75,6 +72,7 @@ version = "0.1.0" dependencies = [ "crsql_core", "crsql_fractindex_core", + "libc-print", "sqlite_nostd", ] @@ -83,9 +81,11 @@ name = "crsql_core" version = "0.1.0" dependencies = [ "bytes", + "libc-print", "num-derive", "num-traits", "sqlite_nostd", + "xxhash-rust", ] [[package]] @@ -108,22 +108,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] -name = "lazy_static" -version = "1.4.0" +name = "itertools" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] [[package]] -name = "lazycell" -version = "1.3.0" +name = "libc" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] -name = "libc" -version = "0.2.151" +name = "libc-print" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "a52163ef340f2845a3973431341ec3afb9a9c3efedc28bce7c93a5eb35c44b3b" +dependencies = [ + "libc", +] [[package]] name = "libloading" @@ -168,9 +174,9 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", @@ -179,25 +185,13 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] -[[package]] -name = "once_cell" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "prettyplease" version = "0.2.15" @@ -210,9 +204,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -243,9 +237,9 @@ checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "shlex" @@ -294,17 +288,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - [[package]] name = "winapi" version = "0.3.9" @@ -326,3 +309,9 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" diff --git a/core/rs/bundle/rust-toolchain.toml b/core/rs/bundle/rust-toolchain.toml index e6bea8b40..0345c3160 100644 --- a/core/rs/bundle/rust-toolchain.toml +++ b/core/rs/bundle/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-10-05" +channel = "nightly-2026-02-01" components = [ "rust-src", "rustfmt", "clippy" ] diff --git a/core/rs/bundle_static/Cargo.lock b/core/rs/bundle_static/Cargo.lock index 55b91c73b..4d925ead3 100644 --- a/core/rs/bundle_static/Cargo.lock +++ b/core/rs/bundle_static/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -19,17 +19,15 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", @@ -37,7 +35,6 @@ dependencies = [ "rustc-hash", "shlex", "syn", - "which", ] [[package]] @@ -52,15 +49,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" -[[package]] -name = "cc" -version = "1.0.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] - [[package]] name = "cexpr" version = "0.6.0" @@ -113,6 +101,7 @@ dependencies = [ "num-derive", "num-traits", "sqlite_nostd", + "xxhash-rust", ] [[package]] @@ -128,27 +117,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" -[[package]] -name = "errno" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "glob" version = "0.3.1" @@ -156,26 +124,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] -name = "home" -version = "0.5.5" +name = "itertools" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ - "windows-sys", + "either", ] -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.148" @@ -201,12 +157,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "linux-raw-sys" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" - [[package]] name = "log" version = "0.4.20" @@ -237,9 +187,9 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", @@ -248,25 +198,13 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "prettyplease" version = "0.2.15" @@ -279,9 +217,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -326,22 +264,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustix" -version = "0.38.13" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "shlex" @@ -390,18 +315,6 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "winapi" version = "0.3.9" @@ -425,67 +338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" +name = "xxhash-rust" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" diff --git a/core/rs/bundle_static/rust-toolchain.toml b/core/rs/bundle_static/rust-toolchain.toml index 4e0cf4784..70bd037bf 100644 --- a/core/rs/bundle_static/rust-toolchain.toml +++ b/core/rs/bundle_static/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-10-05" +channel = "nightly-2026-02-01" diff --git a/core/rs/core/Cargo.lock b/core/rs/core/Cargo.lock index dc45e3470..57468ef86 100644 --- a/core/rs/core/Cargo.lock +++ b/core/rs/core/Cargo.lock @@ -10,17 +10,15 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", @@ -28,7 +26,6 @@ dependencies = [ "rustc-hash", "shlex", "syn", - "which", ] [[package]] @@ -78,6 +75,7 @@ dependencies = [ "num-derive", "num-traits", "sqlite_nostd", + "xxhash-rust", ] [[package]] @@ -93,16 +91,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] -name = "lazy_static" -version = "1.4.0" +name = "itertools" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] [[package]] name = "libc" @@ -162,9 +157,9 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", @@ -173,25 +168,13 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] -[[package]] -name = "once_cell" -version = "1.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "prettyplease" version = "0.2.15" @@ -204,9 +187,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -237,9 +220,9 @@ checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "shlex" @@ -288,17 +271,6 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - [[package]] name = "winapi" version = "0.3.9" @@ -320,3 +292,9 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xxhash-rust" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" diff --git a/core/rs/core/Cargo.toml b/core/rs/core/Cargo.toml index be2a94e16..221e4072b 100644 --- a/core/rs/core/Cargo.toml +++ b/core/rs/core/Cargo.toml @@ -17,6 +17,11 @@ num-traits = { version = "0.2.17", default-features = false } num-derive = "0.4.1" libc-print = "*" +[dependencies.xxhash-rust] +version = "0.8.17" +default-features = false +features = ["xxh3"] + [dev-dependencies] [profile.dev] diff --git a/core/rs/core/rust-toolchain.toml b/core/rs/core/rust-toolchain.toml index f5663da1b..bbba1c38b 100644 --- a/core/rs/core/rust-toolchain.toml +++ b/core/rs/core/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2023-10-05" \ No newline at end of file +channel = "nightly-2026-02-01" \ No newline at end of file diff --git a/core/rs/core/src/alter_v2.rs b/core/rs/core/src/alter_v2.rs new file mode 100644 index 000000000..439ebdc54 --- /dev/null +++ b/core/rs/core/src/alter_v2.rs @@ -0,0 +1,309 @@ +extern crate alloc; + +use alloc::boxed::Box; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec; +use alloc::vec::Vec; +use core::ffi::{c_char, c_int, CStr}; +use sqlite_nostd::{sqlite3, Connection, ResultCode}; + +use crate::c::crsql_ExtData; +use crate::consts; +use crate::tableinfo::{crsql_ensure_table_infos_are_up_to_date, TableInfo, SchemaVersion}; +use core::mem; + +/// Compact V2 metadata tables after an ALTER TABLE operation. +/// Handles: +/// - New columns: add to v2_col_map +/// - Removed columns: remove from v2_col_map, delete clock entries +/// - PK changes: drop and recreate all V2 metadata tables + backfill +#[no_mangle] +pub unsafe extern "C" fn crsql_compact_post_alter_v2( + db: *mut sqlite3, + tbl_name: *const c_char, + ext_data: *mut crsql_ExtData, + errmsg: *mut *mut c_char, +) -> c_int { + match compact_post_alter_v2(db, tbl_name, ext_data, errmsg) { + Ok(rc) | Err(rc) => rc as c_int, + } +} + +unsafe fn compact_post_alter_v2( + db: *mut sqlite3, + tbl_name: *const c_char, + ext_data: *mut crsql_ExtData, + errmsg: *mut *mut c_char, +) -> Result { + let tbl_name_str = CStr::from_ptr(tbl_name).to_str()?; + let escaped = crate::util::escape_ident(tbl_name_str); + let tbl_name_val = crate::util::escape_ident_as_value(tbl_name_str); + + // Ensure table infos are up to date so we can detect schema changes + let c_rc = crsql_ensure_table_infos_are_up_to_date(db, ext_data, errmsg); + if c_rc != ResultCode::OK as c_int { + return Err(ResultCode::ERROR); + } + + let table_infos = + mem::ManuallyDrop::new(Box::from_raw((*ext_data).tableInfos as *mut Vec)); + let tbl_info = table_infos.iter().find(|x| x.tbl_name == tbl_name_str); + if tbl_info.is_none() { + return Err(ResultCode::ERROR); + } + let tbl_info = tbl_info.unwrap(); + + // Only handle V2 tables + if tbl_info.schema_version != SchemaVersion::V2 && tbl_info.schema_version != SchemaVersion::V2AndV1 { + return Ok(ResultCode::OK); + } + + // Check if PK columns changed by comparing current schema with v2_pks columns + let pk_changed = check_pk_changed_v2(db, tbl_name_str, &escaped)?; + + if pk_changed { + // PK change: drop and recreate all V2 metadata tables, then backfill + crate::bootstrap_v2::drop_v2_tables(db, tbl_name_str)?; + crate::bootstrap_v2::create_v2_tables(db, tbl_info)?; + crate::backfill_v2::backfill_table_v2(db, tbl_name_str, &tbl_info.pks, &tbl_info.non_pks, tbl_info.uses_rowid_key, &tbl_info.rowid_alias, false)?; + } else { + // Sync col_map with current schema + sync_col_map_v2(db, &escaped, tbl_name_str, tbl_info, ext_data)? + } + + Ok(ResultCode::OK) +} + +/// Check if PK columns changed by comparing pragma_table_info with v2_pks schema +unsafe fn check_pk_changed_v2( + db: *mut sqlite3, + tbl_name: &str, + escaped: &str, +) -> Result { + // Check if v2_pks table exists + let stmt = db.prepare_v2(&format!( + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='{}{}'\0", + escaped, consts::V2_PKS_SUFFIX + ))?; + stmt.step()?; + let exists = stmt.column_int(0); + drop(stmt); + + if exists == 0 { + // No V2 tables yet, nothing to compare + return Ok(false); + } + + // For rowid-key tables, v2_pks uses __crsql_key (not the actual PK column name). + // Check if v2_pks has __crsql_key as a column — if so, it's a rowid-key table + // and PK changes are detected by comparing the rowid alias, not column names. + let rowid_check = db.prepare_v2(&format!( + "SELECT count(*) FROM pragma_table_info('{escaped}{pks_suffix}') WHERE name = '__crsql_key'\0", + escaped = escaped, + pks_suffix = consts::V2_PKS_SUFFIX, + ))?; + rowid_check.step()?; + let is_rowid_key = rowid_check.column_int(0) > 0; + drop(rowid_check); + + if is_rowid_key { + // Rowid-key table: v2_pks only has __crsql_key, hashed_pk, cl. + // PK change = the rowid alias column changed. + // We detect this by checking if the current PK column count is still 1 + // and it's an INTEGER PRIMARY KEY (rowid alias). + // For simplicity, just check PK column count hasn't changed. + let stmt = db.prepare_v2(&format!( + "SELECT count(*) FROM pragma_table_info('{tbl_name_val}') WHERE pk > 0\0", + tbl_name_val = crate::util::escape_ident_as_value(tbl_name), + ))?; + stmt.step()?; + let pk_count = stmt.column_int(0); + // Rowid-key tables always have exactly 1 PK column + return Ok(pk_count != 1); + } + + // Non-rowid tables: compare PK column names directly + let stmt = db.prepare_v2(&format!( + "SELECT count(*) FROM ( + SELECT name FROM pragma_table_info('{tbl_name_val}') WHERE pk > 0 + EXCEPT + SELECT name FROM pragma_table_info('{escaped}{pks_suffix}') + WHERE name NOT IN ('__crsql_key', 'hashed_pk', 'cl') + ) UNION ALL + SELECT count(*) FROM ( + SELECT name FROM pragma_table_info('{escaped}{pks_suffix}') + WHERE name NOT IN ('__crsql_key', 'hashed_pk', 'cl') + EXCEPT + SELECT name FROM pragma_table_info('{tbl_name_val}') WHERE pk > 0 + )\0", + tbl_name_val = crate::util::escape_ident_as_value(tbl_name), + escaped = escaped, + pks_suffix = consts::V2_PKS_SUFFIX, + ))?; + stmt.step()?; + let diff = stmt.column_int(0); + Ok(diff > 0) +} + +/// Sync v2_col_map with the current table schema. +/// Adds new non-PK columns and removes deleted ones. +unsafe fn sync_col_map_v2( + db: *mut sqlite3, + escaped: &str, + tbl_name: &str, + tbl_info: &TableInfo, + ext_data: *mut crsql_ExtData, +) -> Result<(), ResultCode> { + // Get current columns in col_map + let stmt = db.prepare_v2(&format!( + "SELECT col_id, col_name FROM \"{}{}\"\0", + escaped, consts::V2_COL_MAP_SUFFIX + ))?; + + let mut existing_cols: Vec<(i64, String)> = vec![]; + while stmt.step()? == ResultCode::ROW { + existing_cols.push((stmt.column_int64(0), stmt.column_text(1)?.to_string())); + } + drop(stmt); + + // Current non-PK column names from schema + let current_names: Vec = tbl_info.non_pks.iter().map(|c| c.name.clone()).collect(); + let current_set: Vec<&str> = current_names.iter().map(|s| s.as_str()).collect(); + + // Remove deleted columns from col_map. + // If the table will become PK-only, migrate the last dropped column's clock + // entries to col_id=0 (preserving db_version, seq, ts, site_id) instead of + // deleting them. Other dropped columns' clock entries are deleted normally. + let will_be_pk_only = tbl_info.non_pks.is_empty(); + let mut dropped_col_ids: Vec = vec![]; + for (col_id, col_name) in &existing_cols { + if !current_set.contains(&col_name.as_str()) { + dropped_col_ids.push(*col_id); + // Delete from col_map + let stmt = db.prepare_v2(&format!( + "DELETE FROM \"{}{}\" WHERE col_id = ?\0", + escaped, consts::V2_COL_MAP_SUFFIX + ))?; + stmt.bind_int64(1, *col_id)?; + stmt.step()?; + drop(stmt); + } + } + + if will_be_pk_only && !dropped_col_ids.is_empty() { + // Migrate the last dropped column's clock entries to col_id=0. + // This preserves the row modification history (db_version, seq, ts, site_id). + let migrate_col_id = *dropped_col_ids.last().unwrap(); + let col_id_mask = consts::CRSQL_COL_ID_MASK as i64; + + if migrate_col_id != 0 { + // Update cell_key: replace col_id part with 0 for only this column's entries + // cell_key = (pk_key << COL_ID_BITS) | col_id + // new_cell_key = (cell_key >> COL_ID_BITS) << COL_ID_BITS = cell_key & ~col_id_mask + let stmt = db.prepare_v2(&format!( + "UPDATE \"{}{}\" SET cell_key = cell_key & ~{} WHERE cell_key & {} = ?\0", + escaped, consts::V2_CLOCK_SUFFIX, col_id_mask, col_id_mask + ))?; + stmt.bind_int64(1, migrate_col_id)?; + stmt.step()?; + drop(stmt); + } + + // Delete clock entries for all other dropped columns (not the migrated one) + for col_id in &dropped_col_ids { + if *col_id != migrate_col_id { + let stmt = db.prepare_v2(&format!( + "DELETE FROM \"{}{}\" WHERE cell_key & {} = ?\0", + escaped, consts::V2_CLOCK_SUFFIX, col_id_mask + ))?; + stmt.bind_int64(1, *col_id)?; + stmt.step()?; + drop(stmt); + } + } + } else { + // Normal case: delete clock entries for all dropped columns + let col_id_mask = consts::CRSQL_COL_ID_MASK; + for col_id in &dropped_col_ids { + let stmt = db.prepare_v2(&format!( + "DELETE FROM \"{}{}\" WHERE cell_key & {} = ?\0", + escaped, consts::V2_CLOCK_SUFFIX, col_id_mask + ))?; + stmt.bind_int64(1, *col_id)?; + stmt.step()?; + drop(stmt); + } + } + + // Add new columns to col_map. + // Always try col_id=0 first (important for PK-only → normal transition + // where sentinel entries at col_id=0 become regular clock entries). + let existing_names: Vec = existing_cols.iter().map(|(_, n)| n.clone()).collect(); + let remaining_col_ids: Vec = existing_cols.iter() + .filter(|(_, n)| current_set.contains(&n.as_str())) + .map(|(id, _)| *id) + .collect(); + let has_col_id_0 = remaining_col_ids.contains(&0); + let max_col_id = remaining_col_ids.iter().max().copied().unwrap_or(-1); + + let mut next_col_id: i64 = if !has_col_id_0 { 0 } else { max_col_id + 1 }; + for col in &tbl_info.non_pks { + if !existing_names.contains(&col.name) { + let stmt = db.prepare_v2(&format!( + "INSERT INTO \"{}{}\" (col_id, col_name) VALUES (?, ?)\0", + escaped, consts::V2_COL_MAP_SUFFIX + ))?; + stmt.bind_int64(1, next_col_id)?; + stmt.bind_text(2, &col.name, sqlite_nostd::Destructor::STATIC)?; + stmt.step()?; + drop(stmt); + // After using slot 0, continue from max_col_id + 1 + if next_col_id == 0 && !has_col_id_0 { + next_col_id = max_col_id + 1; + } else { + next_col_id += 1; + } + } + } + + // If the table became PK-only, create sentinel clock entries at col_id=0 + // for any rows in v2_pks that don't already have one (e.g., rows that had + // no clock entries at all before the column was dropped). + if will_be_pk_only { + let col_id_bits = consts::CRSQL_COL_ID_BITS as i64; + let db_version = crate::db_version::next_db_version(db, ext_data)?; + let ts_val = { + let ts_str = (*ext_data).timestamp.to_string(); + let ts = ts_str.parse::().map_err(|_| ResultCode::ERROR)?; + if ts == 0 { + crate::debug::debug_log("commit_alter: timestamp not set — call crsql_set_ts() first"); + return Err(ResultCode::ERROR); + } + ts + }; + let seq = (*ext_data).seq; + + let sql = format!( + "INSERT OR IGNORE INTO \"{}{}\" (cell_key, col_version, site_id, db_version, seq, ts) + SELECT (p.__crsql_key << {}) | 0, 1, 0, ?, ?, ? + FROM \"{}{}\" p + WHERE NOT EXISTS ( + SELECT 1 FROM \"{}{}\" c + WHERE c.cell_key = (p.__crsql_key << {}) | 0 + )\0", + escaped, consts::V2_CLOCK_SUFFIX, + col_id_bits, + escaped, consts::V2_PKS_SUFFIX, + escaped, consts::V2_CLOCK_SUFFIX, + col_id_bits, + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, db_version)?; + stmt.bind_int(2, seq)?; + stmt.bind_int64(3, ts_val)?; + stmt.step()?; + } + + Ok(()) +} diff --git a/core/rs/core/src/automigrate.rs b/core/rs/core/src/automigrate.rs index 464e413c5..c8957f842 100644 --- a/core/rs/core/src/automigrate.rs +++ b/core/rs/core/src/automigrate.rs @@ -70,21 +70,27 @@ fn automigrate_impl( let result = sqlite::open(strlit!(":memory:")); if let Ok(mem_db) = result { - if let Err(_) = mem_db.exec_safe(&stripped_schema) { - let mem_db_err_msg = mem_db.errmsg()?; - ctx.result_error(&mem_db_err_msg); + if let Err(e) = mem_db.exec_safe(&stripped_schema) { + let mem_db_err_msg = mem_db.errmsg().unwrap_or_else(|_| format!("exec failed: {:?}", e)); + ctx.result_error(&format!("automigrate: mem_db exec stripped_schema failed: {}", mem_db_err_msg)); ctx.result_error_code(mem_db.errcode()); cleanup(mem_db)?; return Err(ResultCode::OK); } - local_db.exec_safe("SAVEPOINT automigrate_tables;")?; + if let Err(e) = local_db.exec_safe("SAVEPOINT automigrate_tables;") { + let errmsg = local_db.errmsg().unwrap_or_else(|_| format!("SAVEPOINT failed: {:?}", e)); + ctx.result_error(&format!("automigrate: SAVEPOINT failed: {}", errmsg)); + cleanup(mem_db)?; + return Err(ResultCode::OK); + } + local_db.exec_safe("SELECT crsql_set_ts('1700000000')")?; let migrate_result = migrate_to(local_db, &mem_db); - if let Err(_) = migrate_result { - local_db.exec_safe("ROLLBACK")?; - let mem_db_err_msg = mem_db.errmsg()?; - ctx.result_error(&mem_db_err_msg); + if let Err(e) = migrate_result { + let _ = local_db.exec_safe("ROLLBACK"); + let mem_db_err_msg = mem_db.errmsg().unwrap_or_else(|_| format!("migrate_to failed: {:?}", e)); + ctx.result_error(&format!("automigrate: migrate_to failed: {}", mem_db_err_msg)); ctx.result_error_code(mem_db.errcode()); cleanup(mem_db)?; return Err(ResultCode::OK); @@ -93,9 +99,20 @@ fn automigrate_impl( } if !desired_schema.is_empty() { - local_db.exec_safe(desired_schema)?; + let schema_with_ts = format!("SELECT crsql_set_ts('1700000000');\n{}", desired_schema); + if let Err(e) = local_db.exec_safe(&schema_with_ts) { + let errmsg = local_db.errmsg().unwrap_or_else(|_| format!("exec failed: {:?}", e)); + ctx.result_error(&format!("automigrate: exec desired_schema failed: {}", errmsg)); + let _ = local_db.exec_safe("ROLLBACK"); + return Err(ResultCode::OK); + } + } + if let Err(e) = local_db.exec_safe("RELEASE automigrate_tables") { + let errmsg = local_db.errmsg().unwrap_or_else(|_| format!("RELEASE failed: {:?}", e)); + ctx.result_error(&format!("automigrate: RELEASE failed: {}", errmsg)); + return Err(ResultCode::OK); } - local_db.exec_safe("RELEASE automigrate_tables") + Ok(ResultCode::OK) } else { ctx.result_error("could not open the temporary migration db"); ctx.result_error_code(ResultCode::CANTOPEN); @@ -161,6 +178,7 @@ fn strip_crr_statements(schema: &str) -> String { .filter(|line| { !line.to_lowercase().contains("crsql_as_crr") && !line.to_lowercase().contains("crsql_fract_as_ordered") + && !line.to_lowercase().contains("crsql_set_ts") }) .collect::>() .join("\n") diff --git a/core/rs/core/src/backfill_v2.rs b/core/rs/core/src/backfill_v2.rs new file mode 100644 index 000000000..7cfdee545 --- /dev/null +++ b/core/rs/core/src/backfill_v2.rs @@ -0,0 +1,173 @@ +extern crate alloc; + +use alloc::format; +use alloc::vec::Vec; + +use sqlite_nostd as sqlite; +use sqlite_nostd::{sqlite3, Connection, Destructor, ResultCode}; + +use crate::consts; +use crate::hash_pk::hash_pk_values; +use crate::tableinfo::ColumnInfo; + +/// Backfill V2 tables for a newly registered V2 CRR. +/// For each row in the main table not yet in v2_pks: +/// 1. Compute hashed_pk from PK values. +/// 2. INSERT into v2_pks with cl=1. +/// 3. For each non-PK column, INSERT into v2_clock. +pub fn backfill_table_v2( + db: *mut sqlite3, + table: &str, + pk_cols: &Vec, + non_pk_cols: &Vec, + uses_rowid_key: bool, + rowid_alias: &str, + no_tx: bool, +) -> Result { + // V2 clock tables require a non-zero ts. Error early if not set. + let ts_check = db.prepare_v2("SELECT crsql_get_ts()\0"); + if let Ok(stmt) = ts_check { + if stmt.step().is_err() { + crate::debug::debug_log("backfill_table_v2: timestamp not set — call crsql_set_ts() first"); + return Err(ResultCode::ERROR); + } + } + + if !no_tx { + db.exec_safe("SAVEPOINT backfill_v2")?; + } + + let escaped = crate::util::escape_ident(table); + let pk_cols_list = pk_cols.iter() + .map(|f| format!("\"{}\"", crate::util::escape_ident(&f.name))) + .collect::>() + .join(", "); + + // Find rows in main table not yet in v2_pks + // For rowid-key tables, compare rowid alias vs __crsql_key, but also select PK cols for hashing + let sql = if uses_rowid_key { + format!( + "SELECT t1.\"{alias}\", {pk_cols} FROM \"{table}\" AS t1 + WHERE t1.\"{alias}\" NOT IN (SELECT __crsql_key FROM \"{table}{pks_suffix}\")", + alias = crate::util::escape_ident(rowid_alias), + table = escaped, + pk_cols = pk_cols_list, + pks_suffix = consts::V2_PKS_SUFFIX, + ) + } else { + format!( + "SELECT {pk_cols} FROM \"{table}\" AS t1 + EXCEPT SELECT {pk_cols} FROM \"{table}{pks_suffix}\" AS t2", + table = escaped, + pk_cols = pk_cols_list, + pks_suffix = consts::V2_PKS_SUFFIX, + ) + }; + let read_stmt = db.prepare_v2(&sql)?; + + // Prepare insert into v2_pks + let insert_pks_sql = if uses_rowid_key { + format!( + "INSERT INTO \"{escaped}{suffix}\" (__crsql_key, hashed_pk, cl) VALUES (?, ?, 1) RETURNING __crsql_key", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + ) + } else { + let pk_values = pk_cols.iter() + .map(|_| "?") + .collect::>() + .join(", "); + format!( + "INSERT INTO \"{escaped}{suffix}\" ({pk_cols}, hashed_pk, cl) VALUES ({pk_values}, ?, 1) RETURNING __crsql_key", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + pk_cols = pk_cols_list, + pk_values = pk_values, + ) + }; + let insert_pks_stmt = db.prepare_v2(&insert_pks_sql)?; + + // Prepare insert into v2_clock + let clock_sql = format!( + "INSERT OR REPLACE INTO \"{escaped}{suffix}\" (cell_key, col_version, site_id, db_version, seq, ts) VALUES (?, 1, 0, crsql_next_db_version(), crsql_increment_and_get_seq(), crsql_get_ts())", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + ); + let clock_stmt = db.prepare_v2(&clock_sql)?; + + let result = (|| { + while read_stmt.step()? == ResultCode::ROW { + // For rowid-key tables: col 0 = rowid, cols 1..n = PK columns + // For non-rowid tables: cols 0..n = PK columns + let (rowid, pk_values) = if uses_rowid_key { + let r = read_stmt.column_int64(0); + let pks: Vec<*mut sqlite::value> = (0..pk_cols.len()) + .map(|i| read_stmt.column_value(i as i32 + 1)) + .collect::, _>>()?; + (r, pks) + } else { + let pks: Vec<*mut sqlite::value> = (0..pk_cols.len()) + .map(|i| read_stmt.column_value(i as i32)) + .collect::, _>>()?; + (0i64, pks) + }; + + // Compute hashed_pk from PK values (not rowid) + let hashed_pk = hash_pk_values(&pk_values)?; + + // Insert into v2_pks + if uses_rowid_key { + // __crsql_key = rowid, hashed_pk = hash(PK values) + insert_pks_stmt.bind_int64(1, rowid)?; + insert_pks_stmt.bind_blob(2, &hashed_pk, Destructor::STATIC)?; + } else { + for (i, val) in pk_values.iter().enumerate() { + insert_pks_stmt.bind_value(i as i32 + 1, *val)?; + } + insert_pks_stmt.bind_blob(pk_values.len() as i32 + 1, &hashed_pk, Destructor::STATIC)?; + } + + match insert_pks_stmt.step()? { + ResultCode::ROW => { + let key = insert_pks_stmt.column_int64(0); + insert_pks_stmt.reset()?; + + // Insert clock entries for each non-PK column + for (col_id, _col) in non_pk_cols.iter().enumerate() { + let cell_key = (key << consts::CRSQL_COL_ID_BITS as i64) | col_id as i64; + clock_stmt.bind_int64(1, cell_key)?; + clock_stmt.step()?; + clock_stmt.reset()?; + } + + // For PK-only tables, write a sentinel clock entry at col_id=0 + if non_pk_cols.is_empty() { + let cell_key = (key << consts::CRSQL_COL_ID_BITS as i64) | 0; + clock_stmt.bind_int64(1, cell_key)?; + clock_stmt.step()?; + clock_stmt.reset()?; + } + } + _ => { + insert_pks_stmt.reset()?; + } + } + } + Ok(ResultCode::OK) + })(); + + read_stmt.reset()?; + + if let Err(e) = result { + if !no_tx { + db.exec_safe("ROLLBACK")?; + } + return Err(e); + } + + if !no_tx { + db.exec_safe("RELEASE backfill_v2") + } else { + Ok(ResultCode::OK) + } +} diff --git a/core/rs/core/src/bootstrap.rs b/core/rs/core/src/bootstrap.rs index 75918d567..e14685a79 100644 --- a/core/rs/core/src/bootstrap.rs +++ b/core/rs/core/src/bootstrap.rs @@ -184,6 +184,14 @@ fn maybe_update_db_inner( return Err(ResultCode::ERROR); } } + // TODO(0.19): Disallow opening 0.17 DBs (recorded_version < CRSQLITE_VERSION_0_18_0). + // Only accept fully migrated 0.18 DBs (all CRRs migrated to V2 metadata, no V1 tables). + // 0.19 will drop V1 metadata + V1 wire format entirely, so any DB still on V1 + // metadata must be migrated on 0.18 before upgrading. + + // Validate compile-time format constants against stored values. + // If stored: refuse to operate if they differ. If not stored: store them. + validate_or_store_compile_constants(db, err_msg)?; // write the db version if we migrated to a new one or we are a blank slate db if recorded_version < consts::CRSQLITE_VERSION || is_blank_slate { @@ -196,6 +204,50 @@ fn maybe_update_db_inner( Ok(ResultCode::OK) } +/// Compile-time constants that affect on-disk data format. +/// Stored in crsql_master on first load and validated on subsequent loads. +const COMPILE_CONST_KEYS: &[(&str, i32)] = &[ + ("crsql_col_id_bits", consts::CRSQL_COL_ID_BITS as i32), + ("crsql_pk_hash_size", consts::PK_HASH_SIZE as i32), + ("crsql_site_id_len", consts::SITE_ID_LEN), +]; + +fn validate_or_store_compile_constants( + db: *mut sqlite3, + err_msg: *mut *mut c_char, +) -> Result { + for (key, compile_val) in COMPILE_CONST_KEYS { + let stmt = db.prepare_v2(&format!( + "SELECT value FROM crsql_master WHERE key = '{}'", + key + ))?; + let step_result = stmt.step()?; + if step_result == ResultCode::ROW { + let stored_val = stmt.column_int(0); + if stored_val != *compile_val { + let cstring = CString::new(format!( + "cr-sqlite compile-time constant mismatch: '{}' was compiled as {} but the database expects {}. \ + The on-disk data format is incompatible with this build of the extension.", + key, compile_val, stored_val + ))?; + unsafe { + (*err_msg) = cstring.into_raw(); + } + return Err(ResultCode::ERROR); + } + } else { + // Key not stored yet (blank slate or upgrading from 0.17): store the compile-time value + let stmt = db.prepare_v2(&format!( + "INSERT OR REPLACE INTO crsql_master VALUES ('{}', ?)", + key + ))?; + stmt.bind_int(1, *compile_val)?; + stmt.step()?; + } + } + Ok(ResultCode::OK) +} + /** * The clock table holds the versions for each column of a given row. * diff --git a/core/rs/core/src/bootstrap_v2.rs b/core/rs/core/src/bootstrap_v2.rs new file mode 100644 index 000000000..c62c3a4b8 --- /dev/null +++ b/core/rs/core/src/bootstrap_v2.rs @@ -0,0 +1,230 @@ +extern crate alloc; + +use alloc::format; +use alloc::string::String; + +use sqlite_nostd as sqlite; +use sqlite_nostd::{Connection, ResultCode}; + +use crate::tableinfo::TableInfo; +use crate::consts; + +/// Create all V2 metadata tables for a given CRR table. +/// Tables created: +/// 1. __crsql_v2_col_map — col_name → col_id mapping +/// 2.
__crsql_v2_clock — packed clock (cells only) +/// 3.
__crsql_v2_pks — alive PKs with hash + CL +/// 4.
__crsql_v2_tombstones — dead PKs with hash only +/// 5.
__crsql_v2_tombstone_pks — hash → original PK cols (V1 compat) +pub fn create_v2_tables( + db: *mut sqlite::sqlite3, + table_info: &TableInfo, +) -> Result { + let table_name = &table_info.tbl_name; + let escaped = crate::util::escape_ident(table_name); + + // 1. Column map + db.exec_safe(&format!( + "CREATE TABLE IF NOT EXISTS \"{escaped}{suffix}\" ( + col_id INTEGER PRIMARY KEY, + col_name TEXT NOT NULL + ) STRICT;", + escaped = escaped, + suffix = consts::V2_COL_MAP_SUFFIX + ))?; + + db.exec_safe(&format!( + "CREATE UNIQUE INDEX IF NOT EXISTS \"idx_{escaped}_v2_col_map_name\" + ON \"{escaped}{suffix}\"(col_name);", + escaped = escaped, + suffix = consts::V2_COL_MAP_SUFFIX + ))?; + + // 2. Clock table + db.exec_safe(&format!( + "CREATE TABLE IF NOT EXISTS \"{escaped}{suffix}\" ( + cell_key INTEGER PRIMARY KEY, + col_version INTEGER NOT NULL, + site_id INTEGER NOT NULL, + db_version INTEGER NOT NULL, + seq INTEGER NOT NULL, + ts INTEGER NOT NULL CHECK (ts > 0) + ) STRICT;", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + ))?; + + db.exec_safe(&format!( + "CREATE INDEX IF NOT EXISTS \"{escaped}{suffix}_feed_idx\" + ON \"{escaped}{suffix}\"(site_id, db_version, seq);", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + ))?; + + // 3. Alive PKs + // If uses_rowid_key (single INTEGER PRIMARY KEY on rowid table), __crsql_key = rowid. + // No need to store PK columns separately since rowid IS the PK. + // STRICT always used. For non-rowid tables, dynamic PK columns use ANY type + // (https://www.sqlite.org/stricttables.html) so STRICT works regardless of base table type. + if table_info.uses_rowid_key { + db.exec_safe(&format!( + "CREATE TABLE IF NOT EXISTS \"{escaped}{suffix}\" ( + __crsql_key INTEGER PRIMARY KEY, + hashed_pk BLOB NOT NULL, + cl INTEGER NOT NULL DEFAULT 1 CHECK (cl % 2 = 1) + ) STRICT;", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + ))?; + } else { + let mut pk_cols_sql = String::new(); + for (i, pk) in table_info.pks.iter().enumerate() { + if i > 0 { pk_cols_sql.push_str(", "); } + pk_cols_sql.push_str(&format!( + "\"{}\" ANY NOT NULL", + crate::util::escape_ident(&pk.name) + )); + } + + let create_sql = format!( + "CREATE TABLE IF NOT EXISTS \"{escaped}{suffix}\" ( + __crsql_key INTEGER PRIMARY KEY, + {pk_cols}, + hashed_pk BLOB NOT NULL, + cl INTEGER NOT NULL DEFAULT 1 CHECK (cl % 2 = 1) + ) STRICT;", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + pk_cols = pk_cols_sql + ); + db.exec_safe(&create_sql)?; + } + + let index_sql = format!( + "CREATE UNIQUE INDEX IF NOT EXISTS \"idx_{escaped}_v2_pks_hash\" + ON \"{escaped}{suffix}\"(hashed_pk);", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX + ); + db.exec_safe(&index_sql)?; + + // 4. Tombstones + db.exec_safe(&format!( + "CREATE TABLE IF NOT EXISTS \"{escaped}{suffix}\" ( + site_id INTEGER NOT NULL, + db_version INTEGER NOT NULL, + seq INTEGER NOT NULL, + hashed_pk BLOB NOT NULL, + cl INTEGER NOT NULL CHECK (cl % 2 = 0), + ts INTEGER NOT NULL CHECK (ts > 0), + PRIMARY KEY (site_id, db_version, seq) + ) WITHOUT ROWID, STRICT;", + escaped = escaped, + suffix = consts::V2_TOMBSTONES_SUFFIX + ))?; + + db.exec_safe(&format!( + "CREATE UNIQUE INDEX IF NOT EXISTS \"idx_{escaped}_v2_tombstones_hash\" + ON \"{escaped}{suffix}\"(hashed_pk);", + escaped = escaped, + suffix = consts::V2_TOMBSTONES_SUFFIX + ))?; + + // 5. Tombstone PKs (V1 compat) + // STRICT always; dynamic PK columns use ANY type. + let mut tombstone_pk_cols_sql = String::new(); + for (i, pk) in table_info.pks.iter().enumerate() { + if i > 0 { tombstone_pk_cols_sql.push_str(", "); } + tombstone_pk_cols_sql.push_str(&format!( + "\"{}\" ANY NOT NULL", + crate::util::escape_ident(&pk.name) + )); + } + + let tombstone_pks_sql = format!( + "CREATE TABLE IF NOT EXISTS \"{escaped}{suffix}\" ( + hashed_pk BLOB PRIMARY KEY, + {pk_cols} + ) WITHOUT ROWID, STRICT;", + escaped = escaped, + suffix = consts::V2_TOMBSTONE_PKS_SUFFIX, + pk_cols = tombstone_pk_cols_sql + ); + db.exec_safe(&tombstone_pks_sql)?; + + // Populate col_map with existing non-PK columns + populate_col_map(db, table_info)?; + + Ok(ResultCode::OK) +} + +/// Populate the col_map table with column names from the table info. +/// Assigns 0-based col_id to each non-PK column. +fn populate_col_map( + db: *mut sqlite::sqlite3, + table_info: &TableInfo, +) -> Result { + let escaped = crate::util::escape_ident(&table_info.tbl_name); + let stmt = db.prepare_v2(&format!( + "INSERT OR IGNORE INTO \"{escaped}{suffix}\" (col_id, col_name) VALUES (?, ?)", + escaped = escaped, + suffix = consts::V2_COL_MAP_SUFFIX + ))?; + + for (i, col) in table_info.non_pks.iter().enumerate() { + stmt.bind_int(1, i as i32)?; + stmt.bind_text(2, &col.name, sqlite::Destructor::STATIC)?; + stmt.step()?; + stmt.reset()?; + } + + Ok(ResultCode::OK) +} + +/// Drop all V2 metadata tables for a given table. +pub fn drop_v2_tables( + db: *mut sqlite::sqlite3, + table: &str, +) -> Result { + let escaped = crate::util::escape_ident(table); + + db.exec_safe(&format!( + "DROP TABLE IF EXISTS \"{escaped}{suffix}\"", + escaped = escaped, + suffix = consts::V2_COL_MAP_SUFFIX + ))?; + db.exec_safe(&format!( + "DROP TABLE IF EXISTS \"{escaped}{suffix}\"", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + ))?; + db.exec_safe(&format!( + "DROP TABLE IF EXISTS \"{escaped}{suffix}\"", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX + ))?; + db.exec_safe(&format!( + "DROP TABLE IF EXISTS \"{escaped}{suffix}\"", + escaped = escaped, + suffix = consts::V2_TOMBSTONES_SUFFIX + ))?; + db.exec_safe(&format!( + "DROP TABLE IF EXISTS \"{escaped}{suffix}\"", + escaped = escaped, + suffix = consts::V2_TOMBSTONE_PKS_SUFFIX + )) +} + +/// Check if V2 tables exist for a given table. +pub fn has_v2_tables( + db: *mut sqlite::sqlite3, + table: &str, +) -> Result { + let stmt = db.prepare_v2(&format!( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND tbl_name = '{escaped}{suffix}'", + escaped = crate::util::escape_ident_as_value(table), + suffix = consts::V2_CLOCK_SUFFIX + ))?; + let rc = stmt.step()?; + Ok(rc == ResultCode::ROW) +} diff --git a/core/rs/core/src/c.rs b/core/rs/core/src/c.rs index b9d59e440..36bde09e9 100644 --- a/core/rs/core/src/c.rs +++ b/core/rs/core/src/c.rs @@ -38,13 +38,18 @@ pub enum ClockUnionColumn { Seq = 7, Cl = 8, Ts = 9, + Cval = 10, } +// TODO(0.19): Remove ChangeRowType entirely once V1 wire format is dropped. +// In V2-only mode, all rows are packed and the row type discrimination +// (Update vs PackedUpdate, lazy cval fetch, etc.) is no longer needed. #[derive(FromPrimitive, PartialEq, Debug)] pub enum ChangeRowType { Update = 0, Delete = 1, PkOnly = 2, + PackedUpdate = 3, } #[repr(C)] @@ -72,6 +77,9 @@ pub struct crsql_ExtData { pub pSelectSiteIdOrdinalStmt: *mut sqlite::stmt, pub pSelectClockTablesStmt: *mut sqlite::stmt, pub mergeEqualValues: ::core::ffi::c_int, + pub metadataWriteVersion: ::core::ffi::c_int, + pub metadataUseVersion: ::core::ffi::c_int, + pub syncLogVersion: ::core::ffi::c_int, pub timestamp: ::core::ffi::c_ulonglong, pub ordinalMap: *mut ::core::ffi::c_void, } @@ -271,7 +279,7 @@ fn bindgen_test_layout_crsql_ExtData() { let ptr = UNINIT.as_ptr(); assert_eq!( ::core::mem::size_of::(), - 168usize, + 176usize, concat!("Size of: ", stringify!(crsql_ExtData)) ); assert_eq!( @@ -492,8 +500,38 @@ fn bindgen_test_layout_crsql_ExtData() { ) ); assert_eq!( - unsafe { ::core::ptr::addr_of!((*ptr).timestamp) as usize - ptr as usize }, + unsafe { ::core::ptr::addr_of!((*ptr).metadataWriteVersion) as usize - ptr as usize }, + 148usize, + concat!( + "Offset of field: ", + stringify!(crsql_ExtData), + "::", + stringify!(metadataWriteVersion) + ) + ); + assert_eq!( + unsafe { ::core::ptr::addr_of!((*ptr).metadataUseVersion) as usize - ptr as usize }, 152usize, + concat!( + "Offset of field: ", + stringify!(crsql_ExtData), + "::", + stringify!(metadataUseVersion) + ) + ); + assert_eq!( + unsafe { ::core::ptr::addr_of!((*ptr).syncLogVersion) as usize - ptr as usize }, + 156usize, + concat!( + "Offset of field: ", + stringify!(crsql_ExtData), + "::", + stringify!(syncLogVersion) + ) + ); + assert_eq!( + unsafe { ::core::ptr::addr_of!((*ptr).timestamp) as usize - ptr as usize }, + 160usize, concat!( "Offset of field: ", stringify!(crsql_ExtData), @@ -503,7 +541,7 @@ fn bindgen_test_layout_crsql_ExtData() { ); assert_eq!( unsafe { ::core::ptr::addr_of!((*ptr).ordinalMap) as usize - ptr as usize }, - 160usize, + 168usize, concat!( "Offset of field: ", stringify!(crsql_ExtData), diff --git a/core/rs/core/src/changes_vtab.rs b/core/rs/core/src/changes_vtab.rs index 47e760b79..39f6a531e 100644 --- a/core/rs/core/src/changes_vtab.rs +++ b/core/rs/core/src/changes_vtab.rs @@ -21,6 +21,7 @@ use sqlite_nostd::ResultCode; use crate::c::{ crsql_Changes_cursor, crsql_Changes_vtab, ChangeRowType, ClockUnionColumn, CrsqlChangesColumn, }; +use crate::consts; use crate::changes_vtab_read::changes_union_query; use crate::pack_columns::bind_package_to_stmt; use crate::pack_columns::unpack_columns; @@ -289,7 +290,9 @@ unsafe fn changes_filter( return Ok(ResultCode::OK); } - let sql = changes_union_query(&tbl_infos, idx_str)?; + let metadata_use_version = unsafe { (*(*tab).pExtData).metadataUseVersion }; + let sync_log_version = unsafe { (*(*tab).pExtData).syncLogVersion }; + let sql = changes_union_query(&tbl_infos, idx_str, metadata_use_version, sync_log_version)?; let stmt = db.prepare_v2(&sql)?; for (i, arg) in args.iter().enumerate() { @@ -387,16 +390,29 @@ unsafe fn changes_next( return Err(ResultCode::ERROR); } - if cid == crate::c::DELETE_SENTINEL { + if cid == crate::c::DELETE_SENTINEL || cid == consts::V2_HASH_TOMBSTONE_CID { (*cursor).rowType = ChangeRowType::Delete as c_int; return Ok(ResultCode::OK); } else if cid == crate::c::INSERT_SENTINEL { (*cursor).rowType = ChangeRowType::PkOnly as c_int; return Ok(ResultCode::OK); } else { + let sync_log_version = (*(*(*cursor).pTab).pExtData).syncLogVersion; + if sync_log_version == crate::consts::SYNC_LOG_V2 { + // Packed (v2 sync-log) row: all update rows are packed in V2 wire format. + // cval is already in the query result (ClockUnionColumn::Cval), no lazy fetch needed. + (*cursor).rowType = ChangeRowType::PackedUpdate as c_int; + return Ok(ResultCode::OK); + } (*cursor).rowType = ChangeRowType::Update as c_int; } + // V2 metadata fetches cval inline in the query — no lazy fetch needed. + let metadata_use_version = (*(*(*cursor).pTab).pExtData).metadataUseVersion; + if metadata_use_version == crate::consts::META_USE_V2 { + return Ok(ResultCode::OK); + } + let row_stmt_ref = tbl_info.get_row_patch_data_stmt((*(*cursor).pTab).db, cid)?; let row_stmt = row_stmt_ref.as_ref().ok_or(ResultCode::ERROR)?; @@ -457,20 +473,62 @@ fn column_impl( ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Pks as i32)); } Some(CrsqlChangesColumn::Cval) => unsafe { - if (*cursor).pRowStmt.is_null() { - ctx.result_null(); - } else { - ctx.result_value((*cursor).pRowStmt.column_value(0)); + let row_type = ChangeRowType::from_i32((*cursor).rowType); + match row_type { + Some(ChangeRowType::PackedUpdate) => { + // Packed format: cval is in the query result directly. + // Copy blob data to a local Vec before passing to result_blob_transient + // to avoid use-after-free when other columns are read from the underlying stmt. + let val = changes_stmt.column_value(ClockUnionColumn::Cval as i32); + let blob_copy: Vec = val.blob().to_vec(); + ctx.result_blob_transient(&blob_copy); + } + Some(ChangeRowType::Update) => { + // V2 metadata: cval is inline in the query (pRowStmt is null, lazy fetch skipped). + // Use result_value to preserve the original type (text, int, float, blob). + // V1 metadata: cval is fetched lazily via pRowStmt. + if (*cursor).pRowStmt.is_null() { + let val = changes_stmt.column_value(ClockUnionColumn::Cval as i32); + ctx.result_value(val); + } else { + ctx.result_value((*cursor).pRowStmt.column_value(0)); + } + } + _ => { + // V1 metadata: lazy fetch from main table via pRowStmt + if (*cursor).pRowStmt.is_null() { + ctx.result_null(); + } else { + ctx.result_value((*cursor).pRowStmt.column_value(0)); + } + } } }, Some(CrsqlChangesColumn::Cid) => unsafe { let row_type = ChangeRowType::from_i32((*cursor).rowType); match row_type { Some(ChangeRowType::PkOnly) => ctx.result_text_static(crate::c::INSERT_SENTINEL), - Some(ChangeRowType::Delete) => ctx.result_text_static(crate::c::DELETE_SENTINEL), + Some(ChangeRowType::Delete) => { + // Could be -1 (V1 delete sentinel) or -2 (V2 hash tombstone) + ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Cid as i32)); + } + Some(ChangeRowType::PackedUpdate) => { + // Packed format: cid is group_concat with char(0) separators, cast to blob. + // Read as blob explicitly to preserve null-byte separators. + let blob_copy: Vec = changes_stmt.column_blob(ClockUnionColumn::Cid as i32).to_vec(); + ctx.result_blob_transient(&blob_copy); + } Some(ChangeRowType::Update) => { + // V2 metadata: pRowStmt is null (no lazy fetch), cid is in the query result. + // V1 metadata: pRowStmt is non-null if the row exists in the main table. + // If pRowStmt is null in V1, the row was deleted — return DELETE_SENTINEL. if (*cursor).pRowStmt.is_null() { - ctx.result_text_static(crate::c::DELETE_SENTINEL); + let metadata_use_version = (*(*(*cursor).pTab).pExtData).metadataUseVersion; + if metadata_use_version == crate::consts::META_USE_V2 { + ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Cid as i32)); + } else { + ctx.result_text_static(crate::c::DELETE_SENTINEL); + } } else { ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Cid as i32)); } @@ -478,8 +536,14 @@ fn column_impl( None => return Err(ResultCode::ABORT), } }, - Some(CrsqlChangesColumn::ColVrsn) => { - ctx.result_value(changes_stmt.column_value(ClockUnionColumn::ColVrsn as i32)); + Some(CrsqlChangesColumn::ColVrsn) => unsafe { + let row_type = ChangeRowType::from_i32((*cursor).rowType); + if row_type == Some(ChangeRowType::PackedUpdate) { + let blob_copy: Vec = changes_stmt.column_blob(ClockUnionColumn::ColVrsn as i32).to_vec(); + ctx.result_blob_transient(&blob_copy); + } else { + ctx.result_value(changes_stmt.column_value(ClockUnionColumn::ColVrsn as i32)); + } } Some(CrsqlChangesColumn::DbVrsn) => { ctx.result_value(changes_stmt.column_value(ClockUnionColumn::DbVrsn as i32)); @@ -489,8 +553,14 @@ fn column_impl( // sholdn't matter.. ctx.result_value(changes_stmt.column_value(ClockUnionColumn::SiteId as i32)); } - Some(CrsqlChangesColumn::Seq) => { - ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Seq as i32)); + Some(CrsqlChangesColumn::Seq) => unsafe { + let row_type = ChangeRowType::from_i32((*cursor).rowType); + if row_type == Some(ChangeRowType::PackedUpdate) { + let blob_copy: Vec = changes_stmt.column_blob(ClockUnionColumn::Seq as i32).to_vec(); + ctx.result_blob_transient(&blob_copy); + } else { + ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Seq as i32)); + } } Some(CrsqlChangesColumn::Cl) => { ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Cl as i32)) diff --git a/core/rs/core/src/changes_vtab_read.rs b/core/rs/core/src/changes_vtab_read.rs index 9debf7e19..56951b7ab 100644 --- a/core/rs/core/src/changes_vtab_read.rs +++ b/core/rs/core/src/changes_vtab_read.rs @@ -1,28 +1,24 @@ extern crate alloc; -use crate::tableinfo::TableInfo; +use crate::tableinfo::{TableInfo, SchemaVersion}; use alloc::format; -use alloc::string::String; +use alloc::string::{String, ToString}; use alloc::vec; use alloc::vec::Vec; use sqlite::ResultCode; use sqlite_nostd as sqlite; +use crate::consts; + +// Metadata version constants are in consts.rs + fn crsql_changes_query_for_table(table_info: &TableInfo) -> Result { if table_info.pks.is_empty() { - // no primary keys? We can't get changes for a table w/o primary keys... - // this should be an impossible case. return Err(ResultCode::ABORT); } let pk_list = crate::util::as_identifier_list(&table_info.pks, Some("pk_tbl."))?; - // TODO: we can remove the self join if we put causal length in the primary key table - // We LEFT JOIN and COALESCE the causal length - // since we incorporated an optimization to not store causal length records - // until they're required. I.e., do not store them until a delete - // is actually issued. This cuts data weight quite a bit for - // rows that never get removed. Ok(format!( "SELECT '{table_name_val}' as tbl, @@ -47,22 +43,381 @@ fn crsql_changes_query_for_table(table_info: &TableInfo) -> Result Result { + if table_info.pks.is_empty() { + return Err(ResultCode::ABORT); + } + + let pk_list = crate::util::as_identifier_list(&table_info.pks, Some("pk_tbl."))?; + let pk_list_tomb = crate::util::as_identifier_list(&table_info.pks, Some("tpk_tbl."))?; + let escaped = crate::util::escape_ident(&table_info.tbl_name); + let table_name_val = crate::util::escape_ident_as_value(&table_info.tbl_name); + let col_id_bits = consts::CRSQL_COL_ID_BITS; + + // Always JOIN main table — needed to fetch column values via CASE (cval). + // For rowid-key tables: join on rowid alias = pk_tbl.__crsql_key. + // For non-rowid tables: join on PK columns. + let (pk_expr, main_join) = if table_info.uses_rowid_key { + let mt_pk_list = crate::util::as_identifier_list(&table_info.pks, Some("mt."))?; + let alias = crate::util::escape_ident(&table_info.rowid_alias); + (mt_pk_list, format!("JOIN \"{escaped}\" AS mt ON mt.\"{alias}\" = pk_tbl.__crsql_key", alias = alias)) + } else { + let pk_join_conds: Vec = table_info.pks.iter().map(|c| { + format!("mt.\"{col}\" = pk_tbl.\"{col}\"", col = crate::util::escape_ident(&c.name)) + }).collect(); + (pk_list.clone(), format!("JOIN \"{escaped}\" AS mt ON {conds}", escaped = escaped, conds = pk_join_conds.join(" AND "))) + }; + + let col_val_case = build_col_val_case(table_info)?; + + // Part 1: Cell changes from v2_clock joined with v2_pks and v2_col_map + // cval is fetched inline via CASE expression — no lazy per-row fetch needed. + let cell_changes = format!( + "SELECT + '{table_name_val}' as tbl, + crsql_pack_columns({pk_expr}) as pks, + cm.col_name as cid, + c.col_version as col_vrsn, + c.db_version as db_vrsn, + site_tbl.site_id as site_id, + c.cell_key >> {col_id_bits} as key, + c.seq as seq, + pk_tbl.cl as cl, + c.ts as ts, + {col_val_case} as cval + FROM \"{escaped}{clock_suffix}\" AS c + JOIN \"{escaped}{pks_suffix}\" AS pk_tbl ON (c.cell_key >> {col_id_bits}) = pk_tbl.__crsql_key + {main_join} + JOIN \"{escaped}{col_map_suffix}\" AS cm ON (c.cell_key & {col_id_mask}) = cm.col_id + LEFT JOIN crsql_site_id AS site_tbl ON c.site_id = site_tbl.ordinal", + table_name_val = table_name_val, + pk_expr = pk_expr, + main_join = main_join, + col_val_case = col_val_case, + escaped = escaped, + clock_suffix = consts::V2_CLOCK_SUFFIX, + pks_suffix = consts::V2_PKS_SUFFIX, + col_map_suffix = consts::V2_COL_MAP_SUFFIX, + col_id_bits = col_id_bits, + col_id_mask = consts::CRSQL_COL_ID_MASK, + ); + + // Part 2: Tombstone rows from v2_tombstones joined with v2_tombstone_pks + let tombstone_rows = format!( + "SELECT + '{table_name_val}' as tbl, + crsql_pack_columns({pk_list_tomb}) as pks, + '{delete_sentinel}' as cid, + t.cl as col_vrsn, + t.db_version as db_vrsn, + site_tbl.site_id as site_id, + (1 << 62) | (t.site_id << 46) | (t.db_version << 22) | t.seq as key, + t.seq as seq, + t.cl as cl, + t.ts as ts, + NULL as cval + FROM \"{escaped}{tomb_suffix}\" AS t + JOIN \"{escaped}{tomb_pks_suffix}\" AS tpk_tbl ON t.hashed_pk = tpk_tbl.hashed_pk + LEFT JOIN crsql_site_id AS site_tbl ON t.site_id = site_tbl.ordinal", + table_name_val = table_name_val, + pk_list_tomb = pk_list_tomb, + delete_sentinel = crate::c::DELETE_SENTINEL, + escaped = escaped, + tomb_suffix = consts::V2_TOMBSTONES_SUFFIX, + tomb_pks_suffix = consts::V2_TOMBSTONE_PKS_SUFFIX, + ); + + Ok(format!( + "{cell_changes} UNION ALL {tombstone_rows}", + cell_changes = cell_changes, + tombstone_rows = tombstone_rows, + )) +} + +/// Build a CASE expression that maps col_id → main_table column value. +/// Used in packed (v2 sync-log) format to fetch all column values in one query. +/// Example: CASE (c.cell_key & mask) WHEN 0 THEN mt."col0" WHEN 1 THEN mt."col1" END +fn build_col_val_case(table_info: &TableInfo) -> Result { + let mask = consts::CRSQL_COL_ID_MASK; + + let mut when_clauses = vec![]; + for (col_id, col) in table_info.non_pks.iter().enumerate() { + when_clauses.push(format!( + "WHEN {col_id} THEN mt.\"{col_name}\"", + col_id = col_id, + col_name = crate::util::escape_ident(&col.name) + )); + } + + if when_clauses.is_empty() { + // No non-PK columns — return NULL + Ok("NULL".to_string()) + } else { + Ok(format!( + "CASE (c.cell_key & {mask}) {whens} END", + mask = mask, + whens = when_clauses.join(" ") + )) + } +} + +/// V2 packed feed query (V2 wire format): coalesces clock rows that share +/// (row_key, db_version, site_id) into a single packed event. +/// - cid = GROUP_CONCAT(col_name, char(0)) +/// - col_vrsn = GROUP_CONCAT(col_version, char(0)) +/// - cval = crsql_pack_agg(col_val) (column values fetched from main table via CASE) +/// Sentinels and tombstones are always single events (no packing). +fn crsql_changes_query_for_table_v2_v2wire(table_info: &TableInfo) -> Result { + if table_info.pks.is_empty() { + return Err(ResultCode::ABORT); + } + + let pk_list = crate::util::as_identifier_list(&table_info.pks, Some("pk_tbl."))?; + let escaped = crate::util::escape_ident(&table_info.tbl_name); + let table_name_val = crate::util::escape_ident_as_value(&table_info.tbl_name); + let col_id_bits = consts::CRSQL_COL_ID_BITS; + let col_val_case = build_col_val_case(table_info)?; + + // Always JOIN main table for packed query — needed to fetch column values via CASE. + // For rowid-key tables: join on rowid alias = pk_tbl.__crsql_key. + // For non-rowid tables: join on PK columns. + let (pk_expr, main_join) = if table_info.uses_rowid_key { + let mt_pk_list = crate::util::as_identifier_list(&table_info.pks, Some("mt."))?; + let alias = crate::util::escape_ident(&table_info.rowid_alias); + (mt_pk_list, format!("JOIN \"{escaped}\" AS mt ON mt.\"{alias}\" = pk_tbl.__crsql_key", alias = alias)) + } else { + let pk_join_conds: Vec = table_info.pks.iter().map(|c| { + format!("mt.\"{col}\" = pk_tbl.\"{col}\"", col = crate::util::escape_ident(&c.name)) + }).collect(); + (pk_list.clone(), format!("JOIN \"{escaped}\" AS mt ON {conds}", escaped = escaped, conds = pk_join_conds.join(" AND "))) + }; + + // Part 1: Packed cell changes — GROUP BY (key expression, db_version, site_id) + // No subquery needed with SQLite 3.44+ — ORDER BY inside aggregates ensures alignment. + // GROUP BY must use the full expression, not the column alias, for SQLite 3.44 compatibility. + let cell_changes = format!( + "SELECT + '{table_name_val}' as tbl, + crsql_pack_columns({pk_expr}) as pks, + cast(group_concat(cm.col_name, char(0) ORDER BY cm.col_id) as blob) as cid, + cast(group_concat(c.col_version, char(0) ORDER BY cm.col_id) as blob) as col_vrsn, + c.db_version as db_vrsn, + site_tbl.site_id as site_id, + c.cell_key >> {col_id_bits} as key, + cast(group_concat(c.seq, char(0) ORDER BY cm.col_id) as blob) as seq, + pk_tbl.cl as cl, + c.ts as ts, + crsql_pack_agg(({col_val_case}) ORDER BY cm.col_id) as cval + FROM \"{escaped}{clock_suffix}\" AS c + JOIN \"{escaped}{pks_suffix}\" AS pk_tbl ON (c.cell_key >> {col_id_bits}) = pk_tbl.__crsql_key + {main_join} + JOIN \"{escaped}{col_map_suffix}\" AS cm ON (c.cell_key & {col_id_mask}) = cm.col_id + LEFT JOIN crsql_site_id AS site_tbl ON c.site_id = site_tbl.ordinal + GROUP BY c.cell_key >> {col_id_bits}, c.db_version, site_tbl.site_id", + table_name_val = table_name_val, + pk_expr = pk_expr, + main_join = main_join, + col_val_case = col_val_case, + escaped = escaped, + clock_suffix = consts::V2_CLOCK_SUFFIX, + pks_suffix = consts::V2_PKS_SUFFIX, + col_map_suffix = consts::V2_COL_MAP_SUFFIX, + col_id_bits = col_id_bits, + col_id_mask = consts::CRSQL_COL_ID_MASK, + ); + + // Part 2: Tombstone rows — V2 wire format: hashed_pk as pks, '-2' as cid + let tombstone_rows = format!( + "SELECT + '{table_name_val}' as tbl, + t.hashed_pk as pks, + '{hash_tombstone_cid}' as cid, + NULL as col_vrsn, + t.db_version as db_vrsn, + site_tbl.site_id as site_id, + (1 << 62) | (t.site_id << 46) | (t.db_version << 22) | t.seq as key, + t.seq as seq, + t.cl as cl, + t.ts as ts, + NULL as cval + FROM \"{escaped}{tomb_suffix}\" AS t + LEFT JOIN crsql_site_id AS site_tbl ON t.site_id = site_tbl.ordinal", + table_name_val = table_name_val, + hash_tombstone_cid = consts::V2_HASH_TOMBSTONE_CID, + escaped = escaped, + tomb_suffix = consts::V2_TOMBSTONES_SUFFIX, + ); + + Ok(format!( + "{cell_changes} UNION ALL {tombstone_rows}", + cell_changes = cell_changes, + tombstone_rows = tombstone_rows, + )) +} + +/// PK-only table query: reads sentinel clock entries at col_id=0. +/// No v2_col_map JOIN needed. Emits cid='-1', cval=NULL. +/// Tombstone part is the same as the normal v1wire query. +fn crsql_changes_query_for_table_v2_pkonly(table_info: &TableInfo) -> Result { + if table_info.pks.is_empty() { + return Err(ResultCode::ABORT); + } + + let pk_list_tomb = crate::util::as_identifier_list(&table_info.pks, Some("tpk_tbl."))?; + let escaped = crate::util::escape_ident(&table_info.tbl_name); + let table_name_val = crate::util::escape_ident_as_value(&table_info.tbl_name); + let col_id_bits = consts::CRSQL_COL_ID_BITS; + + // For rowid-key tables: PK columns come from main table (v2_pks only has __crsql_key). + // For non-rowid tables: PK columns come from v2_pks directly. + let (pk_expr, main_join) = if table_info.uses_rowid_key { + let mt_pk_list = crate::util::as_identifier_list(&table_info.pks, Some("mt."))?; + let alias = crate::util::escape_ident(&table_info.rowid_alias); + (mt_pk_list, format!("JOIN \"{escaped}\" AS mt ON mt.\"{alias}\" = pk_tbl.__crsql_key", alias = alias)) + } else { + let pk_list = crate::util::as_identifier_list(&table_info.pks, Some("pk_tbl."))?; + let pk_join_conds: Vec = table_info.pks.iter().map(|c| { + format!("mt.\"{col}\" = pk_tbl.\"{col}\"", col = crate::util::escape_ident(&c.name)) + }).collect(); + (pk_list, format!("JOIN \"{escaped}\" AS mt ON {conds}", escaped = escaped, conds = pk_join_conds.join(" AND "))) + }; + + // Sentinel clock entries at col_id=0 + let cell_changes = format!( + "SELECT + '{table_name_val}' as tbl, + crsql_pack_columns({pk_expr}) as pks, + '{sentinel}' as cid, + c.col_version as col_vrsn, + c.db_version as db_vrsn, + site_tbl.site_id as site_id, + c.cell_key >> {col_id_bits} as key, + c.seq as seq, + pk_tbl.cl as cl, + c.ts as ts, + NULL as cval + FROM \"{escaped}{clock_suffix}\" AS c + JOIN \"{escaped}{pks_suffix}\" AS pk_tbl ON (c.cell_key >> {col_id_bits}) = pk_tbl.__crsql_key + {main_join} + LEFT JOIN crsql_site_id AS site_tbl ON c.site_id = site_tbl.ordinal", + table_name_val = table_name_val, + pk_expr = pk_expr, + sentinel = crate::c::INSERT_SENTINEL, + main_join = main_join, + escaped = escaped, + clock_suffix = consts::V2_CLOCK_SUFFIX, + pks_suffix = consts::V2_PKS_SUFFIX, + col_id_bits = col_id_bits, + ); + + // Tombstone rows (same as v1wire) + let tombstone_rows = format!( + "SELECT + '{table_name_val}' as tbl, + crsql_pack_columns({pk_list_tomb}) as pks, + '{delete_sentinel}' as cid, + t.cl as col_vrsn, + t.db_version as db_vrsn, + site_tbl.site_id as site_id, + (1 << 62) | (t.site_id << 46) | (t.db_version << 22) | t.seq as key, + t.seq as seq, + t.cl as cl, + t.ts as ts, + NULL as cval + FROM \"{escaped}{tomb_suffix}\" AS t + JOIN \"{escaped}{tomb_pks_suffix}\" AS tpk_tbl ON t.hashed_pk = tpk_tbl.hashed_pk + LEFT JOIN crsql_site_id AS site_tbl ON t.site_id = site_tbl.ordinal", + table_name_val = table_name_val, + pk_list_tomb = pk_list_tomb, + delete_sentinel = crate::c::DELETE_SENTINEL, + escaped = escaped, + tomb_suffix = consts::V2_TOMBSTONES_SUFFIX, + tomb_pks_suffix = consts::V2_TOMBSTONE_PKS_SUFFIX, + ); + + Ok(format!( + "{cell_changes} UNION ALL {tombstone_rows}", + cell_changes = cell_changes, + tombstone_rows = tombstone_rows, + )) +} + +/// Decide which query to use for a table based on metadata-use-version and sync-log-version. +/// - metadata_use_version=1: always use V1 query (reads from V1 tables) +/// - metadata_use_version=2 + sync_log_version=1: V2 per-column query (reads from V2 tables, V1 wire) +/// - metadata_use_version=2 + sync_log_version=2: V2 packed query (reads from V2 tables, V2 wire) +/// - PK-only tables (non_pks empty): use PK-only query with sentinel at col_id=0 +fn query_for_table( + table_info: &TableInfo, + metadata_use_version: i32, + sync_log_version: i32, +) -> Result { + // If metadata-use-version is V1, always read from V1 tables (even if V2 tables exist) + if metadata_use_version == consts::META_USE_V1 { + // Only use V1 query if V1 tables exist + if table_info.schema_version == SchemaVersion::V1 + || table_info.schema_version == SchemaVersion::V2AndV1 + { + return crsql_changes_query_for_table(table_info); + } + // V2-only tables but use-version=1 shouldn't happen (config guards prevent it) + // Fall through to V2 query as safety + } + + // metadata-use-version is V2 (or safety fallback) + match table_info.schema_version { + SchemaVersion::V2 | SchemaVersion::V2AndV1 => { + // PK-only tables use a dedicated query with sentinel at col_id=0 + if table_info.non_pks.is_empty() { + return crsql_changes_query_for_table_v2_pkonly(table_info); + } + if sync_log_version == consts::SYNC_LOG_V2 { + crsql_changes_query_for_table_v2_v2wire(table_info) + } else { + crsql_changes_query_for_table_v2_v1wire(table_info) + } + } + SchemaVersion::V1 => crsql_changes_query_for_table(table_info), + } +} + +/// Returns true if the query includes a cval column. +/// V2 metadata always fetches cval inline (both V1 and V2 wire formats). +/// V1 metadata fetches column values lazily in changes_next. +pub fn query_has_cval(metadata_use_version: i32) -> bool { + metadata_use_version == consts::META_USE_V2 +} + pub fn changes_union_query( table_infos: &Vec, idx_str: &str, + metadata_use_version: i32, + sync_log_version: i32, ) -> Result { let mut sub_queries = vec![]; + let has_cval = query_has_cval(metadata_use_version); for table_info in table_infos { - let query_part = crsql_changes_query_for_table(table_info)?; + let query_part = query_for_table(table_info, metadata_use_version, sync_log_version)?; sub_queries.push(query_part); } - // Manually null-terminate the string so we don't have to copy it to create a CString. - // We can just extract the raw bytes of the Rust string. - Ok(format!( - "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id, key, seq, cl, ts FROM ({unions}) {idx_str}\0", - unions = sub_queries.join(" UNION ALL "), - idx_str = idx_str, - )) + if has_cval { + Ok(format!( + "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id, key, seq, cl, ts, cval FROM ({unions}) {idx_str}\0", + unions = sub_queries.join(" UNION ALL "), + idx_str = idx_str, + )) + } else { + Ok(format!( + "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id, key, seq, cl, ts FROM ({unions}) {idx_str}\0", + unions = sub_queries.join(" UNION ALL "), + idx_str = idx_str, + )) + } } diff --git a/core/rs/core/src/changes_vtab_write.rs b/core/rs/core/src/changes_vtab_write.rs index bb95b2fab..821ea792a 100644 --- a/core/rs/core/src/changes_vtab_write.rs +++ b/core/rs/core/src/changes_vtab_write.rs @@ -1,22 +1,26 @@ use alloc::boxed::Box; use alloc::ffi::CString; use alloc::format; +use alloc::string::String; +use alloc::vec; use alloc::vec::Vec; use core::ffi::{c_char, c_int}; use core::mem; use sqlite::Stmt; use sqlite_nostd as sqlite; -use sqlite_nostd::{sqlite3, ResultCode, Value}; +use sqlite_nostd::{sqlite3, Connection, ResultCode, Value}; use crate::c::crsql_ExtData; use crate::c::{crsql_Changes_vtab, CrsqlChangesColumn}; use crate::compare_values::crsql_compare_sqlite_values; +use crate::config; use crate::db_version::{get_or_set_site_ordinal, insert_db_version}; use crate::pack_columns::bind_package_to_stmt; use crate::pack_columns::{unpack_columns, ColumnValue}; use crate::stmt_cache::reset_cached_stmt; -use crate::tableinfo::{crsql_ensure_table_infos_are_up_to_date, TableInfo}; +use crate::tableinfo::{crsql_ensure_table_infos_are_up_to_date, TableInfo, SchemaVersion}; use crate::util::slab_rowid; +use crate::consts; /** * did_cid_win does not take into account the causal length. @@ -176,7 +180,7 @@ fn set_winner_clock( insert_db_vrsn: sqlite::int64, insert_site_id: &[u8], insert_seq: sqlite::int64, - insert_ts: &str, + insert_ts: sqlite::int64, ) -> Result { // set the site_id ordinal // get the returned ordinal @@ -204,7 +208,10 @@ fn set_winner_clock( Some(ordinal) => set_stmt.bind_int64(6, ordinal), None => set_stmt.bind_null(6), }) - .and_then(|_| set_stmt.bind_text(7, insert_ts, sqlite::Destructor::STATIC)); + .and_then(|_| { + let ts_str = format!("{}", insert_ts); + set_stmt.bind_text(7, &ts_str, sqlite::Destructor::TRANSIENT) + }); if let Err(rc) = bind_result { reset_cached_stmt(set_stmt.stmt)?; @@ -236,7 +243,7 @@ fn merge_sentinel_only_insert( remote_db_vsn: sqlite::int64, remote_site_id: &[u8], remote_seq: sqlite::int64, - remote_ts: &str, + remote_ts: sqlite::int64, ) -> Result { let merge_stmt_ref = tbl_info.get_merge_pk_only_insert_stmt(db)?; let merge_stmt = merge_stmt_ref.as_ref().ok_or(ResultCode::ERROR)?; @@ -311,7 +318,7 @@ unsafe fn merge_delete( remote_db_vrsn: sqlite::int64, remote_site_id: &[u8], remote_seq: sqlite::int64, - remote_ts: &str, + remote_ts: sqlite::int64, ) -> Result { let delete_stmt_ref = tbl_info.get_merge_delete_stmt(db)?; let delete_stmt = delete_stmt_ref.as_ref().ok_or(ResultCode::ERROR)?; @@ -421,6 +428,41 @@ fn get_local_cl( Ok(cl) } +/// Post-merge processing shared by all V2 merge paths: +/// 1. Update db_version tracking (if site_id is present) +/// 2. Dual-write: copy V2 metadata to V1 metadata tables (if in dual-write mode) +unsafe fn post_v2_merge( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + unpacked_pks: Option<&Vec>, + hashed_pk: &[u8], + insert_site_id: &[u8], + insert_db_vrsn: sqlite::int64, +) -> Result<(), ResultCode> { + // Update db_version tracking + if !insert_site_id.is_empty() { + let _ = insert_db_version(ext_data, insert_site_id, insert_db_vrsn); + } + // Dual-write: copy V2 metadata to V1 metadata tables + let mwv = unsafe { (*ext_data).metadataWriteVersion }; + if mwv == config::METADATA_VERSION_V2_AND_V1 { + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + let (v2_key_opt, v2_cl) = + v2_lookup_key_and_cl(db, &escaped, hashed_pk).unwrap_or((None, 0)); + let _ = v2_to_v1_mirror_metadata( + db, + ext_data, + tbl_info, + unpacked_pks, + hashed_pk, + v2_key_opt, + v2_cl, + ); + } + Ok(()) +} + unsafe fn merge_insert( vtab: *mut sqlite::vtab, argc: c_int, @@ -449,20 +491,18 @@ unsafe fn merge_insert( let insert_tbl = insert_tbl.text(); let insert_pks = args[2 + CrsqlChangesColumn::Pk as usize]; let insert_col = args[2 + CrsqlChangesColumn::Cid as usize]; - if insert_col.bytes() > crate::consts::MAX_TBL_NAME_LEN { - let err = CString::new("crsql - column name exceeded max length")?; - *errmsg = err.into_raw(); - return Err(ResultCode::ERROR); - } + let insert_col_vrsn_raw = args[2 + CrsqlChangesColumn::ColVrsn as usize]; + // V2 wire packed format: col_version is Text/Blob (packed), and col names + // are null-separated — skip the single-column length check for those. + let col_vrsn_type = insert_col_vrsn_raw.value_type(); let insert_col = insert_col.text(); let insert_val = args[2 + CrsqlChangesColumn::Cval as usize]; - let insert_col_vrsn = args[2 + CrsqlChangesColumn::ColVrsn as usize].int64(); let insert_db_vrsn = args[2 + CrsqlChangesColumn::DbVrsn as usize].int64(); let insert_site_id = args[2 + CrsqlChangesColumn::SiteId as usize]; let insert_cl = args[2 + CrsqlChangesColumn::Cl as usize].int64(); - let insert_seq = args[2 + CrsqlChangesColumn::Seq as usize].int64(); - let insert_ts = args[2 + CrsqlChangesColumn::Ts as usize].text(); + let insert_seq_raw = args[2 + CrsqlChangesColumn::Seq as usize]; + let insert_ts_raw = args[2 + CrsqlChangesColumn::Ts as usize].int64(); if insert_site_id.bytes() > crate::consts::SITE_ID_LEN { let err = CString::new("crsql - site id exceeded max length")?; @@ -472,13 +512,45 @@ unsafe fn merge_insert( let insert_site_id = insert_site_id.blob(); + // Detect V2 wire packed row from the incoming data, not from local syncLogVersion config. + // syncLogVersion controls what we emit, not what we accept. A node with sync-log-version=1 + // can receive V2 wire format changes from a peer with sync-log-version=2. + // V1 wire always has INTEGER col_vrsn (raw c.col_version). + // V2 wire always has BLOB col_vrsn (cast(group_concat(...) as blob)). + // So col_vrsn type alone distinguishes the two formats. + // Tombstone rows (cid='-1' or cid='-2') are never packed regardless of wire format. + let is_v2_hash_tombstone = insert_col == crate::consts::V2_HASH_TOMBSTONE_CID; + let is_tombstone = insert_col == crate::c::DELETE_SENTINEL || is_v2_hash_tombstone; + let is_v2_wire_packed = !is_tombstone + && (col_vrsn_type == sqlite::ColumnType::Text + || col_vrsn_type == sqlite::ColumnType::Blob); + + // Skip column name length check for V2 packed rows (col names are null-separated) + if !is_v2_wire_packed && insert_col.len() > crate::consts::MAX_TBL_NAME_LEN as usize { + let err = CString::new("crsql - column name exceeded max length")?; + *errmsg = err.into_raw(); + return Err(ResultCode::ERROR); + } + + // Reject V2 wire format changes if this node is not on V2 metadata. + // Per spec: "Nodes with metadata-use-version set to 1 will emit an error + // if they receive V2 wire format changes." + if (is_v2_wire_packed || is_v2_hash_tombstone) + && (*(*tab).pExtData).metadataUseVersion != consts::META_USE_V2 + { + let err = CString::new( + "crsql - received V2 wire format change but metadata-use-version is not 2. \ + Set metadata-use-version=2 before accepting V2 wire format changes.", + )?; + *errmsg = err.into_raw(); + return Err(ResultCode::ERROR); + } + + // Look up table info — needed by all branches below. let mut tbl_infos = mem::ManuallyDrop::new(Box::from_raw( (*(*tab).pExtData).tableInfos as *mut Vec, )); - - // TODO: will this work given `insert_tbl` is null termed? let tbl_info_index = tbl_infos.iter().position(|x| x.tbl_name == insert_tbl); - if tbl_info_index.is_none() { let err = CString::new(format!( "crsql - could not find the schema information for table {}", @@ -487,10 +559,191 @@ unsafe fn merge_insert( *errmsg = err.into_raw(); return Err(ResultCode::ERROR); } - // TODO: technically safe since we checked `is_none` but this should be more idiomatic let tbl_info_index = tbl_info_index.unwrap(); - let tbl_info = &mut tbl_infos[tbl_info_index]; + + // V2 clock tables require a non-zero ts. If we get a legacy row with + // ts=0 then we fall back to the current ts. Error early if current ts is not set. + if unsafe { (*(*tab).pExtData).timestamp } == 0 { + let err = CString::new("crsql - timestamp not set — call crsql_set_ts() before syncing changes")?; + unsafe { *errmsg = err.into_raw() }; + return Err(ResultCode::ERROR); + } + + // If incoming ts is 0, fall back to current set timestamp. + let insert_ts = if insert_ts_raw > 0 { + insert_ts_raw + } else { + unsafe { (*(*tab).pExtData).timestamp as i64 } + }; + + // V1-only write mode: use the old V1 merge path + let mwv = unsafe { (*(*tab).pExtData).metadataWriteVersion }; + if mwv == config::METADATA_VERSION_V1 { + return v1_merge_insert( + db, + (*tab).pExtData, + tbl_info, + insert_tbl, + insert_pks, + insert_col, + insert_col_vrsn_raw, + insert_val, + insert_db_vrsn, + insert_site_id, + insert_cl, + insert_seq_raw, + insert_ts_raw, + rowid, + tbl_info_index, + errmsg, + ); + } + + // --- V2 or V2AndV1 mode --- + // Convert any V1 wire format to V2 wire format on the fly, then dispatch + // to v2_packed_merge or v2_merge_insert_hash_tombstone. + + // Compute common PK data. + // V2 hash tombstone: pks blob IS the hashed_pk, no unpacked pks needed. + let (unpacked_pks_opt, hashed_pk): (Option>, Vec) = if is_v2_hash_tombstone { + (None, insert_pks.blob().to_vec()) + } else { + let packed_pks = insert_pks.blob(); + let unpacked_pks = unpack_columns(&packed_pks)?; + let hashed_pk = crate::hash_pk::hash_packed_blob(&packed_pks); + (Some(unpacked_pks), hashed_pk) + }; + + // Hydrate V2 from V1 metadata in dual-write mode for V1 wire format + if !is_v2_wire_packed && !is_v2_hash_tombstone && mwv == config::METADATA_VERSION_V2_AND_V1 { + if let Some(ref unpacked_pks) = unpacked_pks_opt { + v1_to_v2_hydrate_row( + db, + (*tab).pExtData, + tbl_info, + unpacked_pks, + &hashed_pk, + )?; + } + } + + // Delete if V2 hash tombstone, or V1 wire with even CL. + let is_delete = is_v2_hash_tombstone || (!is_v2_wire_packed && insert_cl % 2 == 0); + + let result = if is_delete { + let col_vrsn = insert_col_vrsn_raw.int64(); + let seq = insert_seq_raw.int64(); + + v2_merge_insert_hash_tombstone( + db, + (*tab).pExtData, + tbl_info, + &hashed_pk, + insert_tbl, + insert_col, + insert_val, + col_vrsn, + insert_db_vrsn, + insert_site_id, + insert_cl, + seq, + insert_ts, + rowid, + tbl_info_index, + errmsg, + ) + } else { + // Packed merge: compute vectors based on wire format + let (col_names, col_vrsns, seqs, unpacked_vals) = if is_v2_wire_packed { + let col_names: Vec<&str> = insert_col.split('\0').collect(); + let col_vrsns: Vec = insert_col_vrsn_raw.text().split('\0').map(|s| s.parse::().unwrap_or(0)).collect(); + let seqs: Vec = insert_seq_raw.text().split('\0').map(|s| s.parse::().unwrap_or(0)).collect(); + let unpacked_vals = unpack_columns(insert_val.blob())?; + + let n_cols = col_names.len(); + if col_vrsns.len() != n_cols || seqs.len() != n_cols || unpacked_vals.len() != n_cols { + let err = CString::new(format!( + "crsql - V2 wire packed row has mismatched lengths: col_names={}, col_vrsns={}, seqs={}, vals={}", + n_cols, col_vrsns.len(), seqs.len(), unpacked_vals.len() + ))?; + *errmsg = err.into_raw(); + return Err(ResultCode::ERROR); + } + (col_names, col_vrsns, seqs, unpacked_vals) + } else { + // V1 wire format: convert to single-element (or empty) vectors + let insert_col_vrsn = insert_col_vrsn_raw.int64(); + let insert_seq = insert_seq_raw.int64(); + + if insert_col == crate::c::INSERT_SENTINEL { + (Vec::new(), Vec::new(), Vec::new(), Vec::new()) + } else { + let col_val = sqlite_value_to_column_value(insert_val); + ( + vec![insert_col], + vec![insert_col_vrsn], + vec![insert_seq], + vec![col_val], + ) + } + }; + + v2_packed_merge( + db, + (*tab).pExtData, + tbl_info, + unpacked_pks_opt.as_ref().unwrap(), + &hashed_pk, + &col_names, + &col_vrsns, + &seqs, + &unpacked_vals, + insert_db_vrsn, + insert_site_id, + insert_cl, + insert_ts, + ) + }; + + // Post-merge: db_version + dual-write V1 metadata + if result.is_ok() { + let _ = post_v2_merge( + db, + (*tab).pExtData, + tbl_info, + unpacked_pks_opt.as_ref(), + &hashed_pk, + insert_site_id, + insert_db_vrsn, + ); + } + + return result; +} + +/// V1 merge path: handles incoming changes for tables using V1 metadata only. +#[allow(clippy::too_many_arguments)] +unsafe fn v1_merge_insert( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &mut TableInfo, + insert_tbl: &str, + insert_pks: *mut sqlite::value, + insert_col: &str, + insert_col_vrsn_raw: *mut sqlite::value, + insert_val: *mut sqlite::value, + insert_db_vrsn: sqlite::int64, + insert_site_id: &[u8], + insert_cl: sqlite::int64, + insert_seq_raw: *mut sqlite::value, + insert_ts: sqlite::int64, + rowid: *mut sqlite::int64, + tbl_info_index: usize, + errmsg: *mut *mut c_char, +) -> Result { + let insert_col_vrsn = insert_col_vrsn_raw.int64(); + let insert_seq = insert_seq_raw.int64(); let unpacked_pks = unpack_columns(insert_pks.blob())?; // Get or create key as the first thing we do. @@ -519,7 +772,7 @@ unsafe fn merge_insert( // We got a delete event but we've already processed a delete at that version. // Just bail. if insert_cl == local_cl { - if unsafe { (*(*tab).pExtData).mergeEqualValues == 1 } + if unsafe { (*ext_data).mergeEqualValues == 1 } && did_site_id_win( db, insert_tbl, @@ -533,7 +786,7 @@ unsafe fn merge_insert( // here we set the same winner for the clock if incoming site_id won set_winner_clock( db, - (*tab).pExtData, + ext_data, &tbl_info, key, insert_col, @@ -549,7 +802,7 @@ unsafe fn merge_insert( // else, it is a delete and the cl is > than ours. Drop the row. let merge_result = merge_delete( db, - (*tab).pExtData, + ext_data, &tbl_info, &unpacked_pks, key, @@ -564,7 +817,7 @@ unsafe fn merge_insert( return Err(rc); } Ok(inner_rowid) => { - (*(*tab).pExtData).rowsImpacted += 1; + (*ext_data).rowsImpacted += 1; *rowid = slab_rowid(tbl_info_index as i32, inner_rowid); return Ok(ResultCode::OK); } @@ -587,7 +840,7 @@ unsafe fn merge_insert( } let inner_rowid = merge_sentinel_only_insert( db, - (*tab).pExtData, + ext_data, &tbl_info, &unpacked_pks, key, @@ -599,7 +852,7 @@ unsafe fn merge_insert( )?; // a success & rowid of -1 means the merge was a no-op if inner_rowid != -1 { - (*(*tab).pExtData).rowsImpacted += 1; + (*ext_data).rowsImpacted += 1; *rowid = slab_rowid(tbl_info_index as i32, inner_rowid); } return Ok(ResultCode::OK); @@ -616,7 +869,7 @@ unsafe fn merge_insert( // and the version to set to is the cl not col_vrsn of current insert let inner_rowid = merge_sentinel_only_insert( db, - (*tab).pExtData, + ext_data, &tbl_info, &unpacked_pks, key, @@ -628,7 +881,7 @@ unsafe fn merge_insert( )?; // a success & rowid of -1 means the merge was a no-op if inner_rowid != -1 { - (*(*tab).pExtData).rowsImpacted += 1; + (*ext_data).rowsImpacted += 1; *rowid = slab_rowid(tbl_info_index as i32, inner_rowid); } } @@ -640,7 +893,7 @@ unsafe fn merge_insert( || !row_exists_locally || did_cid_win( db, - (*tab).pExtData, + ext_data, insert_tbl, &tbl_info, &unpacked_pks, @@ -670,25 +923,25 @@ unsafe fn merge_insert( return Err(rc); } - let rc = (*(*tab).pExtData) + let rc = (*ext_data) .pSetSyncBitStmt .step() - .and_then(|_| (*(*tab).pExtData).pSetSyncBitStmt.reset()) + .and_then(|_| (*ext_data).pSetSyncBitStmt.reset()) .and_then(|_| merge_stmt.step()); reset_cached_stmt(merge_stmt.stmt)?; - let sync_rc = (*(*tab).pExtData) + let sync_rc = (*ext_data) .pClearSyncBitStmt .step() - .and_then(|_| (*(*tab).pExtData).pClearSyncBitStmt.reset()); + .and_then(|_| (*ext_data).pClearSyncBitStmt.reset()); rc?; sync_rc?; let merge_result = set_winner_clock( db, - (*tab).pExtData, + ext_data, &tbl_info, key, insert_col, @@ -703,7 +956,7 @@ unsafe fn merge_insert( return Err(rc); } Ok(inner_rowid) => { - (*(*tab).pExtData).rowsImpacted += 1; + (*ext_data).rowsImpacted += 1; *rowid = slab_rowid(tbl_info_index as i32, inner_rowid); return Ok(ResultCode::OK); } @@ -712,7 +965,7 @@ unsafe fn merge_insert( // Update the received db_version whether the change won or not. if res.is_ok() && !insert_site_id.is_empty() { - if let Err(rc) = insert_db_version((*tab).pExtData, insert_site_id, insert_db_vrsn) { + if let Err(rc) = insert_db_version(ext_data, insert_site_id, insert_db_vrsn) { let err = CString::new(format!( "Unable to insert db version {} for site id {:?}: {:?}", insert_db_vrsn, insert_site_id, rc @@ -729,3 +982,1018 @@ unsafe fn merge_insert( res } + +/// Ensure an alive row exists in v2_pks at incoming_cl. +/// Handles: stale CL bail, skipped-delete cleanup, resurrection cleanup, new row creation. +/// Returns (local_key, local_cl) where local_cl is the CL *before* any modification. +/// Returns None if incoming_cl < local_cl (stale, caller should bail). +unsafe fn v2_ensure_alive_row_at_cl( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + unpacked_pks: &Vec, + hashed_pk: &[u8], + incoming_cl: i64, + incoming_db_version: i64, + incoming_site_id: i64, +) -> Result, ResultCode> { + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + + let (local_key_opt, local_cl) = v2_lookup_key_and_cl(db, &escaped, hashed_pk)?; + + if incoming_cl < local_cl { + return Ok(None); + } + + let local_key = if incoming_cl > local_cl { + if let Some(key) = local_key_opt { + // Row exists in v2_pks but CL jumped — we skipped a delete. + // Nuke local clocks, v2_pks, and base table row before re-creating. + v2_nuke_local_row(db, ext_data, &escaped, key, unpacked_pks, tbl_info)?; + } else if local_cl > 0 { + // Was in tombstones — resurrection. Clean up tombstone entries. + v2_nuke_tombstone(db, &escaped, hashed_pk)?; + } + // Create fresh v2_pks entry with new CL + let new_key = v2_insert_pk_row(db, ext_data, &escaped, tbl_info, unpacked_pks, hashed_pk, incoming_cl)?; + // Create zero-version clock entries for all mapped columns so they appear in sync logs. + // Use the incoming change's site_id and db_version since it created this row. + // seq=0 for placeholders; the packed column updates will overwrite specific entries. + let col_id_bits = consts::CRSQL_COL_ID_BITS as i64; + let base = new_key << col_id_bits; + let clock_sql = format!( + "INSERT INTO \"{}{}\" (cell_key, col_version, site_id, db_version, seq, ts)\n SELECT ? + col_id, 0, ?, ?, 0, crsql_get_ts()\n FROM \"{}{}\"\0", + escaped, consts::V2_CLOCK_SUFFIX, + escaped, consts::V2_COL_MAP_SUFFIX + ); + let clock_stmt = db.prepare_v2(&clock_sql)?; + clock_stmt.bind_int64(1, base)?; + clock_stmt.bind_int64(2, incoming_site_id)?; + clock_stmt.bind_int64(3, incoming_db_version)?; + clock_stmt.step()?; + new_key + } else { + // incoming_cl == local_cl — row must already exist + local_key_opt.unwrap_or(0) + }; + + Ok(Some((local_key, local_cl))) +} + +/// Look up key and CL from v2_pks or v2_tombstones by hashed_pk. +/// A row is in exactly one of v2_pks (alive) or v2_tombstones (dead), never both. +unsafe fn v2_lookup_key_and_cl( + db: *mut sqlite3, + escaped: &str, + hashed_pk: &[u8], +) -> Result<(Option, i64), ResultCode> { + let sql = format!( + "SELECT __crsql_key, cl FROM \"{}{}\" WHERE hashed_pk = ? \ + UNION ALL \ + SELECT NULL, cl FROM \"{}{}\" WHERE hashed_pk = ? \ + LIMIT 1\0", + escaped, consts::V2_PKS_SUFFIX, + escaped, consts::V2_TOMBSTONES_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_blob(1, hashed_pk, sqlite::Destructor::STATIC)?; + stmt.bind_blob(2, hashed_pk, sqlite::Destructor::STATIC)?; + if stmt.step()? == ResultCode::ROW { + let key = stmt.column_int64(0); + let cl = stmt.column_int64(1); + // key is NULL if the row was found in v2_tombstones + if stmt.column_type(0)? == sqlite::ColumnType::Null { + return Ok((None, cl)); + } + return Ok((Some(key), cl)); + } + + // No existing entry + Ok((None, 0)) +} + +/// On-demand V1→V2 hydration for a single row. +/// In dual-write mode with incomplete migration, a row may exist in V1 tables +/// but not yet in V2. This function migrates that row's V1 metadata to V2 +/// so the V2 merge/local-write paths can see the correct CL and clock state. +/// +/// `unpacked_pks` is used to look up the V1 key. For rowid-key integer PK tables, the +/// first PK column should be the rowid alias value. +/// `hashed_pk` is the pre-computed hash of the PKs for V2 table insertion. +pub unsafe fn v1_to_v2_hydrate_row( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + unpacked_pks: &Vec, + hashed_pk: &[u8], +) -> Result<(), ResultCode> { + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + crate::debug::debug_log("v1_to_v2_hydrate_row: timestamp not set — call crsql_set_ts() first"); + return Err(ResultCode::ERROR); + } + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + let col_id_bits = consts::CRSQL_COL_ID_BITS as i64; + let ts_fallback = unsafe { (*ext_data).timestamp as i64 }; + + // Guard: if V2 already has an entry for this row, skip hydration. + // The row was either already migrated or written via dual-write triggers. + let (existing_v2_key, existing_v2_cl) = v2_lookup_key_and_cl(db, &escaped, hashed_pk)?; + if existing_v2_key.is_some() || existing_v2_cl != 0 { + return Ok(()); + } + + // 1. Look up V1 key (SELECT only — do not create) + let v1_key = { + let stmt_ref = tbl_info.get_select_key_stmt(db)?; + let stmt = stmt_ref.as_ref().ok_or(ResultCode::ERROR)?; + bind_package_to_stmt(stmt.stmt, unpacked_pks, 0)?; + let result = stmt.step(); + let v1_key = match result { + Ok(ResultCode::ROW) => Some(stmt.column_int64(0)), + _ => None, + }; + reset_cached_stmt(stmt.stmt)?; + match v1_key { + Some(k) => k, + None => return Ok(()), // No V1 entry — nothing to hydrate + } + }; + + // 2. Look up V1 CL from sentinel clock entry + let (v1_cl, is_dead) = { + let sql = format!( + "SELECT col_version FROM \"{}__crsql_clock\" WHERE key = ? AND col_name = '{}'\0", + escaped, crate::c::DELETE_SENTINEL + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, v1_key)?; + if stmt.step()? == ResultCode::ROW { + let cl = stmt.column_int64(0); + (cl, cl % 2 == 0) + } else { + // No sentinel — row is alive at CL=1 (or whatever clocks exist) + // Check if any clock entries exist at all + let sql = format!( + "SELECT 1 FROM \"{}__crsql_clock\" WHERE key = ? LIMIT 1\0", + escaped + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, v1_key)?; + if stmt.step()? == ResultCode::ROW { + (1, false) // Alive with no explicit sentinel → CL=1 + } else { + return Ok(()); // No V1 metadata at all — nothing to hydrate + } + } + }; + + if is_dead { + // 3a. Row is dead — insert into v2_tombstones + v2_tombstone_pks + // Get tombstone metadata from V1 sentinel + let sql = format!( + "SELECT site_id, db_version, seq, ts FROM \"{}__crsql_clock\" WHERE key = ? AND col_name = '{}'\0", + escaped, crate::c::DELETE_SENTINEL + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, v1_key)?; + if stmt.step()? == ResultCode::ROW { + let site_id = stmt.column_int64(0); + let db_version = stmt.column_int64(1); + let seq = stmt.column_int64(2); + let ts = stmt.column_int64(3); + let ts = if ts > 0 { ts } else { ts_fallback }; + + // Insert tombstone + let sql = format!( + "INSERT OR REPLACE INTO \"{}{}\" (site_id, db_version, seq, hashed_pk, cl, ts) VALUES (?, ?, ?, ?, ?, ?)\0", + escaped, consts::V2_TOMBSTONES_SUFFIX + ); + let ins_stmt = db.prepare_v2(&sql)?; + ins_stmt.bind_int64(1, site_id)?; + ins_stmt.bind_int64(2, db_version)?; + ins_stmt.bind_int64(3, seq)?; + ins_stmt.bind_blob(4, hashed_pk, sqlite::Destructor::STATIC)?; + ins_stmt.bind_int64(5, v1_cl)?; + ins_stmt.bind_int64(6, ts)?; + ins_stmt.step()?; + + // Insert tombstone PKs + let pk_cols: Vec<&str> = tbl_info.pks.iter().map(|c| c.name.as_str()).collect(); + let placeholders: Vec<&str> = unpacked_pks.iter().map(|_| "?").collect(); + let sql = format!( + "INSERT OR REPLACE INTO \"{}{}\" ({}, hashed_pk) VALUES ({}, ?)\0", + escaped, consts::V2_TOMBSTONE_PKS_SUFFIX, + pk_cols.join(", "), + placeholders.join(", ") + ); + let ins_stmt = db.prepare_v2(&sql)?; + bind_package_to_stmt(ins_stmt.stmt, unpacked_pks, 0)?; + ins_stmt.bind_blob(unpacked_pks.len() as i32 + 1, hashed_pk, sqlite::Destructor::STATIC)?; + ins_stmt.step()?; + } + } else { + // 3b. Row is alive — insert into v2_pks, then copy clock entries to v2_clock + let v2_key = if tbl_info.uses_rowid_key { + let pk_cols: Vec<&str> = tbl_info.pks.iter().map(|c| c.name.as_str()).collect(); + let placeholders: Vec<&str> = unpacked_pks.iter().map(|_| "?").collect(); + let alias = crate::util::escape_ident(&tbl_info.rowid_alias); + let sql = format!( + "SELECT \"{}\" FROM \"{}\" WHERE {}\0", + alias, escaped, + pk_cols.iter().zip(placeholders.iter()) + .map(|(c, p)| format!("\"{}\" = {}", c, p)) + .collect::>().join(" AND ") + ); + let stmt = db.prepare_v2(&sql)?; + bind_package_to_stmt(stmt.stmt, unpacked_pks, 0)?; + if stmt.step()? != ResultCode::ROW { + return Ok(()); // Row not found in base table — skip hydration + } + let rowid_val = stmt.column_int64(0); + let sql = format!( + "INSERT INTO \"{}{}\" (__crsql_key, hashed_pk, cl) VALUES (?, ?, ?)\0", + escaped, consts::V2_PKS_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, rowid_val)?; + stmt.bind_blob(2, hashed_pk, sqlite::Destructor::STATIC)?; + stmt.bind_int64(3, v1_cl)?; + stmt.step()?; + rowid_val + } else { + // Non-rowid table: insert PK columns into v2_pks + let pk_cols: Vec<&str> = tbl_info.pks.iter().map(|c| c.name.as_str()).collect(); + let placeholders: Vec<&str> = unpacked_pks.iter().map(|_| "?").collect(); + let sql = format!( + "INSERT INTO \"{}{}\" ({}, hashed_pk, cl) VALUES ({}, ?, ?) RETURNING __crsql_key\0", + escaped, consts::V2_PKS_SUFFIX, + pk_cols.join(", "), + placeholders.join(", ") + ); + let stmt = db.prepare_v2(&sql)?; + bind_package_to_stmt(stmt.stmt, unpacked_pks, 0)?; + stmt.bind_blob(unpacked_pks.len() as i32 + 1, hashed_pk, sqlite::Destructor::STATIC)?; + stmt.bind_int64(unpacked_pks.len() as i32 + 2, v1_cl)?; + stmt.step()?; + stmt.column_int64(0) + }; + + // Copy clock entries from V1 to V2 + let sql = format!( + "INSERT INTO \"{}{}\" (cell_key, col_version, site_id, db_version, seq, ts) + SELECT (? << {} | m.col_id), c.col_version, c.site_id, c.db_version, c.seq, + CASE WHEN c.ts > 0 THEN c.ts ELSE {ts_fallback} END + FROM \"{}__crsql_clock\" c + JOIN \"{}{}\" m ON c.col_name = m.col_name + WHERE c.key = ?\0", + escaped, consts::V2_CLOCK_SUFFIX, + col_id_bits, + escaped, + escaped, consts::V2_COL_MAP_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, v2_key)?; + stmt.bind_int64(2, v1_key)?; + stmt.step()?; + } + + Ok(()) +} + +/// Convenience wrapper for local write paths that have raw sqlite::value pointers. +/// Converts PKs to ColumnValue, computes hashed_pk, and calls v1_to_v2_hydrate_row. +pub unsafe fn v1_to_v2_hydrate_row_from_values( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + pks: &[*mut sqlite::value], +) -> Result<(), ResultCode> { + let unpacked_pks: Vec = pks.iter().map(|v| sqlite_value_to_column_value(*v)).collect(); + let packed = crate::pack_columns::pack_column_values(&unpacked_pks)?; + let hashed_pk = crate::hash_pk::hash_packed_blob(&packed); + v1_to_v2_hydrate_row(db, ext_data, tbl_info, &unpacked_pks, &hashed_pk) +} + +/// Insert a row into the base table with PK columns only (OR IGNORE). +/// Sync bit is set to suppress after_insert triggers. +unsafe fn v2_insert_base_row( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + escaped: &str, + tbl_info: &TableInfo, + unpacked_pks: &Vec, +) -> Result<(), ResultCode> { + let pk_cols: Vec<&str> = tbl_info.pks.iter().map(|c| c.name.as_str()).collect(); + let placeholders: Vec<&str> = unpacked_pks.iter().map(|_| "?").collect(); + let base_sql = format!( + "INSERT INTO \"{}\" ({}) VALUES ({})\0", + escaped, + pk_cols.join(", "), + placeholders.join(", ") + ); + (*ext_data).pSetSyncBitStmt.step()?; + (*ext_data).pSetSyncBitStmt.reset()?; + let base_stmt = db.prepare_v2(&base_sql)?; + bind_package_to_stmt(base_stmt.stmt, unpacked_pks, 0)?; + base_stmt.step()?; + reset_cached_stmt(base_stmt.stmt)?; + (*ext_data).pClearSyncBitStmt.step()?; + (*ext_data).pClearSyncBitStmt.reset()?; + Ok(()) +} + +/// Insert a new PK row into v2_pks and return the __crsql_key. +/// For rowid-key tables: INSERT into base table first, get the rowid, +/// then use it as __crsql_key. +/// For non-rowid tables: INSERT into base table, then INSERT PK columns + +/// hashed_pk + cl into v2_pks (auto-increment __crsql_key) with RETURNING. +unsafe fn v2_insert_pk_row( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + escaped: &str, + tbl_info: &TableInfo, + unpacked_pks: &Vec, + hashed_pk: &[u8], + cl: i64, +) -> Result { + // First create the row in the base table + v2_insert_base_row(db, ext_data, escaped, tbl_info, unpacked_pks)?; + + // Then insert the PK row into v2_pks + if tbl_info.uses_rowid_key { + let rowid = sqlite::last_insert_rowid(db); + let sql = format!( + "INSERT INTO \"{}{}\" (__crsql_key, hashed_pk, cl) VALUES (?, ?, ?)\0", + escaped, + consts::V2_PKS_SUFFIX, + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, rowid)?; + stmt.bind_blob(2, hashed_pk, sqlite::Destructor::STATIC)?; + stmt.bind_int64(3, cl)?; + stmt.step()?; + Ok(rowid) + } else { + let pk_cols: Vec<&str> = tbl_info.pks.iter().map(|c| c.name.as_str()).collect(); + let placeholders: Vec<&str> = unpacked_pks.iter().map(|_| "?").collect(); + let sql = format!( + "INSERT INTO \"{}{}\" ({}, hashed_pk, cl) VALUES ({}, ?, ?) RETURNING __crsql_key\0", + escaped, + consts::V2_PKS_SUFFIX, + pk_cols.join(", "), + placeholders.join(", ") + ); + let stmt = db.prepare_v2(&sql)?; + bind_package_to_stmt(stmt.stmt, unpacked_pks, 0)?; + stmt.bind_blob(unpacked_pks.len() as i32 + 1, hashed_pk, sqlite::Destructor::STATIC)?; + stmt.bind_int64(unpacked_pks.len() as i32 + 2, cl)?; + stmt.step()?; + Ok(stmt.column_int64(0)) + } +} + +/// Get col_id from v2_col_map by col_name +unsafe fn v2_get_col_id( + db: *mut sqlite3, + escaped: &str, + col_name: &str, +) -> Result, ResultCode> { + let sql = format!( + "SELECT col_id FROM \"{}{}\" WHERE col_name = ?\0", + escaped, consts::V2_COL_MAP_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_text(1, col_name, sqlite::Destructor::STATIC)?; + if stmt.step()? == ResultCode::ROW { + return Ok(Some(stmt.column_int64(0))); + } + Ok(None) +} + +/// Handle V2 wire tombstone (cid=-2): we only have hashed_pk, not packed PK values. +/// Look up PK values from local v2_pks if the row exists there. +/// If the row doesn't exist locally, insert a bare tombstone with hash only. +#[allow(clippy::too_many_arguments)] +unsafe fn v2_merge_insert_hash_tombstone( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &mut TableInfo, + hashed_pk: &[u8], + insert_tbl: &str, + _insert_col: &str, + _insert_val: *mut sqlite::value, + _insert_col_vrsn: sqlite::int64, + insert_db_vrsn: sqlite::int64, + insert_site_id: &[u8], + insert_cl: sqlite::int64, + insert_seq: sqlite::int64, + insert_ts: sqlite::int64, + _rowid: *mut sqlite::int64, + _tbl_info_index: usize, + _errmsg: *mut *mut c_char, +) -> Result { + // ts check is done at the top of merge_insert + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + + // Look up key and CL from v2_pks (alive) or v2_tombstones (dead) + let (local_key, local_cl) = v2_lookup_key_and_cl(db, &escaped, hashed_pk)?; + + // Bail early if incoming CL can't beat local CL + if insert_cl < local_cl { + return Ok(ResultCode::OK); + } + + // If insert_cl == local_cl then the row should be already deleted + // If insert_cl > local_cl then this deletion always wins + // Therefore we can always run this upsert + let merge_equal = unsafe { (*ext_data).mergeEqualValues }; + let where_clause = if merge_equal == 1 { + format!( + "WHERE excluded.cl > \"{}{}\".cl \ + OR (excluded.cl = \"{}{}\".cl \ + AND ? > (SELECT site_id FROM crsql_site_id WHERE ordinal = \"{}{}\".site_id))", + escaped, consts::V2_TOMBSTONES_SUFFIX, + escaped, consts::V2_TOMBSTONES_SUFFIX, + escaped, consts::V2_TOMBSTONES_SUFFIX + ) + } else { + format!( + "WHERE excluded.cl > \"{}{}\".cl", + escaped, consts::V2_TOMBSTONES_SUFFIX + ) + }; + let sql = format!( + "INSERT INTO \"{}{}\" (site_id, db_version, seq, hashed_pk, cl, ts) \ + VALUES (?, ?, ?, ?, ?, ?) \ + ON CONFLICT(hashed_pk) DO UPDATE SET \ + site_id = excluded.site_id, \ + db_version = excluded.db_version, \ + seq = excluded.seq, \ + cl = excluded.cl, \ + ts = excluded.ts \ + {}\0", + escaped, consts::V2_TOMBSTONES_SUFFIX, + where_clause + ); + let stmt = db.prepare_v2(&sql)?; + let site_ordinal = if insert_site_id.is_empty() { + 0 + } else { + get_or_set_site_ordinal(ext_data, insert_site_id)? + }; + stmt.bind_int64(1, site_ordinal as i64)?; + stmt.bind_int64(2, insert_db_vrsn)?; + stmt.bind_int64(3, insert_seq)?; + stmt.bind_blob(4, hashed_pk, sqlite::Destructor::STATIC)?; + stmt.bind_int64(5, insert_cl)?; + stmt.bind_int64(6, insert_ts)?; + if merge_equal == 1 { + stmt.bind_blob(7, insert_site_id, sqlite::Destructor::STATIC)?; + } + stmt.step()?; + + // If the row was alive, nuke its local state (clocks, v2_pks, base table row). + // Also save PK values into v2_tombstone_pks for future lookups. + if let Some(local_key) = local_key { + let pk_col_names: Vec = tbl_info.pks.iter().map(|c| c.name.clone()).collect(); + let pk_cols: Vec<&str> = pk_col_names.iter().map(|s| s.as_str()).collect(); + + // Look up PK values once — used for both tombstone_pks insert and v2_nuke_local_row. + // For rowid-key tables, PKs live in the base table (v2_pks only has __crsql_key). + // For non-rowid tables, PKs are stored directly in v2_pks. + let lookup_sql = if tbl_info.uses_rowid_key { + format!( + "SELECT {} FROM \"{}\" WHERE \"{}\" = ?\0", + pk_cols.join(", "), + escaped, + crate::util::escape_ident(&tbl_info.rowid_alias) + ) + } else { + format!( + "SELECT {} FROM \"{}{}\" WHERE __crsql_key = ?\0", + pk_cols.join(", "), + escaped, consts::V2_PKS_SUFFIX + ) + }; + let stmt = db.prepare_v2(&lookup_sql)?; + stmt.bind_int64(1, local_key)?; + let mut unpacked_pks: Vec = Vec::new(); + if stmt.step()? == ResultCode::ROW { + for i in 0..pk_cols.len() { + let val = stmt.column_value(i as i32)?; + unpacked_pks.push(sqlite_value_to_column_value(val)); + } + } + + // Save PK values into v2_tombstone_pks for future lookups + if !unpacked_pks.is_empty() { + let placeholders: Vec<&str> = unpacked_pks.iter().map(|_| "?").collect(); + let sql = format!( + "INSERT INTO \"{}{}\" ({}, hashed_pk) VALUES ({}, ?)\0", + escaped, + consts::V2_TOMBSTONE_PKS_SUFFIX, + pk_cols.join(", "), + placeholders.join(", ") + ); + let stmt = db.prepare_v2(&sql)?; + bind_package_to_stmt(stmt.stmt, &unpacked_pks, 0)?; + stmt.bind_blob(unpacked_pks.len() as i32 + 1, hashed_pk, sqlite::Destructor::STATIC)?; + stmt.step()?; + } + + // Nuke clocks, v2_pks, and base table row + v2_nuke_local_row(db, ext_data, &escaped, local_key, &unpacked_pks, tbl_info)?; + } + + Ok(ResultCode::OK) +} + +/// Convert a sqlite value pointer to a ColumnValue for binding. +pub unsafe fn sqlite_value_to_column_value(val: *mut sqlite::value) -> ColumnValue { + use sqlite_nostd::Value; + let val_type = val.value_type(); + match val_type { + sqlite::ColumnType::Integer => ColumnValue::Integer(val.int64()), + sqlite::ColumnType::Float => ColumnValue::Float(val.double()), + sqlite::ColumnType::Text => { + ColumnValue::Text(alloc::string::ToString::to_string(val.text())) + } + sqlite::ColumnType::Blob => { + ColumnValue::Blob(val.blob().to_vec()) + } + sqlite::ColumnType::Null => ColumnValue::Null, + } +} + +/// V2 packed merge: process a full V2 wire packed row in one pass. +/// Looks up local key/CL once, handles resurrection and skipped-delete cleanup, +/// then applies each column change. +#[allow(clippy::too_many_arguments)] +unsafe fn v2_packed_merge( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &mut TableInfo, + unpacked_pks: &Vec, + hashed_pk: &[u8], + col_names: &[&str], + col_vrsns: &[i64], + seqs: &[i64], + unpacked_vals: &[ColumnValue], + db_vrsn: i64, + site_id: &[u8], + incoming_cl: i64, + ts: i64, +) -> Result { + // ts check is done at the top of merge_insert + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + let col_id_bits = consts::CRSQL_COL_ID_BITS; + + // Ensure alive row exists at incoming_cl. Handles stale bail, resurrection, + // skipped-delete cleanup, and new row creation in one shot. + let site_ordinal = if site_id.is_empty() { + 0 + } else { + get_or_set_site_ordinal(ext_data, site_id)? + }; + let (local_key, local_cl) = match v2_ensure_alive_row_at_cl( + db, ext_data, tbl_info, unpacked_pks, hashed_pk, incoming_cl, + db_vrsn, site_ordinal, + )? { + Some(result) => result, + None => return Ok(ResultCode::OK), // stale CL + }; + + // Apply each column change. + // The upsert in v2_apply_value_change_colval handles conflict resolution: + // - When CL won, local clocks are at col_version=0, so incoming always wins. + // - When CL is equal, the WHERE clause compares col_version and values. + for i in 0..col_names.len() { + v2_apply_value_change_colval( + db, ext_data, tbl_info, &escaped, local_key, col_names[i], + &unpacked_vals[i], col_vrsns[i], db_vrsn, site_id, seqs[i], ts, + unpacked_pks, col_id_bits, + )?; + } + + Ok(ResultCode::OK) +} + +/// Remove all local state for a row: clocks, v2_pks entry, and base table row. +unsafe fn v2_nuke_local_row( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + escaped: &str, + key: i64, + unpacked_pks: &Vec, + tbl_info: &TableInfo, +) -> Result<(), ResultCode> { + let col_id_bits = consts::CRSQL_COL_ID_BITS as i64; + let col_id_mask = consts::CRSQL_COL_ID_MASK as i64; + + // Delete clocks for this key (range scan on INTEGER PRIMARY KEY index) + let base = key << col_id_bits; + let sql = format!( + "DELETE FROM \"{}{}\" WHERE cell_key >= ? AND cell_key <= ?\0", + escaped, consts::V2_CLOCK_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, base)?; + stmt.bind_int64(2, base | col_id_mask)?; + stmt.step()?; + + // Delete from v2_pks + let sql = format!( + "DELETE FROM \"{}{}\" WHERE __crsql_key = ?\0", + escaped, consts::V2_PKS_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_int64(1, key)?; + stmt.step()?; + + // Delete from base table (with sync bit to prevent trigger recursion) + let sql = if tbl_info.uses_rowid_key { + format!( + "DELETE FROM \"{}\" WHERE \"{}\" = ?\0", + escaped, + crate::util::escape_ident(&tbl_info.rowid_alias) + ) + } else { + let pk_cols: Vec<&str> = tbl_info.pks.iter().map(|c| c.name.as_str()).collect(); + let where_conds: Vec = pk_cols.iter().map(|c| { + format!("\"{}\" = ?", crate::util::escape_ident(c)) + }).collect(); + format!( + "DELETE FROM \"{}\" WHERE {}\0", + escaped, + where_conds.join(" AND ") + ) + }; + // Set sync bit to suppress after_delete trigger + (*ext_data).pSetSyncBitStmt.step()?; + (*ext_data).pSetSyncBitStmt.reset()?; + let stmt = db.prepare_v2(&sql)?; + if tbl_info.uses_rowid_key { + stmt.bind_int64(1, key)?; + } else { + bind_package_to_stmt(stmt.stmt, unpacked_pks, 0)?; + } + stmt.step()?; + // Clear sync bit + (*ext_data).pClearSyncBitStmt.step()?; + (*ext_data).pClearSyncBitStmt.reset()?; + + Ok(()) +} + +/// Remove tombstone entries for a hashed_pk (resurrection cleanup). +unsafe fn v2_nuke_tombstone( + db: *mut sqlite3, + escaped: &str, + hashed_pk: &[u8], +) -> Result<(), ResultCode> { + let del_tomb = db.prepare_v2(&format!( + "DELETE FROM \"{}{}\" WHERE hashed_pk = ?\0", + escaped, consts::V2_TOMBSTONES_SUFFIX + ))?; + del_tomb.bind_blob(1, hashed_pk, sqlite::Destructor::STATIC)?; + del_tomb.step()?; + + let del_tpk = db.prepare_v2(&format!( + "DELETE FROM \"{}{}\" WHERE hashed_pk = ?\0", + escaped, consts::V2_TOMBSTONE_PKS_SUFFIX + ))?; + del_tpk.bind_blob(1, hashed_pk, sqlite::Destructor::STATIC)?; + del_tpk.step()?; + + Ok(()) +} + +/// Look up PK values from V2 metadata tables (v2_pks or v2_tombstone_pks) by hashed_pk. +/// Used when unpacked PKs are not available from the wire (e.g., hash tombstone case). +unsafe fn v2_lookup_pks_for_v1_copy( + db: *mut sqlite3, + escaped: &str, + tbl_info: &TableInfo, + hashed_pk: &[u8], +) -> Result, ResultCode> { + let pk_col_names: Vec = tbl_info.pks.iter().map(|c| crate::util::escape_ident(&c.name)).collect(); + let pk_list = pk_col_names.join(", "); + + // Try v2_tombstone_pks first (row might have been deleted) + let sql = format!( + "SELECT {} FROM \"{}{}\" WHERE hashed_pk = ?\0", + pk_list, escaped, consts::V2_TOMBSTONE_PKS_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_blob(1, hashed_pk, sqlite::Destructor::STATIC)?; + if stmt.step()? == ResultCode::ROW { + let mut result = Vec::new(); + for i in 0..tbl_info.pks.len() { + result.push(sqlite_value_to_column_value(stmt.column_value(i as i32)?)); + } + return Ok(result); + } + + // Try v2_pks (row might still be alive) + if tbl_info.uses_rowid_key { + let alias = crate::util::escape_ident(&tbl_info.rowid_alias); + let t_pk_list: Vec = pk_col_names.iter().map(|c| format!("t.{}", c)).collect(); + let sql = format!( + "SELECT {} FROM \"{}{}\" p JOIN \"{}\" t ON t.\"{}\" = p.__crsql_key WHERE p.hashed_pk = ?\0", + t_pk_list.join(", "), escaped, consts::V2_PKS_SUFFIX, + escaped, alias + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_blob(1, hashed_pk, sqlite::Destructor::STATIC)?; + if stmt.step()? == ResultCode::ROW { + let mut result = Vec::new(); + for i in 0..tbl_info.pks.len() { + result.push(sqlite_value_to_column_value(stmt.column_value(i as i32)?)); + } + return Ok(result); + } + } else { + let sql = format!( + "SELECT {} FROM \"{}{}\" WHERE hashed_pk = ?\0", + pk_list, escaped, consts::V2_PKS_SUFFIX + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_blob(1, hashed_pk, sqlite::Destructor::STATIC)?; + if stmt.step()? == ResultCode::ROW { + let mut result = Vec::new(); + for i in 0..tbl_info.pks.len() { + result.push(sqlite_value_to_column_value(stmt.column_value(i as i32)?)); + } + return Ok(result); + } + } + + Ok(Vec::new()) +} + +/// Copy V2 metadata state to V1 metadata tables for dual-write mode. +/// Called after V2 merge has completed for SchemaVersion::V2AndV1 tables. +/// Receives the post-merge V2 key/CL from the caller to avoid a redundant lookup. +/// Reads the current V2 clock entries and mirrors them to V1 tables +/// (__crsql_pks + __crsql_clock), ensuring semantic equivalence +/// without relying on trigger-based V1 metadata population. +unsafe fn v2_to_v1_mirror_metadata( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + unpacked_pks: Option<&Vec>, + hashed_pk: &[u8], + v2_key_opt: Option, + v2_cl: i64, +) -> Result<(), ResultCode> { + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + let ts_fallback = unsafe { (*ext_data).timestamp as i64 }; + if v2_cl == 0 { + return Ok(()); + } + + // 1. Get unpacked PKs — from parameter or look up from V2 tables + let looked_up_pks; + let pks: &Vec = match unpacked_pks { + Some(p) => p, + None => { + looked_up_pks = v2_lookup_pks_for_v1_copy(db, &escaped, tbl_info, hashed_pk)?; + &looked_up_pks + } + }; + + if pks.is_empty() { + return Ok(()); + } + + // 3. Get or create V1 PK entry + let v1_key = tbl_info.get_or_create_key(db, pks)?; + + // 4. Delete all existing V1 clock entries and sentinels for this v1 key + let del_sql = format!( + "DELETE FROM \"{}__crsql_clock\" WHERE key = ?\0", + escaped + ); + let del_stmt = db.prepare_v2(&del_sql)?; + del_stmt.bind_int64(1, v1_key)?; + del_stmt.step()?; + + let col_id_bits = consts::CRSQL_COL_ID_BITS as i64; + let col_id_mask = consts::CRSQL_COL_ID_MASK as i64; + + // Set the V1 sentinel if needed + // V1 omits the sentinel for CL=1 (alive, default), so only write it for CL > 1. + // Besides the CL V1 requires (site_id, db_version, seq, ts) + // In V2 we have this extra data for dead rows but we don't have it for alive rows + // For alive rows we improvise by taking the values from col_id=0 + if v2_cl > 1 { + let (source_sql, bind_cell_key) = if let Some(k) = v2_key_opt { + ( + format!( + "SELECT site_id, db_version, seq, ts FROM \"{}{}\" WHERE cell_key = ?\0", + escaped, consts::V2_CLOCK_SUFFIX + ), + (k << col_id_bits) | 0, + ) + } else { + ( + format!( + "SELECT site_id, db_version, seq, ts FROM \"{}{}\" WHERE hashed_pk = ?\0", + escaped, consts::V2_TOMBSTONES_SUFFIX + ), + 0, + ) + }; + let ins_sql = format!( + "INSERT INTO \"{}__crsql_clock\" (key, col_name, col_version, db_version, seq, site_id, ts) + SELECT ?, '-1', {cl}, site_id, db_version, seq, + CASE WHEN ts > 0 THEN ts ELSE {ts_fallback} END + FROM ({src}) LIMIT 1\0", + escaped, cl = v2_cl, ts_fallback = ts_fallback, src = source_sql.trim_end_matches('\0') + ); + let ins_stmt = db.prepare_v2(&ins_sql)?; + ins_stmt.bind_int64(1, v1_key)?; + if v2_key_opt.is_some() { + ins_stmt.bind_int64(2, bind_cell_key)?; + } else { + ins_stmt.bind_blob(2, hashed_pk, sqlite::Destructor::STATIC)?; + } + ins_stmt.step()?; + } + + // Copy clocks from V2 to V1. For PK only tables it's an no op as the + // insert sentinel was created already and the V2_COL_MAP_SUFFIX join will filter it out + if let Some(v2_key) = v2_key_opt { + // cell_key = (v2_key << col_id_bits) | col_id + // All entries for this v2_key are in range [base, base | col_id_mask]. + // Use range scan to leverage the INTEGER PRIMARY KEY index. + let base = v2_key << col_id_bits; + let cell_sql = format!( + "INSERT INTO \"{}__crsql_clock\" (key, col_name, col_version, db_version, seq, site_id, ts) + SELECT ?, m.col_name, c.col_version, c.db_version, c.seq, + CASE WHEN c.ts > 0 THEN c.ts ELSE {ts_fallback} END, c.site_id + FROM \"{}{}\" c + JOIN \"{}{}\" m ON (c.cell_key & ?) = m.col_id + WHERE c.cell_key >= ? AND c.cell_key <= ?\0", + escaped, + escaped, consts::V2_CLOCK_SUFFIX, + escaped, consts::V2_COL_MAP_SUFFIX, + ts_fallback = ts_fallback, + ); + let cell_stmt = db.prepare_v2(&cell_sql)?; + cell_stmt.bind_int64(1, v1_key)?; + cell_stmt.bind_int64(2, col_id_mask)?; + cell_stmt.bind_int64(3, base)?; + cell_stmt.bind_int64(4, base | col_id_mask)?; + cell_stmt.step()?; + } + + Ok(()) +} + +/// Apply a value change using ColumnValue instead of *mut sqlite::value. +/// The clock upsert decides if the change wins via WHERE + RETURNING. +/// The base table is only updated if the clock upsert applied (RETURNING yielded a row). +#[allow(clippy::too_many_arguments)] +unsafe fn v2_apply_value_change_colval( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + escaped: &str, + key: i64, + col_name: &str, + val: &ColumnValue, + col_vrsn: i64, + db_version: i64, + site_id: &[u8], + seq: i64, + ts: i64, + unpacked_pks: &Vec, + col_id_bits: u32, +) -> Result<(), ResultCode> { + // Get site ordinal + let site_ordinal = if site_id.is_empty() { + 0 + } else { + get_or_set_site_ordinal(ext_data, site_id)? + }; + + // Get col_id + let col_id = v2_get_col_id(db, escaped, col_name)?; + if col_id.is_none() { + return Ok(()); + } + let col_id = col_id.unwrap(); + let cell_key = (key << col_id_bits) | (col_id as i64); + + // Build base table subquery for value comparison. + // For rowid-key tables, use rowid alias directly (1 param). + // For non-rowid tables, use PK columns (n params). + let escaped_col = crate::util::escape_ident(col_name); + let merge_equal = unsafe { (*ext_data).mergeEqualValues }; + let pk_len = unpacked_pks.len() as i32; + + let (subquery, subquery_param_count) = if tbl_info.uses_rowid_key { + let alias = crate::util::escape_ident(&tbl_info.rowid_alias); + ( + format!("SELECT \"{}\" FROM \"{}\" WHERE \"{}\" = ?", escaped_col, escaped, alias), + 1 + ) + } else { + let pk_where: String = tbl_info.pks + .iter() + .map(|c| format!("\"{}\" = ?", crate::util::escape_ident(&c.name))) + .collect::>() + .join(" AND "); + ( + format!("SELECT \"{}\" FROM \"{}\" WHERE {}", escaped_col, escaped, pk_where), + pk_len + ) + }; + + // Upsert v2_clock entry with conflict resolution in WHERE clause. + // RETURNING tells us if the change won — only then do we update the base table. + // site_id comparison uses blob comparison via crsql_site_id join, matching V1 semantics. + let sql = format!( + "INSERT INTO \"{}{}\" (cell_key, col_version, site_id, db_version, seq, ts)\n + VALUES (?, ?, ?, ?, ?, ?)\n + ON CONFLICT(cell_key) DO UPDATE SET\n + col_version = excluded.col_version,\n + site_id = excluded.site_id,\n + db_version = excluded.db_version,\n + seq = excluded.seq,\n + ts = excluded.ts\n + WHERE excluded.col_version > col_version\n + OR (excluded.col_version = col_version AND\n + crsql_change_wins(?, ({}),\n + ? > (SELECT site_id FROM crsql_site_id WHERE ordinal = site_id), ?))\n + RETURNING cell_key\0", + escaped, consts::V2_CLOCK_SUFFIX, + subquery + ); + let stmt = db.prepare_v2(&sql)?; + // Bind clock values (params 1-6) + stmt.bind_int64(1, cell_key)?; + stmt.bind_int64(2, col_vrsn)?; + stmt.bind_int64(3, site_ordinal)?; + stmt.bind_int64(4, db_version)?; + stmt.bind_int64(5, seq)?; + stmt.bind_int64(6, ts)?; + // Bind incoming value for crsql_change_wins (param 7) + match val { + ColumnValue::Integer(i) => { stmt.bind_int64(7, *i)?; } + ColumnValue::Float(f) => { stmt.bind_double(7, *f)?; } + ColumnValue::Text(t) => { stmt.bind_text(7, t, sqlite::Destructor::STATIC)?; } + ColumnValue::Blob(b) => { stmt.bind_blob(7, b, sqlite::Destructor::STATIC)?; } + ColumnValue::Null => { stmt.bind_null(7)?; } + } + // Bind subquery params (params 8..8+subquery_param_count) + if tbl_info.uses_rowid_key { + stmt.bind_int64(8, key)?; + } else { + bind_package_to_stmt(stmt.stmt, unpacked_pks, 7)?; + } + // Bind incoming site_id blob for comparison (param 8+subquery_param_count) + stmt.bind_blob(8 + subquery_param_count, site_id, sqlite::Destructor::STATIC)?; + // Bind mergeEqualValues flag (param 9+subquery_param_count) + stmt.bind_int(9 + subquery_param_count, merge_equal)?; + + let won = stmt.step()? == ResultCode::ROW; + reset_cached_stmt(stmt.stmt)?; + + if !won { + return Ok(()); + } + + // Change won — update the actual user table. + // Set sync bit to suppress triggers that would overwrite V2 clock with local site_id. + (*ext_data).pSetSyncBitStmt.step()?; + (*ext_data).pSetSyncBitStmt.reset()?; + + let merge_stmt_ref = tbl_info.get_merge_insert_stmt(db, col_name)?; + let merge_stmt = merge_stmt_ref.as_ref().ok_or(ResultCode::ERROR)?; + bind_package_to_stmt(merge_stmt.stmt, unpacked_pks, 0)?; + let raw_stmt = merge_stmt.stmt; + match val { + ColumnValue::Integer(i) => { raw_stmt.bind_int64(pk_len + 1, *i)?; } + ColumnValue::Float(f) => { raw_stmt.bind_double(pk_len + 1, *f)?; } + ColumnValue::Text(t) => { raw_stmt.bind_text(pk_len + 1, t, sqlite::Destructor::STATIC)?; } + ColumnValue::Blob(b) => { raw_stmt.bind_blob(pk_len + 1, b, sqlite::Destructor::STATIC)?; } + ColumnValue::Null => { raw_stmt.bind_null(pk_len + 1)?; } + } + match val { + ColumnValue::Integer(i) => { raw_stmt.bind_int64(pk_len + 2, *i)?; } + ColumnValue::Float(f) => { raw_stmt.bind_double(pk_len + 2, *f)?; } + ColumnValue::Text(t) => { raw_stmt.bind_text(pk_len + 2, t, sqlite::Destructor::STATIC)?; } + ColumnValue::Blob(b) => { raw_stmt.bind_blob(pk_len + 2, b, sqlite::Destructor::STATIC)?; } + ColumnValue::Null => { raw_stmt.bind_null(pk_len + 2)?; } + } + raw_stmt.step()?; + raw_stmt.reset()?; + + (*ext_data).pClearSyncBitStmt.step()?; + (*ext_data).pClearSyncBitStmt.reset()?; + + Ok(()) +} diff --git a/core/rs/core/src/config.rs b/core/rs/core/src/config.rs index 44ba9511f..1c734d99a 100644 --- a/core/rs/core/src/config.rs +++ b/core/rs/core/src/config.rs @@ -1,5 +1,7 @@ use alloc::format; +use alloc::vec::Vec; +use core::ffi::c_int; use sqlite::{Connection, Context}; use sqlite_nostd as sqlite; use sqlite_nostd::{ManagedStmt, ResultCode, Value}; @@ -7,6 +9,22 @@ use sqlite_nostd::{ManagedStmt, ResultCode, Value}; use crate::c::crsql_ExtData; pub const MERGE_EQUAL_VALUES: &str = "merge-equal-values"; +pub const METADATA_WRITE_VERSION: &str = "metadata-write-version"; +pub const METADATA_USE_VERSION: &str = "metadata-use-version"; +pub const SYNC_LOG_VERSION: &str = "sync-log-version"; + +/// Default metadata write version: 1 = V1 (legacy), 2 = V2&V1 (dual write), 3 = V2 only +pub const METADATA_WRITE_VERSION_DEFAULT: c_int = 1; +/// Default metadata use version: 1 = V1 (legacy), 2 = V2 +pub const METADATA_USE_VERSION_DEFAULT: c_int = 1; +/// Default sync log version: 1 = V1 (per-column rows), 2 = V2 (packed) +pub const SYNC_LOG_VERSION_DEFAULT: c_int = 1; + +// Integer values for metadata-write-version config option +// Migration order: 1 -> 2 -> 3 (forward only, except 2 -> 1 rollback) +pub const METADATA_VERSION_V1: c_int = 1; +pub const METADATA_VERSION_V2_AND_V1: c_int = 2; // dual write, migration in progress +pub const METADATA_VERSION_V2: c_int = 3; // V2 only, V1 tables dropped pub extern "C" fn crsql_config_set( ctx: *mut sqlite::context, @@ -16,14 +34,176 @@ pub extern "C" fn crsql_config_set( let args = sqlite::args!(argc, argv); let name = args[0].text(); + let ext_data = ctx.user_data() as *mut crsql_ExtData; let value = match name { MERGE_EQUAL_VALUES => { let value = args[1]; - let ext_data = ctx.user_data() as *mut crsql_ExtData; unsafe { (*ext_data).mergeEqualValues = value.int() }; value } + METADATA_WRITE_VERSION => { + let new_val = args[1].int(); + let old_val = unsafe { (*ext_data).metadataWriteVersion }; + if !validate_write_version_transition(old_val, new_val) { + ctx.result_error("Invalid metadata-write-version transition"); + ctx.result_error_code(ResultCode::ERROR); + return; + } + let db = ctx.db_handle(); + // Direct 1->3 transition: skip migration/cleanup, just verify no CRR tables exist + if old_val == METADATA_VERSION_V1 && new_val == METADATA_VERSION_V2 { + match has_no_crr_tables(db) { + Ok(true) => { + // No CRR tables — safe to go directly to V2-only + }, + Ok(false) => { + ctx.result_error("Cannot set metadata-write-version to v2 directly: existing CRR tables found. Migrate via v2&v1 first."); + ctx.result_error_code(ResultCode::ERROR); + return; + }, + Err(rc) => { + ctx.result_error("Failed to check for existing CRR tables"); + ctx.result_error_code(rc); + return; + } + } + } else { + // Any other transition requires prior cleanup tasks to be done + match is_cleanup_complete(db) { + Ok(true) => {}, + Ok(false) => { + ctx.result_error("Cannot transition metadata-write-version: cleanup tasks still pending"); + ctx.result_error_code(ResultCode::ERROR); + return; + }, + Err(rc) => { + ctx.result_error("Failed to check cleanup status"); + ctx.result_error_code(rc); + return; + } + } + // Setting to v2&v1 queues migration tasks for all V1 CRR tables + if new_val == METADATA_VERSION_V2_AND_V1 && old_val == METADATA_VERSION_V1 { + // Create V2 tables for all existing CRR tables so dual-write + // triggers have somewhere to write immediately. + if let Err(rc) = create_v2_tables_for_existing_crrs(db) { + ctx.result_error("Failed to create V2 tables during transition"); + ctx.result_error_code(rc); + return; + } + if let Err(rc) = queue_migration_tasks(db) { + ctx.result_error("Failed to queue migration tasks"); + ctx.result_error_code(rc); + return; + } + } + // Transitioning to v2 (dropping V1 tables) requires migration to be complete + if new_val == METADATA_VERSION_V2 { + match is_migration_complete(db) { + Ok(true) => { + // Queue V1 table cleanup tasks + if let Err(rc) = queue_v1_cleanup_tasks(db) { + ctx.result_error("Failed to queue V1 cleanup tasks"); + ctx.result_error_code(rc); + return; + } + }, + Ok(false) => { + ctx.result_error("Cannot set metadata-write-version to v2: migration not complete for all tables"); + ctx.result_error_code(ResultCode::ERROR); + return; + }, + Err(rc) => { + ctx.result_error("Failed to check migration status"); + ctx.result_error_code(rc); + return; + } + } + } + // Rolling back to v1 queues V2 table cleanup tasks and aborts migration + if new_val == METADATA_VERSION_V1 && old_val == METADATA_VERSION_V2_AND_V1 { + // Clear any pending migration markers since we're aborting migration + if let Err(rc) = clear_migration_markers(db) { + ctx.result_error("Failed to clear migration markers"); + ctx.result_error_code(rc); + return; + } + if let Err(rc) = queue_v2_cleanup_tasks(db) { + ctx.result_error("Failed to queue V2 cleanup tasks"); + ctx.result_error_code(rc); + return; + } + } + } + // Auto-cascade dependent config values to prevent invalid states + if new_val == METADATA_VERSION_V1 { + // Rolling back to V1: force use-version and sync-log-version to V1 + unsafe { (*ext_data).metadataUseVersion = 1; } + unsafe { (*ext_data).syncLogVersion = 1; } + } else if new_val == METADATA_VERSION_V2 { + // Moving to V2-only: force use-version to V2 + unsafe { (*ext_data).metadataUseVersion = 2; } + } + unsafe { (*ext_data).metadataWriteVersion = new_val }; + args[1] + } + METADATA_USE_VERSION => { + let new_val = args[1].int(); + let old_val = unsafe { (*ext_data).metadataUseVersion }; + if !validate_use_version_transition(old_val, new_val, unsafe { (*ext_data).metadataWriteVersion }) { + ctx.result_error("Invalid metadata-use-version transition"); + ctx.result_error_code(ResultCode::ERROR); + return; + } + // Setting to v2 requires all migrations to be complete + if new_val == 2 { + let db = ctx.db_handle(); + match is_migration_complete(db) { + Ok(true) => {}, + Ok(false) => { + ctx.result_error("Cannot set metadata-use-version to v2: migration not complete for all tables"); + ctx.result_error_code(ResultCode::ERROR); + return; + }, + Err(rc) => { + ctx.result_error("Failed to check migration status"); + ctx.result_error_code(rc); + return; + } + } + } + unsafe { (*ext_data).metadataUseVersion = new_val }; + args[1] + } + SYNC_LOG_VERSION => { + let new_val = args[1].int(); + let old_val = unsafe { (*ext_data).syncLogVersion }; + if !validate_sync_log_transition(old_val, new_val, unsafe { (*ext_data).metadataUseVersion }, unsafe { (*ext_data).metadataWriteVersion }) { + ctx.result_error("Invalid sync-log-version transition"); + ctx.result_error_code(ResultCode::ERROR); + return; + } + // Setting to v2 requires all migrations to be complete + if new_val == 2 { + let db = ctx.db_handle(); + match is_migration_complete(db) { + Ok(true) => {}, + Ok(false) => { + ctx.result_error("Cannot set sync-log-version to v2: migration not complete for all tables"); + ctx.result_error_code(ResultCode::ERROR); + return; + }, + Err(rc) => { + ctx.result_error("Failed to check migration status"); + ctx.result_error_code(rc); + return; + } + } + } + unsafe { (*ext_data).syncLogVersion = new_val }; + args[1] + } _ => { ctx.result_error("Unknown setting name"); ctx.result_error_code(ResultCode::ERROR); @@ -79,6 +259,18 @@ pub extern "C" fn crsql_config_get( let ext_data = ctx.user_data() as *mut crsql_ExtData; ctx.result_int(unsafe { (*ext_data).mergeEqualValues }); } + METADATA_WRITE_VERSION => { + let ext_data = ctx.user_data() as *mut crsql_ExtData; + ctx.result_int(unsafe { (*ext_data).metadataWriteVersion }); + } + METADATA_USE_VERSION => { + let ext_data = ctx.user_data() as *mut crsql_ExtData; + ctx.result_int(unsafe { (*ext_data).metadataUseVersion }); + } + SYNC_LOG_VERSION => { + let ext_data = ctx.user_data() as *mut crsql_ExtData; + ctx.result_int(unsafe { (*ext_data).syncLogVersion }); + } _ => { ctx.result_error("Unknown setting name"); ctx.result_error_code(ResultCode::ERROR); @@ -86,3 +278,193 @@ pub extern "C" fn crsql_config_get( } } } + +/// Validate metadata-write-version transitions. +/// Allowed: 1->2, 2->3, 2->1 (rollback), 1->3 (only when no CRR tables exist) +/// Forbidden: 3->2, 3->1 +fn validate_write_version_transition(old: c_int, new: c_int) -> bool { + match (old, new) { + // v1 -> v2&v1: forward, starts migration + (METADATA_VERSION_V1, METADATA_VERSION_V2_AND_V1) => true, + // v2&v1 -> v2: forward, V1 tables will be dropped + (METADATA_VERSION_V2_AND_V1, METADATA_VERSION_V2) => true, + // v2&v1 -> v1: rollback, V1 tables were kept in sync + (METADATA_VERSION_V2_AND_V1, METADATA_VERSION_V1) => true, + // v1 -> v2: direct V2-only, only allowed when no CRR tables exist + // The actual check for empty DB is done in crsql_config_set + (METADATA_VERSION_V1, METADATA_VERSION_V2) => true, + // no-op + _ if old == new => true, + // everything else forbidden + _ => false, + } +} + +/// Validate metadata-use-version transitions. +/// v1->v2: forward (guarded by write version being v2 or v2&v1) +/// v2->v1: only if write version is v1 or v2&v1 (V1 tables still active) +fn validate_use_version_transition(old: c_int, new: c_int, write_version: c_int) -> bool { + match (old, new) { + // v1 -> v2: forward, requires V2 tables being written + (1, 2) => write_version == METADATA_VERSION_V2 || write_version == METADATA_VERSION_V2_AND_V1, + // v2 -> v1: rollback, only if V1 tables still active + (2, 1) => write_version == METADATA_VERSION_V1 || write_version == METADATA_VERSION_V2_AND_V1, + // no-op + _ if old == new => true, + _ => false, + } +} + +/// Validate sync-log-version transitions. +/// v1->v2: forward, requires use-version=v2 and write-version=v2 or v2&v1 +/// v2->v1: rollback, requires use-version=v1 +fn validate_sync_log_transition(old: c_int, new: c_int, use_version: c_int, write_version: c_int) -> bool { + match (old, new) { + // v1 -> v2: forward, requires use=v2 and write=v2/v2&v1 + (1, 2) => use_version == 2 && (write_version == METADATA_VERSION_V2 || write_version == METADATA_VERSION_V2_AND_V1), + // v2 -> v1: rollback, requires use=v1 + (2, 1) => use_version == 1, + // no-op + _ if old == new => true, + _ => false, + } +} + +/// Create V2 metadata tables for all existing CRR tables that don't have them yet. +/// Called during the V1→V2AndV1 transition so dual-write triggers have V2 tables ready. +fn create_v2_tables_for_existing_crrs(db: *mut sqlite_nostd::sqlite3) -> Result<(), ResultCode> { + let sql = "SELECT DISTINCT name FROM sqlite_master WHERE name LIKE '%__crsql_clock'\0"; + let stmt = db.prepare_v2(sql)?; + let mut table_names: Vec = Vec::new(); + while stmt.step()? == ResultCode::ROW { + let name = stmt.column_text(0)?; + if let Some(base) = name.strip_suffix("__crsql_clock") { + table_names.push(alloc::string::String::from(base)); + } + } + + for tbl_name in &table_names { + // Check if V2 tables already exist (e.g., table was created in dual-write mode) + let check_sql = format!( + "SELECT 1 FROM sqlite_master WHERE name = '{}__crsql_v2_pks'\0", + tbl_name + ); + let check = db.prepare_v2(&check_sql)?; + if check.step()? == ResultCode::ROW { + continue; // V2 tables already exist + } + + // Pull table info and create V2 tables + let mut err: *mut core::ffi::c_char = core::ptr::null_mut(); + let tbl_info = crate::tableinfo::pull_table_info(db, tbl_name, &mut err)?; + crate::bootstrap_v2::create_v2_tables(db, &tbl_info)?; + } + + Ok(()) +} + +/// Queue migration tasks for all V1 CRR tables. +/// Sets a progress marker of 0 for each table that has __crsql_clock (V1) tables. +fn queue_migration_tasks(db: *mut sqlite_nostd::sqlite3) -> Result<(), ResultCode> { + // Find all V1 clock tables (tables with __crsql_clock suffix) + let sql = "SELECT DISTINCT name FROM sqlite_master WHERE name LIKE '%__crsql_clock'\0"; + let stmt = db.prepare_v2(sql)?; + let mut table_names: Vec = Vec::new(); + while stmt.step()? == ResultCode::ROW { + let name = stmt.column_text(0)?; + // Strip __crsql_clock suffix to get base table name + if let Some(base) = name.strip_suffix("__crsql_clock") { + table_names.push(alloc::string::String::from(base)); + } + } + + // Queue a migration task for each table by setting progress marker to 0 + for tbl_name in &table_names { + let key = format!("migration_v1_to_v2_migration_{}", tbl_name); + let sql = "INSERT OR REPLACE INTO crsql_master (key, value) VALUES (?, 0)\0"; + let stmt = db.prepare_v2(sql)?; + stmt.bind_text(1, &key, sqlite::Destructor::TRANSIENT)?; + stmt.step()?; + } + + Ok(()) +} + +/// Check if all migration tasks are complete (no pending migration markers in crsql_master). +fn is_migration_complete(db: *mut sqlite_nostd::sqlite3) -> Result { + let sql = "SELECT count(*) FROM crsql_master WHERE key LIKE 'migration_v1_to_v2_migration_%'\0"; + let stmt = db.prepare_v2(sql)?; + stmt.step()?; + let count = stmt.column_int64(0); + Ok(count == 0) +} + +/// Clear all pending migration markers. Called when aborting migration (2->1 rollback). +fn clear_migration_markers(db: *mut sqlite_nostd::sqlite3) -> Result<(), ResultCode> { + let sql = "DELETE FROM crsql_master WHERE key LIKE 'migration_v1_to_v2_migration_%'\0"; + db.exec_safe(sql)?; + Ok(()) +} + +/// Queue V1 table cleanup tasks for all CRR tables that have V1 clock tables. +/// Called when transitioning from v2&v1 to v2 (V1 tables no longer needed). +fn queue_v1_cleanup_tasks(db: *mut sqlite_nostd::sqlite3) -> Result<(), ResultCode> { + let sql = "SELECT DISTINCT name FROM sqlite_master WHERE name LIKE '%__crsql_clock'\0"; + let stmt = db.prepare_v2(sql)?; + let mut table_names: Vec = Vec::new(); + while stmt.step()? == ResultCode::ROW { + let name = stmt.column_text(0)?; + if let Some(base) = name.strip_suffix("__crsql_clock") { + table_names.push(alloc::string::String::from(base)); + } + } + for tbl_name in &table_names { + let key = format!("cleanup_v1_tables_{}", tbl_name); + let sql = "INSERT OR REPLACE INTO crsql_master (key, value) VALUES (?, 0)\0"; + let stmt = db.prepare_v2(sql)?; + stmt.bind_text(1, &key, sqlite::Destructor::TRANSIENT)?; + stmt.step()?; + } + Ok(()) +} + +/// Queue V2 table cleanup tasks for all CRR tables that have V2 tables. +/// Called when rolling back from v2&v1 to v1 (V2 tables no longer needed). +fn queue_v2_cleanup_tasks(db: *mut sqlite_nostd::sqlite3) -> Result<(), ResultCode> { + let sql = "SELECT DISTINCT name FROM sqlite_master WHERE name LIKE '%__crsql_v2_clock'\0"; + let stmt = db.prepare_v2(sql)?; + let mut table_names: Vec = Vec::new(); + while stmt.step()? == ResultCode::ROW { + let name = stmt.column_text(0)?; + if let Some(base) = name.strip_suffix("__crsql_v2_clock") { + table_names.push(alloc::string::String::from(base)); + } + } + for tbl_name in &table_names { + let key = format!("cleanup_v2_tables_{}", tbl_name); + let sql = "INSERT OR REPLACE INTO crsql_master (key, value) VALUES (?, 0)\0"; + let stmt = db.prepare_v2(sql)?; + stmt.bind_text(1, &key, sqlite::Destructor::TRANSIENT)?; + stmt.step()?; + } + Ok(()) +} + +/// Check if all cleanup tasks are complete (no pending cleanup markers in crsql_master). +fn is_cleanup_complete(db: *mut sqlite_nostd::sqlite3) -> Result { + let sql = "SELECT count(*) FROM crsql_master WHERE key LIKE 'cleanup_v1_tables_%' OR key LIKE 'cleanup_v2_tables_%'\0"; + let stmt = db.prepare_v2(sql)?; + stmt.step()?; + let count = stmt.column_int64(0); + Ok(count == 0) +} + +/// Check if the database has no existing CRR tables. +/// Used to determine if a direct 1->3 (V2-only) transition is safe. +fn has_no_crr_tables(db: *mut sqlite_nostd::sqlite3) -> Result { + let sql = "SELECT count(*) FROM sqlite_master WHERE type = 'trigger' AND name LIKE '%__crsql_itrig'\0"; + let stmt = db.prepare_v2(sql)?; + stmt.step()?; + let count = stmt.column_int64(0); + Ok(count == 0) +} diff --git a/core/rs/core/src/consts.rs b/core/rs/core/src/consts.rs index 17d27153e..2e9c71fb3 100644 --- a/core/rs/core/src/consts.rs +++ b/core/rs/core/src/consts.rs @@ -11,10 +11,14 @@ pub const TBL_SCHEMA: &'static str = "crsql_master"; // 00_05_01_00 // and, if we ever need it, we can track individual builds of a patch release // 00_05_01_01 -pub const CRSQLITE_VERSION: i32 = 17_00_00; -pub const _CRSQLITE_VERSION_STR: &'static str = "0.17.0"; +pub const CRSQLITE_VERSION: i32 = 18_00_00; +pub const _CRSQLITE_VERSION_STR: &'static str = "0.18.0"; pub const CRSQLITE_VERSION_0_17_0: i32 = 17_00_00; +/// Minimum SQLite version required: 3.44.0 (ORDER BY in aggregates). +/// Format: XYYZZ00 -> 3.44.0 = 3044000 +pub const MIN_SQLITE_VERSION: i32 = 3044000; + pub const SITE_ID_LEN: i32 = 16; pub const ROWID_SLAB_SIZE: i64 = 10000000000000; // db version is a signed 64bit int since sqlite doesn't support saving and @@ -22,3 +26,33 @@ pub const ROWID_SLAB_SIZE: i64 = 10000000000000; // million entries per second for 3,000 centuries. pub const MIN_POSSIBLE_DB_VERSION: i64 = 0; pub const MAX_TBL_NAME_LEN: i32 = 2048; + +// V2 metadata constants +/// Number of bits used for col_id in the packed cell_key. +/// Default: 12 bits = up to 4096 columns per table. +/// cell_key = (pk_key << CRSQL_COL_ID_BITS) | col_id +pub const CRSQL_COL_ID_BITS: u32 = 12; +/// Mask to extract col_id from cell_key: (1 << CRSQL_COL_ID_BITS) - 1 +pub const CRSQL_COL_ID_MASK: i64 = (1i64 << CRSQL_COL_ID_BITS) - 1; +/// Number of bytes for truncated XXH128 hash of PK values. +/// Default: 10 bytes / 80 bits. +pub const PK_HASH_SIZE: usize = 10; +/// Maximum __crsql_key value for rowid tables to keep cell_key positive in signed INT64. +/// 2^(63 - CRSQL_COL_ID_BITS) +pub const MAX_ROWID_KEY: i64 = 1i64 << (63 - CRSQL_COL_ID_BITS); + +// V2 table name suffixes +pub const V2_COL_MAP_SUFFIX: &str = "__crsql_v2_col_map"; +pub const V2_CLOCK_SUFFIX: &str = "__crsql_v2_clock"; +pub const V2_PKS_SUFFIX: &str = "__crsql_v2_pks"; +pub const V2_TOMBSTONES_SUFFIX: &str = "__crsql_v2_tombstones"; +pub const V2_TOMBSTONE_PKS_SUFFIX: &str = "__crsql_v2_tombstone_pks"; + +// V2 wire format sentinels +pub static V2_HASH_TOMBSTONE_CID: &str = "-2"; + +// Metadata and sync log version constants +pub const META_USE_V1: i32 = 1; +pub const META_USE_V2: i32 = 2; +pub const SYNC_LOG_V1: i32 = 1; +pub const SYNC_LOG_V2: i32 = 2; diff --git a/core/rs/core/src/create_cl_set_vtab.rs b/core/rs/core/src/create_cl_set_vtab.rs index a4ba94868..983010481 100644 --- a/core/rs/core/src/create_cl_set_vtab.rs +++ b/core/rs/core/src/create_cl_set_vtab.rs @@ -70,7 +70,7 @@ fn create_impl( let schema = vtab_args.database_name; let table = base_name_from_virtual_name(vtab_args.table_name); - create_crr(db, schema, table, false, true, err) + create_crr(db, schema, table, false, true, false, err) } fn create_clset_storage( diff --git a/core/rs/core/src/create_crr.rs b/core/rs/core/src/create_crr.rs index 311a89367..94a4d935f 100644 --- a/core/rs/core/src/create_crr.rs +++ b/core/rs/core/src/create_crr.rs @@ -1,11 +1,15 @@ +extern crate alloc; +use alloc::format; use core::ffi::c_char; use sqlite_nostd as sqlite; -use sqlite_nostd::ResultCode; +use sqlite_nostd::{Connection, ResultCode, StrRef}; use crate::bootstrap::create_clock_table; +use crate::consts; use crate::tableinfo::{is_table_compatible, pull_table_info}; use crate::triggers::create_triggers; use crate::{backfill_table, is_crr, remove_crr_triggers_if_exist}; +use crate::config; /** * Create a new crr -- @@ -17,6 +21,7 @@ pub fn create_crr( table: &str, is_commit_alter: bool, no_tx: bool, + without_rowid: bool, err: *mut *mut c_char, ) -> Result { if !is_table_compatible(db, table, err)? { @@ -30,20 +35,144 @@ pub fn create_crr( // since nothing would exist in it for a table not yet made into a crr. // TODO: Note: we can optimize out our `ensureTableInfosAreUpToDate` by mutating our ext data // when upgrading stuff to CRRs - let table_info = pull_table_info(db, table, err)?; + let mut table_info = pull_table_info(db, table, err)?; + + // Override uses_rowid_key if without_rowid was requested. + // This only matters on first registration — subsequent pull_table_info calls + // will infer from v2_pks schema. + if without_rowid && table_info.uses_rowid_key { + table_info.uses_rowid_key = false; + // Persist the without_rowid preference so migration path can read it + // when creating v2_tables before v2_pks exists. + let key = format!("without_rowid_{}\0", table); + let stmt = db.prepare_v2("INSERT OR REPLACE INTO crsql_master (key, value) VALUES (?, 1)\0")?; + stmt.bind_text(1, &key, sqlite::Destructor::TRANSIENT)?; + stmt.step()?; + } + + let metadata_write_version = get_metadata_write_version(db); + + // Create V2 tables if metadata write mode is dual-write (2) or V2-only (3) + if metadata_write_version >= config::METADATA_VERSION_V2_AND_V1 { + crate::bootstrap_v2::create_v2_tables(db, &table_info)?; + } + + // Create V1 clock tables unless mode is V2-only (3) + if metadata_write_version != config::METADATA_VERSION_V2 { + create_clock_table(db, &table_info, err)?; + } - create_clock_table(db, &table_info, err)?; remove_crr_triggers_if_exist(db, table)?; create_triggers(db, &table_info, err)?; - backfill_table( - db, - table, - &table_info.pks, - &table_info.non_pks, - is_commit_alter, - no_tx, - )?; + // For rowid tables (not converted to without_rowid), validate rowid range + // and add enforcement triggers. + if table_info.uses_rowid_key { + validate_rowid_range(db, table, &table_info.rowid_alias, err)?; + } + + // Backfill appropriate metadata tables based on write mode + if metadata_write_version == config::METADATA_VERSION_V2 { + crate::backfill_v2::backfill_table_v2( + db, + table, + &table_info.pks, + &table_info.non_pks, + table_info.uses_rowid_key, + &table_info.rowid_alias, + no_tx, + )?; + } else if metadata_write_version == config::METADATA_VERSION_V2_AND_V1 { + // Dual-write: backfill both V1 and V2 + backfill_table( + db, + table, + &table_info.pks, + &table_info.non_pks, + is_commit_alter, + no_tx, + )?; + crate::backfill_v2::backfill_table_v2( + db, + table, + &table_info.pks, + &table_info.non_pks, + table_info.uses_rowid_key, + &table_info.rowid_alias, + no_tx, + )?; + } else { + // V1-only: backfill V1 tables only + backfill_table( + db, + table, + &table_info.pks, + &table_info.non_pks, + is_commit_alter, + no_tx, + )?; + } Ok(ResultCode::OK) } + +/// Validate that existing rowids are within the safe range for cell_key packing. +/// cell_key = (rowid << CRSQL_COL_ID_BITS) | col_id must fit in a signed INT64, +/// so rowid must be >= 0 and < 2^(63 - CRSQL_COL_ID_BITS). +/// Runtime enforcement for new writes is done in the after_insert/after_update +/// trigger handlers in Rust, gated by tbl_info.uses_rowid_key. +fn validate_rowid_range( + db: *mut sqlite::sqlite3, + table: &str, + rowid_alias: &str, + err: *mut *mut c_char, +) -> Result { + let escaped = crate::util::escape_ident(table); + let alias = crate::util::escape_ident(rowid_alias); + + // Scan existing rowids for violations + let stmt = db.prepare_v2(&format!( + "SELECT max(\"{alias}\"), min(\"{alias}\") FROM \"{escaped}\"", + alias = alias, + escaped = escaped, + ))?; + stmt.step()?; + // column_int64 returns 0 for NULL (empty table), which is in range — safe to skip + let max_rowid = stmt.column_int64(0); + let min_rowid = stmt.column_int64(1); + + if max_rowid >= consts::MAX_ROWID_KEY || min_rowid < 0 { + err.set(&format!( + "Table {table} has rowids outside the safe range [0, {max_key}). \ + Found range [{min_rowid}, {max_rowid}]. \ + cell_key = (rowid << {bits}) | col_id must fit in a signed INT64. \ + Pass 'without_rowid' to crsql_as_crr to use the table PK columns instead of rowid as the internal key.", + table = table, + max_key = consts::MAX_ROWID_KEY, + min_rowid = min_rowid, + max_rowid = max_rowid, + bits = consts::CRSQL_COL_ID_BITS, + )); + return Err(ResultCode::ERROR); + } + + Ok(ResultCode::OK) +} + +/// Read the persisted metadata-write-version config from crsql_master. +/// Returns METADATA_WRITE_VERSION_DEFAULT (1) if not set or table doesn't exist. +fn get_metadata_write_version(db: *mut sqlite::sqlite3) -> core::ffi::c_int { + let stmt = db.prepare_v2( + "SELECT value FROM crsql_master WHERE key = 'config.metadata-write-version'" + ); + match stmt { + Ok(stmt) => { + if stmt.step().unwrap_or(ResultCode::DONE) == ResultCode::ROW { + stmt.column_int(0) + } else { + config::METADATA_WRITE_VERSION_DEFAULT + } + } + Err(_) => config::METADATA_WRITE_VERSION_DEFAULT, + } +} diff --git a/core/rs/core/src/hash_pk.rs b/core/rs/core/src/hash_pk.rs new file mode 100644 index 000000000..8b0ad55fe --- /dev/null +++ b/core/rs/core/src/hash_pk.rs @@ -0,0 +1,55 @@ +extern crate alloc; + +use alloc::vec::Vec; +use sqlite_nostd as sqlite; +use sqlite_nostd::{Context, ResultCode}; + +use crate::pack_columns::pack_columns; + +/// Compute a truncated XXH128 hash of packed PK values. +/// Returns PK_HASH_SIZE bytes. +pub fn hash_pk_values(values: &[*mut sqlite::value]) -> Result, ResultCode> { + let packed = pack_columns(values)?; + let hash = xxhash_rust::xxh3::xxh3_128(&packed); + let bytes = hash.to_be_bytes(); + Ok(bytes[..crate::consts::PK_HASH_SIZE].to_vec()) +} + +/// Hash PK values from unpacked ColumnValue list. +/// Packs the values and hashes with XXH128, truncated to PK_HASH_SIZE bytes. +pub fn hash_pk_values_from_column_values( + values: &[crate::pack_columns::ColumnValue], +) -> Result, ResultCode> { + let packed = crate::pack_columns::pack_column_values(values)?; + let hash = xxhash_rust::xxh3::xxh3_128(&packed); + let bytes = hash.to_be_bytes(); + Ok(bytes[..crate::consts::PK_HASH_SIZE].to_vec()) +} + +/// Hash a pre-packed PK blob directly (e.g., from crsql_changes pk column). +/// Avoids the unpack→repack cycle when the packed blob is already available. +pub fn hash_packed_blob(packed: &[u8]) -> Vec { + let hash = xxhash_rust::xxh3::xxh3_128(packed); + let bytes = hash.to_be_bytes(); + bytes[..crate::consts::PK_HASH_SIZE].to_vec() +} + +/// SQL function: crsql_hash_pk(pk1, pk2, ...) +/// Takes variadic PK values, packs them, hashes with XXH128, truncates to PK_HASH_SIZE bytes. +pub extern "C" fn crsql_hash_pk( + ctx: *mut sqlite::context, + argc: i32, + argv: *mut *mut sqlite::value, +) { + let args = sqlite::args!(argc, argv); + + match hash_pk_values(args) { + Ok(hash) => { + ctx.result_blob_owned(hash); + } + Err(code) => { + ctx.result_error("crsql_hash_pk: failed to hash PK values"); + ctx.result_error_code(code); + } + } +} diff --git a/core/rs/core/src/lib.rs b/core/rs/core/src/lib.rs index 8ad852a24..92346f872 100644 --- a/core/rs/core/src/lib.rs +++ b/core/rs/core/src/lib.rs @@ -4,13 +4,22 @@ // TODO: these pub mods are exposed for the integration testing // we should re-export in a `test` mod such that they do not become public apis mod alter; +mod alter_v2; mod automigrate; mod backfill; #[cfg(feature = "test")] +pub mod backfill_v2; +#[cfg(not(feature = "test"))] +mod backfill_v2; +#[cfg(feature = "test")] pub mod bootstrap; #[cfg(not(feature = "test"))] mod bootstrap; #[cfg(feature = "test")] +pub mod bootstrap_v2; +#[cfg(not(feature = "test"))] +mod bootstrap_v2; +#[cfg(feature = "test")] pub mod c; #[cfg(not(feature = "test"))] mod c; @@ -19,8 +28,8 @@ mod changes_vtab_read; mod changes_vtab_write; mod commit; mod compare_values; -mod config; -mod consts; +pub mod config; +pub mod consts; mod create_cl_set_vtab; mod create_crr; #[cfg(feature = "test")] @@ -29,8 +38,13 @@ pub mod db_version; mod db_version; mod debug; mod ext_data; +#[cfg(feature = "test")] +pub mod hash_pk; +#[cfg(not(feature = "test"))] +mod hash_pk; mod is_crr; mod local_writes; +mod migrate; #[cfg(feature = "test")] pub mod pack_columns; #[cfg(not(feature = "test"))] @@ -43,6 +57,10 @@ pub mod tableinfo; mod tableinfo; mod teardown; #[cfg(feature = "test")] +pub mod teardown_v2; +#[cfg(not(feature = "test"))] +mod teardown_v2; +#[cfg(feature = "test")] pub mod test_exports; mod triggers; mod unpack_columns_vtab; @@ -56,6 +74,7 @@ use core::mem; use core::ptr::null_mut; extern crate alloc; use alter::crsql_compact_post_alter; +use alter_v2::crsql_compact_post_alter_v2; use automigrate::*; use backfill::*; use c::{crsql_freeExtData, crsql_initSiteIdExt, crsql_newExtData}; @@ -67,15 +86,14 @@ use db_version::{ insert_db_version, }; use is_crr::*; +use migrate::crsql_incremental_maintenance; use local_writes::after_delete::x_crsql_after_delete; use local_writes::after_insert::x_crsql_after_insert; use local_writes::after_update::x_crsql_after_update; use sqlite::{Destructor, ResultCode}; use sqlite_nostd as sqlite; use sqlite_nostd::{Connection, Context, Value}; -#[cfg(feature = "test")] -use tableinfo::TableInfo; -use tableinfo::{crsql_ensure_table_infos_are_up_to_date, is_table_compatible, pull_table_info}; +use tableinfo::{TableInfo, crsql_ensure_table_infos_are_up_to_date, is_table_compatible, pull_table_info}; use teardown::*; use triggers::create_triggers; @@ -123,6 +141,21 @@ pub extern "C" fn sqlite3_crsqlcore_init( ) -> *mut c_void { sqlite::EXTENSION_INIT2(api); + let sqlite_version = sqlite::libversion_number(); + if sqlite_version < consts::MIN_SQLITE_VERSION { + let msg = format!( + "cr-sqlite requires SQLite 3.44.0 or later (ORDER BY in aggregates). Found version {}.", + sqlite::libversion() + ); + let cstring = alloc::ffi::CString::new(msg).unwrap_or(alloc::ffi::CString::new("cr-sqlite requires SQLite 3.44.0 or later").unwrap()); + unsafe { + if !err_msg.is_null() { + *err_msg = cstring.into_raw(); + } + } + return null_mut(); + } + let rc = db .create_function_v2( "crsql_set_debug", @@ -241,6 +274,11 @@ pub extern "C" fn sqlite3_crsqlcore_init( return null_mut(); } + // Set default metadata write version to V1 + unsafe { (*ext_data).metadataWriteVersion = crate::config::METADATA_WRITE_VERSION_DEFAULT; } + unsafe { (*ext_data).metadataUseVersion = crate::config::METADATA_USE_VERSION_DEFAULT; } + unsafe { (*ext_data).syncLogVersion = crate::config::SYNC_LOG_VERSION_DEFAULT; } + let rc = db .create_function_v2( "crsql_update_site_id", @@ -425,7 +463,7 @@ pub extern "C" fn sqlite3_crsqlcore_init( "crsql_as_crr", -1, sqlite::UTF8 | sqlite::DETERMINISTIC, - None, + Some(ext_data as *mut c_void), Some(x_crsql_as_crr), None, None, @@ -454,6 +492,23 @@ pub extern "C" fn sqlite3_crsqlcore_init( return null_mut(); } + let rc = db + .create_function_v2( + "crsql_change_wins", + 4, + sqlite::UTF8 | sqlite::DETERMINISTIC, + None, + Some(x_crsql_change_wins), + None, + None, + None, + ) + .unwrap_or(ResultCode::ERROR); + if rc != ResultCode::OK { + unsafe { crsql_freeExtData(ext_data) }; + return null_mut(); + } + #[cfg(feature = "test")] if let Err(_) = db.create_function_v2( "crsql_cache_site_ordinal", @@ -538,7 +593,7 @@ pub extern "C" fn sqlite3_crsqlcore_init( "crsql_begin_alter", -1, sqlite::UTF8 | sqlite::DIRECTONLY, - None, + Some(ext_data as *mut c_void), Some(x_crsql_begin_alter), None, None, @@ -686,6 +741,59 @@ pub extern "C" fn sqlite3_crsqlcore_init( return null_mut(); } + // V2 functions: crsql_hash_pk (scalar) and crsql_pack_agg (aggregate) + let rc = db + .create_function_v2( + "crsql_hash_pk", + -1, + sqlite::UTF8 | sqlite::DETERMINISTIC, + None, + Some(hash_pk::crsql_hash_pk), + None, + None, + None, + ) + .unwrap_or(ResultCode::ERROR); + if rc != ResultCode::OK { + unsafe { crsql_freeExtData(ext_data) }; + return null_mut(); + } + + let rc = db + .create_function_v2( + "crsql_pack_agg", + 1, + sqlite::UTF8 | sqlite::DETERMINISTIC, + None, + None, + Some(pack_columns::crsql_pack_agg_step), + Some(pack_columns::crsql_pack_agg_final), + None, + ) + .unwrap_or(ResultCode::ERROR); + if rc != ResultCode::OK { + unsafe { crsql_freeExtData(ext_data) }; + return null_mut(); + } + + // V2: crsql_incremental_maintenance(chunk_size) -> remaining work units + let rc = db + .create_function_v2( + "crsql_incremental_maintenance", + 1, + sqlite::UTF8 | sqlite::DIRECTONLY, + Some(ext_data as *mut c_void), + Some(x_crsql_incremental_maintenance), + None, + None, + None, + ) + .unwrap_or(ResultCode::ERROR); + if rc != ResultCode::OK { + unsafe { crsql_freeExtData(ext_data) }; + return null_mut(); + } + return ext_data as *mut c_void; } @@ -744,6 +852,9 @@ unsafe extern "C" fn x_crsql_finalize( * Takes a table name and turns it into a CRR. * * This allows users to create and modify tables as normal. + * + * Optional flags (after table name): + * 'without_rowid' - treat a rowid table as WITHOUT ROWID for V2 metadata (uses hashed_pk instead of rowid as key) */ unsafe extern "C" fn x_crsql_as_crr( ctx: *mut sqlite::context, @@ -759,13 +870,27 @@ unsafe extern "C" fn x_crsql_as_crr( } let args = sqlite::args!(argc, argv); - let (schema_name, table_name) = if argc == 2 { - (args[0].text(), args[1].text()) + let known_flags = ["without_rowid"]; + + // Parse args: (table) | (schema, table) | (table, flags...) | (schema, table, flags...) + let (schema_name, table_name, flags) = if argc >= 2 && known_flags.contains(&args[1].text()) { + // (table, flags...) + ("main\0", args[0].text(), &args[1..]) + } else if argc >= 3 && known_flags.contains(&args[2].text()) { + // (schema, table, flags...) + (args[0].text(), args[1].text(), &args[2..]) + } else if argc == 2 { + // (schema, table) + (args[0].text(), args[1].text(), &[][..]) } else { - ("main\0", args[0].text()) + // (table) + ("main\0", args[0].text(), &[][..]) }; + let without_rowid = flags.iter().any(|f| f.text() == "without_rowid"); + let db = ctx.db_handle(); + let ext_data = ctx.user_data() as *mut c::crsql_ExtData; let mut err_msg = null_mut(); let rc = db.exec_safe("SAVEPOINT as_crr"); if rc.is_err() { @@ -773,12 +898,19 @@ unsafe extern "C" fn x_crsql_as_crr( return; } + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + ctx.result_error("crsql_as_crr: timestamp not set — call crsql_set_ts() first"); + return; + } + let rc = crsql_create_crr( db, schema_name.as_ptr() as *const c_char, table_name.as_ptr() as *const c_char, 0, 0, + if without_rowid { 1 } else { 0 }, &mut err_msg as *mut _, ); if rc != ResultCode::OK as c_int { @@ -806,6 +938,19 @@ unsafe extern "C" fn x_crsql_rows_impacted( sqlite::result_int(ctx, rows_impacted); } +unsafe extern "C" fn x_crsql_incremental_maintenance( + ctx: *mut sqlite::context, + argc: i32, + argv: *mut *mut sqlite::value, +) { + let ext_data = ctx.user_data() as *mut c::crsql_ExtData; + let args = sqlite::args!(argc, argv); + let chunk_size = if args.len() > 0 { args[0].int() } else { 1000 }; + let db = ctx.db_handle(); + let result = crsql_incremental_maintenance(db, chunk_size, ext_data); + ctx.result_int(result); +} + unsafe extern "C" fn x_crsql_begin_alter( ctx: *mut sqlite::context, argc: i32, @@ -827,6 +972,13 @@ unsafe extern "C" fn x_crsql_begin_alter( ("main\0", args[0].text()) }; + let ext_data = ctx.user_data() as *mut c::crsql_ExtData; + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + ctx.result_error("crsql_begin_alter: timestamp not set — call crsql_set_ts() first"); + return; + } + let db = ctx.db_handle(); let rc = db.exec_safe("SAVEPOINT alter_crr"); if rc.is_err() { @@ -868,6 +1020,12 @@ unsafe extern "C" fn x_crsql_commit_alter( let mut err_msg = null_mut(); let db = ctx.db_handle(); + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + ctx.result_error("crsql_commit_alter: timestamp not set — call crsql_set_ts() first"); + return; + } + let rc = if non_destructive { match pull_table_info(db, table_name, &mut err_msg as *mut _) { Ok(table_info) => { @@ -886,12 +1044,41 @@ unsafe extern "C" fn x_crsql_commit_alter( Err(rc) => rc as c_int, } } else { - let rc = crsql_compact_post_alter( - db, - table_name.as_ptr() as *const c_char, - ext_data, - &mut err_msg as *mut _, - ); + // Check schema version from table infos to decide which compaction to run + let table_infos = + mem::ManuallyDrop::new(Box::from_raw((*ext_data).tableInfos as *mut Vec)); + let tbl_info = table_infos.iter().find(|t| t.tbl_name == table_name); + let has_v1 = tbl_info.map_or(false, |t| { + t.schema_version == tableinfo::SchemaVersion::V1 || t.schema_version == tableinfo::SchemaVersion::V2AndV1 + }); + + let has_v2 = tbl_info.map_or(false, |t| { + t.schema_version == tableinfo::SchemaVersion::V2 || t.schema_version == tableinfo::SchemaVersion::V2AndV1 + }); + + let mut rc = ResultCode::OK as c_int; + if has_v1 { + rc = crsql_compact_post_alter( + db, + table_name.as_ptr() as *const c_char, + ext_data, + &mut err_msg as *mut _, + ); + } + + // Run V2 alter compaction only if V2 tables exist + if rc == ResultCode::OK as c_int && has_v2 { + let rc2 = crsql_compact_post_alter_v2( + db, + table_name.as_ptr() as *const c_char, + ext_data, + &mut err_msg as *mut _, + ); + if rc2 != ResultCode::OK as c_int { + ctx.result_error("crsql_compact_post_alter_v2 failed"); + return; + } + } if rc == ResultCode::OK as c_int { crsql_create_crr( @@ -900,6 +1087,7 @@ unsafe extern "C" fn x_crsql_commit_alter( table_name.as_ptr() as *const c_char, 1, 0, + 0, &mut err_msg as *mut _, ) } else { @@ -1068,6 +1256,8 @@ unsafe extern "C" fn x_crsql_cache_db_version( /** * Return the timestamp for the current transaction. + * Errors if timestamp has not been set via crsql_set_ts(). + * Returns an integer (not text) for STRICT table compatibility. */ unsafe extern "C" fn x_crsql_get_ts( ctx: *mut sqlite::context, @@ -1075,8 +1265,43 @@ unsafe extern "C" fn x_crsql_get_ts( _argv: *mut *mut sqlite::value, ) { let ext_data = ctx.user_data() as *mut c::crsql_ExtData; - let ts = (*ext_data).timestamp.to_string(); - ctx.result_text_transient(&ts); + let ts = (*ext_data).timestamp; + if ts == 0 { + ctx.result_error("crsql_get_ts: timestamp not set. Call crsql_set_ts() before writing to V2 clock tables."); + return; + } + ctx.result_int64(ts as sqlite::int64); +} + +/** + * Compare incoming vs local value for LWW merge resolution. + * Args: (incoming_val, local_val, site_id_wins, merge_equal_values) + * site_id_wins is a boolean (1/0) computed by the caller via blob comparison. + * Returns 1 if incoming wins, 0 otherwise. + */ +unsafe extern "C" fn x_crsql_change_wins( + ctx: *mut sqlite::context, + argc: i32, + argv: *mut *mut sqlite::value, +) { + if argc != 4 { + ctx.result_error("crsql_change_wins expects 4 args"); + return; + } + let args = sqlite::args!(argc, argv); + let cmp = crate::compare_values::crsql_compare_sqlite_values(args[0], args[1]); + if cmp > 0 { + ctx.result_int(1); + } else if cmp < 0 { + ctx.result_int(0); + } else { + let merge_equal = args[3].int(); + if merge_equal == 1 { + ctx.result_int(args[2].int()); + } else { + ctx.result_int(0); + } + } } /** @@ -1278,6 +1503,7 @@ pub extern "C" fn crsql_create_crr( table: *const c_char, is_commit_alter: c_int, no_tx: c_int, + without_rowid: c_int, err: *mut *mut c_char, ) -> c_int { let schema = unsafe { CStr::from_ptr(schema).to_str() }; @@ -1285,7 +1511,7 @@ pub extern "C" fn crsql_create_crr( match (table, schema) { (Ok(table), Ok(schema)) => { - create_crr(db, schema, table, is_commit_alter != 0, no_tx != 0, err) + create_crr(db, schema, table, is_commit_alter != 0, no_tx != 0, without_rowid != 0, err) .unwrap_or_else(|err| err) as c_int } _ => ResultCode::NOMEM as c_int, diff --git a/core/rs/core/src/local_writes/after_delete.rs b/core/rs/core/src/local_writes/after_delete.rs index 724a4b386..1df66b87b 100644 --- a/core/rs/core/src/local_writes/after_delete.rs +++ b/core/rs/core/src/local_writes/after_delete.rs @@ -7,7 +7,7 @@ use sqlite::Context; use sqlite::ResultCode; use sqlite_nostd as sqlite; -use crate::{c::crsql_ExtData, tableinfo::TableInfo}; +use crate::{c::crsql_ExtData, tableinfo::{TableInfo, SchemaVersion}, config}; use super::bump_seq; use super::mark_locally_deleted; @@ -41,6 +41,39 @@ fn after_delete( tbl_info: &mut TableInfo, pks_old: &[*mut value], ) -> Result { + let mwv = unsafe { (*ext_data).metadataWriteVersion }; + + // Mode 3 (V2-only): write to V2 only, even if V1 tables still exist during cleanup + if mwv == config::METADATA_VERSION_V2 { + return super::v2::v2_after_delete(db, ext_data, tbl_info, pks_old); + } + + // Mode 1 (V1-only): write to V1 only, even if V2 tables still exist from rollback + let mut v2_ran = false; + let saved_seq = unsafe { (*ext_data).seq }; + if mwv == config::METADATA_VERSION_V1 { + // fall through to V1 write logic below, skipping V2 + } else { + // V2-only schema: just write to V2 tables + if tbl_info.schema_version == SchemaVersion::V2 { + return super::v2::v2_after_delete(db, ext_data, tbl_info, pks_old); + } + + // V2AndV1 mode: write to V2 tables first, then fall through to V1 + if tbl_info.schema_version == SchemaVersion::V2AndV1 { + // Hydrate V2 from V1 on-demand (migration may not have reached this row yet) + unsafe { crate::changes_vtab_write::v1_to_v2_hydrate_row_from_values(db, ext_data, tbl_info, pks_old) } + .map_err(|_| "V1 to V2 hydration failed".to_string())?; + super::v2::v2_after_delete(db, ext_data, tbl_info, pks_old)?; + v2_ran = true; + } + } + + // Restore seq so V1 reuses the same values V2 just bumped. + if v2_ran { + unsafe { (*ext_data).seq = saved_seq; } + } + let ts = unsafe { (*ext_data).timestamp.to_string() }; let db_version = crate::db_version::next_db_version(db, ext_data)?; let seq = bump_seq(ext_data); diff --git a/core/rs/core/src/local_writes/after_insert.rs b/core/rs/core/src/local_writes/after_insert.rs index 2a4cf4e89..4a972093c 100644 --- a/core/rs/core/src/local_writes/after_insert.rs +++ b/core/rs/core/src/local_writes/after_insert.rs @@ -1,3 +1,4 @@ +use alloc::format; use alloc::string::String; use alloc::string::ToString; use core::ffi::c_int; @@ -6,14 +7,16 @@ use sqlite::value; use sqlite::Context; use sqlite::ResultCode; use sqlite_nostd as sqlite; +use sqlite_nostd::Value; -use crate::{c::crsql_ExtData, tableinfo::TableInfo}; +use crate::{c::crsql_ExtData, tableinfo::{TableInfo, SchemaVersion}, config, consts}; use super::bump_seq; use super::trigger_fn_preamble; /** - * crsql_after_insert("table", pk_values...) + * crsql_after_insert("table", pk_values..., [rowid_value]) + * The rowid_value is appended when the table uses_rowid_key. */ pub unsafe extern "C" fn x_crsql_after_insert( ctx: *mut sqlite::context, @@ -21,7 +24,13 @@ pub unsafe extern "C" fn x_crsql_after_insert( argv: *mut *mut sqlite::value, ) { let result = trigger_fn_preamble(ctx, argc, argv, |table_info, values, ext_data| { - after_insert(ctx.db_handle(), ext_data, table_info, &values[1..]) + let (pks_new, rowid_val) = if table_info.uses_rowid_key { + let len = values.len(); + (&values[1..len - 1], Some(values[len - 1])) + } else { + (&values[1..], None) + }; + after_insert(ctx.db_handle(), ext_data, table_info, pks_new, rowid_val) }); match result { @@ -39,7 +48,52 @@ fn after_insert( ext_data: *mut crsql_ExtData, tbl_info: &mut TableInfo, pks_new: &[*mut value], + rowid_val: Option<*mut value>, ) -> Result { + // Enforce rowid range for rowid-key tables + if let Some(rowid) = rowid_val { + let rowid = rowid.int64(); + if rowid < 0 || rowid >= consts::MAX_ROWID_KEY { + return Err(format!( + "rowid out of cr-sqlite safe range [0, {})", + consts::MAX_ROWID_KEY + )); + } + } + + let mwv = unsafe { (*ext_data).metadataWriteVersion }; + + // Mode 3 (V2-only): write to V2 only, even if V1 tables still exist during cleanup + if mwv == config::METADATA_VERSION_V2 { + return super::v2::v2_after_insert(db, ext_data, tbl_info, pks_new); + } + + // Mode 1 (V1-only): write to V1 only, even if V2 tables still exist from rollback + let mut v2_ran = false; + let saved_seq = unsafe { (*ext_data).seq }; + if mwv == config::METADATA_VERSION_V1 { + // fall through to V1 write logic below, skipping V2 + } else { + // V2-only schema: just write to V2 tables + if tbl_info.schema_version == SchemaVersion::V2 { + return super::v2::v2_after_insert(db, ext_data, tbl_info, pks_new); + } + + // V2AndV1 mode: write to V2 tables first, then fall through to V1 + if tbl_info.schema_version == SchemaVersion::V2AndV1 { + // Hydrate V2 from V1 on-demand (migration may not have reached this row yet) + unsafe { crate::changes_vtab_write::v1_to_v2_hydrate_row_from_values(db, ext_data, tbl_info, pks_new) } + .map_err(|_| "V1 to V2 hydration failed".to_string())?; + super::v2::v2_after_insert(db, ext_data, tbl_info, pks_new)?; + v2_ran = true; + } + } + + // Restore seq so V1 reuses the same values V2 just bumped. + if v2_ran { + unsafe { (*ext_data).seq = saved_seq; } + } + let ts = unsafe { (*ext_data).timestamp.to_string() }; let db_version = crate::db_version::next_db_version(db, ext_data)?; diff --git a/core/rs/core/src/local_writes/after_update.rs b/core/rs/core/src/local_writes/after_update.rs index b75b55ef0..17524b0bf 100644 --- a/core/rs/core/src/local_writes/after_update.rs +++ b/core/rs/core/src/local_writes/after_update.rs @@ -3,11 +3,13 @@ use core::ffi::c_int; use alloc::format; use alloc::string::String; use alloc::string::ToString; +use alloc::vec::Vec; use sqlite::{sqlite3, value, Context, ResultCode}; use sqlite_nostd as sqlite; +use sqlite_nostd::Value; use crate::compare_values::crsql_compare_sqlite_values; -use crate::{c::crsql_ExtData, tableinfo::TableInfo}; +use crate::{c::crsql_ExtData, tableinfo::{TableInfo, SchemaVersion}, config, consts}; use super::trigger_fn_preamble; @@ -17,8 +19,9 @@ pub unsafe extern "C" fn x_crsql_after_update( argv: *mut *mut sqlite::value, ) { let result = trigger_fn_preamble(ctx, argc, argv, |table_info, values, ext_data| { - let (pks_new, pks_old, non_pks_new, non_pks_old) = - partition_values(values, 1, table_info.pks.len(), table_info.non_pks.len())?; + let has_rowid = table_info.uses_rowid_key; + let (pks_new, pks_old, non_pks_new, non_pks_old, rowid_val) = + partition_values(values, 1, table_info.pks.len(), table_info.non_pks.len(), has_rowid)?; after_update( ctx.db_handle(), @@ -28,6 +31,7 @@ pub unsafe extern "C" fn x_crsql_after_update( pks_old, non_pks_new, non_pks_old, + rowid_val, ) }); @@ -46,8 +50,9 @@ fn partition_values( offset: usize, num_pks: usize, num_non_pks: usize, -) -> Result<(&[T], &[T], &[T], &[T]), String> { - let expected_len = offset + num_pks * 2 + num_non_pks * 2; + has_rowid: bool, +) -> Result<(&[T], &[T], &[T], &[T], Option<&T>), String> { + let expected_len = offset + num_pks * 2 + num_non_pks * 2 + if has_rowid { 1 } else { 0 }; if values.len() != expected_len { return Err(format!( "expected {} values, got {}", @@ -55,11 +60,14 @@ fn partition_values( values.len() )); } + let core_end = offset + num_pks * 2 + num_non_pks * 2; + let rowid_val = if has_rowid { Some(&values[core_end]) } else { None }; Ok(( &values[offset..num_pks + offset], &values[num_pks + offset..num_pks * 2 + offset], &values[num_pks * 2 + offset..num_pks * 2 + num_non_pks + offset], - &values[num_pks * 2 + num_non_pks + offset..], + &values[num_pks * 2 + num_non_pks + offset..core_end], + rowid_val, )) } @@ -71,7 +79,93 @@ fn after_update( pks_old: &[*mut value], non_pks_new: &[*mut value], non_pks_old: &[*mut value], + rowid_val: Option<&*mut value>, ) -> Result { + // Enforce rowid range for rowid-key tables + if let Some(rowid) = rowid_val { + let rowid = (*rowid).int64(); + if rowid < 0 || rowid >= consts::MAX_ROWID_KEY { + return Err(format!( + "rowid out of cr-sqlite safe range [0, {})", + consts::MAX_ROWID_KEY + )); + } + } + + let mwv = unsafe { (*ext_data).metadataWriteVersion }; + + // Mode 3 (V2-only): write to V2 only, even if V1 tables still exist during cleanup + if mwv == config::METADATA_VERSION_V2 { + let mut changed_indices = Vec::new(); + for (i, (new, old)) in non_pks_new.iter().zip(non_pks_old.iter()).enumerate() { + if crsql_compare_sqlite_values(*new, *old) != 0 { + changed_indices.push(i); + } + } + if crate::compare_values::any_value_changed(pks_new, pks_old)? { + super::v2::v2_after_delete(db, ext_data, tbl_info, pks_old)?; + super::v2::v2_after_insert(db, ext_data, tbl_info, pks_new)?; + return Ok(ResultCode::OK); + } + return super::v2::v2_after_update(db, ext_data, tbl_info, pks_new, &changed_indices); + } + + // Mode 1 (V1-only): write to V1 only, even if V2 tables still exist from rollback + let mut v2_ran = false; + let saved_seq = unsafe { (*ext_data).seq }; + if mwv == config::METADATA_VERSION_V1 { + // fall through to V1 write logic below, skipping V2 + } else { + // V2-only schema: just write to V2 tables + if tbl_info.schema_version == SchemaVersion::V2 { + // Compute changed column indices + let mut changed_indices = Vec::new(); + for (i, (new, old)) in non_pks_new.iter().zip(non_pks_old.iter()).enumerate() { + if crsql_compare_sqlite_values(*new, *old) != 0 { + changed_indices.push(i); + } + } + // Also handle PK changes as delete+insert in V2 + if crate::compare_values::any_value_changed(pks_new, pks_old)? { + // PK changed: do delete of old + insert of new + super::v2::v2_after_delete(db, ext_data, tbl_info, pks_old)?; + super::v2::v2_after_insert(db, ext_data, tbl_info, pks_new)?; + return Ok(ResultCode::OK); + } + return super::v2::v2_after_update(db, ext_data, tbl_info, pks_new, &changed_indices); + } + + // V2AndV1 mode: write to V2 tables first, then fall through to V1 + if tbl_info.schema_version == SchemaVersion::V2AndV1 { + let mut changed_indices = Vec::new(); + for (i, (new, old)) in non_pks_new.iter().zip(non_pks_old.iter()).enumerate() { + if crsql_compare_sqlite_values(*new, *old) != 0 { + changed_indices.push(i); + } + } + if crate::compare_values::any_value_changed(pks_new, pks_old)? { + // Hydrate both old and new PKs before delete+insert + unsafe { crate::changes_vtab_write::v1_to_v2_hydrate_row_from_values(db, ext_data, tbl_info, pks_old) } + .map_err(|_| "V1 to V2 hydration failed".to_string())?; + unsafe { crate::changes_vtab_write::v1_to_v2_hydrate_row_from_values(db, ext_data, tbl_info, pks_new) } + .map_err(|_| "V1 to V2 hydration failed".to_string())?; + super::v2::v2_after_delete(db, ext_data, tbl_info, pks_old)?; + super::v2::v2_after_insert(db, ext_data, tbl_info, pks_new)?; + } else { + // Hydrate the row being updated + unsafe { crate::changes_vtab_write::v1_to_v2_hydrate_row_from_values(db, ext_data, tbl_info, pks_new) } + .map_err(|_| "V1 to V2 hydration failed".to_string())?; + super::v2::v2_after_update(db, ext_data, tbl_info, pks_new, &changed_indices)?; + } + v2_ran = true; + } + } + + // Restore seq so V1 reuses the same values V2 just bumped. + if v2_ran { + unsafe { (*ext_data).seq = saved_seq; } + } + let ts = unsafe { (*ext_data).timestamp.to_string() }; let next_db_version = crate::db_version::peek_next_db_version(db, ext_data)?; let new_key = tbl_info @@ -191,39 +285,43 @@ mod tests { ]; assert_eq!( - partition_values(&values1, 1, 1, 1), + partition_values(&values1, 1, 1, 1, false), Ok(( &["pk.new"] as &[&str], &["pk.old"] as &[&str], &["c.new"] as &[&str], - &["c.old"] as &[&str] + &["c.old"] as &[&str], + None )) ); assert_eq!( - partition_values(&values2, 1, 1, 0), + partition_values(&values2, 1, 1, 0, false), Ok(( &["pk.new"] as &[&str], &["pk.old"] as &[&str], &[] as &[&str], - &[] as &[&str] + &[] as &[&str], + None )) ); assert_eq!( - partition_values(&values3, 1, 2, 0), + partition_values(&values3, 1, 2, 0, false), Ok(( &["pk1.new", "pk2.new"] as &[&str], &["pk1.old", "pk2.old"] as &[&str], &[] as &[&str], - &[] as &[&str] + &[] as &[&str], + None )) ); assert_eq!( - partition_values(&values4, 1, 2, 2), + partition_values(&values4, 1, 2, 2, false), Ok(( &["pk1.new", "pk2.new"] as &[&str], &["pk1.old", "pk2.old"] as &[&str], &["c.new", "d.new"] as &[&str], - &["c.old", "d.old"] as &[&str] + &["c.old", "d.old"] as &[&str], + None )) ); } diff --git a/core/rs/core/src/local_writes/mod.rs b/core/rs/core/src/local_writes/mod.rs index f14100e9f..ffd5ab0f5 100644 --- a/core/rs/core/src/local_writes/mod.rs +++ b/core/rs/core/src/local_writes/mod.rs @@ -18,6 +18,7 @@ use crate::tableinfo::{crsql_ensure_table_infos_are_up_to_date, ColumnInfo, Tabl pub mod after_delete; pub mod after_insert; pub mod after_update; +pub mod v2; fn trigger_fn_preamble( ctx: *mut sqlite::context, @@ -111,7 +112,7 @@ fn mark_new_pk_row_created( result } -fn bump_seq(ext_data: *mut crsql_ExtData) -> c_int { +pub fn bump_seq(ext_data: *mut crsql_ExtData) -> c_int { unsafe { (*ext_data).seq += 1; (*ext_data).seq - 1 diff --git a/core/rs/core/src/local_writes/v2.rs b/core/rs/core/src/local_writes/v2.rs new file mode 100644 index 000000000..bb37304e9 --- /dev/null +++ b/core/rs/core/src/local_writes/v2.rs @@ -0,0 +1,392 @@ +extern crate alloc; + +use alloc::format; +use alloc::string::String; +use alloc::string::ToString; +use alloc::vec::Vec; + +use sqlite_nostd as sqlite; +use sqlite_nostd::{sqlite3, Connection, Destructor, ResultCode}; + +use crate::c::crsql_ExtData; +use crate::consts; +use crate::hash_pk::hash_pk_values; +use crate::tableinfo::TableInfo; +use super::bump_seq; + +/// V2 after_insert: write to v2_pks and v2_clock tables. +pub fn v2_after_insert( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + pks_new: &[*mut sqlite::value], +) -> Result { + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + return Err("v2_after_insert: timestamp not set — call crsql_set_ts() first".to_string()); + } + let ts = unsafe { (*ext_data).timestamp.to_string() }; + let db_version = crate::db_version::next_db_version(db, ext_data) + .map_err(|_| "failed to get next db_version".to_string())?; + + // Compute hashed_pk + let hashed_pk = hash_pk_values(pks_new) + .map_err(|_| "failed to hash PK values".to_string())?; + + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + + // Insert or update v2_pks entry + let uses_rowid_key = tbl_info.uses_rowid_key; + let pk_cols = tbl_info.pks.iter() + .map(|p| format!("\"{}\"", crate::util::escape_ident(&p.name))) + .collect::>() + .join(", "); + let pk_values = tbl_info.pks.iter() + .map(|_| "?") + .collect::>() + .join(", "); + + // Check if row already exists in v2_pks by hashed_pk + let check_stmt = db.prepare_v2(&format!( + "SELECT __crsql_key, cl FROM \"{escaped}{suffix}\" WHERE hashed_pk = ?", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX + )).map_err(|e| format!("failed to prepare check stmt: {:?}", e))?; + check_stmt.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + let (key, existing_cl) = match check_stmt.step().map_err(|e| format!("step: {:?}", e))? { + ResultCode::ROW => { + let k = check_stmt.column_int64(0); + let cl = check_stmt.column_int64(1); + (Some(k), Some(cl)) + } + ResultCode::DONE => { + (None, None) + } + _ => return Err("unexpected result from check stmt".to_string()), + }; + + let key = if let Some(k) = key { + // Row exists in v2_pks — update CL if it was previously dead (even CL) + if let Some(cl) = existing_cl { + if cl % 2 == 0 { + // Resurrection: CL was even (dead), now odd (alive) + // Bump seq once for the CL change, matching V1's create record update. + let _ = bump_seq(ext_data); + let new_cl = cl + 1; + let update_stmt = db.prepare_v2(&format!( + "UPDATE \"{escaped}{suffix}\" SET cl = ? WHERE __crsql_key = ?", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX + )).map_err(|e| format!("failed to prepare update stmt: {:?}", e))?; + update_stmt.bind_int64(1, new_cl).map_err(|e| format!("bind: {:?}", e))?; + update_stmt.bind_int64(2, k).map_err(|e| format!("bind: {:?}", e))?; + update_stmt.step().map_err(|e| format!("step: {:?}", e))?; + } + } + k + } else { + // Row not in v2_pks — check v2_tombstones for resurrection + let tomb_check = db.prepare_v2(&format!( + "SELECT cl FROM \"{escaped}{suffix}\" WHERE hashed_pk = ?", + escaped = escaped, + suffix = consts::V2_TOMBSTONES_SUFFIX + )).map_err(|e| format!("failed to prepare tomb check: {:?}", e))?; + tomb_check.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + match tomb_check.step().map_err(|e| format!("step: {:?}", e))? { + ResultCode::ROW => { + let tomb_cl = tomb_check.column_int64(0); + let new_cl = tomb_cl + 1; // even→odd = resurrection + + // Bump seq once for the CL change, matching V1's create record update. + let _ = bump_seq(ext_data); + + // Remove from v2_tombstones and v2_tombstone_pks + let del_tomb = db.prepare_v2(&format!( + "DELETE FROM \"{escaped}{suffix}\" WHERE hashed_pk = ?", + escaped = escaped, + suffix = consts::V2_TOMBSTONES_SUFFIX + )).map_err(|e| format!("failed to prepare del tomb: {:?}", e))?; + del_tomb.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + del_tomb.step().map_err(|e| format!("step: {:?}", e))?; + + let del_tpk = db.prepare_v2(&format!( + "DELETE FROM \"{escaped}{suffix}\" WHERE hashed_pk = ?", + escaped = escaped, + suffix = consts::V2_TOMBSTONE_PKS_SUFFIX + )).map_err(|e| format!("failed to prepare del tpk: {:?}", e))?; + del_tpk.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + del_tpk.step().map_err(|e| format!("step: {:?}", e))?; + + // Re-insert into v2_pks with resurrected CL + let insert_sql = if uses_rowid_key { + format!( + "INSERT INTO \"{escaped}{suffix}\" (__crsql_key, hashed_pk, cl) VALUES (?, ?, ?) RETURNING __crsql_key", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + ) + } else { + format!( + "INSERT INTO \"{escaped}{suffix}\" ({pk_cols}, hashed_pk, cl) VALUES ({pk_values}, ?, ?) RETURNING __crsql_key", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + pk_cols = pk_cols, + pk_values = pk_values, + ) + }; + let insert_stmt = db.prepare_v2(&insert_sql) + .map_err(|e| format!("failed to prepare resurrect insert: {:?}", e))?; + if uses_rowid_key { + insert_stmt.bind_value(1, pks_new[0]).map_err(|e| format!("bind: {:?}", e))?; + insert_stmt.bind_blob(2, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + insert_stmt.bind_int64(3, new_cl).map_err(|e| format!("bind: {:?}", e))?; + } else { + for (i, pk) in pks_new.iter().enumerate() { + insert_stmt.bind_value(i as i32 + 1, *pk).map_err(|e| format!("bind: {:?}", e))?; + } + insert_stmt.bind_blob(pks_new.len() as i32 + 1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + insert_stmt.bind_int64(pks_new.len() as i32 + 2, new_cl).map_err(|e| format!("bind: {:?}", e))?; + } + insert_stmt.step().map_err(|e| format!("step: {:?}", e))?; + insert_stmt.column_int64(0) + } + _ => { + // Truly new row — insert into v2_pks with cl=1 + let insert_sql = if uses_rowid_key { + format!( + "INSERT INTO \"{escaped}{suffix}\" (__crsql_key, hashed_pk, cl) VALUES (?, ?, 1) RETURNING __crsql_key", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + ) + } else { + format!( + "INSERT INTO \"{escaped}{suffix}\" ({pk_cols}, hashed_pk, cl) VALUES ({pk_values}, ?, 1) RETURNING __crsql_key", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX, + pk_cols = pk_cols, + pk_values = pk_values, + ) + }; + let insert_stmt = db.prepare_v2(&insert_sql) + .map_err(|e| format!("failed to prepare insert stmt: {:?}", e))?; + if uses_rowid_key { + insert_stmt.bind_value(1, pks_new[0]).map_err(|e| format!("bind: {:?}", e))?; + insert_stmt.bind_blob(2, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + } else { + for (i, pk) in pks_new.iter().enumerate() { + insert_stmt.bind_value(i as i32 + 1, *pk).map_err(|e| format!("bind: {:?}", e))?; + } + insert_stmt.bind_blob(pks_new.len() as i32 + 1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + } + insert_stmt.step().map_err(|e| { + let errmsg = db.errmsg().unwrap_or_else(|_| "unknown".to_string()); + format!("step: {:?} - {}", e, errmsg) + })?; + insert_stmt.column_int64(0) + } + } + }; + + // Write clock entries for each non-PK column + for (col_id, _col) in tbl_info.non_pks.iter().enumerate() { + let seq = bump_seq(ext_data); + let cell_key = (key << consts::CRSQL_COL_ID_BITS as i64) | col_id as i64; + let clock_sql = format!( + "INSERT OR REPLACE INTO \"{escaped}{suffix}\" (cell_key, col_version, site_id, db_version, seq, ts) VALUES (?, 1, 0, ?, ?, ?)", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + ); + let clock_stmt = db.prepare_v2(&clock_sql) + .map_err(|e| format!("failed to prepare clock stmt: {:?}", e))?; + clock_stmt.bind_int64(1, cell_key).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.bind_int64(2, db_version).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.bind_int(3, seq).map_err(|e| format!("bind: {:?}", e))?; + let ts_val = ts.parse::().map_err(|_| "invalid ts".to_string())?; + if ts_val == 0 { return Err("zero ts".to_string()); } + clock_stmt.bind_int64(4, ts_val).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.step().map_err(|e| format!("step: {:?}", e))?; + } + + // For pk-only tables (no non-PK columns), write a sentinel clock entry at col_id=0. + // This carries db_version, seq, ts, site_id so the row is visible in the feed. + if tbl_info.non_pks.is_empty() { + let seq = bump_seq(ext_data); + let cell_key = (key << consts::CRSQL_COL_ID_BITS as i64) | 0; + let clock_sql = format!( + "INSERT OR REPLACE INTO \"{escaped}{suffix}\" (cell_key, col_version, site_id, db_version, seq, ts) VALUES (?, 1, 0, ?, ?, ?)", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + ); + let clock_stmt = db.prepare_v2(&clock_sql) + .map_err(|e| format!("failed to prepare sentinel clock stmt: {:?}", e))?; + clock_stmt.bind_int64(1, cell_key).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.bind_int64(2, db_version).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.bind_int(3, seq).map_err(|e| format!("bind: {:?}", e))?; + let ts_val = ts.parse::().map_err(|_| "invalid ts".to_string())?; + if ts_val == 0 { return Err("zero ts".to_string()); } + clock_stmt.bind_int64(4, ts_val).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.step().map_err(|e| format!("step: {:?}", e))?; + } + + Ok(ResultCode::OK) +} + +/// V2 after_update: update v2_clock entries for changed columns. +pub fn v2_after_update( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + pks_new: &[*mut sqlite::value], + changed_col_indices: &[usize], +) -> Result { + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + return Err("v2_after_update: timestamp not set — call crsql_set_ts() first".to_string()); + } + let ts = unsafe { (*ext_data).timestamp.to_string() }; + let db_version = crate::db_version::next_db_version(db, ext_data) + .map_err(|_| "failed to get next db_version".to_string())?; + + let hashed_pk = hash_pk_values(pks_new) + .map_err(|_| "failed to hash PK values".to_string())?; + + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + + let key_stmt = db.prepare_v2(&format!( + "SELECT __crsql_key FROM \"{escaped}{suffix}\" WHERE hashed_pk = ?", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX + )).map_err(|e| format!("failed to prepare key stmt: {:?}", e))?; + key_stmt.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + let key = match key_stmt.step().map_err(|e| format!("step: {:?}", e))? { + ResultCode::ROW => key_stmt.column_int64(0), + _ => return Err("row not found in v2_pks for update".to_string()), + }; + + for (i, &col_idx) in changed_col_indices.iter().enumerate() { + let seq = bump_seq(ext_data); + let cell_key = (key << consts::CRSQL_COL_ID_BITS as i64) | col_idx as i64; + let clock_sql = format!( + "INSERT OR REPLACE INTO \"{escaped}{suffix}\" (cell_key, col_version, site_id, db_version, seq, ts) VALUES (?, COALESCE((SELECT col_version + 1 FROM \"{escaped}{suffix}\" WHERE cell_key = ?), 1), 0, ?, ?, ?)", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + ); + let clock_stmt = db.prepare_v2(&clock_sql) + .map_err(|e| format!("failed to prepare clock stmt: {:?}", e))?; + clock_stmt.bind_int64(1, cell_key).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.bind_int64(2, cell_key).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.bind_int64(3, db_version).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.bind_int(4, seq).map_err(|e| format!("bind: {:?}", e))?; + let ts_val = ts.parse::().map_err(|_| "invalid ts".to_string())?; + if ts_val == 0 { return Err("zero ts".to_string()); } + clock_stmt.bind_int64(5, ts_val).map_err(|e| format!("bind: {:?}", e))?; + clock_stmt.step().map_err(|e| format!("step: {:?}", e))?; + } + + Ok(ResultCode::OK) +} + +/// V2 after_delete: move row from v2_pks to v2_tombstones, delete clock entries. +pub fn v2_after_delete( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + pks_old: &[*mut sqlite::value], +) -> Result { + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + return Err("v2_after_delete: timestamp not set — call crsql_set_ts() first".to_string()); + } + let ts = unsafe { (*ext_data).timestamp.to_string() }; + let db_version = crate::db_version::next_db_version(db, ext_data) + .map_err(|_| "failed to get next db_version".to_string())?; + + let hashed_pk = hash_pk_values(pks_old) + .map_err(|_| "failed to hash PK values".to_string())?; + + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + + let key_stmt = db.prepare_v2(&format!( + "SELECT __crsql_key, cl FROM \"{escaped}{suffix}\" WHERE hashed_pk = ?", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX + )).map_err(|e| format!("failed to prepare key stmt: {:?}", e))?; + key_stmt.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + + let (key, cl) = match key_stmt.step().map_err(|e| format!("step: {:?}", e))? { + ResultCode::ROW => (key_stmt.column_int64(0), key_stmt.column_int64(1)), + _ => { + let tomb_stmt = db.prepare_v2(&format!( + "SELECT cl FROM \"{escaped}{suffix}\" WHERE hashed_pk = ?", + escaped = escaped, + suffix = consts::V2_TOMBSTONES_SUFFIX + )).map_err(|e| format!("failed to prepare tomb stmt: {:?}", e))?; + tomb_stmt.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + match tomb_stmt.step().map_err(|e| format!("step: {:?}", e))? { + ResultCode::ROW => return Ok(ResultCode::OK), + _ => return Ok(ResultCode::OK), + } + } + }; + + let new_cl = cl + 1; + let seq = bump_seq(ext_data); + + let del_stmt = db.prepare_v2(&format!( + "DELETE FROM \"{escaped}{suffix}\" WHERE __crsql_key = ?", + escaped = escaped, + suffix = consts::V2_PKS_SUFFIX + )).map_err(|e| format!("failed to prepare del stmt: {:?}", e))?; + del_stmt.bind_int64(1, key).map_err(|e| format!("bind: {:?}", e))?; + del_stmt.step().map_err(|e| format!("step: {:?}", e))?; + + let del_clock_stmt = db.prepare_v2(&format!( + "DELETE FROM \"{escaped}{suffix}\" WHERE cell_key >= ? AND cell_key <= ?", + escaped = escaped, + suffix = consts::V2_CLOCK_SUFFIX + )).map_err(|e| format!("failed to prepare del clock stmt: {:?}", e))?; + let base = key << consts::CRSQL_COL_ID_BITS; + del_clock_stmt.bind_int64(1, base).map_err(|e| format!("bind: {:?}", e))?; + del_clock_stmt.bind_int64(2, base | consts::CRSQL_COL_ID_MASK as i64).map_err(|e| format!("bind: {:?}", e))?; + del_clock_stmt.step().map_err(|e| format!("step: {:?}", e))?; + + let tomb_sql = format!( + "INSERT OR REPLACE INTO \"{escaped}{suffix}\" (site_id, db_version, seq, hashed_pk, cl, ts) VALUES (0, ?, ?, ?, ?, ?)", + escaped = escaped, + suffix = consts::V2_TOMBSTONES_SUFFIX + ); + let tomb_stmt = db.prepare_v2(&tomb_sql) + .map_err(|e| format!("failed to prepare tomb insert stmt: {:?}", e))?; + tomb_stmt.bind_int64(1, db_version).map_err(|e| format!("bind: {:?}", e))?; + tomb_stmt.bind_int(2, seq).map_err(|e| format!("bind: {:?}", e))?; + tomb_stmt.bind_blob(3, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + tomb_stmt.bind_int64(4, new_cl).map_err(|e| format!("bind: {:?}", e))?; + let ts_val = ts.parse::().map_err(|_| "invalid ts".to_string())?; + if ts_val == 0 { return Err("zero ts".to_string()); } + tomb_stmt.bind_int64(5, ts_val).map_err(|e| format!("bind: {:?}", e))?; + tomb_stmt.step().map_err(|e| format!("step: {:?}", e))?; + + let pk_cols = tbl_info.pks.iter() + .map(|p| format!("\"{}\"", crate::util::escape_ident(&p.name))) + .collect::>() + .join(", "); + let pk_values = tbl_info.pks.iter() + .map(|_| "?") + .collect::>() + .join(", "); + let tpk_sql = format!( + "INSERT OR REPLACE INTO \"{escaped}{suffix}\" (hashed_pk, {pk_cols}) VALUES (?, {pk_values})", + escaped = escaped, + suffix = consts::V2_TOMBSTONE_PKS_SUFFIX, + pk_cols = pk_cols, + pk_values = pk_values, + ); + let tpk_stmt = db.prepare_v2(&tpk_sql) + .map_err(|e| format!("failed to prepare tpk stmt: {:?}", e))?; + tpk_stmt.bind_blob(1, &hashed_pk, Destructor::STATIC).map_err(|e| format!("bind: {:?}", e))?; + for (i, pk) in pks_old.iter().enumerate() { + tpk_stmt.bind_value(i as i32 + 2, *pk).map_err(|e| format!("bind: {:?}", e))?; + } + tpk_stmt.step().map_err(|e| format!("step: {:?}", e))?; + + Ok(ResultCode::OK) +} diff --git a/core/rs/core/src/migrate.rs b/core/rs/core/src/migrate.rs new file mode 100644 index 000000000..c43995f61 --- /dev/null +++ b/core/rs/core/src/migrate.rs @@ -0,0 +1,662 @@ +extern crate alloc; + +use alloc::boxed::Box; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use core::ffi::{c_char, c_int}; +use sqlite_nostd::{sqlite3, Connection, ResultCode}; + +use crate::c::crsql_ExtData; +use crate::consts; +use crate::tableinfo::{TableInfo, SchemaVersion}; +use core::mem; + +/// crsql_incremental_maintenance(chunk_size) -> INTEGER +/// Dispatches to pending maintenance tasks across all CRR tables. +/// Does up to `chunk_size` units of work per call, returns total remaining. +/// When it returns 0, all maintenance is complete. +#[no_mangle] +pub unsafe extern "C" fn crsql_incremental_maintenance( + db: *mut sqlite3, + chunk_size: c_int, + ext_data: *mut crsql_ExtData, +) -> c_int { + match incremental_maintenance(db, chunk_size, ext_data) { + Ok(remaining) => remaining, + Err(_) => -1, + } +} + +unsafe fn incremental_maintenance( + db: *mut sqlite3, + chunk_size: c_int, + ext_data: *mut crsql_ExtData, +) -> Result { + // V2 clock tables require a non-zero ts. Error early if not set. + if unsafe { (*ext_data).timestamp } == 0 { + crate::debug::debug_log("incremental_maintenance: timestamp not set — call crsql_set_ts() first"); + return Err(ResultCode::ERROR); + } + + // Refresh table infos cache — new CRRs may have been created since last refresh + let mut err: *mut c_char = core::ptr::null_mut(); + let rc = crate::tableinfo::crsql_ensure_table_infos_are_up_to_date( + db, + ext_data, + &mut err as *mut *mut c_char, + ); + if rc != ResultCode::OK as c_int { + crate::debug::debug_log("incremental_maintenance: ensure_table_infos failed"); + return Err(ResultCode::ERROR); + } + crate::debug::debug_log("incremental_maintenance: table infos refreshed"); + + let table_infos = + mem::ManuallyDrop::new(Box::from_raw((*ext_data).tableInfos as *mut Vec)); + crate::debug::debug_log(&format!("incremental_maintenance: {} table infos", table_infos.len())); + + let mut total_remaining: c_int = 0; + + // Priority 1: V1 table cleanup tasks (from v2&v1 -> v2 transition) + let cleanup_v1_sql = "SELECT key FROM crsql_master WHERE key LIKE 'cleanup_v1_tables_%'\0"; + let cleanup_stmt = db.prepare_v2(cleanup_v1_sql)?; + let mut v1_cleanup_tables: Vec = Vec::new(); + while cleanup_stmt.step()? == ResultCode::ROW { + let key = cleanup_stmt.column_text(0)?; + if let Some(tbl) = key.strip_prefix("cleanup_v1_tables_") { + v1_cleanup_tables.push(String::from(tbl)); + } + } + for tbl_name in &v1_cleanup_tables { + let remaining = cleanup_v1_tables_chunk(db, tbl_name, chunk_size as i64)?; + if remaining == 0 { + clear_cleanup_marker(db, &format!("cleanup_v1_tables_{}", tbl_name))?; + } else { + total_remaining += remaining as c_int; + } + } + + // Priority 2: V2 table cleanup tasks (from v2&v1 -> v1 rollback) + let cleanup_v2_sql = "SELECT key FROM crsql_master WHERE key LIKE 'cleanup_v2_tables_%'\0"; + let cleanup_stmt = db.prepare_v2(cleanup_v2_sql)?; + let mut v2_cleanup_tables: Vec = Vec::new(); + while cleanup_stmt.step()? == ResultCode::ROW { + let key = cleanup_stmt.column_text(0)?; + if let Some(tbl) = key.strip_prefix("cleanup_v2_tables_") { + v2_cleanup_tables.push(String::from(tbl)); + } + } + for tbl_name in &v2_cleanup_tables { + let remaining = cleanup_v2_tables_chunk(db, tbl_name, chunk_size as i64)?; + if remaining == 0 { + clear_cleanup_marker(db, &format!("cleanup_v2_tables_{}", tbl_name))?; + } else { + total_remaining += remaining as c_int; + } + } + + // Priority 3: V1→V2 migration tasks + for tbl_info in table_infos.iter() { + crate::debug::debug_log(&format!("migration: checking table {} schema_version={:?}", tbl_info.tbl_name, tbl_info.schema_version)); + // Only migrate tables that have V1 tables (V1 or V2AndV1 schema) + if tbl_info.schema_version == SchemaVersion::V2 { + continue; + } + + // Check if V2 tables already exist + let has_v2 = crate::bootstrap_v2::has_v2_tables(db, &tbl_info.tbl_name)?; + crate::debug::debug_log(&format!("migration: {} has_v2={}", tbl_info.tbl_name, has_v2)); + if !has_v2 { + // First call for this table: create V2 tables + match crate::bootstrap_v2::create_v2_tables(db, tbl_info) { + Ok(_) => crate::debug::debug_log(&format!("migration: created v2 tables for {}", tbl_info.tbl_name)), + Err(e) => { + crate::debug::debug_log(&format!("migration: create_v2_tables failed for {}: {:?}", tbl_info.tbl_name, e)); + return Err(e); + } + } + } + + // Migrate a chunk of rows from V1 to V2 + let remaining = migrate_v1_to_v2_chunk(db, ext_data, tbl_info, chunk_size as i64)?; + crate::debug::debug_log(&format!("migration: {} remaining={}", tbl_info.tbl_name, remaining)); + total_remaining += remaining as c_int; + } + + Ok(total_remaining) +} + +/// Chunked V1 table cleanup: DELETE rows in batches, then DROP empty tables. +/// Returns estimated remaining rows across both tables. +unsafe fn cleanup_v1_tables_chunk( + db: *mut sqlite3, + tbl_name: &str, + chunk_size: i64, +) -> Result { + let escaped = crate::util::escape_ident(tbl_name); + db.exec_safe("SAVEPOINT cleanup_v1_chunk")?; + + let result = (|| { + // Delete a chunk from __crsql_clock + db.exec_safe(&format!( + "DELETE FROM \"{escaped}__crsql_clock\" LIMIT {chunk_size}", + escaped = escaped, + chunk_size = chunk_size, + ))?; + let clock_remaining = count_rows(db, &format!("\"{escaped}__crsql_clock\"", escaped = escaped))?; + + // Delete a chunk from __crsql_pks + db.exec_safe(&format!( + "DELETE FROM \"{escaped}__crsql_pks\" LIMIT {chunk_size}", + escaped = escaped, + chunk_size = chunk_size, + ))?; + let pks_remaining = count_rows(db, &format!("\"{escaped}__crsql_pks\"", escaped = escaped))?; + + // When both empty, drop the tables + if clock_remaining == 0 && pks_remaining == 0 { + db.exec_safe(&format!( + "DROP TABLE IF EXISTS \"{escaped}__crsql_clock\";\ + DROP TABLE IF EXISTS \"{escaped}__crsql_pks\";", + escaped = escaped, + ))?; + } + + Ok(clock_remaining + pks_remaining) + })(); + + match result { + Ok(remaining) => { + db.exec_safe("RELEASE cleanup_v1_chunk")?; + Ok(remaining) + } + Err(e) => { + db.exec_safe("ROLLBACK")?; + Err(e) + } + } +} + +/// Chunked V2 table cleanup: DELETE rows in batches, then DROP empty tables. +/// Returns estimated remaining rows across all V2 tables. +unsafe fn cleanup_v2_tables_chunk( + db: *mut sqlite3, + tbl_name: &str, + chunk_size: i64, +) -> Result { + let escaped = crate::util::escape_ident(tbl_name); + db.exec_safe("SAVEPOINT cleanup_v2_chunk")?; + + let v2_suffixes = [ + consts::V2_COL_MAP_SUFFIX, + consts::V2_CLOCK_SUFFIX, + consts::V2_PKS_SUFFIX, + consts::V2_TOMBSTONES_SUFFIX, + consts::V2_TOMBSTONE_PKS_SUFFIX, + ]; + + let result = (|| { + let mut total_remaining: i64 = 0; + let mut all_empty = true; + + for suffix in &v2_suffixes { + let table_name = format!("\"{escaped}{suffix}\"", escaped = escaped, suffix = suffix); + db.exec_safe(&format!( + "DELETE FROM {table_name} LIMIT {chunk_size}", + table_name = table_name, + chunk_size = chunk_size, + ))?; + let remaining = count_rows(db, &table_name)?; + total_remaining += remaining; + if remaining > 0 { + all_empty = false; + } + } + + // When all empty, drop the tables + if all_empty { + for suffix in &v2_suffixes { + db.exec_safe(&format!( + "DROP TABLE IF EXISTS \"{escaped}{suffix}\";", + escaped = escaped, + suffix = suffix, + ))?; + } + } + + Ok(total_remaining) + })(); + + match result { + Ok(remaining) => { + db.exec_safe("RELEASE cleanup_v2_chunk")?; + Ok(remaining) + } + Err(e) => { + db.exec_safe("ROLLBACK")?; + Err(e) + } + } +} + +/// Count rows in a table (quick check). +unsafe fn count_rows(db: *mut sqlite3, table_name: &str) -> Result { + let sql = format!("SELECT count(*) FROM {table_name}\0", table_name = table_name); + let stmt = db.prepare_v2(&sql)?; + stmt.step()?; + Ok(stmt.column_int64(0)) +} + +/// Clear a cleanup marker from crsql_master by exact key. +unsafe fn clear_cleanup_marker(db: *mut sqlite3, key: &str) -> Result<(), ResultCode> { + let sql = "DELETE FROM crsql_master WHERE key = ?\0"; + let stmt = db.prepare_v2(sql)?; + stmt.bind_text(1, key, sqlite_nostd::Destructor::TRANSIENT)?; + stmt.step()?; + Ok(()) +} + +/// Migrate a chunk of V1 PK rows to V2 tables. +/// Returns the number of remaining rows to migrate. +unsafe fn migrate_v1_to_v2_chunk( + db: *mut sqlite3, + ext_data: *mut crsql_ExtData, + tbl_info: &TableInfo, + chunk_size: i64, +) -> Result { + let escaped = crate::util::escape_ident(&tbl_info.tbl_name); + let ts_fallback = unsafe { (*ext_data).timestamp as i64 }; + + db.exec_safe("SAVEPOINT migration_chunk")?; + + // Get progress marker from crsql_master + let progress = get_progress_marker(db, &tbl_info.tbl_name, "v1_to_v2_migration")?; + let start_key = progress.unwrap_or(0); + + // One-time count of total rows to migrate (stored for estimate) + let total_key = format!("migration_v1_to_v2_total_{}\0", tbl_info.tbl_name); + let total: i64 = { + let stmt = db.prepare_v2("SELECT value FROM crsql_master WHERE key = ?\0")?; + stmt.bind_text(1, &total_key, sqlite_nostd::Destructor::TRANSIENT)?; + if stmt.step()? == ResultCode::ROW { + stmt.column_int64(0) + } else { + // First call: count total rows + let count_sql = format!( + "SELECT count(*) FROM \"{escaped}__crsql_pks\"\0", + escaped = escaped, + ); + let count_stmt = db.prepare_v2(&count_sql)?; + count_stmt.step()?; + let total = count_stmt.column_int64(0); + // Store it + let insert_sql = "INSERT OR REPLACE INTO crsql_master (key, value) VALUES (?, ?)\0"; + let insert_stmt = db.prepare_v2(insert_sql)?; + insert_stmt.bind_text(1, &total_key, sqlite_nostd::Destructor::TRANSIENT)?; + insert_stmt.bind_int64(2, total)?; + insert_stmt.step()?; + total + } + }; + + // Track cumulative processed count for estimate + let done_key = format!("migration_v1_to_v2_done_{}\0", tbl_info.tbl_name); + let mut cumulative_done: i64 = { + let stmt = db.prepare_v2("SELECT value FROM crsql_master WHERE key = ?\0")?; + stmt.bind_text(1, &done_key, sqlite_nostd::Destructor::TRANSIENT)?; + if stmt.step()? == ResultCode::ROW { + stmt.column_int64(0) + } else { + 0 + } + }; + + // Process a chunk of rows + let pk_cols: Vec = tbl_info.pks.iter().map(|c| format!("\"{}\"", crate::util::escape_ident(&c.name))).collect(); + let pk_cols_list = pk_cols.join(", "); + let pk_cols_no_quotes: Vec = tbl_info.pks.iter().map(|c| format!("p.\"{}\"", crate::util::escape_ident(&c.name))).collect(); + let pk_cols_p_list = pk_cols_no_quotes.join(", "); + // PK columns qualified with t. (backing table alias) for rowid-key tables + let pk_cols_t: Vec = tbl_info.pks.iter().map(|c| format!("t.\"{}\"", crate::util::escape_ident(&c.name))).collect(); + let pk_cols_t_list = pk_cols_t.join(", "); + + let sentinel = crate::c::INSERT_SENTINEL; + let col_id_bits = consts::CRSQL_COL_ID_BITS as i64; + let uses_rowid_key = tbl_info.uses_rowid_key; + + // Build PK join condition between pks table and backing table. + // The backing table is aliased as `t` in rowid-key queries and un-aliased in non-rowid queries. + // For rowid-key tables: use t."col" = p."col" + // For non-rowid tables: use "{escaped}"."col" = p."col" + let pk_join_conds: Vec = tbl_info.pks.iter().map(|c| { + if uses_rowid_key { + format!("t.\"{col}\" = p.\"{col}\"", col = crate::util::escape_ident(&c.name)) + } else { + format!("\"{escaped}\".\"{col}\" = p.\"{col}\"", escaped = escaped, col = crate::util::escape_ident(&c.name)) + } + }).collect(); + let pk_join_cond = pk_join_conds.join(" AND "); + + let result = (|| { + // Step 1: Batch insert alive PKs into v2_pks + if uses_rowid_key { + db.exec_safe(&format!( + "INSERT OR IGNORE INTO \"{escaped}{v2_pks}\" (__crsql_key, hashed_pk, cl) + SELECT t.\"{rowid_alias}\", crsql_hash_pk({pk_cols_t}), + CASE WHEN s.col_version IS NULL OR s.col_version % 2 != 0 THEN + CASE WHEN s.col_version IS NULL THEN 1 ELSE s.col_version END + ELSE 1 END + FROM \"{escaped}__crsql_pks\" p + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON p.__crsql_key = chunk.__crsql_key + JOIN \"{escaped}\" t ON {pk_join_cond} + LEFT JOIN \"{escaped}__crsql_clock\" s + ON p.__crsql_key = s.key AND s.col_name = '{sentinel}' + WHERE s.col_version IS NULL OR s.col_version % 2 != 0", + escaped = escaped, + v2_pks = consts::V2_PKS_SUFFIX, + rowid_alias = crate::util::escape_ident(&tbl_info.rowid_alias), + pk_cols_t = pk_cols_t_list, + pk_join_cond = pk_join_cond, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ))?; + } else { + db.exec_safe(&format!( + "INSERT OR IGNORE INTO \"{escaped}{v2_pks}\" ({pk_cols}, hashed_pk, cl) + SELECT {pk_cols_p}, crsql_hash_pk({pk_cols_p}), + CASE WHEN s.col_version IS NULL OR s.col_version % 2 != 0 THEN + CASE WHEN s.col_version IS NULL THEN 1 ELSE s.col_version END + ELSE 1 END + FROM \"{escaped}__crsql_pks\" p + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON p.__crsql_key = chunk.__crsql_key + JOIN \"{escaped}\" ON {pk_join_cond} + LEFT JOIN \"{escaped}__crsql_clock\" s + ON p.__crsql_key = s.key AND s.col_name = '{sentinel}' + WHERE s.col_version IS NULL OR s.col_version % 2 != 0", + escaped = escaped, + v2_pks = consts::V2_PKS_SUFFIX, + pk_cols = pk_cols_list, + pk_cols_p = pk_cols_p_list, + pk_join_cond = pk_join_cond, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ))?; + } + + // Step 2: Batch insert tombstones (dead rows) + // Tombstones always use PK columns for hash (peers need PK values to identify rows) + // Use actual site_id, db_version, seq from V1 clock entry — not dummy 0s, + // because v2_tombstones has PRIMARY KEY (site_id, db_version, seq). + db.exec_safe(&format!( + "INSERT OR REPLACE INTO \"{escaped}{v2_tomb}\" + (site_id, db_version, seq, hashed_pk, cl, ts) + SELECT s.site_id, s.db_version, s.seq, crsql_hash_pk({pk_cols_p}), s.col_version, + CASE WHEN s.ts > 0 THEN s.ts ELSE {ts_fallback} END + FROM \"{escaped}__crsql_pks\" p + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON p.__crsql_key = chunk.__crsql_key + JOIN \"{escaped}__crsql_clock\" s + ON p.__crsql_key = s.key AND s.col_name = '{sentinel}' + WHERE s.col_version % 2 = 0", + escaped = escaped, + v2_tomb = consts::V2_TOMBSTONES_SUFFIX, + pk_cols_p = pk_cols_p_list, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ts_fallback = ts_fallback, + ))?; + + // Step 3: Batch insert tombstone PKs + db.exec_safe(&format!( + "INSERT OR REPLACE INTO \"{escaped}{v2_tomb_pks}\" ({pk_cols}, hashed_pk) + SELECT {pk_cols_p}, crsql_hash_pk({pk_cols_p}) + FROM \"{escaped}__crsql_pks\" p + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON p.__crsql_key = chunk.__crsql_key + JOIN \"{escaped}__crsql_clock\" s + ON p.__crsql_key = s.key AND s.col_name = '{sentinel}' + WHERE s.col_version % 2 = 0", + escaped = escaped, + v2_tomb_pks = consts::V2_TOMBSTONE_PKS_SUFFIX, + pk_cols = pk_cols_list, + pk_cols_p = pk_cols_p_list, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ))?; + + // Step 4: Batch migrate clock entries — INNER JOIN on pks chunk + v2_col_map + // For rowid-key tables: cell_key uses t. instead of c.key + if uses_rowid_key { + db.exec_safe(&format!( + "INSERT OR REPLACE INTO \"{escaped}{v2_clock}\" + (cell_key, col_version, site_id, db_version, seq, ts) + SELECT (t.\"{rowid_alias}\" << {col_id_bits}) | m.col_id, + c.col_version, c.site_id, c.db_version, c.seq, + CASE WHEN c.ts > 0 THEN c.ts ELSE {ts_fallback} END + FROM \"{escaped}__crsql_clock\" c + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON c.key = chunk.__crsql_key + JOIN \"{escaped}__crsql_pks\" p ON c.key = p.__crsql_key + JOIN \"{escaped}\" t ON {pk_join_cond} + JOIN \"{escaped}{v2_col_map}\" m ON c.col_name = m.col_name + WHERE c.col_name != '{sentinel}'", + escaped = escaped, + v2_clock = consts::V2_CLOCK_SUFFIX, + v2_col_map = consts::V2_COL_MAP_SUFFIX, + col_id_bits = col_id_bits, + rowid_alias = crate::util::escape_ident(&tbl_info.rowid_alias), + pk_join_cond = pk_join_cond, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ts_fallback = ts_fallback, + ))?; + } else { + db.exec_safe(&format!( + "INSERT OR REPLACE INTO \"{escaped}{v2_clock}\" + (cell_key, col_version, site_id, db_version, seq, ts) + SELECT (vp.__crsql_key << {col_id_bits}) | m.col_id, + c.col_version, c.site_id, c.db_version, c.seq, + CASE WHEN c.ts > 0 THEN c.ts ELSE {ts_fallback} END + FROM \"{escaped}__crsql_clock\" c + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON c.key = chunk.__crsql_key + JOIN \"{escaped}__crsql_pks\" p ON c.key = p.__crsql_key + JOIN \"{escaped}{v2_pks}\" vp ON vp.hashed_pk = crsql_hash_pk({pk_cols_p}) + JOIN \"{escaped}{v2_col_map}\" m ON c.col_name = m.col_name + WHERE c.col_name != '{sentinel}'", + escaped = escaped, + v2_clock = consts::V2_CLOCK_SUFFIX, + v2_pks = consts::V2_PKS_SUFFIX, + v2_col_map = consts::V2_COL_MAP_SUFFIX, + col_id_bits = col_id_bits, + pk_cols_p = pk_cols_p_list, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ts_fallback = ts_fallback, + ))?; + } + + // Step 4b: For PK-only tables, migrate V1 sentinel clock entries to V2 sentinel at col_id=0. + // The normal clock migration (step 4) skips sentinels. For PK-only tables, the sentinel + // is the only clock entry, so we need to migrate it separately. + if tbl_info.non_pks.is_empty() { + if uses_rowid_key { + db.exec_safe(&format!( + "INSERT OR REPLACE INTO \"{escaped}{v2_clock}\" + (cell_key, col_version, site_id, db_version, seq, ts) + SELECT (t.\"{rowid_alias}\" << {col_id_bits}) | 0, + 1, c.site_id, c.db_version, c.seq, + CASE WHEN c.ts > 0 THEN c.ts ELSE {ts_fallback} END + FROM \"{escaped}__crsql_clock\" c + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON c.key = chunk.__crsql_key + JOIN \"{escaped}__crsql_pks\" p ON c.key = p.__crsql_key + JOIN \"{escaped}\" t ON {pk_join_cond} + WHERE c.col_name = '{sentinel}'", + escaped = escaped, + v2_clock = consts::V2_CLOCK_SUFFIX, + col_id_bits = col_id_bits, + rowid_alias = crate::util::escape_ident(&tbl_info.rowid_alias), + pk_join_cond = pk_join_cond, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ts_fallback = ts_fallback, + ))?; + } else { + db.exec_safe(&format!( + "INSERT OR REPLACE INTO \"{escaped}{v2_clock}\" + (cell_key, col_version, site_id, db_version, seq, ts) + SELECT (vp.__crsql_key << {col_id_bits}) | 0, + 1, c.site_id, c.db_version, c.seq, + CASE WHEN c.ts > 0 THEN c.ts ELSE {ts_fallback} END + FROM \"{escaped}__crsql_clock\" c + JOIN (SELECT __crsql_key FROM \"{escaped}__crsql_pks\" WHERE __crsql_key > {start_key} ORDER BY __crsql_key LIMIT {chunk_size}) chunk + ON c.key = chunk.__crsql_key + JOIN \"{escaped}__crsql_pks\" p ON c.key = p.__crsql_key + JOIN \"{escaped}{v2_pks}\" vp ON vp.hashed_pk = crsql_hash_pk({pk_cols_p}) + WHERE c.col_name = '{sentinel}'", + escaped = escaped, + v2_clock = consts::V2_CLOCK_SUFFIX, + v2_pks = consts::V2_PKS_SUFFIX, + col_id_bits = col_id_bits, + pk_cols_p = pk_cols_p_list, + sentinel = sentinel, + start_key = start_key, + chunk_size = chunk_size, + ))?; + } + } + + // Step 5: Get max key from chunk to update cursor. + // NULL means the chunk was empty — migration is done. + let max_key_sql = format!( + "SELECT max(__crsql_key) FROM ( + SELECT __crsql_key FROM \"{escaped}__crsql_pks\" + WHERE __crsql_key > {start_key} + ORDER BY __crsql_key + LIMIT {chunk_size} + )\0", + escaped = escaped, + start_key = start_key, + chunk_size = chunk_size, + ); + let max_key_stmt = db.prepare_v2(&max_key_sql)?; + max_key_stmt.step()?; + let last_key = max_key_stmt.column_int64(0); + + if last_key == 0 { + // Chunk was empty — migration complete + Ok(0i64) + } else { + // Update progress marker to last key processed + set_progress_marker(db, &tbl_info.tbl_name, "v1_to_v2_migration", last_key)?; + // Use chunk_size as processed count for estimate (actual may be less due to orphans/IGNORE) + Ok(chunk_size) + } + })(); + + match result { + Ok(processed) => { + cumulative_done += processed; + crate::debug::debug_log(&format!("migrate_chunk: processed={} cumulative_done={} total={}", processed, cumulative_done, total)); + if processed == 0 || cumulative_done >= total { + // Migration complete for this table (no more rows, or all rows processed) + clear_progress_marker(db, &tbl_info.tbl_name, "v1_to_v2_migration")?; + // Clear total and done markers + let clear_sql = "DELETE FROM crsql_master WHERE key = ?\0"; + let stmt = db.prepare_v2(clear_sql)?; + stmt.bind_text(1, &total_key, sqlite_nostd::Destructor::TRANSIENT)?; + stmt.step()?; + let stmt = db.prepare_v2(clear_sql)?; + stmt.bind_text(1, &done_key, sqlite_nostd::Destructor::TRANSIENT)?; + stmt.step()?; + db.exec_safe("RELEASE migration_chunk")?; + Ok(0) + } else { + // Update done marker + let update_sql = "INSERT OR REPLACE INTO crsql_master (key, value) VALUES (?, ?)\0"; + let stmt = db.prepare_v2(update_sql)?; + stmt.bind_text(1, &done_key, sqlite_nostd::Destructor::TRANSIENT)?; + stmt.bind_int64(2, cumulative_done)?; + stmt.step()?; + db.exec_safe("RELEASE migration_chunk")?; + let remaining = total - cumulative_done; + let remaining = if remaining < 0 { 0i64 } else { remaining }; + crate::debug::debug_log(&format!("migrate_chunk: returning remaining={} (total={} cum_done={})", remaining, total, cumulative_done)); + Ok(remaining) + } + } + Err(e) => { + let errmsg = match db.errmsg() { + Ok(s) => s, + Err(_) => alloc::string::String::from("unknown"), + }; + crate::debug::debug_log(&format!("migrate_v1_to_v2_chunk FAILED: {:?} errmsg={}", e, errmsg)); + db.exec_safe("ROLLBACK")?; + Err(e) + } + } +} + +/// Get progress marker from crsql_master +unsafe fn get_progress_marker( + db: *mut sqlite3, + tbl_name: &str, + task_type: &str, +) -> Result, ResultCode> { + let key = format!("migration_{}_{}", task_type, tbl_name); + let sql = format!( + "SELECT value FROM crsql_master WHERE key = ?\0" + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_text(1, &key, sqlite_nostd::Destructor::STATIC)?; + if stmt.step()? == ResultCode::ROW { + let val = stmt.column_int64(0); + return Ok(Some(val)); + } + Ok(None) +} + +/// Set progress marker in crsql_master +unsafe fn set_progress_marker( + db: *mut sqlite3, + tbl_name: &str, + task_type: &str, + marker: i64, +) -> Result<(), ResultCode> { + let key = format!("migration_{}_{}", task_type, tbl_name); + let sql = format!( + "INSERT OR REPLACE INTO crsql_master (key, value) VALUES (?, ?)\0" + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_text(1, &key, sqlite_nostd::Destructor::STATIC)?; + stmt.bind_int64(2, marker)?; + stmt.step()?; + Ok(()) +} + +/// Clear progress marker from crsql_master +unsafe fn clear_progress_marker( + db: *mut sqlite3, + tbl_name: &str, + task_type: &str, +) -> Result<(), ResultCode> { + let key = format!("migration_{}_{}", task_type, tbl_name); + let sql = format!( + "DELETE FROM crsql_master WHERE key = ?\0" + ); + let stmt = db.prepare_v2(&sql)?; + stmt.bind_text(1, &key, sqlite_nostd::Destructor::STATIC)?; + stmt.step()?; + Ok(()) +} diff --git a/core/rs/core/src/pack_columns.rs b/core/rs/core/src/pack_columns.rs index a1f5bd73e..a48c62a40 100644 --- a/core/rs/core/src/pack_columns.rs +++ b/core/rs/core/src/pack_columns.rs @@ -1,5 +1,6 @@ extern crate alloc; +use alloc::boxed::Box; use alloc::string::String; use alloc::vec; use alloc::vec::Vec; @@ -7,7 +8,8 @@ use bytes::{Buf, BufMut}; #[cfg(not(feature = "std"))] use num_traits::FromPrimitive; use sqlite_nostd as sqlite; -use sqlite_nostd::{ColumnType, Context, ResultCode, Stmt, Value}; +use sqlite_nostd::{aggregate_context, ColumnType, Context, ResultCode, Stmt, Value}; +use core::ffi::c_int; pub extern "C" fn crsql_pack_columns( ctx: *mut sqlite::context, @@ -28,7 +30,7 @@ pub extern "C" fn crsql_pack_columns( } } -fn pack_columns(args: &[*mut sqlite::value]) -> Result, ResultCode> { +pub fn pack_columns(args: &[*mut sqlite::value]) -> Result, ResultCode> { let mut buf = vec![]; /* * Format: @@ -42,9 +44,9 @@ fn pack_columns(args: &[*mut sqlite::value]) -> Result, ResultCode> { * Not packing an integer into the minimal number of bytes required is rather wasteful. * E.g., the number `0` would take 8 bytes rather than 1 byte. */ - let len_result: Result = args.len().try_into(); + let len_result: Result = args.len().try_into(); if let Ok(len) = len_result { - buf.put_u8(len); + put_varint(&mut buf, len); for value in args { match value.value_type() { ColumnType::Blob => { @@ -85,6 +87,52 @@ fn pack_columns(args: &[*mut sqlite::value]) -> Result, ResultCode> { } } +/// Pack a slice of ColumnValue into the same wire format as pack_columns. +/// Used by V2 code paths that work with unpacked values. +pub fn pack_column_values(values: &[ColumnValue]) -> Result, ResultCode> { + let mut buf = vec![]; + let len_result: Result = values.len().try_into(); + if let Ok(len) = len_result { + put_varint(&mut buf, len); + for value in values { + match value { + ColumnValue::Blob(b) => { + let len = b.len() as i32; + let num_bytes_for_len = num_bytes_needed_i32(len); + let type_byte = num_bytes_for_len << 3 | (ColumnType::Blob as u8); + buf.put_u8(type_byte); + buf.put_int(len as i64, num_bytes_for_len as usize); + buf.put_slice(b); + } + ColumnValue::Null => { + buf.put_u8(ColumnType::Null as u8); + } + ColumnValue::Float(f) => { + buf.put_u8(ColumnType::Float as u8); + buf.put_f64(*f); + } + ColumnValue::Integer(val) => { + let num_bytes_for_int = num_bytes_needed_i64(*val); + let type_byte = num_bytes_for_int << 3 | (ColumnType::Integer as u8); + buf.put_u8(type_byte); + buf.put_int(*val, num_bytes_for_int as usize); + } + ColumnValue::Text(t) => { + let len = t.len() as i32; + let num_bytes_for_len = num_bytes_needed_i32(len); + let type_byte = num_bytes_for_len << 3 | (ColumnType::Text as u8); + buf.put_u8(type_byte); + buf.put_int(len as i64, num_bytes_for_len as usize); + buf.put_slice(t.as_bytes()); + } + } + } + Ok(buf) + } else { + Err(ResultCode::ABORT) + } +} + fn num_bytes_needed_i32(val: i32) -> u8 { if val & 0xFF000000u32 as i32 != 0 { return 4; @@ -121,11 +169,70 @@ pub enum ColumnValue { Text(String), } +/// Encode a value as a SQLite varint into the buffer. +/// Values 0-127 encode as a single byte (0x00-0x7F), byte-identical +/// to the old u8 format. Larger values use multi-byte encoding. +fn put_varint(buf: &mut Vec, value: u64) { + if value < 0x80 { + buf.put_u8(value as u8); + return; + } + // SQLite varint: up to 9 bytes, high bit = continuation + let mut bytes = [0u8; 9]; + let mut n = 0; + let mut v = value; + if v == 0 { + buf.put_u8(0); + return; + } + while v > 0 && n < 9 { + bytes[n] = (v & 0x7F) as u8; + v >>= 7; + n += 1; + } + // Set continuation bits on all but the last byte + for i in 1..n { + bytes[i - 1] |= 0x80; + } + // Bytes are stored MSB first (reverse of how we filled them) + for i in (0..n).rev() { + buf.put_u8(bytes[i]); + } +} + +/// Read a SQLite varint from the buffer. Returns the value and number of bytes consumed. +fn get_varint(buf: &[u8]) -> Result<(u64, usize), ResultCode> { + if buf.is_empty() { + return Err(ResultCode::ABORT); + } + let mut result: u64 = 0; + let mut i = 0; + while i < buf.len() && i < 9 { + let byte = buf[i]; + if i == 8 { + // 9th byte uses all 8 bits + result = (result << 8) | byte as u64; + i += 1; + break; + } + result = (result << 7) | (byte & 0x7F) as u64; + i += 1; + if byte & 0x80 == 0 { + break; + } + } + if i > buf.len() { + return Err(ResultCode::ABORT); + } + Ok((result, i)) +} + // TODO: make a table valued function that can be used to extract a row per packed column? pub fn unpack_columns(data: &[u8]) -> Result, ResultCode> { let mut ret = vec![]; - let mut buf = data; - let num_columns = buf.get_u8(); + let (num_columns, varint_len) = get_varint(data)?; + let mut buf = &data[varint_len..]; + let num_columns = num_columns as usize; for _i in 0..num_columns { if !buf.has_remaining() { @@ -207,3 +314,110 @@ fn bind_slot( ColumnValue::Text(t) => stmt.bind_text(slot_num as i32, t, sqlite::Destructor::STATIC), } } + +/// Accumulator state for crsql_pack_agg aggregate function. +/// Stored via sqlite3_aggregate_context. +#[repr(C)] +struct PackAggAcc { + buf: *mut Vec, + count: u64, +} + +impl PackAggAcc { + const SIZE: c_int = core::mem::size_of::() as c_int; +} + +/// xStep callback for crsql_pack_agg. +/// Encodes a single SQLite value into the accumulator buffer using the same +/// TLV encoding as crsql_pack_columns. +pub unsafe extern "C" fn crsql_pack_agg_step( + ctx: *mut sqlite::context, + argc: c_int, + argv: *mut *mut sqlite::value, +) { + let args = sqlite::args!(argc, argv); + if args.is_empty() { + return; + } + let value = args[0]; + + let acc_ptr = aggregate_context(ctx, PackAggAcc::SIZE) as *mut PackAggAcc; + if acc_ptr.is_null() { + ctx.result_error("crsql_pack_agg: failed to allocate aggregate context"); + ctx.result_error_code(ResultCode::NOMEM); + return; + } + + // First call: initialize the buffer + if (*acc_ptr).buf.is_null() { + (*acc_ptr).buf = Box::into_raw(Box::new(Vec::new())); + (*acc_ptr).count = 0; + } + + let buf = &mut *(*acc_ptr).buf; + encode_value(buf, value); + (*acc_ptr).count += 1; +} + +/// xFinal callback for crsql_pack_agg. +/// Prepends the varint count header and returns the packed blob. +pub unsafe extern "C" fn crsql_pack_agg_final(ctx: *mut sqlite::context) { + let acc_ptr = aggregate_context(ctx, 0) as *mut PackAggAcc; + + if acc_ptr.is_null() || (*acc_ptr).buf.is_null() { + // No rows were processed (xStep never called). + // Return an empty packed blob: just varint(0). + let mut empty = Vec::new(); + put_varint(&mut empty, 0); + ctx.result_blob_owned(empty); + return; + } + + let buf_ptr = (*acc_ptr).buf; + let count = (*acc_ptr).count; + let buf = Box::from_raw(buf_ptr); + + // Prepend varint count header + let mut result = Vec::with_capacity(9 + buf.len()); + put_varint(&mut result, count); + result.extend_from_slice(&buf); + + ctx.result_blob_owned(result); +} + +/// Encode a single SQLite value into the buffer using the same TLV encoding +/// as pack_columns. +fn encode_value(buf: &mut Vec, value: *mut sqlite::value) { + match value.value_type() { + ColumnType::Blob => { + let len = value.bytes(); + let num_bytes_for_len = num_bytes_needed_i32(len); + let type_byte = num_bytes_for_len << 3 | (ColumnType::Blob as u8); + buf.put_u8(type_byte); + buf.put_int(len as i64, num_bytes_for_len as usize); + buf.put_slice(value.blob()); + } + ColumnType::Null => { + buf.put_u8(ColumnType::Null as u8); + } + ColumnType::Float => { + buf.put_u8(ColumnType::Float as u8); + buf.put_f64(value.double()); + } + ColumnType::Integer => { + let val = value.int64(); + let num_bytes_for_int = num_bytes_needed_i64(val); + let type_byte = num_bytes_for_int << 3 | (ColumnType::Integer as u8); + buf.put_u8(type_byte); + buf.put_int(val, num_bytes_for_int as usize); + } + ColumnType::Text => { + let len = value.bytes(); + let num_bytes_for_len = num_bytes_needed_i32(len); + let type_byte = num_bytes_for_len << 3 | (ColumnType::Text as u8); + buf.put_u8(type_byte); + buf.put_int(len as i64, num_bytes_for_len as usize); + buf.put_slice(value.blob()); + } + } +} diff --git a/core/rs/core/src/tableinfo.rs b/core/rs/core/src/tableinfo.rs index 081d54b3a..ba761d8c2 100644 --- a/core/rs/core/src/tableinfo.rs +++ b/core/rs/core/src/tableinfo.rs @@ -2,6 +2,7 @@ use crate::alloc::string::ToString; use crate::c::crsql_ExtData; use crate::c::crsql_fetchPragmaSchemaVersion; use crate::c::TABLE_INFO_SCHEMA_VERSION; +use crate::consts; use crate::pack_columns::bind_package_to_stmt; use crate::pack_columns::ColumnValue; use crate::stmt_cache::reset_cached_stmt; @@ -30,10 +31,40 @@ use sqlite_nostd::StrRef; // TODO: make this configurable with a crsql_config_set. const MAX_CL_CACHE_SIZE: usize = 1500; + +/// Which metadata schema version is active for a table. +/// Set at TableInfo creation time based on config flags and which tables physically exist. +#[derive(PartialEq, Debug, Copy, Clone)] +pub enum SchemaVersion { + V1, + V2, + V2AndV1, +} + pub struct TableInfo { pub tbl_name: String, pub pks: Vec, pub non_pks: Vec, + pub schema_version: SchemaVersion, + /// True when the main table is a rowid table with accessible rowid. + /// V2 uses __crsql_key = main_table.rowid and fetches PK values from + /// the main table via SELECT pk_cols WHERE rowid = ?. + /// See design doc §3 "__crsql_key Assignment (Rowid Reuse)". + /// + /// IMPORTANT: This flag is about using the implicit SQLite rowid as + /// __crsql_key in v2_pks (3-column schema). It is NOT the same as + /// "INTEGER PRIMARY KEY" — that is a separate SQLite concept. Currently + /// this is only enabled for single INTEGER PRIMARY KEY tables where the + /// PK column IS the rowid alias, but the concept is independent and may + /// be extended to other rowid tables in the future. When this is true, + /// the rowid must be looked up from the base table via rowid_alias — + /// do NOT assume unpacked_pks[0] is the rowid. + pub uses_rowid_key: bool, + /// The column name to use as the rowid alias for JOINs/ad-hoc queries. + /// For INTEGER PRIMARY KEY tables: the PK column name (e.g. "id"). + /// For plain rowid tables: "rowid" (or first unshadowed built-in alias). + /// Only valid when uses_rowid_key is true. + pub rowid_alias: String, // Lookaside -- // insert returning? @@ -763,6 +794,7 @@ impl Drop for TableInfo { pub struct ColumnInfo { pub cid: i32, pub name: String, + pub col_type: String, // > 0 if it is a primary key columns // the value refers to the position in the `PRIMARY KEY (cols...)` statement pub pk: i32, @@ -937,13 +969,40 @@ fn pull_all_table_infos( } } + // Also discover V2-only CRR tables (mode 3 creates no V1 clock tables) + let v2_stmt = db.prepare_v2( + "SELECT tbl_name FROM sqlite_master WHERE type='table' AND tbl_name LIKE '%__crsql_v2_clock'", + )?; + loop { + match v2_stmt.step() { + Ok(ResultCode::ROW) => { + clock_table_names.push(v2_stmt.column_text(0)?.to_string()); + } + Ok(ResultCode::DONE) => { + break; + } + Ok(rc) | Err(rc) => { + return Err(rc); + } + } + } + + let mut seen = alloc::collections::BTreeSet::new(); let mut ret = vec![]; for name in clock_table_names { - ret.push(pull_table_info( - db, - &name[0..(name.len() - "__crsql_clock".len())], - err, - )?) + // Strip either __crsql_clock (V1) or __crsql_v2_clock (V2) suffix + let base_name = if name.ends_with("__crsql_v2_clock") { + name[0..(name.len() - "__crsql_v2_clock".len())].to_string() + } else if name.ends_with("__crsql_clock") { + name[0..(name.len() - "__crsql_clock".len())].to_string() + } else { + continue; + }; + if seen.contains(base_name.as_str()) { + continue; + } + seen.insert(base_name.clone()); + ret.push(pull_table_info(db, &base_name, err)?); } Ok(ret) @@ -973,7 +1032,7 @@ pub fn pull_table_info( }; let sql = format!( - "SELECT \"cid\", \"name\", \"pk\" + "SELECT \"cid\", \"name\", \"type\", \"pk\" FROM pragma_table_info('{table}') ORDER BY cid ASC" ); let column_infos = match db.prepare_v2(&sql) { @@ -983,8 +1042,9 @@ pub fn pull_table_info( while stmt.step()? == ResultCode::ROW { cols.push(ColumnInfo { name: stmt.column_text(1)?.to_string(), + col_type: stmt.column_text(2)?.to_string(), cid: stmt.column_int(0), - pk: stmt.column_int(2), + pk: stmt.column_int(3), curr_value_stmt: RefCell::new(None), merge_insert_stmt: RefCell::new(None), row_patch_data_stmt: RefCell::new(None), @@ -1003,13 +1063,123 @@ pub fn pull_table_info( } }; + // Check alias shadowing before partition consumes column_infos + let has_rowid_col = column_infos.iter().any(|c| c.name == "rowid"); + let has_oid_col = column_infos.iter().any(|c| c.name == "oid"); + let has_rowid_under_col = column_infos.iter().any(|c| c.name == "_rowid_"); + let all_aliases_shadowed = has_rowid_col && has_oid_col && has_rowid_under_col; + let (mut pks, non_pks): (Vec<_>, Vec<_>) = column_infos.into_iter().partition(|x| x.pk > 0); pks.sort_by_key(|x| x.pk); + // Detect rowid key optimization per design doc §3: + // 1. INTEGER PRIMARY KEY exists (pk > 0 AND type = 'INTEGER') → it IS the rowid alias. + // 2. No INTEGER PRIMARY KEY, but none of rowid/oid/_rowid_ are shadowed → rowid accessible. + // 3. All three aliases shadowed AND no INTEGER PRIMARY KEY → auto-increment fallback. + let integer_pk = pks.iter().find(|pk| { + let type_sql = format!( + "SELECT type FROM pragma_table_info('{table}') WHERE name = '{pk_name}'", + table = crate::util::escape_ident_as_value(table), + pk_name = crate::util::escape_ident_as_value(&pk.name), + ); + db.prepare_v2(&type_sql).and_then(|stmt| { + stmt.step()?; + Ok(stmt.column_text(0)?.to_string() == "INTEGER") + }).unwrap_or(false) + }); + + // Determine the rowid alias to use for ad-hoc queries + let rowid_alias = if let Some(pk) = integer_pk { + // Case 1: INTEGER PRIMARY KEY — the PK column IS the rowid alias + pk.name.clone() + } else if !all_aliases_shadowed { + // Case 2: pick first unshadowed built-in alias + if !has_rowid_col { + "rowid".to_string() + } else if !has_oid_col { + "oid".to_string() + } else { + "_rowid_".to_string() + } + } else { + // Case 3: fallback — no alias + String::new() + }; + + let rowid_accessible = integer_pk.is_some() || !all_aliases_shadowed; + + // Verify rowid is actually accessible (table is not WITHOUT ROWID) + let mut uses_rowid_key = if integer_pk.is_some() { + // INTEGER PRIMARY KEY: the PK column IS the rowid alias. + // Verify it's accessible (table is not WITHOUT ROWID). + db.prepare_v2(&format!( + "SELECT \"{alias}\" FROM \"{escaped}\" LIMIT 0", + alias = crate::util::escape_ident(&rowid_alias), + escaped = crate::util::escape_ident(table), + )).is_ok() + } else { + // No INTEGER PRIMARY KEY — the PK is NOT the rowid. + // We can still use the hidden rowid as __crsql_key, but we must store PK columns separately. + false + }; + + // Detect V2 metadata tables + let has_v2 = crate::bootstrap_v2::has_v2_tables(db, table).unwrap_or(false); + + // If v2_pks table exists, infer uses_rowid_key from its schema. + // Rowid-key v2_pks has 3 columns: __crsql_key, hashed_pk, cl. + // Non-rowid v2_pks has 3 + N PK columns. + // This is the persisted source of truth for the without_rowid decision. + if has_v2 { + let v2_pks_name = format!("{}{}", crate::util::escape_ident_as_value(table), consts::V2_PKS_SUFFIX); + let pks_count_stmt = db.prepare_v2(&format!( + "SELECT count(*) FROM pragma_table_info('{name}')", + name = v2_pks_name, + )); + if let Ok(stmt) = pks_count_stmt { + if stmt.step().unwrap_or(ResultCode::DONE) == ResultCode::ROW { + let col_count = stmt.column_int(0); + // 3 columns = __crsql_key + hashed_pk + cl = rowid-key + // >3 columns = has PK columns = non-rowid + uses_rowid_key = col_count == 3; + } + } + } else { + // v2_pks doesn't exist yet — check crsql_master for without_rowid flag + // persisted by create_crr when the without_rowid option was used. + let without_rowid_key = format!("without_rowid_{}\0", table); + let stmt = db.prepare_v2("SELECT value FROM crsql_master WHERE key = ?\0"); + if let Ok(stmt) = stmt { + if stmt.bind_text(1, &without_rowid_key, sqlite::Destructor::TRANSIENT).is_ok() { + if stmt.step().unwrap_or(ResultCode::DONE) == ResultCode::ROW { + uses_rowid_key = false; + } + } + } + } + let has_v1 = { + let stmt = db.prepare_v2(&format!( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND tbl_name = '{escaped}{suffix}'", + escaped = crate::util::escape_ident_as_value(table), + suffix = "__crsql_clock" + ))?; + stmt.step()? == ResultCode::ROW + }; + let schema_version = if has_v2 && has_v1 { + SchemaVersion::V2AndV1 + } else if has_v2 { + SchemaVersion::V2 + } else { + SchemaVersion::V1 + }; + Ok(TableInfo { tbl_name: table.to_string(), pks, non_pks, + schema_version, + uses_rowid_key, + rowid_alias, set_winner_clock_stmt: RefCell::new(None), local_cl_stmt: RefCell::new(None), col_version_stmt: RefCell::new(None), diff --git a/core/rs/core/src/teardown.rs b/core/rs/core/src/teardown.rs index 301cc551e..8efe4c343 100644 --- a/core/rs/core/src/teardown.rs +++ b/core/rs/core/src/teardown.rs @@ -2,6 +2,8 @@ use sqlite_nostd as sqlite; use sqlite_nostd::{Connection, ResultCode}; extern crate alloc; use alloc::format; +use alloc::string::String; +use alloc::vec::Vec; pub fn remove_crr_clock_table_if_exists( db: *mut sqlite::sqlite3, @@ -34,13 +36,19 @@ pub fn remove_crr_triggers_if_exist( table = escaped_table ))?; - // get all columns of table - // iterate pk cols - // drop triggers against those pk cols - let stmt = db.prepare_v2("SELECT name FROM pragma_table_info(?) WHERE pk > 0")?; - stmt.bind_text(1, table, sqlite::Destructor::STATIC)?; - while stmt.step()? == ResultCode::ROW { - let col_name = stmt.column_text(0)?; + // Collect pk col names first, then drop triggers after statement is finalized + // to avoid LOCKED errors from schema modifications while a stmt is active. + let pk_cols: Vec = { + let stmt = db.prepare_v2("SELECT name FROM pragma_table_info(?) WHERE pk > 0")?; + stmt.bind_text(1, table, sqlite::Destructor::STATIC)?; + let mut cols = Vec::new(); + while stmt.step()? == ResultCode::ROW { + cols.push(String::from(stmt.column_text(0)?)); + } + cols + }; + + for col_name in &pk_cols { db.exec_safe(&format!( "DROP TRIGGER IF EXISTS \"{tbl_name}_{col_name}__crsql_utrig\"", tbl_name = crate::util::escape_ident(table), diff --git a/core/rs/core/src/teardown_v2.rs b/core/rs/core/src/teardown_v2.rs new file mode 100644 index 000000000..16a3d0421 --- /dev/null +++ b/core/rs/core/src/teardown_v2.rs @@ -0,0 +1,14 @@ +extern crate alloc; + +use sqlite_nostd as sqlite; +use sqlite_nostd::ResultCode; + +use crate::bootstrap_v2; + +/// Remove all V2 metadata tables for a CRR table. +pub fn remove_crr_v2_tables( + db: *mut sqlite::sqlite3, + table: &str, +) -> Result { + bootstrap_v2::drop_v2_tables(db, table) +} diff --git a/core/rs/core/src/test_exports.rs b/core/rs/core/src/test_exports.rs index 77f0dba8a..aa0e65b4b 100644 --- a/core/rs/core/src/test_exports.rs +++ b/core/rs/core/src/test_exports.rs @@ -1,5 +1,6 @@ pub use crate::bootstrap; pub use crate::c; +pub use crate::consts; pub use crate::db_version; pub use crate::pack_columns; pub use crate::tableinfo; diff --git a/core/rs/core/src/triggers.rs b/core/rs/core/src/triggers.rs index 3a3cb0183..b99884723 100644 --- a/core/rs/core/src/triggers.rs +++ b/core/rs/core/src/triggers.rs @@ -1,5 +1,6 @@ extern crate alloc; use alloc::format; +use alloc::string::String; use sqlite::Connection; use core::ffi::c_char; @@ -24,14 +25,21 @@ fn create_insert_trigger( table_info: &TableInfo, _err: *mut *mut c_char, ) -> Result { + let rowid_expr = if table_info.uses_rowid_key { + let alias = crate::util::escape_ident(&table_info.rowid_alias); + format!(", NEW.\"{alias}\"") + } else { + String::new() + }; let create_trigger_sql = format!( "CREATE TRIGGER IF NOT EXISTS \"{table_name}__crsql_itrig\" AFTER INSERT ON \"{table_name}\" WHEN crsql_internal_sync_bit() = 0 BEGIN - VALUES (crsql_after_insert('{table_name}', {pk_new_list})); + VALUES (crsql_after_insert('{table_name}', {pk_new_list}{rowid_expr})); END;", table_name = crate::util::escape_ident_as_value(&table_info.tbl_name), - pk_new_list = crate::util::as_identifier_list(&table_info.pks, Some("NEW."))? + pk_new_list = crate::util::as_identifier_list(&table_info.pks, Some("NEW."))?, + rowid_expr = rowid_expr ); db.exec_safe(&create_trigger_sql) @@ -48,21 +56,30 @@ fn create_update_trigger( let pk_new_list = crate::util::as_identifier_list(pk_columns, Some("NEW."))?; let pk_old_list = crate::util::as_identifier_list(pk_columns, Some("OLD."))?; + let rowid_expr = if table_info.uses_rowid_key { + let alias = crate::util::escape_ident(&table_info.rowid_alias); + format!(", NEW.\"{alias}\"") + } else { + String::new() + }; + let trigger_body = if non_pk_columns.is_empty() { format!( - "VALUES (crsql_after_update('{table_name}', {pk_new_list}, {pk_old_list}))", + "VALUES (crsql_after_update('{table_name}', {pk_new_list}, {pk_old_list}{rowid_expr}))", table_name = crate::util::escape_ident_as_value(table_name), pk_new_list = pk_new_list, pk_old_list = pk_old_list, + rowid_expr = rowid_expr, ) } else { format!( - "VALUES (crsql_after_update('{table_name}', {pk_new_list}, {pk_old_list}, {non_pk_new_list}, {non_pk_old_list}))", + "VALUES (crsql_after_update('{table_name}', {pk_new_list}, {pk_old_list}, {non_pk_new_list}, {non_pk_old_list}{rowid_expr}))", table_name = crate::util::escape_ident_as_value(table_name), pk_new_list = pk_new_list, pk_old_list = pk_old_list, non_pk_new_list = crate::util::as_identifier_list(non_pk_columns, Some("NEW."))?, - non_pk_old_list = crate::util::as_identifier_list(non_pk_columns, Some("OLD."))? + non_pk_old_list = crate::util::as_identifier_list(non_pk_columns, Some("OLD."))?, + rowid_expr = rowid_expr ) }; db.exec_safe(&format!( diff --git a/core/rs/fractindex-core/rust-toolchain.toml b/core/rs/fractindex-core/rust-toolchain.toml index f5663da1b..bbba1c38b 100644 --- a/core/rs/fractindex-core/rust-toolchain.toml +++ b/core/rs/fractindex-core/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2023-10-05" \ No newline at end of file +channel = "nightly-2026-02-01" \ No newline at end of file diff --git a/core/rs/integration_check/Cargo.lock b/core/rs/integration_check/Cargo.lock index fd9e431fd..18801e3ed 100644 --- a/core/rs/integration_check/Cargo.lock +++ b/core/rs/integration_check/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -19,17 +19,15 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", @@ -37,7 +35,6 @@ dependencies = [ "rustc-hash", "shlex", "syn", - "which", ] [[package]] @@ -136,6 +133,7 @@ dependencies = [ "num-derive", "num-traits", "sqlite_nostd", + "xxhash-rust", ] [[package]] @@ -195,15 +193,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys", -] - [[package]] name = "is-terminal" version = "0.4.9" @@ -216,16 +205,19 @@ dependencies = [ ] [[package]] -name = "lazy_static" -version = "1.4.0" +name = "itertools" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] [[package]] -name = "lazycell" -version = "1.3.0" +name = "lazy_static" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" @@ -288,9 +280,9 @@ dependencies = [ [[package]] name = "num-derive" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb77679af88f8b125209d354a202862602672222e7f2313fdd6dc349bad4712" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", @@ -299,25 +291,13 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "prettyplease" version = "0.2.15" @@ -330,9 +310,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -377,9 +357,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" @@ -533,18 +513,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "winapi" version = "0.3.9" @@ -638,3 +606,9 @@ name = "xml-rs" version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab77e97b50aee93da431f2cee7cd0f43b4d1da3c408042f2d7d164187774f0a" + +[[package]] +name = "xxhash-rust" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" diff --git a/core/rs/integration_check/rust-toolchain.toml b/core/rs/integration_check/rust-toolchain.toml index f5663da1b..bbba1c38b 100644 --- a/core/rs/integration_check/rust-toolchain.toml +++ b/core/rs/integration_check/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2023-10-05" \ No newline at end of file +channel = "nightly-2026-02-01" \ No newline at end of file diff --git a/core/rs/integration_check/src/lib.rs b/core/rs/integration_check/src/lib.rs index f9cb51cbc..94ecd742e 100644 --- a/core/rs/integration_check/src/lib.rs +++ b/core/rs/integration_check/src/lib.rs @@ -35,6 +35,12 @@ pub extern "C" fn crsql_integration_check() { t::test_cl_set_vtab::run_suite().expect("test cl set vtab suite"); println!("Running db_version"); t::test_db_version::run_suite().expect("test db version suite"); + println!("Running v2_tests"); + t::v2_tests::run_suite().expect("v2 tests suite"); + println!("Running v2_compat_tests"); + t::v2_compat_tests::run_suite().expect("v2 compat tests suite"); + println!("Running rowid_check"); + t::rowid_check::run_suite().expect("rowid check suite"); } pub fn opendb() -> Result { diff --git a/core/rs/integration_check/src/t/automigrate.rs b/core/rs/integration_check/src/t/automigrate.rs index b3f3be2c4..dbc53d1cd 100644 --- a/core/rs/integration_check/src/t/automigrate.rs +++ b/core/rs/integration_check/src/t/automigrate.rs @@ -8,13 +8,20 @@ use sqlite_nostd as sqlite; fn idempotent() { let db = crate::opendb().expect("db opened"); let schema = " + CREATE TABLE IF NOT EXISTS item (id integer primary key not null, data any) strict; + CREATE TABLE IF NOT EXISTS container (id integer primary key, contained integer); + CREATE INDEX IF NOT EXISTS container_contained ON container (contained); + SELECT crsql_set_ts('1700000000'); + SELECT crsql_as_crr('item'); + "; + let migrate_schema = " CREATE TABLE IF NOT EXISTS item (id integer primary key not null, data any) strict; CREATE TABLE IF NOT EXISTS container (id integer primary key, contained integer); CREATE INDEX IF NOT EXISTS container_contained ON container (contained); SELECT crsql_as_crr('item'); "; db.db.exec_safe(schema).expect("schema made"); - invoke_automigrate(&db.db, schema).expect("migrated"); + invoke_automigrate(&db.db, migrate_schema).expect("migrated"); assert!(expect_tables(&db.db, vec!["item", "container"]).expect("compared tables")); assert!( @@ -369,6 +376,7 @@ fn to_empty_from_something() -> Result<(), ResultCode> { db.db.exec_safe("CREATE TABLE bar (a, b, c);")?; db.db .exec_safe("CREATE TABLE item (id1 not null, id2 not null, x, primary key (id1, id2));")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('item')")?; db.db .exec_safe("SELECT crsql_automigrate('', 'SELECT crsql_finalize();')")?; diff --git a/core/rs/integration_check/src/t/backfill.rs b/core/rs/integration_check/src/t/backfill.rs index 3d0467b4b..09dcb22e7 100644 --- a/core/rs/integration_check/src/t/backfill.rs +++ b/core/rs/integration_check/src/t/backfill.rs @@ -10,6 +10,7 @@ fn new_empty_table() -> Result<(), ResultCode> { // Just testing that we can execute these statements without error db.db .exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name);")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; db.db.exec_safe("SELECT * FROM foo__crsql_clock;")?; Ok(()) @@ -21,9 +22,11 @@ fn new_nonempty_table(apply_twice: bool) -> Result<(), ResultCode> { .exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name);")?; db.db .exec_safe("INSERT INTO foo VALUES (1, 'one'), (2, 'two');")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; let stmt = db.db.prepare_v2("SELECT * FROM foo__crsql_clock;")?; if apply_twice { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; } @@ -65,8 +68,10 @@ fn reapplied_empty_table() -> Result<(), ResultCode> { // Just testing that we can execute these statements without error db.db .exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name);")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; db.db.exec_safe("SELECT * FROM foo__crsql_clock;")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; db.db.exec_safe("SELECT * FROM foo__crsql_clock;")?; Ok(()) diff --git a/core/rs/integration_check/src/t/mod.rs b/core/rs/integration_check/src/t/mod.rs index 21b674fc6..4a72c2800 100644 --- a/core/rs/integration_check/src/t/mod.rs +++ b/core/rs/integration_check/src/t/mod.rs @@ -4,8 +4,11 @@ pub mod fract; pub mod pack_columns; pub mod pk_only_tables; pub mod pk_update; +pub mod rowid_check; pub mod sync_bit_honored; pub mod tableinfo; pub mod teardown; pub mod test_cl_set_vtab; pub mod test_db_version; +pub mod v2_compat_tests; +pub mod v2_tests; diff --git a/core/rs/integration_check/src/t/pk_only_tables.rs b/core/rs/integration_check/src/t/pk_only_tables.rs index 3e8135d1c..1b2987521 100644 --- a/core/rs/integration_check/src/t/pk_only_tables.rs +++ b/core/rs/integration_check/src/t/pk_only_tables.rs @@ -18,6 +18,9 @@ fn sync_left_to_right(l: &dyn Connection, r: &dyn Connection, since: sqlite::int .bind_blob(2, siteid, Destructor::STATIC) .expect("bound site id"); + r.exec_safe("BEGIN").expect("begin"); + r.exec_safe("SELECT crsql_set_ts('1700000000')").expect("set ts"); + while stmt_l.step().expect("pulled change set") == ResultCode::ROW { let stmt_r = r .prepare_v2("INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") @@ -29,6 +32,7 @@ fn sync_left_to_right(l: &dyn Connection, r: &dyn Connection, since: sqlite::int } stmt_r.step().expect("inserted change"); } + r.exec_safe("COMMIT").expect("commit"); } // fn print_changes( @@ -72,6 +76,8 @@ fn sync_left_to_right(l: &dyn Connection, r: &dyn Connection, since: sqlite::int fn setup_schema(db: &ManagedConnection) { db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL);") .expect("created foo"); + db.exec_safe("SELECT crsql_set_ts('1700000000')") + .expect("set ts"); db.exec_safe("SELECT crsql_as_crr('foo');") .expect("converted to crr"); } @@ -90,6 +96,8 @@ fn insert_pkonly_row() { fn setup_schema(db: &ManagedConnection) { db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL);") .expect("created foo"); + db.exec_safe("SELECT crsql_set_ts('1700000000')") + .expect("set ts"); db.exec_safe("SELECT crsql_as_crr('foo');") .expect("upgraded to crr"); } @@ -132,6 +140,8 @@ fn modify_pkonly_row() -> Result<(), ResultCode> { fn setup_schema(db: &ManagedConnection) { db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL);") .expect("create foo"); + db.exec_safe("SELECT crsql_set_ts('1700000000')") + .expect("set ts"); db.exec_safe("SELECT crsql_as_crr('foo');") .expect("upgrade foo"); } @@ -178,6 +188,7 @@ fn junction_table() -> Result<(), ResultCode> { fn setup_schema(db: &ManagedConnection) -> Result { db.exec_safe("CREATE TABLE jx (id1 NOT NULL, id2 NOT NULL, PRIMARY KEY(id1, id2));")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.exec_safe("SELECT crsql_as_crr('jx');") } @@ -234,6 +245,7 @@ fn discord_report_1() -> Result<(), ResultCode> { let db_a = crate::opendb()?; db_a.db .exec_safe("CREATE TABLE IF NOT EXISTS data (id NUMBER PRIMARY KEY NOT NULL);")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db_a.db.exec_safe("SELECT crsql_as_crr('data')")?; db_a.db .exec_safe("INSERT INTO data VALUES (42) ON CONFLICT DO NOTHING;")?; diff --git a/core/rs/integration_check/src/t/rowid_check.rs b/core/rs/integration_check/src/t/rowid_check.rs new file mode 100644 index 000000000..24d185d34 --- /dev/null +++ b/core/rs/integration_check/src/t/rowid_check.rs @@ -0,0 +1,523 @@ +extern crate alloc; +use alloc::format; +use libc_print::std_name::println; +use sqlite::{Connection, ManagedConnection, ResultCode}; +use sqlite_nostd as sqlite; + +/// Run incremental maintenance until V2 migration is complete. +fn migrate_to_v2(db: &ManagedConnection) -> Result<(), ResultCode> { + db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + let mut iterations = 0; + while remaining > 0 && iterations < 100 { + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0); + iterations += 1; + } + Ok(()) +} + +/// Count columns in v2_pks table for a given table name. +/// Returns 0 if v2_pks doesn't exist. +fn v2_pks_col_count(db: &ManagedConnection, table: &str) -> i32 { + let stmt = db.prepare_v2(&format!( + "SELECT count(*) FROM pragma_table_info('{table}__crsql_v2_pks')", + table = table + )); + match stmt { + Ok(s) => { + s.step().unwrap_or(ResultCode::DONE); + s.column_int(0) + } + Err(_) => 0, + } +} + +/// Test that as_crr on a rowid table with safe rowids succeeds. +/// After V2 migration, v2_pks should have rowid-key schema (3 columns). +fn test_safe_rowids_get_triggers() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'a'), (100, 'b'), (1000000, 'c')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Migrate to V2 and verify v2_pks has rowid-key schema (3 columns) + migrate_to_v2(&db.db)?; + let col_count = v2_pks_col_count(&db.db, "foo"); + assert!(col_count == 3, "expected 3 columns in v2_pks (rowid-key schema), got {}", col_count); + Ok(()) +} + +/// Test that as_crr on an empty rowid table succeeds (rowid check is enforced at runtime in Rust). +fn test_empty_rowid_table_gets_triggers() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + // Verify auto-generated rowid insert works + db.db.exec_safe("INSERT INTO foo (data) VALUES ('auto')")?; + Ok(()) +} + +/// Test that as_crr rejects a table with rowids exceeding MAX_ROWID_KEY. +fn test_rowid_too_large_rejected() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + // Insert a rowid that exceeds MAX_ROWID_KEY (2^51) + db.db.exec_safe(&format!( + "INSERT INTO foo VALUES ({}, 'too_big')", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + ))?; + + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_as_crr('foo')"); + assert!( + rc.is_err(), + "expected as_crr to fail for table with oversized rowid" + ); + Ok(()) +} + +/// Test that as_crr rejects a table with negative rowids. +fn test_negative_rowid_rejected() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("INSERT INTO foo VALUES (-1, 'negative')")?; + + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_as_crr('foo')"); + assert!( + rc.is_err(), + "expected as_crr to fail for table with negative rowid" + ); + Ok(()) +} + +/// Test that without_rowid flag allows as_crr even with oversized rowids, +/// and after migration uses the non-rowid V2 schema (v2_pks has PK columns). +fn test_without_rowid_allows_oversized() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe(&format!( + "INSERT INTO foo VALUES ({}, 'too_big')", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + ))?; + + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_as_crr('foo', 'without_rowid')"); + assert!( + rc.is_ok(), + "expected as_crr with without_rowid to succeed" + ); + + // Migrate to V2 and verify v2_pks has non-rowid schema (4 columns for 1 PK) + migrate_to_v2(&db.db)?; + let col_count = v2_pks_col_count(&db.db, "foo"); + assert!(col_count == 4, "expected 4 columns in v2_pks (non-rowid schema), got {}", col_count); + Ok(()) +} + +/// Test that the rowid range check blocks inserts of oversized rowids. +fn test_trigger_blocks_oversized_insert() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Insert within range should work + db.db.exec_safe("INSERT INTO foo VALUES (1, 'ok')")?; + + // Insert above MAX_ROWID_KEY should fail + let rc = db.db.exec_safe(&format!( + "INSERT INTO foo VALUES ({}, 'too_big')", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + )); + assert!( + rc.is_err(), + "expected insert with oversized rowid to be blocked by trigger" + ); + Ok(()) +} + +/// Test that the rowid range check blocks updates to oversized rowids. +fn test_trigger_blocks_oversized_update() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'ok')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Update to oversized rowid should fail + let rc = db.db.exec_safe(&format!( + "UPDATE foo SET id = {} WHERE id = 1", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + )); + assert!( + rc.is_err(), + "expected update with oversized rowid to be blocked by trigger" + ); + Ok(()) +} + +/// Test that the rowid range check blocks negative rowid inserts. +fn test_trigger_blocks_negative_insert() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + let rc = db.db.exec_safe("INSERT INTO foo VALUES (-1, 'negative')"); + assert!( + rc.is_err(), + "expected insert with negative rowid to be blocked by trigger" + ); + Ok(()) +} + +/// Test that non-rowid tables (WITHOUT ROWID or non-integer PK) are not subject to rowid range checks. +fn test_non_rowid_table_no_check_triggers() -> Result<(), ResultCode> { + let db = crate::opendb()?; + // Text PK — not a rowid alias + db.db.exec_safe("CREATE TABLE foo (id TEXT PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("INSERT INTO foo VALUES ('a', 'b')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + // Non-rowid table should allow inserts without rowid range issues + db.db.exec_safe("INSERT INTO foo VALUES ('b', 'c')")?; + Ok(()) +} + +/// Test that downgrade (crsql_as_table) removes rowid range enforcement. +fn test_downgrade_drops_check_triggers() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // While CRR, oversized insert should fail + let rc = db.db.exec_safe(&format!( + "INSERT INTO foo VALUES ({}, 'too_big')", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + )); + assert!(rc.is_err(), "expected oversized insert to be blocked while CRR"); + + // Downgrade + let rc = db.db.exec_safe("SELECT crsql_as_table('foo')"); + if rc.is_err() { + println!("test_downgrade: crsql_as_table failed!"); + return Err(ResultCode::ERROR); + } + + // After downgrade, oversized insert should succeed (no rowid check) + let rc = db.db.exec_safe(&format!( + "INSERT INTO foo VALUES ({}, 'too_big')", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + )); + assert!(rc.is_ok(), "expected oversized insert to succeed after downgrade"); + Ok(()) +} + +/// Test that ALTER preserves rowid range enforcement (it's in the Rust trigger code, not separate triggers). +fn test_alter_preserves_check_triggers() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Alter: add a column + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_begin_alter('foo')"); + if rc.is_err() { + println!("test_alter_preserves_check: crsql_begin_alter failed!"); + return Err(ResultCode::ERROR); + } + let rc = db.db.exec_safe("ALTER TABLE foo ADD COLUMN extra TEXT"); + if rc.is_err() { + println!("test_alter_preserves_check: ALTER TABLE failed!"); + return Err(ResultCode::ERROR); + } + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_commit_alter('foo')"); + if rc.is_err() { + println!("test_alter_preserves_check: crsql_commit_alter failed!"); + return Err(ResultCode::ERROR); + } + + // After alter, rowid range check should still be enforced + let rc = db.db.exec_safe(&format!( + "INSERT INTO foo VALUES ({}, 'too_big', 'extra')", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + )); + assert!(rc.is_err(), "expected oversized insert to be blocked after ALTER"); + Ok(()) +} + +/// Test that ALTER preserves the without_rowid preference (no triggers, non-rowid v2_pks schema). +fn test_alter_preserves_without_rowid() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo', 'without_rowid')")?; + + // Alter: add a column + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_begin_alter('foo')")?; + db.db.exec_safe("ALTER TABLE foo ADD COLUMN extra TEXT")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_commit_alter('foo')")?; + + // Migrate to V2 and verify v2_pks still has non-rowid schema (4 columns for 1 PK) + migrate_to_v2(&db.db)?; + let col_count = v2_pks_col_count(&db.db, "foo"); + assert!(col_count == 4, "expected 4 columns in v2_pks after ALTER (non-rowid preserved), got {}", col_count); + Ok(()) +} + +/// Test that schema, table name form works with without_rowid. +fn test_schema_table_without_rowid_form() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe(&format!( + "INSERT INTO foo VALUES ({}, 'big')", + crsql_bundle::test_exports::consts::MAX_ROWID_KEY as i64 + ))?; + + // (schema, table, flag) form + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_as_crr('main', 'foo', 'without_rowid')"); + assert!(rc.is_ok(), "expected (schema, table, flag) form to work"); + Ok(()) +} + +/// Test that direct V2 mode (metadata-write-version=3 on empty DB) creates V2 tables +/// directly without V1 clock tables. +fn test_direct_v2_mode_creates_v2_only() -> Result<(), ResultCode> { + let db = crate::opendb()?; + // Set metadata-write-version to 3 (V2-only) on empty DB with no CRR tables + let rc = db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)"); + if rc.is_err() { + println!("test_direct_v2: config_set failed"); + return Err(ResultCode::ERROR); + } + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'a'), (2, 'b')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_as_crr('foo')"); + if rc.is_err() { + println!("test_direct_v2: crsql_as_crr failed"); + return Err(ResultCode::ERROR); + } + + // V2 tables should exist + { + let v2_clock = db.db.prepare_v2( + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='foo__crsql_v2_clock'" + )?; + v2_clock.step()?; + let count = v2_clock.column_int(0); + if count != 1 { + println!("test_direct_v2: v2_clock table missing, count={}", count); + return Err(ResultCode::CONSTRAINT); + } + } + + // V1 clock table should NOT exist + { + let v1_clock = db.db.prepare_v2( + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='foo__crsql_clock'" + )?; + v1_clock.step()?; + let count = v1_clock.column_int(0); + if count != 0 { + println!("test_direct_v2: V1 clock table exists, count={}", count); + return Err(ResultCode::CONSTRAINT); + } + } + + // v2_pks should have rowid-key schema (3 columns) + let col_count = v2_pks_col_count(&db.db, "foo"); + if col_count != 3 { + println!("test_direct_v2: v2_pks col_count={}, expected 3", col_count); + return Err(ResultCode::CONSTRAINT); + } + + // Verify data was backfilled into v2_pks + { + let pks_stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_pks")?; + pks_stmt.step()?; + let count = pks_stmt.column_int(0); + if count != 2 { + println!("test_direct_v2: v2_pks rows={}, expected 2", count); + return Err(ResultCode::CONSTRAINT); + } + } + + // Verify a write goes to V2 tables (insert) + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (3, 'c')")?; + { + let pks_after = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_pks")?; + pks_after.step()?; + let count = pks_after.column_int(0); + if count != 3 { + println!("test_direct_v2: v2_pks rows after insert={}, expected 3", count); + return Err(ResultCode::CONSTRAINT); + } + } + Ok(()) +} + +/// Test that dual-write mode (metadata-write-version=2) creates both V1 and V2 tables. +fn test_dual_write_mode_creates_both() -> Result<(), ResultCode> { + let db = crate::opendb()?; + // Set metadata-write-version to 2 (dual-write) on empty DB + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'a'), (2, 'b')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Both V1 and V2 tables should exist + let v1_clock = db.db.prepare_v2( + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='foo__crsql_clock'" + )?; + v1_clock.step()?; + assert!(v1_clock.column_int(0) == 1, "expected V1 clock table in mode 2"); + + let v2_clock = db.db.prepare_v2( + "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='foo__crsql_v2_clock'" + )?; + v2_clock.step()?; + assert!(v2_clock.column_int(0) == 1, "expected V2 clock table in mode 2"); + + // Both should have backfilled data + let v1_pks = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_pks")?; + v1_pks.step()?; + assert!(v1_pks.column_int(0) == 2, "expected 2 rows in V1 pks after backfill"); + + let v2_pks = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_pks")?; + v2_pks.step()?; + assert!(v2_pks.column_int(0) == 2, "expected 2 rows in V2 pks after backfill"); + + // A write should go to both V1 and V2 + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (3, 'c')")?; + let v1_after = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_pks")?; + v1_after.step()?; + assert!(v1_after.column_int(0) == 3, "expected 3 rows in V1 pks after insert"); + + let v2_after = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_pks")?; + v2_after.step()?; + assert!(v2_after.column_int(0) == 3, "expected 3 rows in V2 pks after insert"); + Ok(()) +} + +/// Test that mode 3 cannot be set when CRR tables already exist. +fn test_mode3_rejected_with_existing_crrs() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Now try to set mode 3 — should fail since CRR tables exist + let rc = db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)"); + assert!(rc.is_err(), "expected mode 3 to be rejected when CRR tables exist"); + Ok(()) +} + +/// Test that in mode 3, triggers write only to V2 even if V1 tables exist. +/// This simulates the cleanup phase where V1 tables haven't been dropped yet. +fn test_mode3_writes_v2_only_with_v1_present() -> Result<(), ResultCode> { + let db = crate::opendb()?; + // Start in mode 1, create CRR with V1 tables + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, data TEXT)")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'a')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Migrate to dual-write, then to V2-only + let rc = db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)"); + if rc.is_err() { + println!("test_mode3: failed to set mode 2"); + return Err(ResultCode::ERROR); + } + // Run migration to create V2 tables + let mut remaining = 1; + let mut iterations = 0; + while remaining > 0 && iterations < 100 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0); + iterations += 1; + } + println!("test_mode3: migration done, iterations={}", iterations); + + // Now transition to V2-only (mode 3) + // V1 tables still exist but won't be written to + let rc = db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)"); + if rc.is_err() { + println!("test_mode3: failed to set mode 3"); + return Err(ResultCode::ERROR); + } + println!("test_mode3: mode 3 set successfully"); + + // Insert a new row — should only go to V2 + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 'b')")?; + println!("test_mode3: insert done"); + + // V2 pks should have the new row + let v2_count = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_pks")?; + v2_count.step()?; + let v2_rows = v2_count.column_int(0); + assert!(v2_rows >= 2, "expected at least 2 rows in v2_pks, got {}", v2_rows); + + // V1 pks should NOT have the new row (mode 3 = V2-only writes) + let v1_count = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_pks WHERE __crsql_key = 2")?; + v1_count.step()?; + assert!(v1_count.column_int(0) == 0, "expected V1 pks to NOT have new row in mode 3"); + Ok(()) +} + +pub fn run_suite() -> Result<(), ResultCode> { + println!("rowid_check: test_safe_rowids_get_triggers"); + test_safe_rowids_get_triggers()?; + println!("rowid_check: test_empty_rowid_table_gets_triggers"); + test_empty_rowid_table_gets_triggers()?; + println!("rowid_check: test_rowid_too_large_rejected"); + test_rowid_too_large_rejected()?; + println!("rowid_check: test_negative_rowid_rejected"); + test_negative_rowid_rejected()?; + println!("rowid_check: test_without_rowid_allows_oversized"); + test_without_rowid_allows_oversized()?; + println!("rowid_check: test_trigger_blocks_oversized_insert"); + test_trigger_blocks_oversized_insert()?; + println!("rowid_check: test_trigger_blocks_oversized_update"); + test_trigger_blocks_oversized_update()?; + println!("rowid_check: test_trigger_blocks_negative_insert"); + test_trigger_blocks_negative_insert()?; + println!("rowid_check: test_non_rowid_table_no_check_triggers"); + test_non_rowid_table_no_check_triggers()?; + println!("rowid_check: test_downgrade_drops_check_triggers"); + test_downgrade_drops_check_triggers()?; + println!("rowid_check: test_alter_preserves_check_triggers"); + test_alter_preserves_check_triggers()?; + println!("rowid_check: test_alter_preserves_without_rowid"); + test_alter_preserves_without_rowid()?; + println!("rowid_check: test_schema_table_without_rowid_form"); + test_schema_table_without_rowid_form()?; + println!("rowid_check: test_direct_v2_mode_creates_v2_only"); + test_direct_v2_mode_creates_v2_only()?; + println!("rowid_check: test_dual_write_mode_creates_both"); + test_dual_write_mode_creates_both()?; + println!("rowid_check: test_mode3_rejected_with_existing_crrs"); + test_mode3_rejected_with_existing_crrs()?; + println!("rowid_check: test_mode3_writes_v2_only_with_v1_present"); + test_mode3_writes_v2_only_with_v1_present()?; + Ok(()) +} diff --git a/core/rs/integration_check/src/t/sync_bit_honored.rs b/core/rs/integration_check/src/t/sync_bit_honored.rs index 292d96063..81096f550 100644 --- a/core/rs/integration_check/src/t/sync_bit_honored.rs +++ b/core/rs/integration_check/src/t/sync_bit_honored.rs @@ -7,6 +7,7 @@ fn sync_bit_honored() -> Result<(), ResultCode> { let db = crate::opendb()?; let conn = &db.db; conn.exec_safe("CREATE TABLE foo (a primary key not null, b);")?; + conn.exec_safe("SELECT crsql_set_ts('1700000000')")?; conn.exec_safe("SELECT crsql_as_crr('foo');")?; conn.exec_safe("SELECT crsql_internal_sync_bit(1)")?; conn.exec_safe("INSERT INTO foo VALUES (1, 2);")?; diff --git a/core/rs/integration_check/src/t/tableinfo.rs b/core/rs/integration_check/src/t/tableinfo.rs index f7b9f982c..d21fa0658 100644 --- a/core/rs/integration_check/src/t/tableinfo.rs +++ b/core/rs/integration_check/src/t/tableinfo.rs @@ -346,6 +346,7 @@ fn test_leak_condition() { c1.exec_safe("CREATE TABLE foo (a not null, b not null, primary key (a, b));") .expect("made foo"); + c1.exec_safe("SELECT crsql_set_ts('1700000000')").expect("set ts"); c1.exec_safe("SELECT crsql_as_crr('foo')") .expect("made foo a crr"); c1.exec_safe("INSERT INTO foo VALUES (1, 2)") diff --git a/core/rs/integration_check/src/t/teardown.rs b/core/rs/integration_check/src/t/teardown.rs index aeadfe8f9..bc4499ef5 100644 --- a/core/rs/integration_check/src/t/teardown.rs +++ b/core/rs/integration_check/src/t/teardown.rs @@ -6,6 +6,7 @@ fn tear_down() -> Result<(), ResultCode> { let db = crate::opendb()?; db.db .exec_safe("CREATE TABLE foo (a primary key not null, b);")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; db.db.exec_safe("SELECT crsql_as_table('foo');")?; let stmt = db diff --git a/core/rs/integration_check/src/t/test_db_version.rs b/core/rs/integration_check/src/t/test_db_version.rs index 2dfe7b4e9..98a5917ba 100644 --- a/core/rs/integration_check/src/t/test_db_version.rs +++ b/core/rs/integration_check/src/t/test_db_version.rs @@ -48,6 +48,8 @@ fn test_fetch_db_version_from_storage() -> Result { // create some schemas db.exec_safe("CREATE TABLE foo (a primary key not null, b);") .expect("made foo"); + db.exec_safe("SELECT crsql_set_ts('1700000000')") + .expect("set ts"); db.exec_safe("SELECT crsql_as_crr('foo');") .expect("made foo crr"); test_exports::db_version::fetch_db_version_from_storage(raw_db, ext_data)?; @@ -62,6 +64,8 @@ fn test_fetch_db_version_from_storage() -> Result { db.exec_safe("CREATE TABLE bar (a primary key not null, b);") .expect("created bar"); + db.exec_safe("SELECT crsql_set_ts('1700000000')") + .expect("set ts"); db.exec_safe("SELECT crsql_as_crr('bar');") .expect("bar as crr"); db.exec_safe("INSERT INTO bar VALUES (1, 2)") @@ -123,9 +127,11 @@ fn test_get_or_set_site_ordinal() -> Result<(), ResultCode> { db.db .exec_safe("CREATE TABLE foo (a primary key not null, b);")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; db.db.exec_safe("BEGIN TRANSACTION;")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; let other_site_id = "other_site_id".as_bytes(); @@ -189,6 +195,7 @@ fn test_get_or_set_pk_cl() -> Result<(), ResultCode> { db.db .exec_safe("CREATE TABLE foo (a primary key not null, b);")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; let insert_foo_stmt = db.db.prepare_v2("INSERT INTO foo VALUES (?, ?);")?; @@ -256,6 +263,7 @@ fn test_get_or_set_pk_cl() -> Result<(), ResultCode> { assert_eq!(-1, get_cache_cl(&get_cache_cl_stmt, "foo", key2)?); db.db.exec_safe("BEGIN TRANSACTION;")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SAVEPOINT test;")?; // new site_id in crsql_changes table @@ -378,11 +386,13 @@ fn test_insert_db_version_cache() -> Result<(), ResultCode> { let db = &c.db; db.db .exec_safe("CREATE TABLE foo (a primary key not null, b);")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; db.db.exec_safe("SELECT crsql_as_crr('foo');")?; let remote_site = "remote_site".as_bytes(); db.db.exec_safe("BEGIN TRANSACTION;")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; // cache should be empty before any remote inserts assert_eq!(-1, get_cache_db_version(db.db, remote_site)?); @@ -431,6 +441,7 @@ fn test_insert_db_version_cache() -> Result<(), ResultCode> { // new transaction: inserting a lower db_version (2) should be short-circuited // because we fetch the latest db_version from DB on cache miss db.db.exec_safe("BEGIN TRANSACTION;")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; let pk4: [u8; 3] = [1, 9, 4]; let stmt4 = db.db.prepare_v2( "INSERT INTO crsql_changes VALUES ('foo', ?, 'b', 4, 1, 2, ?, 1, 0, 0);", diff --git a/core/rs/integration_check/src/t/v2_compat_tests.rs b/core/rs/integration_check/src/t/v2_compat_tests.rs new file mode 100644 index 000000000..03c2cba3c --- /dev/null +++ b/core/rs/integration_check/src/t/v2_compat_tests.rs @@ -0,0 +1,861 @@ +extern crate crsql_bundle; +use libc_print::libc_println; +use sqlite::{Connection, Destructor, ManagedConnection, ResultCode}; +use sqlite_nostd as sqlite; + +/// Migrate a DB to V2&V1 and run maintenance until complete. +fn migrate_to_v2(db: &ManagedConnection) -> Result<(), ResultCode> { + db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + let mut iterations = 0; + while remaining > 0 && iterations < 100 { + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + return Err(ResultCode::ERROR); + } + iterations += 1; + } + db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + Ok(()) +} + +/// Sync all changes from left to right, filtering out the right db's own site_id. +fn sync_left_to_right( + l: &dyn Connection, + r: &dyn Connection, + since: sqlite::int64, +) -> Result<(), ResultCode> { + let siteid_stmt = r.prepare_v2("SELECT crsql_site_id()")?; + siteid_stmt.step()?; + let siteid = siteid_stmt.column_blob(0)?; + + let stmt_l = l.prepare_v2( + "SELECT * FROM crsql_changes WHERE db_version >= ? AND site_id IS NOT ?", + )?; + stmt_l.bind_int64(1, since)?; + stmt_l.bind_blob(2, siteid, Destructor::STATIC)?; + + r.exec_safe("BEGIN")?; + r.exec_safe("SELECT crsql_set_ts('1700000000')")?; + while stmt_l.step()? == ResultCode::ROW { + let stmt_r = r + .prepare_v2("INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")?; + for x in 0..10 { + stmt_r.bind_value(x + 1, stmt_l.column_value(x)?)?; + } + match stmt_r.step() { + Ok(_) => {} + Err(e) => { + let msg = r.errmsg().unwrap_or_else(|_| alloc::string::ToString::to_string("unknown")); + libc_println!("V2 SYNC ERROR: {:?} - {}", e, msg); + libc_println!(" row: tbl={:?} pk={:?} cid={:?}", + stmt_l.column_text(0), stmt_l.column_text(1), stmt_l.column_text(2)); + let _ = r.exec_safe("ROLLBACK"); + return Err(e); + } + } + } + r.exec_safe("COMMIT")?; + Ok(()) +} + +/// Compare all rows in a table between two databases, returning true if identical. +fn tables_match( + l: &dyn Connection, + r: &dyn Connection, + table: &str, + order_col: &str, +) -> Result { + let sql = alloc::format!( + "SELECT * FROM \"{}\" ORDER BY \"{}\" ASC", + table, order_col + ); + let stmt_l = l.prepare_v2(&sql)?; + let stmt_r = r.prepare_v2(&sql)?; + + loop { + let rc_l = stmt_l.step()?; + let rc_r = stmt_r.step()?; + if rc_l != rc_r { + libc_println!("MISMATCH: step results differ ({:?} vs {:?})", rc_l, rc_r); + return Ok(false); + } + if rc_l == ResultCode::DONE { + return Ok(true); + } + let n = stmt_l.column_count(); + for i in 0..n { + let lt = stmt_l.column_type(i)?; + let rt = stmt_r.column_type(i)?; + if lt != rt { + libc_println!("MISMATCH: column {} type {:?} vs {:?}", i, lt, rt); + return Ok(false); + } + match lt { + sqlite::ColumnType::Integer => { + if stmt_l.column_int64(i) != stmt_r.column_int64(i) { + libc_println!("MISMATCH: column {} int {} vs {}", i, stmt_l.column_int64(i), stmt_r.column_int64(i)); + return Ok(false); + } + } + sqlite::ColumnType::Float => { + if stmt_l.column_double(i) != stmt_r.column_double(i) { + libc_println!("MISMATCH: column {} float {} vs {}", i, stmt_l.column_double(i), stmt_r.column_double(i)); + return Ok(false); + } + } + sqlite::ColumnType::Text => { + if stmt_l.column_text(i)? != stmt_r.column_text(i)? { + libc_println!("MISMATCH: column {} text {:?} vs {:?}", i, stmt_l.column_text(i)?, stmt_r.column_text(i)?); + return Ok(false); + } + } + sqlite::ColumnType::Blob => { + if stmt_l.column_blob(i)? != stmt_r.column_blob(i)? { + libc_println!("MISMATCH: column {} blob {:?} vs {:?}", i, stmt_l.column_blob(i)?, stmt_r.column_blob(i)?); + return Ok(false); + } + } + sqlite::ColumnType::Null => {} + } + } + } +} + +/// Test: basic insert + sync in V2 mode +fn v2_basic_insert_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_basic_insert_sync START ==="); + let db_a = crate::opendb()?; + db_a.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT, val INTEGER)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db_a.db)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'one', 100)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (2, 'two', 200)")?; + + let db_b = crate::opendb()?; + db_b.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT, val INTEGER)")?; + db_b.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_b.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db_b.db)?; + + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + libc_println!("=== v2_basic_insert_sync PASS ==="); + Ok(()) +} + +/// Test: update + sync in V2 mode +fn v2_update_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_update_sync START ==="); + let db_a = crate::opendb()?; + db_a.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT, val INTEGER)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db_a.db)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'one', 100)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("UPDATE foo SET name = 'updated', val = 150 WHERE id = 1")?; + + let db_b = crate::opendb()?; + db_b.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT, val INTEGER)")?; + db_b.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_b.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db_b.db)?; + + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + libc_println!("=== v2_update_sync PASS ==="); + Ok(()) +} + +/// Test: delete + sync in V2 mode +fn v2_delete_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_delete_sync START ==="); + let db_a = crate::opendb()?; + db_a.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db_a.db)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'one')")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (2, 'two')")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + + let db_b = crate::opendb()?; + db_b.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT)")?; + db_b.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_b.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db_b.db)?; + + // First sync to get initial state + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + + // Verify row 1 is deleted in destination + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 1")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 0, "row 1 should be deleted in destination"); + + libc_println!("=== v2_delete_sync PASS ==="); + Ok(()) +} + +/// Test: PK-only table insert + sync in V2 mode +fn v2_pk_only_insert_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_pk_only_insert_sync START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (1)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (2)")?; + + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + libc_println!("=== v2_pk_only_insert_sync PASS ==="); + Ok(()) +} + +/// Test: composite PK table + sync in V2 mode +fn v2_composite_pk_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_composite_pk_sync START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE jx (id1 NOT NULL, id2 NOT NULL, val TEXT, PRIMARY KEY(id1, id2))")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('jx')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO jx VALUES (1, 2, 'a')")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO jx VALUES (3, 4, 'b')")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("UPDATE jx SET val = 'updated' WHERE id1 = 1 AND id2 = 2")?; + + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + assert!(tables_match(&db_a.db, &db_b.db, "jx", "id1")?); + libc_println!("=== v2_composite_pk_sync PASS ==="); + Ok(()) +} + +/// Test: sync bit honored in V2 mode (writes with sync bit should not create clock entries) +fn v2_sync_bit_honored() -> Result<(), ResultCode> { + libc_println!("=== v2_sync_bit_honored START ==="); + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db.db)?; + + // Enable sync bit + db.db.exec_safe("SELECT crsql_internal_sync_bit(1)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 2)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("UPDATE foo SET b = 5 WHERE a = 1")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 2)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("DELETE FROM foo WHERE a = 2")?; + + // V2 clock table should be empty + let stmt = db.db.prepare_v2("SELECT 1 FROM foo__crsql_v2_clock")?; + let result = stmt.step()?; + assert!(result == ResultCode::DONE, "V2 clock table should be empty when sync bit is set"); + + libc_println!("=== v2_sync_bit_honored PASS ==="); + Ok(()) +} + +/// Test: bidirectional sync in V2 mode +fn v2_bidirectional_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_bidirectional_sync START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE items (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('items')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + // A inserts row 1, B inserts row 2 + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO items VALUES (1, 'from_a', 10)")?; + db_b.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_b.db.exec_safe("INSERT INTO items VALUES (2, 'from_b', 20)")?; + + // Sync A -> B + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + // Sync B -> A + sync_left_to_right(&db_b.db, &db_a.db, 0)?; + + assert!(tables_match(&db_a.db, &db_b.db, "items", "id")?); + + // Both should have both rows + let stmt = db_a.db.prepare_v2("SELECT count(*) FROM items")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 2, "db_a should have 2 rows"); + + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM items")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 2, "db_b should have 2 rows"); + + libc_println!("=== v2_bidirectional_sync PASS ==="); + Ok(()) +} + +/// Test: crsql_changes produces correct output in V2 mode +fn v2_changes_output() -> Result<(), ResultCode> { + libc_println!("=== v2_changes_output START ==="); + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT, val INTEGER)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db.db)?; + + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'one', 100)")?; + + // Check crsql_changes produces results + let stmt = db.db.prepare_v2("SELECT * FROM crsql_changes")?; + let mut count = 0; + while stmt.step()? == ResultCode::ROW { + count += 1; + let tbl = stmt.column_text(0)?; + assert_eq!(tbl, "foo", "table name should be foo"); + // cid should be a column name or sentinel + let cid = stmt.column_text(2)?; + assert!(cid == "-1" || cid == "name" || cid == "val" || cid.contains('\0'), + "cid should be a valid column name, got: {:?}", cid); + } + assert!(count > 0, "crsql_changes should produce at least one row"); + + libc_println!("=== v2_changes_output PASS ==="); + Ok(()) +} + +/// Test: delete then re-insert (resurrect) in V2 mode +fn v2_delete_then_reinsert() -> Result<(), ResultCode> { + libc_println!("=== v2_delete_then_reinsert START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + // Insert, sync, delete, re-insert, sync again + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'first')")?; + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'second')")?; + + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + // Destination should have the re-inserted row + let stmt = db_b.db.prepare_v2("SELECT name FROM foo WHERE id = 1")?; + stmt.step()?; + assert_eq!(stmt.column_text(0)?, "second", "destination should have re-inserted value"); + + libc_println!("=== v2_delete_then_reinsert PASS ==="); + Ok(()) +} + +/// Test: PK-only table delete sync in V2 mode +fn v2_pk_only_delete_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_pk_only_delete_sync START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (1)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (2)")?; + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + + // Delete one row and sync + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + // db_b should also have deleted the row + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 1")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 0, "destination should have deleted id=1"); + + // db_b should still have id=2 + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 2")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 1, "destination should still have id=2"); + + libc_println!("=== v2_pk_only_delete_sync PASS ==="); + Ok(()) +} + +/// Test: PK-only table delete-then-reinsert in V2 mode +fn v2_pk_only_reinsert() -> Result<(), ResultCode> { + libc_println!("=== v2_pk_only_reinsert START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (1)")?; + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + // Delete then reinsert + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (1)")?; + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + // db_b should have the row back + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 1")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 1, "destination should have resurrected id=1"); + + libc_println!("=== v2_pk_only_reinsert PASS ==="); + Ok(()) +} + +/// Test: ALTER TABLE add column to PK-only table (sentinel becomes regular clock entry) +fn v2_alter_add_column_to_pk_only() -> Result<(), ResultCode> { + libc_println!("=== v2_alter_add_column_to_pk_only START ==="); + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db.db)?; + + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo (id) VALUES (1)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo (id) VALUES (2)")?; + + // Verify sentinel entries exist at col_id=0 + let stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_clock WHERE cell_key & 255 = 0")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 2, "should have 2 sentinel clock entries at col_id=0"); + + // Add a non-PK column via crsql_begin_alter / crsql_commit_alter + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_begin_alter('foo')")?; + db.db.exec_safe("ALTER TABLE foo ADD COLUMN name TEXT")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_commit_alter('foo')")?; + + // Now col_id=0 should be mapped to 'name' in v2_col_map + let stmt = db.db.prepare_v2("SELECT col_name FROM foo__crsql_v2_col_map WHERE col_id = 0")?; + stmt.step()?; + assert_eq!(stmt.column_text(0)?, "name", "col_id=0 should be mapped to 'name'"); + + // The old sentinel entries should still exist (now as regular clock entries for 'name') + let stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_clock WHERE cell_key & 255 = 0")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 2, "clock entries at col_id=0 should still exist"); + + // Updating the new column should work + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("UPDATE foo SET name = 'hello' WHERE id = 1")?; + + // Changes feed should now emit 'name' column changes (not sentinel cid=-1) + let stmt = db.db.prepare_v2("SELECT cid FROM crsql_changes WHERE \"table\" = 'foo'")?; + let mut found_name = false; + while stmt.step()? == ResultCode::ROW { + if stmt.column_text(0)? == "name" { + found_name = true; + } + } + assert!(found_name, "changes feed should emit 'name' column changes"); + + libc_println!("=== v2_alter_add_column_to_pk_only PASS ==="); + Ok(()) +} + +/// Test: ALTER TABLE drop last non-PK column (table becomes PK-only, clock entries migrated) +fn v2_alter_drop_column_becomes_pk_only() -> Result<(), ResultCode> { + libc_println!("=== v2_alter_drop_column_becomes_pk_only START ==="); + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, name TEXT)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + migrate_to_v2(&db.db)?; + + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'hello')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 'world')")?; + + // Record the db_version of the clock entries before the drop + let stmt = db.db.prepare_v2("SELECT min(db_version), max(db_version) FROM foo__crsql_v2_clock")?; + stmt.step()?; + let min_db_ver_before = stmt.column_int64(0); + let max_db_ver_before = stmt.column_int64(1); + + // Drop the last non-PK column + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_begin_alter('foo')")?; + db.db.exec_safe("ALTER TABLE foo DROP COLUMN name")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_commit_alter('foo')")?; + + // v2_col_map should be empty (no non-PK columns) + let stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_col_map")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 0, "v2_col_map should be empty after dropping last non-PK column"); + + // Sentinel clock entries should exist at col_id=0 + let stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_clock WHERE cell_key & 255 = 0")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 2, "should have 2 sentinel clock entries at col_id=0"); + + // Clock entries should be migrated (preserving db_version), not freshly created + let stmt = db.db.prepare_v2("SELECT min(db_version), max(db_version) FROM foo__crsql_v2_clock WHERE cell_key & 255 = 0")?; + stmt.step()?; + let min_db_ver_after = stmt.column_int64(0); + let max_db_ver_after = stmt.column_int64(1); + assert_eq!(min_db_ver_after, min_db_ver_before, "migrated sentinels should preserve min db_version"); + assert_eq!(max_db_ver_after, max_db_ver_before, "migrated sentinels should preserve max db_version"); + + // Changes feed should emit sentinel rows (cid=-1) for PK-only table + let stmt = db.db.prepare_v2("SELECT cid FROM crsql_changes WHERE \"table\" = 'foo'")?; + let mut found_sentinel = false; + while stmt.step()? == ResultCode::ROW { + if stmt.column_text(0)? == "-1" { + found_sentinel = true; + } + } + assert!(found_sentinel, "changes feed should emit sentinel rows for PK-only table"); + + libc_println!("=== v2_alter_drop_column_becomes_pk_only PASS ==="); + Ok(()) +} + +/// Test: PK-only bidirectional sync +fn v2_pk_only_bidirectional() -> Result<(), ResultCode> { + libc_println!("=== v2_pk_only_bidirectional START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, name TEXT)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + // Both insert different rows + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (1)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (2)")?; + db_b.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_b.db.exec_safe("INSERT INTO foo (id) VALUES (3)")?; + db_b.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_b.db.exec_safe("INSERT INTO foo (id) VALUES (4)")?; + + // Sync both directions + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + sync_left_to_right(&db_b.db, &db_a.db, 0)?; + + // Both should have all 4 rows + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + + let stmt = db_a.db.prepare_v2("SELECT count(*) FROM foo")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 4, "db_a should have all 4 rows"); + + libc_println!("=== v2_pk_only_bidirectional PASS ==="); + Ok(()) +} + +/// Helper: sync with V2 wire format (sync-log-version=2 on both sides) +fn sync_v2_wire(l: &dyn Connection, r: &dyn Connection, since: sqlite::int64) -> Result<(), ResultCode> { + let siteid_stmt = r.prepare_v2("SELECT crsql_site_id()")?; + siteid_stmt.step()?; + let siteid = siteid_stmt.column_blob(0)?; + + let stmt_l = l.prepare_v2( + "SELECT * FROM crsql_changes WHERE db_version >= ? AND site_id IS NOT ?", + )?; + stmt_l.bind_int64(1, since)?; + stmt_l.bind_blob(2, siteid, Destructor::STATIC)?; + + r.exec_safe("BEGIN")?; + r.exec_safe("SELECT crsql_set_ts('1700000000')")?; + while stmt_l.step()? == ResultCode::ROW { + let stmt_r = r + .prepare_v2("INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")?; + for x in 0..10 { + stmt_r.bind_value(x + 1, stmt_l.column_value(x)?)?; + } + match stmt_r.step() { + Ok(_) => {} + Err(e) => { + let msg = r.errmsg().unwrap_or_else(|_| alloc::string::ToString::to_string("unknown")); + libc_println!("V2 WIRE SYNC ERROR: {:?} - {}", e, msg); + let _ = r.exec_safe("ROLLBACK"); + return Err(e); + } + } + } + r.exec_safe("COMMIT")?; + Ok(()) +} + +/// Test: V2 wire format delete sync (cid=-2 hash tombstone) +fn v2_wire_delete_sync() -> Result<(), ResultCode> { + libc_println!("=== v2_wire_delete_sync START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, name TEXT)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + // Enable V2 wire format on both sides + for db in [&db_a.db, &db_b.db] { + db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + } + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'first')")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (2, 'second')")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + + // Delete and sync via V2 wire + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 1")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 0, "v2 wire: destination should have deleted id=1"); + + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 2")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 1, "v2 wire: destination should still have id=2"); + + libc_println!("=== v2_wire_delete_sync PASS ==="); + Ok(()) +} + +/// Test: V2 wire format delete-then-reinsert (resurrection via hash tombstone) +fn v2_wire_delete_then_reinsert() -> Result<(), ResultCode> { + libc_println!("=== v2_wire_delete_then_reinsert START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, name TEXT)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + } + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'first')")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + + // Delete, then re-insert with different value + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES (1, 'second')")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + + let stmt = db_b.db.prepare_v2("SELECT name FROM foo WHERE id = 1")?; + stmt.step()?; + assert_eq!(stmt.column_text(0)?, "second", "v2 wire: destination should have resurrected row with 'second'"); + + libc_println!("=== v2_wire_delete_then_reinsert PASS ==="); + Ok(()) +} + +/// Test: V2 wire format PK-only delete sync +fn v2_wire_pk_only_delete() -> Result<(), ResultCode> { + libc_println!("=== v2_wire_pk_only_delete START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + } + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (1)")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo (id) VALUES (2)")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + + // Delete and sync + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 1")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 0, "v2 wire pk-only: destination should have deleted id=1"); + + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 2")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 1, "v2 wire pk-only: destination should still have id=2"); + + libc_println!("=== v2_wire_pk_only_delete PASS ==="); + Ok(()) +} + +/// Test: V2 wire format delete sync for non-rowid table (text PK) +fn v2_wire_non_rowid_delete() -> Result<(), ResultCode> { + libc_println!("=== v2_wire_non_rowid_delete START ==="); + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE foo (id TEXT PRIMARY KEY NOT NULL, name TEXT)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('foo')")?; + } + + migrate_to_v2(&db_a.db)?; + migrate_to_v2(&db_b.db)?; + + for db in [&db_a.db, &db_b.db] { + db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + } + + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES ('a', 'first')")?; + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO foo VALUES ('b', 'second')")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + assert!(tables_match(&db_a.db, &db_b.db, "foo", "id")?); + + // Delete and sync + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM foo WHERE id = 'a'")?; + sync_v2_wire(&db_a.db, &db_b.db, 0)?; + + let stmt = db_b.db.prepare_v2("SELECT count(*) FROM foo WHERE id = 'a'")?; + stmt.step()?; + assert_eq!(stmt.column_int(0), 0, "v2 wire non-rowid: destination should have deleted id='a'"); + + let stmt = db_b.db.prepare_v2("SELECT name FROM foo WHERE id = 'b'")?; + stmt.step()?; + assert_eq!(stmt.column_text(0)?, "second", "v2 wire non-rowid: destination should still have id='b'"); + + libc_println!("=== v2_wire_non_rowid_delete PASS ==="); + Ok(()) +} + +pub fn run_suite() -> Result<(), ResultCode> { + v2_basic_insert_sync()?; + v2_update_sync()?; + v2_delete_sync()?; + v2_pk_only_insert_sync()?; + v2_pk_only_delete_sync()?; + v2_pk_only_reinsert()?; + v2_pk_only_bidirectional()?; + v2_composite_pk_sync()?; + v2_sync_bit_honored()?; + v2_bidirectional_sync()?; + v2_changes_output()?; + v2_delete_then_reinsert()?; + v2_alter_add_column_to_pk_only()?; + v2_alter_drop_column_becomes_pk_only()?; + v2_wire_delete_sync()?; + v2_wire_delete_then_reinsert()?; + v2_wire_pk_only_delete()?; + v2_wire_non_rowid_delete()?; + Ok(()) +} diff --git a/core/rs/integration_check/src/t/v2_tests.rs b/core/rs/integration_check/src/t/v2_tests.rs new file mode 100644 index 000000000..522d155ce --- /dev/null +++ b/core/rs/integration_check/src/t/v2_tests.rs @@ -0,0 +1,1977 @@ +extern crate alloc; +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec; +use alloc::vec::Vec; +use crsql_bundle::test_exports::pack_columns::{unpack_columns, ColumnValue}; +use libc_print::libc_println; +use sqlite::{Connection, Destructor, ManagedConnection, ResultCode}; +use sqlite_nostd as sqlite; + +/// Test that crsql_pack_agg produces byte-identical output to crsql_pack_columns +/// for the same values in the same order. +fn test_pack_agg_matches_pack_columns() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY, x, y, z)")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'hello', 42, x'01020304')")?; + + // Get crsql_pack_columns output + let select_pack = db.db.prepare_v2( + "SELECT crsql_pack_columns(id, x, y, z) FROM foo WHERE id = 1" + )?; + select_pack.step()?; + let packed = select_pack.column_blob(0)?; + + // Get crsql_pack_agg output via aggregate over a grouped query + // We use a subquery to feed the same values to crsql_pack_agg + let select_agg = db.db.prepare_v2( + "SELECT crsql_pack_agg(v) FROM (SELECT id AS v FROM foo WHERE id = 1 UNION ALL SELECT x FROM foo WHERE id = 1 UNION ALL SELECT y FROM foo WHERE id = 1 UNION ALL SELECT z FROM foo WHERE id = 1)" + )?; + select_agg.step()?; + let agged = select_agg.column_blob(0)?; + + assert!( + packed == agged, + "crsql_pack_agg output should match crsql_pack_columns output" + ); + + // Verify unpacking works + let unpacked = unpack_columns(agged)?; + assert!(unpacked.len() == 4); + if let ColumnValue::Integer(i) = unpacked[0] { + assert!(i == 1); + } else { + panic!("expected integer"); + } + if let ColumnValue::Text(s) = &unpacked[1] { + assert!(s == "hello"); + } else { + panic!("expected text"); + } + if let ColumnValue::Integer(i) = unpacked[2] { + assert!(i == 42); + } else { + panic!("expected integer"); + } + if let ColumnValue::Blob(b) = &unpacked[3] { + assert!(b[..] == [1, 2, 3, 4]); + } else { + panic!("expected blob"); + } + + Ok(()) +} + +/// Test crsql_pack_agg with NULL values +fn test_pack_agg_with_nulls() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE bar (id PRIMARY KEY, a, b)")?; + db.db.exec_safe("INSERT INTO bar VALUES (1, NULL, 'text')")?; + + let select_agg = db.db.prepare_v2( + "SELECT crsql_pack_agg(v) FROM (SELECT a AS v FROM bar WHERE id = 1 UNION ALL SELECT b FROM bar WHERE id = 1)" + )?; + select_agg.step()?; + let agged = select_agg.column_blob(0)?; + + let unpacked = unpack_columns(agged)?; + assert!(unpacked.len() == 2); + assert!(matches!(unpacked[0], ColumnValue::Null)); + if let ColumnValue::Text(s) = &unpacked[1] { + assert!(s == "text"); + } else { + panic!("expected text"); + } + + Ok(()) +} + +/// Test crsql_pack_agg with empty input (no rows) +fn test_pack_agg_empty() -> Result<(), ResultCode> { + let db = crate::opendb()?; + + // crsql_pack_agg over empty set should produce varint(0) = single byte 0x00 + let select_agg = db.db.prepare_v2( + "SELECT crsql_pack_agg(v) FROM (SELECT 1 AS v WHERE 0)" + )?; + select_agg.step()?; + let agged = select_agg.column_blob(0)?; + + // Should be a single byte 0x00 (varint encoding of count=0) + assert!(agged.len() == 1, "empty pack_agg should be 1 byte, got {}", agged.len()); + assert!(agged[0] == 0x00, "empty pack_agg should be 0x00"); + + // Unpacking should give 0 columns + let unpacked = unpack_columns(agged)?; + assert!(unpacked.is_empty()); + + Ok(()) +} + +/// Test crsql_hash_pk produces deterministic, truncated hashes +fn test_hash_pk_deterministic() -> Result<(), ResultCode> { + let db = crate::opendb()?; + + // Same input should produce same hash + let select1 = db.db.prepare_v2("SELECT crsql_hash_pk(1, 'abc')")?; + select1.step()?; + let hash1 = select1.column_blob(0)?; + + let select2 = db.db.prepare_v2("SELECT crsql_hash_pk(1, 'abc')")?; + select2.step()?; + let hash2 = select2.column_blob(0)?; + + assert!(hash1 == hash2, "same input should produce same hash"); + assert!( + hash1.len() == 10, + "hash should be PK_HASH_SIZE=10 bytes, got {}", + hash1.len() + ); + + // Different input should produce different hash + let select3 = db.db.prepare_v2("SELECT crsql_hash_pk(2, 'abc')")?; + select3.step()?; + let hash3 = select3.column_blob(0)?; + + assert!(hash1 != hash3, "different input should produce different hash"); + + Ok(()) +} + +/// Test varint count header: values 0-127 should be byte-identical to old u8 format +fn test_varint_count_backward_compat() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE baz (id PRIMARY KEY, a, b)")?; + db.db.exec_safe("INSERT INTO baz VALUES (1, 'x', 'y')")?; + + // 3 columns -> count byte should be 0x03 (same as old u8 format for <128) + let select = db.db.prepare_v2("SELECT crsql_pack_columns(id, a, b) FROM baz WHERE id = 1")?; + select.step()?; + let packed = select.column_blob(0)?; + + assert!( + packed[0] == 3, + "first byte should be count=3 (varint single byte, same as old u8)" + ); + + // Unpack should work correctly + let unpacked = unpack_columns(packed)?; + assert!(unpacked.len() == 3); + + Ok(()) +} + +/// Test that crsql_pack_agg and crsql_pack_columns produce identical output +/// for integer-only values including negative and large numbers +fn test_pack_agg_integers() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE nums (id PRIMARY KEY, a, b, c)")?; + db.db.exec_safe("INSERT INTO nums VALUES (1, -1, 0, 9999999999999)")?; + + let select_pack = db.db.prepare_v2( + "SELECT crsql_pack_columns(a, b, c) FROM nums WHERE id = 1" + )?; + select_pack.step()?; + let packed = select_pack.column_blob(0)?; + + let select_agg = db.db.prepare_v2( + "SELECT crsql_pack_agg(v) FROM (SELECT a AS v FROM nums WHERE id = 1 UNION ALL SELECT b FROM nums WHERE id = 1 UNION ALL SELECT c FROM nums WHERE id = 1)" + )?; + select_agg.step()?; + let agged = select_agg.column_blob(0)?; + + assert!( + packed == agged, + "pack_agg should match pack_columns for integers" + ); + + let unpacked = unpack_columns(agged)?; + assert!(unpacked.len() == 3); + if let ColumnValue::Integer(i) = unpacked[0] { + assert!(i == -1); + } else { + panic!("expected integer"); + } + if let ColumnValue::Integer(i) = unpacked[1] { + assert!(i == 0); + } else { + panic!("expected integer"); + } + if let ColumnValue::Integer(i) = unpacked[2] { + assert!(i == 9999999999999); + } else { + panic!("expected integer"); + } + + Ok(()) +} + +/// Test crsql_hash_pk with different types (integer, text, blob) +fn test_hash_pk_different_types() -> Result<(), ResultCode> { + let db = crate::opendb()?; + + // Integer PK + let s1 = db.db.prepare_v2("SELECT crsql_hash_pk(42)")?; + s1.step()?; + let h1 = s1.column_blob(0)?; + + // Text PK + let s2 = db.db.prepare_v2("SELECT crsql_hash_pk('hello')")?; + s2.step()?; + let h2 = s2.column_blob(0)?; + + // Blob PK + let s3 = db.db.prepare_v2("SELECT crsql_hash_pk(x'deadbeef')")?; + s3.step()?; + let h3 = s3.column_blob(0)?; + + // All should be 10 bytes and different from each other + assert!(h1.len() == 10 && h2.len() == 10 && h3.len() == 10); + assert!(h1 != h2 && h1 != h3 && h2 != h3); + + Ok(()) +} + +/// Test that metadata-use-version=1 reads from V1 tables even when V2 tables exist. +/// Setup: create CRR, insert data, migrate to V2&V1, then query with use-version=1 and use-version=2. +/// Both should produce the same per-column rows (V1 wire format). +fn test_metadata_use_version_dispatch() -> Result<(), ResultCode> { + libc_println!("DEBUG: test_metadata_use_version_dispatch START"); + let db = crate::opendb()?; + libc_println!("DEBUG: db opened"); + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, x, y)")?; + libc_println!("DEBUG: table created"); + db.db.exec_safe("INSERT INTO foo VALUES (1, 'a', 10)")?; + libc_println!("DEBUG: data inserted"); + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let rc = db.db.exec_safe("SELECT crsql_as_crr('foo')"); + match rc { + Ok(code) => libc_println!("DEBUG: as_crr ok: {:?}", code), + Err(code) => { + libc_println!("DEBUG: as_crr failed: {:?}", code); + // Try to get error message + let err_stmt = db.db.prepare_v2("SELECT sqlite3_errmsg(0)\0"); + if let Ok(s) = err_stmt { + let _ = s.step(); + if let Ok(msg) = s.column_text(0) { + libc_println!("DEBUG: errmsg: {}", msg); + } + } + return Err(code); + } + } + + // Check crsql_master exists + let check = db.db.prepare_v2("SELECT 1 FROM sqlite_master WHERE name = 'crsql_master'\0")?; + let has_master = check.step()? == ResultCode::ROW; + libc_println!("crsql_master exists: {}", has_master); + + // Start migration to V2&V1 + let rc = db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)"); + libc_println!("config_set rc: {:?}", rc); + if let Err(e) = rc { + libc_println!("config_set failed: {:?}", e); + return Err(e); + } + + // Enable debug logging + db.db.exec_safe("SELECT crsql_set_debug(1)")?; + + // Check config was set + let get_stmt = db.db.prepare_v2("SELECT crsql_config_get('metadata-write-version')\0")?; + get_stmt.step()?; + libc_println!("metadata-write-version is: {}", get_stmt.column_int(0)); + + // Run maintenance until complete + let mut remaining = 1; + let mut iterations = 0; + while remaining > 0 && iterations < 100 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)\0")?; + let rc = stmt.step()?; + if rc != ResultCode::ROW { + libc_println!("maintenance step returned {:?}, not ROW", rc); + break; + } + remaining = stmt.column_int(0) as i32; + if iterations < 3 { + libc_println!("maintenance iteration {} remaining: {}", iterations, remaining); + } + if remaining < 0 { + panic!("migration failed with remaining={}", remaining); + } + iterations += 1; + } + libc_println!("migration done after {} iterations, remaining={}", iterations, remaining); + + // Verify V2 tables exist + let check = db.db.prepare_v2("SELECT 1 FROM sqlite_master WHERE tbl_name = 'foo__crsql_v2_clock'\0")?; + let has_v2 = check.step()? == ResultCode::ROW; + libc_println!("V2 clock table exists: {}", has_v2); + assert!(has_v2, "V2 clock table should exist"); + + // Set metadata-use-version=1 → should read from V1 tables + libc_println!("DEBUG: setting metadata-use-version=1"); + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 1)")?; + libc_println!("DEBUG: metadata-use-version=1 set ok"); + + let stmt = match db.db.prepare_v2( + "SELECT [table], [pk], [cid], [col_version], [db_version], [site_id], [seq], [cl], [ts] FROM crsql_changes" + ) { + Ok(s) => { + libc_println!("DEBUG: feed query prepared"); + s + } + Err(e) => { + libc_println!("DEBUG: feed query prepare FAILED: {:?}", e); + let errmsg = db.db.errmsg().unwrap_or_else(|_| "unknown".to_string()); + libc_println!("DEBUG: errmsg: {}", errmsg); + return Err(e); + } + }; + let mut rows_v1 = vec![]; + while stmt.step()? == ResultCode::ROW { + rows_v1.push(( + stmt.column_text(0)?.to_string(), + stmt.column_text(2)?.to_string(), + stmt.column_int64(3), + )); + } + + // Should have: 2 cell changes (x, y) — no sentinel row for newly created CRRs (sentinel-omission optimization) + libc_println!("DEBUG: rows_v1 count = {}", rows_v1.len()); + for (i, row) in rows_v1.iter().enumerate() { + libc_println!("DEBUG: rows_v1[{}] = ({}, {}, {})", i, row.0, row.1, row.2); + } + assert!(rows_v1.len() == 2, "use-version=1 should produce 2 rows, got {}", rows_v1.len()); + + // Set metadata-use-version=2 → should read from V2 tables (per-column format) + libc_println!("DEBUG: setting metadata-use-version=2"); + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + libc_println!("DEBUG: metadata-use-version=2 set ok"); + + let stmt = match db.db.prepare_v2( + "SELECT [table], [pk], [cid], [col_version], [db_version], [site_id], [seq], [cl], [ts] FROM crsql_changes" + ) { + Ok(s) => { + libc_println!("DEBUG: v2 feed query prepared"); + s + } + Err(e) => { + libc_println!("DEBUG: v2 feed query prepare FAILED: {:?}", e); + let errmsg = db.db.errmsg().unwrap_or_else(|_| "unknown".to_string()); + libc_println!("DEBUG: errmsg: {}", errmsg); + return Err(e); + } + }; + let mut rows_v2 = vec![]; + while stmt.step()? == ResultCode::ROW { + rows_v2.push(( + stmt.column_text(0)?.to_string(), + stmt.column_text(2)?.to_string(), + stmt.column_int64(3), + )); + } + + // V2 per-column format: 2 cell changes (x, y) — same as V1, no sentinel rows + libc_println!("DEBUG: rows_v2 count = {}", rows_v2.len()); + for (i, row) in rows_v2.iter().enumerate() { + libc_println!("DEBUG: rows_v2[{}] = ({}, {}, {})", i, row.0, row.1, row.2); + } + assert!(rows_v2.len() == 2, "use-version=2 should produce 2 rows, got {}", rows_v2.len()); + + // Both should have the same table name + for r in &rows_v1 { + assert!(r.0 == "foo", "use-version=1 table name should be foo"); + } + for r in &rows_v2 { + assert!(r.0 == "foo", "use-version=2 table name should be foo"); + } + + // Both should have the same column changes + let v1_cids: Vec<_> = rows_v1.iter().map(|r| r.1.clone()).collect(); + let v2_cids: Vec<_> = rows_v2.iter().map(|r| r.1.clone()).collect(); + assert!(v1_cids.contains(&"x".to_string()), "use-version=1 should have x column"); + assert!(v2_cids.contains(&"x".to_string()), "use-version=2 should have x column"); + assert!(v1_cids.contains(&"y".to_string()), "use-version=1 should have y column"); + assert!(v2_cids.contains(&"y".to_string()), "use-version=2 should have y column"); + + Ok(()) +} + +/// Test packed wire format (sync-log-version=2): +/// Multiple column changes for the same row/db_version/site_id are coalesced into one row +/// with char(0)-separated cid and col_vrsn, and a packed cval blob. +fn test_packed_wire_format() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE bar (id PRIMARY KEY NOT NULL, a, b, c)")?; + db.db.exec_safe("INSERT INTO bar VALUES (1, 'x', 42, 3.14)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('bar')")?; + + // Migrate to V2&V1 + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + panic!("migration failed"); + } + } + + // Set use-version=2 and sync-log-version=2 → packed format + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + let stmt = db.db.prepare_v2( + "SELECT [table], [pk], [cid], [val], [cl] FROM crsql_changes" + )?; + + let mut packed_rows = vec![]; + loop { + let rc = stmt.step(); + match rc { + Ok(ResultCode::ROW) => {}, + Ok(ResultCode::DONE) => break, + Ok(rc) => { + return Err(rc); + } + Err(e) => { + let errmsg = db.db.errmsg().unwrap_or_else(|_| "unknown".to_string()); + libc_println!("packed wire format: step failed: {:?} - {}", e, errmsg); + return Err(e); + } + } + // Read blob column before text columns to avoid sqlite3_value invalidation + let cval_raw = stmt.column_blob(3).unwrap_or(&[]); + let tbl = stmt.column_text(0)?.to_string(); + let cid = stmt.column_text(2)?.to_string(); + let cl = stmt.column_int64(4); + + if cid.contains('\0') { + // Packed row: cid has char(0) separators + let cids: Vec = cid.split('\0').map(|s| s.to_string()).collect(); + let cval = cval_raw.to_vec(); + packed_rows.push((tbl, cids, cval, cl)); + } + } + + // Should have 1 packed row (all 3 columns coalesced) + assert!(packed_rows.len() == 1, "should have 1 packed row, got {}", packed_rows.len()); + + let (tbl, cids, cval, _cl) = &packed_rows[0]; + assert!(tbl == "bar", "table should be bar"); + assert!(cids.len() == 3, "packed row should have 3 columns, got {}", cids.len()); + assert!(cids.contains(&"a".to_string()), "should contain column a"); + assert!(cids.contains(&"b".to_string()), "should contain column b"); + assert!(cids.contains(&"c".to_string()), "should contain column c"); + + // cval should be a valid packed blob (unpackable) + let unpacked = unpack_columns(cval)?; + assert!(unpacked.len() == 3, "cval should unpack to 3 values, got {}", unpacked.len()); + // Values should be 'x', 42, 3.14 in col_id order (a=0, b=1, c=2) + if let ColumnValue::Text(s) = &unpacked[0] { + assert!(s == "x", "first cval should be 'x', got {}", s); + } else { + panic!("expected text for first cval"); + } + if let ColumnValue::Integer(i) = unpacked[1] { + assert!(i == 42, "second cval should be 42, got {}", i); + } else { + panic!("expected integer for second cval"); + } + if let ColumnValue::Float(f) = unpacked[2] { + // no_std: manual abs comparison + let diff = if f > 3.14 { f - 3.14 } else { 3.14 - f }; + assert!(diff < 0.001, "third cval should be ~3.14, got {}", f); + } else { + panic!("expected float for third cval"); + } + + Ok(()) +} + +/// Test V2 hash tombstone wire format: +/// When sync-log-version=2, deleted rows should emit with cid='-2' and pks=hashed_pk (blob), +/// not cid='-1' and pks=crsql_pack_columns(real_pks). +fn test_v2_hash_tombstone() -> Result<(), ResultCode> { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE baz (id PRIMARY KEY NOT NULL, a, b)")?; + db.db.exec_safe("INSERT INTO baz VALUES (1, 'x', 'y')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('baz')")?; + + // Migrate to V2 + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + panic!("migration failed"); + } + } + + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + // Set timestamp before delete (v2_tombstones has CHECK (ts > 0)) + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + + // Delete the row to create a tombstone + db.db.exec_safe("DELETE FROM baz WHERE id = 1")?; + + // Query crsql_changes — should get 1 tombstone row with cid=-2 and hashed_pk blob + let stmt = db.db.prepare_v2( + "SELECT [table], [pk], [cid], [val], [cl] FROM crsql_changes" + )?; + + let mut tombstone_row: Option<(String, Vec, String)> = None; + loop { + let rc = stmt.step()?; + if rc == ResultCode::DONE { break; } + if rc != ResultCode::ROW { return Err(rc); } + + // Read blob column first to avoid invalidation + let pk_blob = stmt.column_blob(1).unwrap_or(&[]).to_vec(); + let tbl = stmt.column_text(0)?.to_string(); + let cid = stmt.column_text(2)?.to_string(); + + if cid == "-2" { + // V2 hash tombstone: pks should be hashed_pk blob, val should be NULL + tombstone_row = Some((tbl, pk_blob, cid)); + let val_type = stmt.column_type(3)?; + assert!(val_type == sqlite::ColumnType::Null, + "tombstone val should be NULL, got {:?}", val_type); + } + } + + // Clock entries are deleted on delete, so only the tombstone row should remain + let (tbl, pk_blob, cid) = tombstone_row.expect("should have a tombstone row with cid=-2"); + assert!(tbl == "baz", "tombstone table should be baz, got {}", tbl); + assert!(cid == "-2", "tombstone cid should be -2, got {}", cid); + assert!(pk_blob.len() > 0, "tombstone pk (hashed_pk) should be non-empty blob, got {} bytes", pk_blob.len()); + + Ok(()) +} + +/// Sync all changes from left to right, filtering out the right db's own site_id. +fn sync_left_to_right( + l: &dyn Connection, + r: &dyn Connection, + since: sqlite::int64, +) -> Result<(), ResultCode> { + let siteid_stmt = r.prepare_v2("SELECT crsql_site_id()")?; + siteid_stmt.step()?; + let siteid = siteid_stmt.column_blob(0)?; + + let stmt_l = l.prepare_v2( + "SELECT * FROM crsql_changes WHERE db_version >= ? AND site_id IS NOT ?", + )?; + stmt_l.bind_int64(1, since)?; + stmt_l.bind_blob(2, siteid, Destructor::STATIC)?; + + // Use a transaction so ts is only set once for all merge inserts + r.exec_safe("BEGIN")?; + r.exec_safe("SELECT crsql_set_ts('1700000000')")?; + while stmt_l.step()? == ResultCode::ROW { + let stmt_r = r + .prepare_v2("INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")?; + for x in 0..10 { + stmt_r.bind_value(x + 1, stmt_l.column_value(x)?)?; + } + match stmt_r.step() { + Ok(_) => {} + Err(e) => { + let msg = r.errmsg().unwrap_or_else(|_| "unknown".to_string()); + libc_println!("SYNC ERROR: {:?} - {}", e, msg); + libc_println!(" row: tbl={:?}", stmt_l.column_text(0)); + let _ = r.exec_safe("ROLLBACK"); + return Err(e); + } + } + } + r.exec_safe("COMMIT")?; + Ok(()) +} + +/// Compare all rows in a table between two databases, returning true if identical. +fn tables_match( + l: &dyn Connection, + r: &dyn Connection, + table: &str, + order_col: &str, +) -> Result { + let sql = alloc::format!( + "SELECT * FROM \"{}\" ORDER BY \"{}\" ASC", + table, order_col + ); + let stmt_l = l.prepare_v2(&sql)?; + let stmt_r = r.prepare_v2(&sql)?; + + loop { + let rc_l = stmt_l.step()?; + let rc_r = stmt_r.step()?; + if rc_l != rc_r { + libc_println!("MISMATCH: step results differ ({:?} vs {:?})", rc_l, rc_r); + return Ok(false); + } + if rc_l == ResultCode::DONE { + return Ok(true); + } + let n = stmt_l.column_count(); + for i in 0..n { + let type_l = stmt_l.column_type(i)?; + let type_r = stmt_r.column_type(i)?; + if type_l != type_r { + libc_println!("MISMATCH: column {} type {:?} vs {:?}", i, type_l, type_r); + return Ok(false); + } + match type_l { + sqlite::ColumnType::Null => {} + sqlite::ColumnType::Integer => { + let v_l = stmt_l.column_int64(i); + let v_r = stmt_r.column_int64(i); + if v_l != v_r { + libc_println!("MISMATCH: column {} int {} vs {}", i, v_l, v_r); + return Ok(false); + } + } + sqlite::ColumnType::Float => { + let f_l = stmt_l.column_double(i); + let f_r = stmt_r.column_double(i); + let diff = if f_l > f_r { f_l - f_r } else { f_r - f_l }; + if diff > 0.0001 { + libc_println!("MISMATCH: column {} float {} vs {}", i, f_l, f_r); + return Ok(false); + } + } + sqlite::ColumnType::Text => { + let t_l = stmt_l.column_text(i)?; + let t_r = stmt_r.column_text(i)?; + if t_l != t_r { + libc_println!("MISMATCH: column {} text '{}' vs '{}'", i, t_l, t_r); + return Ok(false); + } + } + sqlite::ColumnType::Blob => { + let b_l = stmt_l.column_blob(i)?; + let b_r = stmt_r.column_blob(i)?; + if b_l != b_r { + libc_println!("MISMATCH: column {} blob len {} vs {}", i, b_l.len(), b_r.len()); + return Ok(false); + } + } + } + } + } +} + +/// Migrate a DB to V2&V1 and run maintenance until complete. +fn migrate_to_v2(db: &ManagedConnection) -> Result<(), ResultCode> { + db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + let mut iterations = 0; + while remaining > 0 && iterations < 100 { + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + return Err(ResultCode::ERROR); + } + iterations += 1; + } + Ok(()) +} + +/// Cross-mode sync round-trip test covering all 3 metadata/wire combinations: +/// 1. V1 metadata (V1 wire) +/// 2. V2 metadata + V1 wire (dual-write source, V1 wire sync) +/// 3. V2 metadata + V2 wire (dual-write source, V2 packed wire sync) +fn test_cross_mode_sync_roundtrip() -> Result<(), ResultCode> { + libc_println!("=== test_cross_mode_sync_roundtrip START ==="); + + let db_src = crate::opendb()?; + db_src.db.exec_safe("CREATE TABLE items (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER, price REAL)")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("SELECT crsql_as_crr('items')")?; + + // Insert rows + db_src.db.exec_safe("INSERT INTO items VALUES (1, 'apple', 10, 1.5)")?; + db_src.db.exec_safe("INSERT INTO items VALUES (2, 'banana', 20, 0.75)")?; + db_src.db.exec_safe("INSERT INTO items VALUES (3, 'cherry', 5, 3.25)")?; + + // Update some rows + db_src.db.exec_safe("UPDATE items SET qty = 15, price = 1.75 WHERE id = 1")?; + db_src.db.exec_safe("UPDATE items SET name = 'banana_split' WHERE id = 2")?; + + // Delete a row + db_src.db.exec_safe("DELETE FROM items WHERE id = 3")?; + + // Insert another row after delete + db_src.db.exec_safe("INSERT INTO items VALUES (4, 'date', 8, 2.0)")?; + + libc_println!("Source DB has 3 rows (1, 2, 4) after operations"); + + // --- Mode 1: V1 metadata sync --- + libc_println!("--- Mode 1: V1 metadata ---"); + { + let db_dst = crate::opendb()?; + db_dst.db.exec_safe("CREATE TABLE items (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER, price REAL)")?; + db_dst.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_dst.db.exec_safe("SELECT crsql_as_crr('items')")?; + + sync_left_to_right(&db_src.db, &db_dst.db, 0)?; + + let match_result = tables_match(&db_src.db, &db_dst.db, "items", "id")?; + assert!(match_result, "V1 meta: destination data should match source"); + libc_println!("V1 metadata: PASS"); + } + + // --- Mode 2: V2 metadata + V1 wire --- + // Migrate source to V2&V1, then sync with V1 wire to a V1 destination + libc_println!("--- Mode 2: V2 metadata + V1 wire ---"); + { + // Migrate source to V2&V1 + db_src.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db_src.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + panic!("migration failed"); + } + } + // Use V1 wire format for sync + db_src.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db_src.db.exec_safe("SELECT crsql_config_set('sync-log-version', 1)")?; + + let db_dst = crate::opendb()?; + db_dst.db.exec_safe("CREATE TABLE items (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER, price REAL)")?; + db_dst.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_dst.db.exec_safe("SELECT crsql_as_crr('items')")?; + + sync_left_to_right(&db_src.db, &db_dst.db, 0)?; + + let match_result = tables_match(&db_src.db, &db_dst.db, "items", "id")?; + assert!(match_result, "V2 meta + V1 wire: destination data should match source"); + libc_println!("V2 metadata + V1 wire: PASS"); + } + + // --- Mode 3: V2 metadata + V2 wire --- + libc_println!("--- Mode 3: V2 metadata + V2 wire ---"); + { + // Use V2 wire format for sync + db_src.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + let db_dst = crate::opendb()?; + db_dst.db.exec_safe("CREATE TABLE items (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER, price REAL)")?; + db_dst.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_dst.db.exec_safe("SELECT crsql_as_crr('items')")?; + // Destination also needs V2 metadata to accept V2 wire + db_dst.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db_dst.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db_dst.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + panic!("destination migration failed"); + } + } + db_dst.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db_dst.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + sync_left_to_right(&db_src.db, &db_dst.db, 0)?; + + let match_result = tables_match(&db_src.db, &db_dst.db, "items", "id")?; + assert!(match_result, "V2 meta + V2 wire: destination data should match source"); + libc_println!("V2 metadata + V2 wire: PASS"); + } + + libc_println!("=== test_cross_mode_sync_roundtrip ALL PASS ==="); + Ok(()) +} + +/// Test that dual-write (metadata-write-version=2) works correctly: +/// - Insert multiple rows after migration +/// - Verify both V1 and V2 tables have correct data +/// - Verify crsql_changes produces same results from both V1 and V2 metadata +fn test_dual_write_multiple_rows() -> Result<(), ResultCode> { + libc_println!("=== test_dual_write_multiple_rows START ==="); + + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a, b)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Migrate to V2&V1 + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + panic!("migration failed"); + } + } + + // Insert multiple rows after migration — exercises dual-write triggers + // Each INSERT auto-commits, so ts must be set before each one + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x', 10)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 'y', 20)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (3, 'z', 30)")?; + + // Verify V2 pks has all 3 rows + let pks_stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_pks")?; + pks_stmt.step()?; + let pks_count = pks_stmt.column_int64(0); + assert!(pks_count == 3, "V2 pks should have 3 rows, got {}", pks_count); + + // Verify V2 clock has entries for all 3 rows (2 cols each = 6) + let clock_stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_clock")?; + clock_stmt.step()?; + let clock_count = clock_stmt.column_int64(0); + assert!(clock_count == 6, "V2 clock should have 6 entries, got {}", clock_count); + + // Verify V1 tables also have the data (dual-write) + let v1_clock_stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_clock")?; + v1_clock_stmt.step()?; + let v1_clock_count = v1_clock_stmt.column_int64(0); + assert!(v1_clock_count == 6, "V1 clock should have 6 entries, got {}", v1_clock_count); + + // Verify crsql_changes produces same row count from V1 and V2 metadata + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 1)")?; + let v1_stmt = db.db.prepare_v2("SELECT [table], [cid], [val] FROM crsql_changes")?; + let mut v1_rows = vec![]; + while v1_stmt.step()? == ResultCode::ROW { + v1_rows.push(( + v1_stmt.column_text(0)?.to_string(), + v1_stmt.column_text(1)?.to_string(), + )); + } + + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db.db.exec_safe("SELECT crsql_config_set('sync-log-version', 1)")?; + let v2_stmt = db.db.prepare_v2("SELECT [table], [cid], [val] FROM crsql_changes")?; + let mut v2_rows = vec![]; + while v2_stmt.step()? == ResultCode::ROW { + v2_rows.push(( + v2_stmt.column_text(0)?.to_string(), + v2_stmt.column_text(1)?.to_string(), + )); + } + + assert!(v1_rows.len() == v2_rows.len(), + "V1 and V2 should produce same number of change rows: {} vs {}", v1_rows.len(), v2_rows.len()); + + libc_println!("=== test_dual_write_multiple_rows PASS ==="); + Ok(()) +} + +/// Test that migration of a table with existing data produces correct V2 metadata: +/// - V2 clock cell_keys must reference V2 pks __crsql_key (not V1 keys) +/// - crsql_changes from V2 metadata must return the same data as V1 +fn test_migration_with_data() -> Result<(), ResultCode> { + libc_println!("=== test_migration_with_data START ==="); + + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a, b)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Insert data BEFORE migration (V1 metadata) + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x', 10)")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 'y', 20)")?; + db.db.exec_safe("INSERT INTO foo VALUES (3, 'z', 30)")?; + + // Migrate to V2&V1 + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + panic!("migration failed"); + } + } + + // Verify V2 pks has all 3 rows + let pks_stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_pks")?; + pks_stmt.step()?; + let pks_count = pks_stmt.column_int64(0); + assert!(pks_count == 3, "V2 pks should have 3 rows, got {}", pks_count); + + // Verify V2 clock has entries for all non-sentinel columns (2 cols * 3 rows = 6) + let clock_stmt = db.db.prepare_v2("SELECT count(*) FROM foo__crsql_v2_clock")?; + clock_stmt.step()?; + let clock_count = clock_stmt.column_int64(0); + assert!(clock_count == 6, "V2 clock should have 6 entries, got {}", clock_count); + + // Critical check: every V2 clock cell_key must reference a V2 pks __crsql_key + // cell_key >> col_id_bits should equal __crsql_key in v2_pks + let join_stmt = db.db.prepare_v2( + "SELECT count(*) FROM foo__crsql_v2_clock c + LEFT JOIN foo__crsql_v2_pks p + ON (c.cell_key >> 12) = p.__crsql_key + WHERE p.__crsql_key IS NULL" + )?; + join_stmt.step()?; + let orphan_count = join_stmt.column_int64(0); + assert!(orphan_count == 0, + "V2 clock has {} orphan entries not matching any V2 pks __crsql_key", orphan_count); + + // Verify crsql_changes from V2 metadata matches V1 metadata + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 1)")?; + let v1_stmt = db.db.prepare_v2("SELECT [table], [cid], [val] FROM crsql_changes")?; + let mut v1_rows = vec![]; + while v1_stmt.step()? == ResultCode::ROW { + v1_rows.push(( + v1_stmt.column_text(0)?.to_string(), + v1_stmt.column_text(1)?.to_string(), + )); + } + + db.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db.db.exec_safe("SELECT crsql_config_set('sync-log-version', 1)")?; + let v2_stmt = db.db.prepare_v2("SELECT [table], [cid], [val] FROM crsql_changes")?; + let mut v2_rows = vec![]; + while v2_stmt.step()? == ResultCode::ROW { + v2_rows.push(( + v2_stmt.column_text(0)?.to_string(), + v2_stmt.column_text(1)?.to_string(), + )); + } + + assert!(v1_rows.len() == v2_rows.len(), + "V1 and V2 should produce same number of change rows: {} vs {}", v1_rows.len(), v2_rows.len()); + + libc_println!("=== test_migration_with_data PASS ==="); + Ok(()) +} + +/// Helper: read crsql_changes and return sorted (tbl, pk, cid, seq) rows. +fn read_changes_sorted(db: &crate::CRConnection, use_version: i32, sync_log: i32) -> Result, String, i64)>, ResultCode> { + db.db.exec_safe(&format!("SELECT crsql_config_set('metadata-use-version', {})", use_version))?; + if use_version == 2 { + db.db.exec_safe(&format!("SELECT crsql_config_set('sync-log-version', {})", sync_log))?; + } + let stmt = db.db.prepare_v2("SELECT [table], [pk], [cid], [seq] FROM crsql_changes")?; + let mut rows: Vec<(String, Vec, String, i64)> = vec![]; + while stmt.step()? == ResultCode::ROW { + rows.push(( + stmt.column_text(0)?.to_string(), + stmt.column_blob(1)?.to_vec(), + stmt.column_text(2)?.to_string(), + stmt.column_int64(3), + )); + } + rows.sort_by(|a, b| (&a.0, &a.1[..], &a.2).cmp(&(&b.0, &b.1[..], &b.2))); + Ok(rows) +} + +/// Helper: compare V1 and V2 change rows and assert equality. +fn assert_changes_match(v1: &[(String, Vec, String, i64)], v2: &[(String, Vec, String, i64)]) { + assert!(v1.len() == v2.len(), + "V1 and V2 should produce same number of change rows: {} vs {}", v1.len(), v2.len()); + for (i, (a, b)) in v1.iter().zip(v2.iter()).enumerate() { + assert!(a.0 == b.0, "Row {} tbl mismatch: V1='{}' V2='{}'", i, a.0, b.0); + assert!(a.1 == b.1, "Row {} pk mismatch for tbl='{}' cid='{}'", i, a.0, a.2); + assert!(a.2 == b.2, "Row {} cid mismatch: V1='{}' V2='{}'", i, a.2, b.2); + assert!(a.3 == b.3, "Row {} seq mismatch for tbl='{}' cid='{}': V1={} V2={}", i, a.0, a.2, a.3, b.3); + } +} + +/// Test that in dual-write mode, seq values are identical between V1 and V2 metadata reads +/// across a comprehensive set of operations: inserts, updates (single + multi-col), +/// deletes, resurrections, split across multiple transactions. +fn test_dual_write_seq_consistency() -> Result<(), ResultCode> { + libc_println!("=== test_dual_write_seq_consistency START ==="); + + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a, b)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Insert some data before migration (V1 mode) + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x', 10)")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 'y', 20)")?; + + // Migrate to V2&V1 (dual write) + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { panic!("migration failed"); } + } + + // --- Transaction 1: inserts --- + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("INSERT INTO foo VALUES (3, 'z', 30)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("INSERT INTO foo VALUES (4, 'w', 40)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("INSERT INTO foo VALUES (5, 'v', 50)")?; + + // --- Transaction 2: single-column updates --- + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("UPDATE foo SET a = 'x2' WHERE id = 1")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("UPDATE foo SET b = 99 WHERE id = 2")?; + + // --- Transaction 3: multi-column updates --- + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("UPDATE foo SET a = 'z2', b = 31 WHERE id = 3")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("UPDATE foo SET a = 'w2', b = 41 WHERE id = 4")?; + + // Compare V1 vs V2 — inserts and updates should match exactly + let v1_rows = read_changes_sorted(&db, 1, 1)?; + let v2_rows = read_changes_sorted(&db, 2, 1)?; + assert_changes_match(&v1_rows, &v2_rows); + + libc_println!("=== test_dual_write_seq_consistency PASS ({} rows) ===", v1_rows.len()); + Ok(()) +} + +/// Test deletes and resurrections in dual-write mode. +/// V1 and V2 have a known semantic difference: when a row is deleted then resurrected, +/// V1 keeps the delete sentinel in __crsql_clock (cid=-1), but V2 removes the tombstone +/// from v2_tombstones. So row counts may differ. This test verifies that for rows +/// that exist in both (matching tbl+pk+cid), seq values are identical. +fn test_dual_write_delete_resurrect() -> Result<(), ResultCode> { + libc_println!("=== test_dual_write_delete_resurrect START ==="); + + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a, b)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x', 10)")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 'y', 20)")?; + + // Migrate to dual-write + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { panic!("migration failed"); } + } + + // Delete a row + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("DELETE FROM foo WHERE id = 2")?; + + // Resurrect it + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("INSERT INTO foo VALUES (2, 'resurrected', 200)")?; + + // Update the resurrected row + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("UPDATE foo SET a = 'resurrected2', b = 201 WHERE id = 2")?; + + // Delete + resurrect in same transaction (ts set once covers both ops) + db.db.exec_safe("SELECT crsql_set_ts('1700000001')")?; + db.db.exec_safe("BEGIN")?; + db.db.exec_safe("DELETE FROM foo WHERE id = 1")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'same_tx_back', 100)")?; + db.db.exec_safe("COMMIT")?; + + // Compare V1 vs V2 — row counts may differ due to tombstone removal on resurrection. + // For rows that exist in both, seq values must match. + let v1_rows = read_changes_sorted(&db, 1, 1)?; + let v2_rows = read_changes_sorted(&db, 2, 1)?; + + let mut mismatches = 0; + let mut matched = 0; + for a in &v1_rows { + for b in &v2_rows { + if a.0 == b.0 && a.1 == b.1 && a.2 == b.2 { + if a.3 != b.3 { + libc_println!("SEQ MISMATCH: tbl={} cid={} V1.seq={} V2.seq={}", a.0, a.2, a.3, b.3); + mismatches += 1; + } + matched += 1; + break; + } + } + } + assert!(mismatches == 0, "Found {} seq mismatches in {} matched rows", mismatches, matched); + + libc_println!("=== test_dual_write_delete_resurrect PASS (matched={}, v1={}, v2={}) ===", + matched, v1_rows.len(), v2_rows.len()); + Ok(()) +} + +/// Randomized fuzz test: perform N random operations in dual-write mode, +/// then verify V1 and V2 metadata reads produce identical seq values. +/// Uses a simple LCG for deterministic reproducibility. +fn test_dual_write_seq_fuzz() -> Result<(), ResultCode> { + libc_println!("=== test_dual_write_seq_fuzz START ==="); + + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a, b)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + + // Seed some data before migration + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + for i in 1..=5 { + db.db.exec_safe(&format!("INSERT INTO foo VALUES ({}, 'init{}', {}00)", i, i, i))?; + } + + // Migrate to dual-write + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { panic!("migration failed"); } + } + + // Simple LCG random number generator (deterministic) + let mut state: u64 = 12345; + let mut next_rand = || { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (state >> 33) as u32 + }; + + let num_ops = 50; + let max_id = 10; // ids 1..max_id, some may not exist (deleted) + + for op_num in 0..num_ops { + let rand = next_rand(); + let op_type = rand % 4; // 0=insert, 1=update, 2=delete, 3=multi-update + let id = (next_rand() % max_id) + 1; + let val_a = next_rand() % 1000; + let val_b = next_rand() % 1000; + + let sql = match op_type { + 0 => format!("INSERT INTO foo VALUES ({}, 'a{}', {})", id, val_a, val_b), + 1 => format!("UPDATE foo SET a = 'u{}' WHERE id = {}", val_a, id), + 2 => format!("DELETE FROM foo WHERE id = {}", id), + 3 => format!("UPDATE foo SET a = 'm{}', b = {} WHERE id = {}", val_a, val_b, id), + _ => unreachable!(), + }; + + // Set ts before each op (auto-commit resets it) + let _ = db.db.exec_safe("SELECT crsql_set_ts('1800000000')"); + // Ignore errors (e.g., update on non-existent row is a no-op, not an error) + let _ = db.db.exec_safe(&sql); + } + + // Compare V1 vs V2 — row counts may differ due to tombstone removal on resurrection. + // For rows that exist in both, seq values must match. + let v1_rows = read_changes_sorted(&db, 1, 1)?; + let v2_rows = read_changes_sorted(&db, 2, 1)?; + + let mut mismatches = 0; + for a in &v1_rows { + for b in &v2_rows { + if a.0 == b.0 && a.1 == b.1 && a.2 == b.2 { + if a.3 != b.3 { + libc_println!("FUZZ SEQ MISMATCH: tbl={} cid={} V1.seq={} V2.seq={}", a.0, a.2, a.3, b.3); + mismatches += 1; + } + break; + } + } + } + assert!(mismatches == 0, "Fuzz: found {} seq mismatches in matched rows", mismatches); + + libc_println!("=== test_dual_write_seq_fuzz PASS (v1={} v2={}, {} ops) ===", v1_rows.len(), v2_rows.len(), num_ops); + Ok(()) +} + +fn test_compile_const_mismatch() -> Result<(), ResultCode> { + libc_println!("=== test_compile_const_mismatch START ==="); + + let path = "crsql_const_test.db\0"; + let path_str = path.trim_end_matches('\0'); + + #[cfg(not(target_os = "windows"))] + extern "C" { + fn unlink(pathname: *const core::ffi::c_char) -> core::ffi::c_int; + } + #[cfg(target_os = "windows")] + extern "C" { + fn _unlink(pathname: *const core::ffi::c_char) -> core::ffi::c_int; + } + + // Remove any leftover file from previous runs + unsafe { + #[cfg(not(target_os = "windows"))] + unlink(path.as_ptr() as *const core::ffi::c_char); + #[cfg(target_os = "windows")] + _unlink(path.as_ptr() as *const core::ffi::c_char); + } + + // Create a fresh DB and initialize crsql + { + let db = crate::opendb_file(path_str)?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'hello')")?; + // Verify the constant was stored + let stmt = db.db.prepare_v2("SELECT value FROM crsql_master WHERE key = 'crsql_col_id_bits'")?; + stmt.step()?; + let stored = stmt.column_int(0); + assert!(stored > 0, "crsql_col_id_bits should be stored in crsql_master"); + libc_println!("Stored crsql_col_id_bits = {}", stored); + } + // Connection A is now closed (dropped) + + // Tamper with the constant — open the DB, modify crsql_master, close. + // Bootstrap validation runs on extension init (connection open), so + // this connection will pass validation (constants still match). + // We tamper AFTER validation, then close. + { + let db = crate::opendb_file(path_str)?; + db.db.exec_safe("UPDATE crsql_master SET value = 999 WHERE key = 'crsql_col_id_bits'")?; + libc_println!("Tampered crsql_col_id_bits to 999"); + } + // Connection is closed + + // Now reopen — bootstrap should detect the mismatch and fail + let result = crate::opendb_file(path_str); + match result { + Ok(db) => { + // If open succeeded, try a crsql operation to see if it fails + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let cr = db.db.exec_safe("SELECT crsql_as_crr('foo')"); + match cr { + Err(rc) => { + libc_println!("=== test_compile_const_mismatch PASS (crsql_as_crr rejected mismatched constant, rc={:?}) ===", rc); + } + Ok(_) => { + // Try querying crsql_changes + let cr2 = db.db.exec_safe("SELECT * FROM crsql_changes"); + match cr2 { + Err(rc) => { + libc_println!("=== test_compile_const_mismatch PASS (crsql_changes rejected mismatched constant, rc={:?}) ===", rc); + } + Ok(_) => { + panic!("FAIL: DB with tampered compile constant was accepted without error"); + } + } + } + } + } + Err(rc) => { + libc_println!("=== test_compile_const_mismatch PASS (bootstrap rejected mismatched constant on open, rc={:?}) ===", rc); + } + } + + // Clean up + unsafe { + #[cfg(not(target_os = "windows"))] + unlink(path.as_ptr() as *const core::ffi::c_char); + #[cfg(target_os = "windows")] + _unlink(path.as_ptr() as *const core::ffi::c_char); + } + + Ok(()) +} + +/// Test V2 wire format sync with a table that has a single PK and single non-PK column. +/// In this case, packed cid won't contain null separators (only one column), +/// but cval is still a packed blob from crsql_pack_agg (with 1 column). +/// This verifies the write path correctly detects V2 wire packed rows +/// when cid has no '\0' but col_vrsn is text/blob type. +fn test_v2_wire_single_col_sync() -> Result<(), ResultCode> { + libc_println!("=== test_v2_wire_single_col_sync START ==="); + + let db_src = crate::opendb()?; + let db_dst = crate::opendb()?; + + // Single PK, single non-PK column + db_src.db.exec_safe("CREATE TABLE item (id PRIMARY KEY NOT NULL, name TEXT)")?; + db_src.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("SELECT crsql_as_crr('item')")?; + db_dst.db.exec_safe("CREATE TABLE item (id PRIMARY KEY NOT NULL, name TEXT)")?; + db_dst.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db_dst.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_dst.db.exec_safe("SELECT crsql_as_crr('item')")?; + + // Insert and update data + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("INSERT INTO item VALUES (1, 'apple')")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("INSERT INTO item VALUES (2, 'banana')")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("UPDATE item SET name = 'cherry' WHERE id = 1")?; + + // Enable V2 wire format on source + db_src.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + // Verify the source emits V2 wire format + let check = db_src.db.prepare_v2( + "SELECT [cid], typeof([cid]), [col_version], typeof([col_version]), [val], typeof([val]) FROM crsql_changes" + )?; + while check.step()? == ResultCode::ROW { + let cid = check.column_text(0)?; + let cid_type = check.column_text(1)?; + let cval_type = check.column_text(5)?; + libc_println!(" src row: cid={:?} cid_type={:?} cval_type={:?}", cid, cid_type, cval_type); + } + + // Sync from src to dst + sync_left_to_right(&db_src.db, &db_dst.db, 0)?; + + // Verify data matches + let match_result = tables_match(&db_src.db, &db_dst.db, "item", "id")?; + assert!(match_result, "V2 wire single-col: destination data should match source"); + + libc_println!("=== test_v2_wire_single_col_sync PASS ==="); + Ok(()) +} + +/// Test that a V1 metadata node rejects V2 wire format changes per spec. +/// The README states: "Nodes with metadata-use-version set to 1 will emit an error +/// if they receive V2 wire format changes." +/// This covers both packed update rows (cid with col_vrsn as blob) and +/// hash tombstone rows (cid='-2' with hashed_pk instead of packed PKs). +fn test_v1_rejects_v2_wire() -> Result<(), ResultCode> { + libc_println!("=== test_v1_rejects_v2_wire START ==="); + + let db_src = crate::opendb()?; + let db_dst = crate::opendb()?; + + // Source: V2 metadata + V2 wire + db_src.db.exec_safe("CREATE TABLE item (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER)")?; + db_src.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("SELECT crsql_as_crr('item')")?; + db_src.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + // Destination: V1 metadata only (default, no metadata-write-version set) + db_dst.db.exec_safe("CREATE TABLE item (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER)")?; + db_dst.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_dst.db.exec_safe("SELECT crsql_as_crr('item')")?; + + // Insert + delete to produce both packed updates and a tombstone + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("INSERT INTO item VALUES (1, 'apple', 10)")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("INSERT INTO item VALUES (2, 'banana', 20)")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("DELETE FROM item WHERE id = 2")?; + + // Attempt sync - should fail with error + let result = sync_left_to_right(&db_src.db, &db_dst.db, 0); + assert!( + result.is_err(), + "V1 metadata node should reject V2 wire format changes, but sync succeeded" + ); + + libc_println!("=== test_v1_rejects_v2_wire PASS ==="); + Ok(()) +} + +/// Test V2 wire packed resurrection: row is deleted on dst (tombstone), +/// then src resurrects it via INSERT and syncs via V2 wire packed format. +/// Dst should resurrect the row (remove tombstone, insert into v2_pks, apply values). +fn test_v2_wire_packed_resurrection() -> Result<(), ResultCode> { + let db_src = crate::opendb()?; + let db_dst = crate::opendb()?; + + // Both nodes: V2 metadata + for db in [&db_src.db, &db_dst.db] { + db.exec_safe("CREATE TABLE item (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER)")?; + db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('item')")?; + db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + } + + // Src inserts a row (CL=1) + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("INSERT INTO item VALUES (1, 'apple', 10)")?; + + // Sync src -> dst, then dst -> src so CLs converge + sync_left_to_right(&db_src.db, &db_dst.db, 0)?; + sync_left_to_right(&db_dst.db, &db_src.db, 0)?; + + // Dst deletes the row (creates tombstone with CL=2) + db_dst.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_dst.db.exec_safe("DELETE FROM item WHERE id = 1")?; + + // Verify dst has tombstone + let tomb_check = db_dst.db.prepare_v2( + "SELECT COUNT(*) FROM item__crsql_v2_tombstones" + )?; + tomb_check.step()?; + assert_eq!(tomb_check.column_int64(0), 1, "dst should have tombstone after delete"); + + // Sync dst -> src so src learns about the delete (src CL becomes 2 = dead) + sync_left_to_right(&db_dst.db, &db_src.db, 0)?; + + // Src resurrects the row via INSERT (CL bumps to 3 = alive) + // After receiving the delete, the row is gone from src's main table, + // so UPDATE won't work — must use INSERT. + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("INSERT INTO item VALUES (1, 'cherry', 20)")?; + + // Enable V2 wire format on src + db_src.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + // Sync src -> dst via V2 wire packed format (triggers resurrection on dst) + sync_left_to_right(&db_src.db, &db_dst.db, 0)?; + + // Verify dst no longer has tombstone + let tomb_after = db_dst.db.prepare_v2( + "SELECT COUNT(*) FROM item__crsql_v2_tombstones" + )?; + tomb_after.step()?; + assert_eq!(tomb_after.column_int64(0), 0, "tombstone should be removed after resurrection"); + + // Verify dst has the row in v2_pks + let pks_after = db_dst.db.prepare_v2( + "SELECT COUNT(*) FROM item__crsql_v2_pks" + )?; + pks_after.step()?; + assert_eq!(pks_after.column_int64(0), 1, "v2_pks should have the row after resurrection"); + + // Verify dst has the actual data + let data_check = db_dst.db.prepare_v2("SELECT name, qty FROM item WHERE id = 1")?; + assert_eq!(data_check.step()?, ResultCode::ROW); + assert_eq!(data_check.column_text(0)?, "cherry", "name should be 'cherry' after resurrection"); + assert_eq!(data_check.column_int64(1), 20, "qty should be 20 after resurrection"); + + libc_println!("=== test_v2_wire_packed_resurrection PASS ==="); + Ok(()) +} + +/// Test that a node in dual-write mode (V2AndV1) arrives at semantically the same +/// metadata and data regardless of whether it receives changes via V1 wire or V2 wire. +/// Covers: inserts, updates, deletes, resurrections, and PK-only tables. +/// For each scenario, two destination nodes are synced from the same source — +/// one via V1 wire, one via V2 wire — and both must match the source and each other. +fn test_dual_write_wire_convergence() -> Result<(), ResultCode> { + libc_println!("=== test_dual_write_wire_convergence START ==="); + + // --- Source node: dual-write mode --- + let db_src = crate::opendb()?; + // Regular table with data columns + db_src.db.exec_safe("CREATE TABLE prod (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER)")?; + // PK-only table + db_src.db.exec_safe("CREATE TABLE tag (id PRIMARY KEY NOT NULL)")?; + // Migrate to dual-write and do all setup + writes in one transaction + db_src.db.exec_safe("BEGIN")?; + db_src.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("SELECT crsql_as_crr('prod')")?; + db_src.db.exec_safe("SELECT crsql_as_crr('tag')")?; + db_src.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + + // --- Phase 1: Inserts --- + db_src.db.exec_safe("INSERT INTO prod VALUES (1, 'apple', 10)")?; + db_src.db.exec_safe("INSERT INTO prod VALUES (2, 'banana', 20)")?; + db_src.db.exec_safe("INSERT INTO prod VALUES (3, 'cherry', 5)")?; + db_src.db.exec_safe("INSERT INTO tag VALUES (100)")?; + db_src.db.exec_safe("INSERT INTO tag VALUES (200)")?; + + // --- Phase 2: Updates --- + db_src.db.exec_safe("UPDATE prod SET name = 'apple2', qty = 15 WHERE id = 1")?; + db_src.db.exec_safe("UPDATE prod SET qty = 25 WHERE id = 2")?; + + // --- Phase 3: Delete --- + db_src.db.exec_safe("DELETE FROM prod WHERE id = 3")?; + db_src.db.exec_safe("DELETE FROM tag WHERE id = 200")?; + db_src.db.exec_safe("COMMIT")?; + + // --- Phase 4: Resurrection --- + // Sync so dst nodes learn about the deletes, then resurrect on src + // We need two dst nodes: one for V1 wire, one for V2 wire. + // But first we need to sync the pre-delete state, then the delete, + // then the resurrection — to properly test resurrection via each wire format. + + // Create two destination nodes in dual-write mode + let make_dst = || -> Result { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE prod (id PRIMARY KEY NOT NULL, name TEXT, qty INTEGER)")?; + db.db.exec_safe("CREATE TABLE tag (id PRIMARY KEY NOT NULL)")?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + db.db.exec_safe("BEGIN")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('prod')")?; + db.db.exec_safe("SELECT crsql_as_crr('tag')")?; + db.db.exec_safe("COMMIT")?; + // Run migration to create V2 tables (each iteration auto-commits) + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { + panic!("dst migration failed"); + } + } + Ok(db) + }; + + // --- Destination A: will receive V1 wire --- + let db_dst_v1 = make_dst()?; + db_dst_v1.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db_dst_v1.db.exec_safe("SELECT crsql_config_set('sync-log-version', 1)")?; + + // --- Destination B: will receive V2 wire --- + let db_dst_v2 = make_dst()?; + db_dst_v2.db.exec_safe("SELECT crsql_config_set('metadata-use-version', 2)")?; + db_dst_v2.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + + // Sync pre-delete state to both destinations + // Source uses V1 wire for first sync + db_src.db.exec_safe("SELECT crsql_config_set('sync-log-version', 1)")?; + sync_left_to_right(&db_src.db, &db_dst_v1.db, 0)?; + sync_left_to_right(&db_src.db, &db_dst_v2.db, 0)?; + + // Verify data matches after initial sync + assert!(tables_match(&db_src.db, &db_dst_v1.db, "prod", "id")?, "V1 wire: prod mismatch after initial sync"); + assert!(tables_match(&db_src.db, &db_dst_v2.db, "prod", "id")?, "V2 wire: prod mismatch after initial sync"); + assert!(tables_match(&db_src.db, &db_dst_v1.db, "tag", "id")?, "V1 wire: tag mismatch after initial sync"); + assert!(tables_match(&db_src.db, &db_dst_v2.db, "tag", "id")?, "V2 wire: tag mismatch after initial sync"); + libc_println!(" Phase 1-2 (inserts + updates): PASS"); + + // Now sync the deletes + sync_left_to_right(&db_src.db, &db_dst_v1.db, 0)?; + sync_left_to_right(&db_src.db, &db_dst_v2.db, 0)?; + + // Verify deletes propagated + let check_v1 = db_dst_v1.db.prepare_v2("SELECT COUNT(*) FROM prod WHERE id = 3")?; + check_v1.step()?; + assert_eq!(check_v1.column_int64(0), 0, "V1 wire: row 3 should be deleted"); + let check_v2 = db_dst_v2.db.prepare_v2("SELECT COUNT(*) FROM prod WHERE id = 3")?; + check_v2.step()?; + assert_eq!(check_v2.column_int64(0), 0, "V2 wire: row 3 should be deleted"); + let check_tag_v1 = db_dst_v1.db.prepare_v2("SELECT COUNT(*) FROM tag WHERE id = 200")?; + check_tag_v1.step()?; + assert_eq!(check_tag_v1.column_int64(0), 0, "V1 wire: tag 200 should be deleted"); + let check_tag_v2 = db_dst_v2.db.prepare_v2("SELECT COUNT(*) FROM tag WHERE id = 200")?; + check_tag_v2.step()?; + assert_eq!(check_tag_v2.column_int64(0), 0, "V2 wire: tag 200 should be deleted"); + libc_println!(" Phase 3 (deletes): PASS"); + + // --- Resurrection: bring back row 3 and tag 200 --- + db_src.db.exec_safe("BEGIN")?; + db_src.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_src.db.exec_safe("INSERT INTO prod VALUES (3, 'cherry2', 50)")?; + db_src.db.exec_safe("INSERT INTO tag VALUES (200)")?; + db_src.db.exec_safe("COMMIT")?; + + // Sync resurrection via V1 wire to dst_v1 + sync_left_to_right(&db_src.db, &db_dst_v1.db, 0)?; + + // Sync resurrection via V2 wire to dst_v2 + db_src.db.exec_safe("SELECT crsql_config_set('sync-log-version', 2)")?; + sync_left_to_right(&db_src.db, &db_dst_v2.db, 0)?; + + // Verify resurrected data matches + let r_v1 = db_dst_v1.db.prepare_v2("SELECT name, qty FROM prod WHERE id = 3")?; + assert_eq!(r_v1.step()?, ResultCode::ROW); + assert_eq!(r_v1.column_text(0)?, "cherry2", "V1 wire: resurrected name mismatch"); + assert_eq!(r_v1.column_int64(1), 50, "V1 wire: resurrected qty mismatch"); + + let r_v2 = db_dst_v2.db.prepare_v2("SELECT name, qty FROM prod WHERE id = 3")?; + assert_eq!(r_v2.step()?, ResultCode::ROW); + assert_eq!(r_v2.column_text(0)?, "cherry2", "V2 wire: resurrected name mismatch"); + assert_eq!(r_v2.column_int64(1), 50, "V2 wire: resurrected qty mismatch"); + + // Verify resurrected PK-only row + let r_tag_v1 = db_dst_v1.db.prepare_v2("SELECT COUNT(*) FROM tag WHERE id = 200")?; + r_tag_v1.step()?; + assert_eq!(r_tag_v1.column_int64(0), 1, "V1 wire: tag 200 should be resurrected"); + let r_tag_v2 = db_dst_v2.db.prepare_v2("SELECT COUNT(*) FROM tag WHERE id = 200")?; + r_tag_v2.step()?; + assert_eq!(r_tag_v2.column_int64(0), 1, "V2 wire: tag 200 should be resurrected"); + libc_println!(" Phase 4 (resurrection): PASS"); + + // --- Final convergence: all three nodes should have identical data --- + assert!(tables_match(&db_src.db, &db_dst_v1.db, "prod", "id")?, "Final: prod mismatch src vs V1 dst"); + assert!(tables_match(&db_src.db, &db_dst_v2.db, "prod", "id")?, "Final: prod mismatch src vs V2 dst"); + assert!(tables_match(&db_dst_v1.db, &db_dst_v2.db, "prod", "id")?, "Final: prod mismatch V1 dst vs V2 dst"); + assert!(tables_match(&db_src.db, &db_dst_v1.db, "tag", "id")?, "Final: tag mismatch src vs V1 dst"); + assert!(tables_match(&db_src.db, &db_dst_v2.db, "tag", "id")?, "Final: tag mismatch src vs V2 dst"); + assert!(tables_match(&db_dst_v1.db, &db_dst_v2.db, "tag", "id")?, "Final: tag mismatch V1 dst vs V2 dst"); + + // --- Verify V1 and V2 metadata consistency on each destination --- + // Both destinations are in dual-write mode, so both have V1 and V2 metadata. + // Check that V1 clock table and V2 clock table have consistent column data. + for (db, label) in [(&db_dst_v1.db, "V1-wire dst"), (&db_dst_v2.db, "V2-wire dst")] { + // V1 clock entries for prod + let v1_clock = db.prepare_v2( + "SELECT COUNT(*) FROM prod__crsql_clock" + )?; + v1_clock.step()?; + let v1_count = v1_clock.column_int64(0); + + // V2 clock entries for prod + let v2_clock = db.prepare_v2( + "SELECT COUNT(*) FROM prod__crsql_v2_clock" + )?; + v2_clock.step()?; + let v2_count = v2_clock.column_int64(0); + + libc_println!(" {} metadata: V1 clock={}, V2 clock={}", label, v1_count, v2_count); + + // V1 pks + let v1_pks = db.prepare_v2( + "SELECT COUNT(*) FROM prod__crsql_pks" + )?; + v1_pks.step()?; + let v1_pks_count = v1_pks.column_int64(0); + + // V2 pks + let v2_pks = db.prepare_v2( + "SELECT COUNT(*) FROM prod__crsql_v2_pks" + )?; + v2_pks.step()?; + let v2_pks_count = v2_pks.column_int64(0); + + libc_println!(" {} metadata: V1 pks={}, V2 pks={}", label, v1_pks_count, v2_pks_count); + + // V2 pks should have alive rows (id 1, 2, 3) + assert_eq!(v2_pks_count, 3, "{}: V2 pks should have 3 alive rows", label); + + // V1 pks should also have 3 rows (V1 keeps all keys including deleted ones) + // V1 may keep deleted rows in pks, so just check >= 3 + assert!(v1_pks_count >= 3, "{}: V1 pks should have at least 3 rows, got {}", label, v1_pks_count); + + // Check that V1 clock entries have the correct remote site_id, not just local. + // If V1 metadata is only populated by triggers (accidentally from V2 merge upserts), + // the site_id will be the local site, not the remote source site. + let v1_clock_sites = db.prepare_v2( + "SELECT DISTINCT s.site_id FROM prod__crsql_clock c JOIN crsql_site_id s ON c.site_id = s.ordinal ORDER BY s.site_id" + )?; + let mut sites = Vec::new(); + while v1_clock_sites.step()? == ResultCode::ROW { + sites.push(v1_clock_sites.column_blob(0)?.to_vec()); + } + libc_println!(" {} V1 clock site_ids: {} distinct", label, sites.len()); + + // The source site_id should appear in V1 clock entries (from sync). + // If only local site_id appears, V1 metadata was populated by triggers, not merge. + let src_siteid_stmt = db_src.db.prepare_v2("SELECT crsql_site_id()")?; + src_siteid_stmt.step()?; + let src_site_id = src_siteid_stmt.column_blob(0)?.to_vec(); + + let has_src_site = sites.iter().any(|s| s.as_slice() == src_site_id.as_slice()); + assert!(has_src_site, "{}: V1 clock should contain source site_id (remote sync), not just local", label); + } + + // --- Verify V2 tombstones are clean on both destinations --- + for (db, label) in [(&db_dst_v1.db, "V1-wire dst"), (&db_dst_v2.db, "V2-wire dst")] { + let tomb_prod = db.prepare_v2( + "SELECT COUNT(*) FROM prod__crsql_v2_tombstones" + )?; + tomb_prod.step()?; + assert_eq!(tomb_prod.column_int64(0), 0, "{}: no prod tombstones after resurrection", label); + + // tag 200 was resurrected, so no tombstone should remain + let tomb_tag = db.prepare_v2( + "SELECT COUNT(*) FROM tag__crsql_v2_tombstones" + )?; + tomb_tag.step()?; + assert_eq!(tomb_tag.column_int64(0), 0, "{}: no tag tombstones after resurrection", label); + } + + // --- Bidirectional sync: dst_v1 -> src and dst_v2 -> src to converge CLs --- + // dst_v1 sends via V1 wire + sync_left_to_right(&db_dst_v1.db, &db_src.db, 0)?; + // dst_v2 sends via V2 wire + sync_left_to_right(&db_dst_v2.db, &db_src.db, 0)?; + + // Final data check after bidirectional sync + assert!(tables_match(&db_src.db, &db_dst_v1.db, "prod", "id")?, "Post-bidi: prod mismatch src vs V1 dst"); + assert!(tables_match(&db_src.db, &db_dst_v2.db, "prod", "id")?, "Post-bidi: prod mismatch src vs V2 dst"); + assert!(tables_match(&db_dst_v1.db, &db_dst_v2.db, "prod", "id")?, "Post-bidi: prod mismatch V1 dst vs V2 dst"); + assert!(tables_match(&db_src.db, &db_dst_v1.db, "tag", "id")?, "Post-bidi: tag mismatch src vs V1 dst"); + assert!(tables_match(&db_src.db, &db_dst_v2.db, "tag", "id")?, "Post-bidi: tag mismatch src vs V2 dst"); + assert!(tables_match(&db_dst_v1.db, &db_dst_v2.db, "tag", "id")?, "Post-bidi: tag mismatch V1 dst vs V2 dst"); + + libc_println!("=== test_dual_write_wire_convergence ALL PASS ==="); + Ok(()) +} + +/// Test tombstone conflict resolution: when two nodes delete the same row +/// and sync, the tombstone with the higher CL wins. If CLs are equal, +/// site_id is used as tiebreaker. This verifies the ON CONFLICT upsert +/// on v2_tombstones.hashed_pk unique index works per the design doc. +fn test_tombstone_conflict_resolution() -> Result<(), ResultCode> { + libc_println!("=== test_tombstone_conflict_resolution START ==="); + + let db_a = crate::opendb()?; + let db_b = crate::opendb()?; + + // Both nodes: V2 metadata + for db in [&db_a.db, &db_b.db] { + db.exec_safe("CREATE TABLE item (id PRIMARY KEY NOT NULL, name TEXT)")?; + db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.exec_safe("SELECT crsql_as_crr('item')")?; + } + + // Both nodes insert the same row (same CL=1 after sync) + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("INSERT INTO item VALUES (1, 'apple')")?; + // Sync A -> B so both have the row + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + // Sync B -> A so CLs converge + sync_left_to_right(&db_b.db, &db_a.db, 0)?; + + // Both nodes delete the row (各自删除) + db_a.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_a.db.exec_safe("DELETE FROM item WHERE id = 1")?; + db_b.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db_b.db.exec_safe("DELETE FROM item WHERE id = 1")?; + + // Both should have tombstones with CL=2 (even = dead) + // Sync A -> B: B receives A's tombstone. CLs are equal (both 2), + // so site_id tiebreaker applies. The upsert should not error. + sync_left_to_right(&db_a.db, &db_b.db, 0)?; + + // Sync B -> A: A receives B's tombstone. Same situation. + sync_left_to_right(&db_b.db, &db_a.db, 0)?; + + // Verify both nodes still have the row deleted + let check_a = db_a.db.prepare_v2("SELECT COUNT(*) FROM item WHERE id = 1")?; + check_a.step()?; + assert_eq!(check_a.column_int64(0), 0, "row should be deleted on A after tombstone sync"); + + let check_b = db_b.db.prepare_v2("SELECT COUNT(*) FROM item WHERE id = 1")?; + check_b.step()?; + assert_eq!(check_b.column_int64(0), 0, "row should be deleted on B after tombstone sync"); + + // Verify tombstone exists exactly once (no duplicate inserts) + let tomb_count_a = db_a.db.prepare_v2( + "SELECT COUNT(*) FROM item__crsql_v2_tombstones WHERE hashed_pk = crsql_hash_pk(1)" + )?; + tomb_count_a.step()?; + assert_eq!(tomb_count_a.column_int64(0), 1, "exactly one tombstone on A"); + + let tomb_count_b = db_b.db.prepare_v2( + "SELECT COUNT(*) FROM item__crsql_v2_tombstones WHERE hashed_pk = crsql_hash_pk(1)" + )?; + tomb_count_b.step()?; + assert_eq!(tomb_count_b.column_int64(0), 1, "exactly one tombstone on B"); + + libc_println!("=== test_tombstone_conflict_resolution PASS ==="); + Ok(()) +} + +/// Test that all V2 write paths fail with descriptive errors when ts is not set. +/// Each operation should return an error (not a SIGSEGV or silent corruption). +fn test_ts_not_set_errors() -> Result<(), ResultCode> { + libc_println!("=== test_ts_not_set_errors START ==="); + + // --- crsql_as_crr without ts in V2 mode --- + { + let db = crate::opendb()?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + // Do NOT call crsql_set_ts — should error + let rc = db.db.exec_safe("SELECT crsql_as_crr('foo')"); + assert!(rc.is_err(), "crsql_as_crr should fail when ts not set in V2 mode"); + libc_println!(" crsql_as_crr without ts: correctly rejected"); + } + + // --- crsql_begin_alter without ts in V2 mode --- + { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + // Migrate to V2 + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { panic!("migration failed"); } + } + // Do NOT call crsql_set_ts — should error + let rc = db.db.exec_safe("SELECT crsql_begin_alter('foo')"); + assert!(rc.is_err(), "crsql_begin_alter should fail when ts not set"); + libc_println!(" crsql_begin_alter without ts: correctly rejected"); + } + + // --- crsql_commit_alter without ts in V2 mode --- + // crsql_begin_alter creates a savepoint, so ALTER TABLE within it doesn't auto-commit. + // The ts check in crsql_commit_alter fires when ts was never set in this transaction. + { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + let mut remaining = 1; + while remaining > 0 { + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + stmt.step()?; + remaining = stmt.column_int(0) as i32; + if remaining < 0 { panic!("migration failed"); } + } + // Set ts, begin_alter, do the ALTER, then commit — ts is still set + // because begin_alter creates a savepoint (no auto-commit). + // To test the ts=0 path, we use a nested savepoint that releases first, + // causing an auto-commit that resets ts. + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_begin_alter('foo')")?; + db.db.exec_safe("ALTER TABLE foo ADD COLUMN b TEXT")?; + // Release the begin_alter savepoint — this commits and resets ts + db.db.exec_safe("RELEASE SAVEPOINT alter_crr")?; + // Now ts=0, commit_alter should error + let rc = db.db.exec_safe("SELECT crsql_commit_alter('foo')"); + assert!(rc.is_err(), "crsql_commit_alter should fail when ts not set after savepoint release"); + libc_println!(" crsql_commit_alter without ts: correctly rejected"); + } + + // --- INSERT on V2 CRR table without ts --- + { + let db = crate::opendb()?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + // Do NOT call crsql_set_ts — INSERT trigger should error + let rc = db.db.exec_safe("INSERT INTO foo VALUES (1, 'x')"); + assert!(rc.is_err(), "INSERT on V2 CRR should fail when ts not set"); + libc_println!(" INSERT without ts: correctly rejected"); + } + + // --- UPDATE on V2 CRR table without ts --- + { + let db = crate::opendb()?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x')")?; + // Do NOT call crsql_set_ts — UPDATE trigger should error + let rc = db.db.exec_safe("UPDATE foo SET a = 'y' WHERE id = 1"); + assert!(rc.is_err(), "UPDATE on V2 CRR should fail when ts not set"); + libc_println!(" UPDATE without ts: correctly rejected"); + } + + // --- DELETE on V2 CRR table without ts --- + { + let db = crate::opendb()?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x')")?; + // Do NOT call crsql_set_ts — DELETE trigger should error + let rc = db.db.exec_safe("DELETE FROM foo WHERE id = 1"); + assert!(rc.is_err(), "DELETE on V2 CRR should fail when ts not set"); + libc_println!(" DELETE without ts: correctly rejected"); + } + + // --- crsql_incremental_maintenance without ts --- + { + let db = crate::opendb()?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x')")?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 2)")?; + // Do NOT call crsql_set_ts — should error + let stmt = db.db.prepare_v2("SELECT crsql_incremental_maintenance(1000)")?; + let rc = stmt.step(); + // incremental_maintenance returns -1 via result_int when ts not set, + // so step() succeeds but returns a negative value + let result = stmt.column_int(0); + assert!(result < 0, "crsql_incremental_maintenance should return < 0 when ts not set, got {}", result); + libc_println!(" crsql_incremental_maintenance without ts: correctly returned error ({})", result); + } + + // --- INSERT INTO crsql_changes (sync) without ts --- + { + let db = crate::opendb()?; + db.db.exec_safe("SELECT crsql_config_set('metadata-write-version', 3)")?; + db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY NOT NULL, a)")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("SELECT crsql_as_crr('foo')")?; + db.db.exec_safe("SELECT crsql_set_ts('1700000000')")?; + db.db.exec_safe("INSERT INTO foo VALUES (1, 'x')")?; + // Read changes from source + let read_stmt = db.db.prepare_v2("SELECT * FROM crsql_changes")?; + read_stmt.step()?; + // Try to merge back without ts — should error + let merge_stmt = db.db.prepare_v2( + "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + )?; + for i in 0..10 { + merge_stmt.bind_value(i + 1, read_stmt.column_value(i)?)?; + } + let rc = merge_stmt.step(); + assert!(rc.is_err(), "INSERT INTO crsql_changes should fail when ts not set"); + libc_println!(" INSERT INTO crsql_changes without ts: correctly rejected"); + } + + libc_println!("=== test_ts_not_set_errors PASS ==="); + Ok(()) +} + +pub fn run_suite() -> Result<(), ResultCode> { + test_pack_agg_matches_pack_columns()?; + test_pack_agg_with_nulls()?; + test_pack_agg_empty()?; + test_pack_agg_integers()?; + test_hash_pk_deterministic()?; + test_hash_pk_different_types()?; + test_varint_count_backward_compat()?; + test_metadata_use_version_dispatch()?; + test_packed_wire_format()?; + test_v2_hash_tombstone()?; + test_migration_with_data()?; + test_dual_write_multiple_rows()?; + test_dual_write_seq_consistency()?; + test_dual_write_delete_resurrect()?; + test_dual_write_seq_fuzz()?; + test_cross_mode_sync_roundtrip()?; + test_compile_const_mismatch()?; + test_v2_wire_single_col_sync()?; + test_v1_rejects_v2_wire()?; + test_v2_wire_packed_resurrection()?; + test_dual_write_wire_convergence()?; + test_tombstone_conflict_resolution()?; + test_ts_not_set_errors()?; + Ok(()) +} diff --git a/core/rs/sqlite-rs-embedded b/core/rs/sqlite-rs-embedded index aba562843..a8e12fa5d 160000 --- a/core/rs/sqlite-rs-embedded +++ b/core/rs/sqlite-rs-embedded @@ -1 +1 @@ -Subproject commit aba562843b26642e2242f8e3388b83a1a2625031 +Subproject commit a8e12fa5dc156d0a57e7eaec59978b5e4f8e96f4 diff --git a/core/src/changes-vtab-rowid.test.c b/core/src/changes-vtab-rowid.test.c index fef376f0e..7c40c55c9 100644 --- a/core/src/changes-vtab-rowid.test.c +++ b/core/src/changes-vtab-rowid.test.c @@ -57,6 +57,7 @@ static void testRowidsForReads() { rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key not null, b);", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0); assert(rc == SQLITE_OK); sqlite3_exec(db, "INSERT INTO foo VALUES (1,2);", 0, 0, 0); @@ -74,12 +75,14 @@ static void testRowidsForReads() { rc = sqlite3_exec(db, "CREATE TABLE bar (a primary key not null, b)", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('bar');", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO bar VALUES (1,2);", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO bar VALUES (2,3);", 0, 0, 0); rc += sqlite3_exec(db, "CREATE TABLE baz (a primary key not null, b)", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('baz');", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO baz VALUES (1,2);", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO baz VALUES (2,3);", 0, 0, 0); diff --git a/core/src/changes-vtab.c b/core/src/changes-vtab.c index 6b041a923..ef4e4f7ff 100644 --- a/core/src/changes-vtab.c +++ b/core/src/changes-vtab.c @@ -25,10 +25,10 @@ static int changesConnect(sqlite3 *db, void *pAux, int argc, rc = sqlite3_declare_vtab( db, - "CREATE TABLE x([table] TEXT NOT NULL, [pk] BLOB NOT NULL, [cid] TEXT " + "CREATE TABLE x([table] TEXT NOT NULL, [pk] BLOB NOT NULL, [cid] BLOB " "NOT NULL, [val] ANY, [col_version] INTEGER NOT NULL, [db_version] " "INTEGER NOT NULL, [site_id] BLOB NOT NULL, [cl] INTEGER NOT NULL, [seq] " - "INTEGER NOT NULL, [ts] TEXT NOT NULL)"); + "BLOB NOT NULL, [ts] TEXT NOT NULL)"); if (rc != SQLITE_OK) { *pzErr = sqlite3_mprintf("Could not define the table"); return rc; diff --git a/core/src/changes-vtab.test.c b/core/src/changes-vtab.test.c index 7f2e52bcd..d5a2a41a7 100644 --- a/core/src/changes-vtab.test.c +++ b/core/src/changes-vtab.test.c @@ -21,6 +21,7 @@ static void testManyPkTable() { rc = sqlite3_exec( db, "CREATE TABLE foo (a not null, b not null, c, primary key (a, b));", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0); assert(rc == SQLITE_OK); rc += sqlite3_exec(db, "INSERT INTO foo VALUES (4,5,6);", 0, 0, 0); @@ -67,6 +68,7 @@ static void testFilters() { rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key not null, b);", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0); assert(rc == SQLITE_OK); rc += sqlite3_exec(db, "INSERT INTO foo VALUES (1,2);", 0, 0, 0); diff --git a/core/src/consts.h b/core/src/consts.h index 60bc1649d..a62a96dc8 100644 --- a/core/src/consts.h +++ b/core/src/consts.h @@ -36,6 +36,6 @@ // 00 00 00 00 // Given we can't prefix an int with 0s, read from right to left. // Rightmost is always `bb` -#define CRSQLITE_VERSION 130000 +#define CRSQLITE_VERSION 180000 #endif diff --git a/core/src/crsqlite.test.c b/core/src/crsqlite.test.c index 2dce7cc55..9dcdb4e78 100644 --- a/core/src/crsqlite.test.c +++ b/core/src/crsqlite.test.c @@ -62,6 +62,10 @@ int syncLeftToRight(sqlite3 *db1, sqlite3 *db2, sqlite3_int64 since) { assert(rc == SQLITE_OK); // printf("err: %s\n", err); + // Set ts so merge_insert doesn't reject the sync + sqlite3_exec(db2, "BEGIN", 0, 0, 0); + sqlite3_exec(db2, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); + while (sqlite3_step(pStmtRead) == SQLITE_ROW) { for (int i = 0; i < 9; ++i) { assert(sqlite3_bind_value(pStmtWrite, i + 1, @@ -72,6 +76,8 @@ int syncLeftToRight(sqlite3 *db1, sqlite3 *db2, sqlite3_int64 since) { sqlite3_reset(pStmtWrite); } + sqlite3_exec(db2, "COMMIT", 0, 0, 0); + sqlite3_finalize(pStmtWrite); sqlite3_finalize(pStmtRead); sqlite3_finalize(pStmt); @@ -106,6 +112,7 @@ static int createSimpleSchema(sqlite3 *db, char **err) { int rc = SQLITE_OK; rc += sqlite3_exec(db, "create table foo (a primary key not null, b);", 0, 0, err); + rc += sqlite3_exec(db, "select crsql_set_ts('1700000000')", 0, 0, err); rc += sqlite3_exec(db, "select crsql_as_crr('foo');", 0, 0, err); return rc; @@ -287,6 +294,7 @@ static void testSelectChangesAfterChangingColumnName() { rc += sqlite3_exec(db, "CREATE TABLE foo(a primary key not null, b);", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo')", 0, 0, 0); assert(rc == SQLITE_OK); @@ -294,11 +302,13 @@ static void testSelectChangesAfterChangingColumnName() { rc += sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2);", 0, 0, 0); assert(rc == SQLITE_OK); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc = sqlite3_exec(db, "SELECT crsql_begin_alter('foo')", 0, 0, &err); rc += sqlite3_exec(db, "ALTER TABLE foo DROP COLUMN b", 0, 0, 0); rc += sqlite3_exec(db, "ALTER TABLE foo ADD COLUMN c", 0, 0, 0); assert(rc == SQLITE_OK); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc = sqlite3_exec(db, "SELECT crsql_commit_alter('foo')", 0, 0, &err); printf("rc: %d, err: %s\n", rc, err); assert(rc == SQLITE_OK); @@ -495,7 +505,9 @@ static void testLamportCondition() { rc += sqlite3_exec( db2, "CREATE TABLE \"hoot\" (\"a\", \"b\" primary key not null, \"c\")", 0, 0, 0); + rc += sqlite3_exec(db1, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db1, "SELECT crsql_as_crr('hoot');", 0, 0, 0); + rc += sqlite3_exec(db2, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db2, "SELECT crsql_as_crr('hoot');", 0, 0, 0); assert(rc == SQLITE_OK); @@ -566,7 +578,9 @@ static void noopsDoNotMoveClocks() { rc += sqlite3_exec( db2, "CREATE TABLE \"hoot\" (\"a\", \"b\" primary key not null, \"c\")", 0, 0, 0); + rc += sqlite3_exec(db1, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db1, "SELECT crsql_as_crr('hoot');", 0, 0, 0); + rc += sqlite3_exec(db2, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db2, "SELECT crsql_as_crr('hoot');", 0, 0, 0); assert(rc == SQLITE_OK); @@ -615,6 +629,7 @@ static void testPullingOnlyLocalChanges() { rc = sqlite3_open(":memory:", &db); rc += sqlite3_exec(db, "CREATE TABLE node (id primary key not null, content)", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('node')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO node VALUES (1, 'some str')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO node VALUES (2, 'other str')", 0, 0, 0); diff --git a/core/src/ext-data.c b/core/src/ext-data.c index 5ae04582d..5959199ac 100644 --- a/core/src/ext-data.c +++ b/core/src/ext-data.c @@ -91,6 +91,9 @@ crsql_ExtData *crsql_newExtData(sqlite3 *db) { // set defaults! pExtData->mergeEqualValues = 0; + pExtData->metadataWriteVersion = 1; // V1 + pExtData->metadataUseVersion = 1; // V1 + pExtData->syncLogVersion = 1; // V1 while (sqlite3_step(pStmt) == SQLITE_ROW) { const unsigned char *name = sqlite3_column_text(pStmt, 0); @@ -105,6 +108,18 @@ crsql_ExtData *crsql_newExtData(sqlite3 *db) { crsql_freeExtData(pExtData); return 0; } + } else if (strcmp("metadata-write-version", (char *)name) == 0) { + if (colType == SQLITE_INTEGER) { + pExtData->metadataWriteVersion = sqlite3_column_int(pStmt, 1); + } + } else if (strcmp("metadata-use-version", (char *)name) == 0) { + if (colType == SQLITE_INTEGER) { + pExtData->metadataUseVersion = sqlite3_column_int(pStmt, 1); + } + } else if (strcmp("sync-log-version", (char *)name) == 0) { + if (colType == SQLITE_INTEGER) { + pExtData->syncLogVersion = sqlite3_column_int(pStmt, 1); + } } else { // unhandled config setting } diff --git a/core/src/ext-data.h b/core/src/ext-data.h index 2d8dd9b41..6eac19208 100644 --- a/core/src/ext-data.h +++ b/core/src/ext-data.h @@ -47,6 +47,9 @@ struct crsql_ExtData { sqlite3_stmt *pSelectClockTablesStmt; int mergeEqualValues; + int metadataWriteVersion; + int metadataUseVersion; + int syncLogVersion; unsigned long long timestamp; void *ordinalMap; }; diff --git a/core/src/is-crr.test.c b/core/src/is-crr.test.c index 14a61ac69..2c39391c4 100644 --- a/core/src/is-crr.test.c +++ b/core/src/is-crr.test.c @@ -30,6 +30,8 @@ static void testCrrIsCrr() { rc = sqlite3_exec(db, "CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)", 0, 0, 0); assert(rc == SQLITE_OK); + rc = sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); + assert(rc == SQLITE_OK); rc = sqlite3_exec(db, "SELECT crsql_as_crr('foo')", 0, 0, 0); assert(rc == SQLITE_OK); @@ -48,6 +50,8 @@ static void testDestroyedCrrIsNotCrr() { rc = sqlite3_exec(db, "CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)", 0, 0, 0); assert(rc == SQLITE_OK); + rc = sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); + assert(rc == SQLITE_OK); rc = sqlite3_exec(db, "SELECT crsql_as_crr('foo')", 0, 0, 0); assert(rc == SQLITE_OK); rc = sqlite3_exec(db, "SELECT crsql_as_table('foo')", 0, 0, 0); diff --git a/core/src/rows-impacted.test.c b/core/src/rows-impacted.test.c index 8ad8dc50b..b8d6ba4e7 100644 --- a/core/src/rows-impacted.test.c +++ b/core/src/rows-impacted.test.c @@ -11,8 +11,8 @@ static sqlite3 *createDb() { int rc = SQLITE_OK; sqlite3 *db; rc = sqlite3_open(":memory:", &db); - rc += - sqlite3_exec(db, "CREATE TABLE foo (a primary key not null, b)", 0, 0, 0); + rc += sqlite3_exec(db, "CREATE TABLE foo (a primary key not null, b)", 0, 0, 0); + rc += sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo')", 0, 0, 0); assert(rc == SQLITE_OK); return db; @@ -26,6 +26,7 @@ static void testSingleInsertSingleTx() { sqlite3_stmt *pStmt = 0; rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', " "2, 1, 1, NULL, 1, 1, '0')", @@ -57,6 +58,7 @@ static void testManyInsertsInATx() { sqlite3_stmt *pStmt = 0; rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', " "2, 1, 1, NULL, 1, 1, '0')", @@ -94,6 +96,7 @@ static void testMultipartInsertInTx() { sqlite3_stmt *pStmt = 0; rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', " "2, 1, 1, NULL, 1, 1, '0'), " @@ -126,6 +129,7 @@ static void testManyTxns() { sqlite3_stmt *pStmt = 0; rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', " "2, 1, 1, NULL, 1, 1, '0')", @@ -138,6 +142,7 @@ static void testManyTxns() { assert(rc == SQLITE_OK); rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010902', 'b', " "2, 1, 1, NULL, 1, 1, '0')", @@ -168,6 +173,7 @@ static void testManyTxns() { // sqlite3_stmt *pStmt = 0; // rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); +// sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); // rc += sqlite3_prepare_v2( // db, // "INSERT INTO crsql_changes VALUES ('foo', 1, 'b', 2, 1, 1, NULL), " @@ -196,6 +202,7 @@ static void testUpdateThatDoesNotChangeAnything() { rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0); rc += sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', " "crsql_pack_columns(1), 'b', 2, 1, 1, NULL, 1, 1, '0')", @@ -209,6 +216,7 @@ static void testUpdateThatDoesNotChangeAnything() { // now test value < rc += sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', " "crsql_pack_columns(1), 'b', 0, 1, 1, NULL, 1, 1, '0')", @@ -222,6 +230,7 @@ static void testUpdateThatDoesNotChangeAnything() { // now test clock < rc += sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', " "crsql_pack_columns(1), 'b', 2, 0, 0, NULL, 1, 1, '0')", @@ -248,6 +257,7 @@ static void testDeleteThatDoesNotChangeAnything() { rc = sqlite3_exec(db, "DELETE FROM foo", 0, 0, 0); rc += sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec( db, "INSERT INTO crsql_changes VALUES ('foo', crsql_pack_columns(1), " @@ -274,6 +284,7 @@ static void testDelete() { rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0); rc += sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', " "'-1', NULL, 2, 2, NULL, 2, 1, '0')", //__crsql_del @@ -299,6 +310,7 @@ static void testCreateThatDoesNotChangeAnything() { rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0); rc += sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', " "2, 1, 1, NULL, 1, 1, '0')", @@ -324,6 +336,7 @@ static void testValueWin() { rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0); rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', " "3, 1, 1, X'00000000000000000000000000000000', 1, 1, '0')", @@ -349,6 +362,7 @@ static void testClockWin() { rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0); rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + sqlite3_exec(db, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db, "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', " "2, 2, 2, NULL, 1, 1, '0')", diff --git a/core/src/sandbox.test.c b/core/src/sandbox.test.c index 61a790d85..ab98475fb 100644 --- a/core/src/sandbox.test.c +++ b/core/src/sandbox.test.c @@ -19,7 +19,9 @@ static void testSandbox() { sqlite3_exec(db1, "CREATE TABLE foo (a primary key not null);", 0, 0, 0); rc += sqlite3_exec(db2, "CREATE TABLE foo (a primary key not null);", 0, 0, 0); + rc += sqlite3_exec(db1, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db1, "SELECT crsql_as_crr('foo')", 0, 0, 0); + rc += sqlite3_exec(db2, "SELECT crsql_set_ts('1700000000')", 0, 0, 0); rc += sqlite3_exec(db2, "SELECT crsql_as_crr('foo')", 0, 0, 0); rc += sqlite3_exec(db1, "INSERT INTO foo VALUES (1)", 0, 0, 0); assert(rc == SQLITE_OK); diff --git a/core/src/sqlite/shell.c b/core/src/sqlite/shell.c index 647a21422..e357f9e9a 100644 --- a/core/src/sqlite/shell.c +++ b/core/src/sqlite/shell.c @@ -246,6 +246,7 @@ typedef unsigned char u8; #if SQLITE_OS_WINRT #include #endif +#undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #include @@ -466,20 +467,25 @@ static int bail_on_error = 0; */ static int stdin_is_interactive = 1; +/* +** If build is for non-RT Windows, without 3rd-party line editing, +** console input and output may be done in a UTF-8 compatible way, +** if the OS is capable of it and the --no-utf8 option is not seen. +*/ #if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \ - && !defined(SHELL_OMIT_WIN_UTF8) + && !defined(SHELL_OMIT_WIN_UTF8) && !SQLITE_OS_WINRT # define SHELL_WIN_UTF8_OPT 1 +/* Record whether to do UTF-8 console I/O translation per stream. */ + static int console_utf8_in = 0; + static int console_utf8_out = 0; +/* Record whether can do UTF-8 or --no-utf8 seen in invocation. */ + static int mbcs_opted = 1; /* Assume cannot do until shown otherwise. */ #else +# define console_utf8_in 0 +# define console_utf8_out 0 # define SHELL_WIN_UTF8_OPT 0 #endif -#if SHELL_WIN_UTF8_OPT -/* -** Setup console for UTF-8 input/output when following variable true. -*/ -static int console_utf8 = 0; -#endif - /* ** On Windows systems we have to know if standard output is a console ** in order to translate UTF-8 into MBCS. The following variable is @@ -613,13 +619,13 @@ static char *dynamicContinuePrompt(void){ #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ #if SHELL_WIN_UTF8_OPT -/* Following struct is used for -utf8 operation. */ +/* Following struct is used for UTF-8 console I/O. */ static struct ConsoleState { int stdinEof; /* EOF has been seen on console input */ int infsMode; /* Input file stream mode upon shell start */ UINT inCodePage; /* Input code page upon shell start */ UINT outCodePage; /* Output code page upon shell start */ - HANDLE hConsoleIn; /* Console input handle */ + HANDLE hConsole; /* Console input or output handle */ DWORD consoleMode; /* Console mode upon shell start */ } conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; @@ -628,50 +634,123 @@ static struct ConsoleState { #endif /* -** Prepare console, (if known to be a WIN32 console), for UTF-8 -** input (from either typing or suitable paste operations) and for -** UTF-8 rendering. This may "fail" with a message to stderr, where -** the preparation is not done and common "code page" issues occur. +** If given stream number is a console, return 1 and get some attributes, +** else return 0 and set the output attributes to invalid values. */ -static void console_prepare(void){ - HANDLE hCI = GetStdHandle(STD_INPUT_HANDLE); - DWORD consoleMode = 0; - if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR - && GetConsoleMode( hCI, &consoleMode) ){ - if( !IsValidCodePage(CP_UTF8) ){ - fprintf(stderr, "Cannot use UTF-8 code page.\n"); - console_utf8 = 0; - return; - } - conState.hConsoleIn = hCI; - conState.consoleMode = consoleMode; - conState.inCodePage = GetConsoleCP(); - conState.outCodePage = GetConsoleOutputCP(); - SetConsoleCP(CP_UTF8); +static short console_attrs(unsigned stnum, HANDLE *pH, DWORD *pConsMode){ + static int stid[3] = { STD_INPUT_HANDLE,STD_OUTPUT_HANDLE,STD_ERROR_HANDLE }; + HANDLE h; + *pH = INVALID_HANDLE_VALUE; + *pConsMode = 0; + if( stnum > 2 ) return 0; + h = GetStdHandle(stid[stnum]); + if( h!=*pH && GetFileType(h)==FILE_TYPE_CHAR && GetConsoleMode(h,pConsMode) ){ + *pH = h; + return 1; + } + return 0; +} + +/* +** Perform a runtime test of Windows console to determine if it can +** do char-stream I/O correctly when the code page is set to CP_UTF8. +** Returns are: 1 => yes it can, 0 => no it cannot +** +** The console's output code page is momentarily set, then restored. +** So this should only be run when the process is given use of the +** console for either input or output. +*/ +static short ConsoleDoesUTF8(void){ + UINT ocp = GetConsoleOutputCP(); + const char TrialUtf8[] = { '\xC8', '\xAB' }; /* "ȫ" or 2 MBCS characters */ + WCHAR aReadBack[1] = { 0 }; /* Read back as 0x022B when decoded as UTF-8. */ + CONSOLE_SCREEN_BUFFER_INFO csbInfo = {0}; + /* Create an inactive screen buffer with which to do the experiment. */ + HANDLE hCSB = CreateConsoleScreenBuffer(GENERIC_READ|GENERIC_WRITE, 0, 0, + CONSOLE_TEXTMODE_BUFFER, NULL); + if( hCSB!=INVALID_HANDLE_VALUE ){ + COORD cpos = {0,0}; + DWORD rbc; + SetConsoleCursorPosition(hCSB, cpos); SetConsoleOutputCP(CP_UTF8); - consoleMode |= ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; - SetConsoleMode(conState.hConsoleIn, consoleMode); + /* Write 2 chars which are a single character in UTF-8 but more in MBCS. */ + WriteConsoleA(hCSB, TrialUtf8, sizeof(TrialUtf8), NULL, NULL); + ReadConsoleOutputCharacterW(hCSB, &aReadBack[0], 1, cpos, &rbc); + GetConsoleScreenBufferInfo(hCSB, &csbInfo); + SetConsoleOutputCP(ocp); + CloseHandle(hCSB); + } + /* Return 1 if cursor advanced by 1 position, else 0. */ + return (short)(csbInfo.dwCursorPosition.X == 1 && aReadBack[0] == 0x022B); +} + +static short in_console = 0; +static short out_console = 0; + +/* +** Determine whether either normal I/O stream is the console, +** and whether it can do UTF-8 translation, setting globals +** in_console, out_console and mbcs_opted accordingly. +*/ +static void probe_console(void){ + HANDLE h; + DWORD cMode; + in_console = console_attrs(0, &h, &cMode); + out_console = console_attrs(1, &h, &cMode); + if( in_console || out_console ) mbcs_opted = !ConsoleDoesUTF8(); +} + +/* +** If console is used for normal I/O, absent a --no-utf8 option, +** prepare console for UTF-8 input (from either typing or suitable +** paste operations) and/or for UTF-8 output rendering. +** +** The console state upon entry is preserved, in conState, so that +** console_restore() can later restore the same console state. +** +** The globals console_utf8_in and console_utf8_out are set, for +** later use in selecting UTF-8 or MBCS console I/O translations. +** This routine depends upon globals set by probe_console(). +*/ +static void console_prepare_utf8(void){ + struct ConsoleState csWork = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; + + console_utf8_in = console_utf8_out = 0; + if( (!in_console && !out_console) || mbcs_opted ) return; + console_attrs((in_console)? 0 : 1, &conState.hConsole, &conState.consoleMode); + conState.inCodePage = GetConsoleCP(); + conState.outCodePage = GetConsoleOutputCP(); + if( in_console ){ + SetConsoleCP(CP_UTF8); + DWORD newConsoleMode = conState.consoleMode + | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; + SetConsoleMode(conState.hConsole, newConsoleMode); conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); - console_utf8 = 1; - }else{ - console_utf8 = 0; + console_utf8_in = 1; + } + if( out_console ){ + SetConsoleOutputCP(CP_UTF8); + console_utf8_out = 1; } } /* -** Undo the effects of console_prepare(), if any. +** Undo the effects of console_prepare_utf8(), if any. */ static void SQLITE_CDECL console_restore(void){ - if( console_utf8 && conState.inCodePage!=0 - && conState.hConsoleIn!=INVALID_HANDLE_VALUE ){ - _setmode(_fileno(stdin), conState.infsMode); - SetConsoleCP(conState.inCodePage); - SetConsoleOutputCP(conState.outCodePage); - SetConsoleMode(conState.hConsoleIn, conState.consoleMode); + if( (console_utf8_in||console_utf8_out) + && conState.hConsole!=INVALID_HANDLE_VALUE ){ + if( console_utf8_in ){ + SetConsoleCP(conState.inCodePage); + _setmode(_fileno(stdin), conState.infsMode); + } + if( console_utf8_out ) SetConsoleOutputCP(conState.outCodePage); + SetConsoleMode(conState.hConsole, conState.consoleMode); /* Avoid multiple calls. */ - conState.hConsoleIn = INVALID_HANDLE_VALUE; + conState.hConsole = INVALID_HANDLE_VALUE; conState.consoleMode = 0; - console_utf8 = 0; + console_utf8_in = 0; + console_utf8_out = 0; } } @@ -679,11 +758,11 @@ static void SQLITE_CDECL console_restore(void){ ** Collect input like fgets(...) with special provisions for input ** from the Windows console to get around its strange coding issues. ** Defers to plain fgets() when input is not interactive or when the -** startup option, -utf8, has not been provided or taken effect. +** UTF-8 input is unavailable or opted out. */ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ if( fin==0 ) fin = stdin; - if( fin==stdin && stdin_is_interactive && console_utf8 ){ + if( fin==stdin && stdin_is_interactive && console_utf8_in ){ # define SQLITE_IALIM 150 wchar_t wbuf[SQLITE_IALIM]; int lend = 0; @@ -696,7 +775,7 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ ? SQLITE_IALIM : (ncmax-1 - noc)/4; # undef SQLITE_IALIM DWORD nbr = 0; - BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0); + BOOL bRC = ReadConsoleW(conState.hConsole, wbuf, na, &nbr, 0); if( !bRC || (noc==0 && nbr==0) ) return 0; if( nbr > 0 ){ int nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, @@ -742,20 +821,16 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ /* ** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, and if the -utf8 -** option is unavailable or (available and inactive), translate the +** console and if this is running on a Windows machine, and if UTF-8 +** output unavailable (or available but opted out), translate the ** output from UTF-8 into MBCS for output through 8-bit stdout stream. -** (With -utf8 active, no translation is needed and must not be done.) +** (Without -no-utf8, no translation is needed and must not be done.) */ #if defined(_WIN32) || defined(WIN32) void utf8_printf(FILE *out, const char *zFormat, ...){ va_list ap; va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) -# if SHELL_WIN_UTF8_OPT - && !console_utf8 -# endif - ){ + if( stdout_is_console && (out==stdout || out==stderr) && !console_utf8_out ){ char *z1 = sqlite3_vmprintf(zFormat, ap); char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); sqlite3_free(z1); @@ -966,14 +1041,10 @@ static char *local_getline(char *zLine, FILE *in){ } } #if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, without -utf8, + /* For interactive input on Windows systems, with -no-utf8, ** translate the multi-byte characterset characters into UTF-8. - ** This is the translation that predates the -utf8 option. */ - if( stdin_is_interactive && in==stdin -# if SHELL_WIN_UTF8_OPT - && !console_utf8 -# endif /* SHELL_WIN_UTF8_OPT */ - ){ + ** This is the translation that predates console UTF-8 input. */ + if( stdin_is_interactive && in==stdin && !console_utf8_in ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ i64 nTrans = strlen(zTrans)+1; @@ -1224,6 +1295,46 @@ static char *shellFakeSchema( return s.z; } +/* +** SQL function: strtod(X) +** +** Use the C-library strtod() function to convert string X into a double. +** Used for comparing the accuracy of SQLite's internal text-to-float conversion +** routines against the C-library. +*/ +static void shellStrtod( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + char *z = (char*)sqlite3_value_text(apVal[0]); + UNUSED_PARAMETER(nVal); + if( z==0 ) return; + sqlite3_result_double(pCtx, strtod(z,0)); +} + +/* +** SQL function: dtostr(X) +** +** Use the C-library printf() function to convert real value X into a string. +** Used for comparing the accuracy of SQLite's internal float-to-text conversion +** routines against the C-library. +*/ +static void shellDtostr( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + double r = sqlite3_value_double(apVal[0]); + int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; + char z[400]; + if( n<1 ) n = 1; + if( n>350 ) n = 350; + snprintf(z, sizeof(z)-1, "%#+.*e", n, r); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); +} + + /* ** SQL function: shell_module_schema(X) ** @@ -1473,7 +1584,7 @@ struct DIR { #endif /* -** Provide the function prototype for the POSIX compatiable getenv() +** Provide the function prototype for the POSIX compatible getenv() ** function. This function is not thread-safe. */ @@ -1799,6 +1910,188 @@ int sqlite3MemTraceDeactivate(void){ } /************************* End ../ext/misc/memtrace.c ********************/ +/************************* Begin ../ext/misc/pcachetrace.c ******************/ +/* +** 2023-06-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_PCACHE2 +** mechanism to add a tracing layer on top of pluggable page cache of +** SQLite. If this extension is registered prior to sqlite3_initialize(), +** it will cause all page cache activities to be logged on standard output, +** or to some other FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --pcachetrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original page cache routines */ +static sqlite3_pcache_methods2 pcacheBase; +static FILE *pcachetraceOut; + +/* Methods that trace pcache activity */ +static int pcachetraceInit(void *pArg){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p)\n", pArg); + } + nRes = pcacheBase.xInit(pArg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p) -> %d\n", pArg, nRes); + } + return nRes; +} +static void pcachetraceShutdown(void *pArg){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShutdown(%p)\n", pArg); + } + pcacheBase.xShutdown(pArg); +} +static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ + sqlite3_pcache *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d)\n", + szPage, szExtra, bPurge); + } + pRes = pcacheBase.xCreate(szPage, szExtra, bPurge); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d) -> %p\n", + szPage, szExtra, bPurge, pRes); + } + return pRes; +} +static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCachesize(%p, %d)\n", p, nCachesize); + } + pcacheBase.xCachesize(p, nCachesize); +} +static int pcachetracePagecount(sqlite3_pcache *p){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p)\n", p); + } + nRes = pcacheBase.xPagecount(p); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p) -> %d\n", p, nRes); + } + return nRes; +} +static sqlite3_pcache_page *pcachetraceFetch( + sqlite3_pcache *p, + unsigned key, + int crFg +){ + sqlite3_pcache_page *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d)\n", p, key, crFg); + } + pRes = pcacheBase.xFetch(p, key, crFg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d) -> %p\n", + p, key, crFg, pRes); + } + return pRes; +} +static void pcachetraceUnpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int bDiscard +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xUnpin(%p, %p, %d)\n", + p, pPg, bDiscard); + } + pcacheBase.xUnpin(p, pPg, bDiscard); +} +static void pcachetraceRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned oldKey, + unsigned newKey +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xRekey(%p, %p, %u, %u)\n", + p, pPg, oldKey, newKey); + } + pcacheBase.xRekey(p, pPg, oldKey, newKey); +} +static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xTruncate(%p, %u)\n", p, n); + } + pcacheBase.xTruncate(p, n); +} +static void pcachetraceDestroy(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xDestroy(%p)\n", p); + } + pcacheBase.xDestroy(p); +} +static void pcachetraceShrink(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShrink(%p)\n", p); + } + pcacheBase.xShrink(p); +} + +/* The substitute pcache methods */ +static sqlite3_pcache_methods2 ersaztPcacheMethods = { + 0, + 0, + pcachetraceInit, + pcachetraceShutdown, + pcachetraceCreate, + pcachetraceCachesize, + pcachetracePagecount, + pcachetraceFetch, + pcachetraceUnpin, + pcachetraceRekey, + pcachetraceTruncate, + pcachetraceDestroy, + pcachetraceShrink +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3PcacheTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &ersaztPcacheMethods); + } + } + pcachetraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3PcacheTraceDeactivate(void){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + memset(&pcacheBase, 0, sizeof(pcacheBase)); + } + } + pcachetraceOut = 0; + return rc; +} + +/************************* End ../ext/misc/pcachetrace.c ********************/ /************************* Begin ../ext/misc/shathree.c ******************/ /* ** 2017-03-08 @@ -2683,41 +2976,24 @@ static void decimal_free(Decimal *p){ } /* -** Allocate a new Decimal object. Initialize it to the number given -** by the input string. +** Allocate a new Decimal object initialized to the text in zIn[]. +** Return NULL if any kind of error occurs. */ -static Decimal *decimal_new( - sqlite3_context *pCtx, - sqlite3_value *pIn, - int nAlt, - const unsigned char *zAlt -){ - Decimal *p; - int n, i; - const unsigned char *zIn; +static Decimal *decimalNewFromText(const char *zIn, int n){ + Decimal *p = 0; + int i; int iExp = 0; + p = sqlite3_malloc( sizeof(*p) ); - if( p==0 ) goto new_no_mem; + if( p==0 ) goto new_from_text_failed; p->sign = 0; p->oom = 0; p->isInit = 1; p->isNull = 0; p->nDigit = 0; p->nFrac = 0; - if( zAlt ){ - n = nAlt, - zIn = zAlt; - }else{ - if( sqlite3_value_type(pIn)==SQLITE_NULL ){ - p->a = 0; - p->isNull = 1; - return p; - } - n = sqlite3_value_bytes(pIn); - zIn = sqlite3_value_text(pIn); - } p->a = sqlite3_malloc64( n+1 ); - if( p->a==0 ) goto new_no_mem; + if( p->a==0 ) goto new_from_text_failed; for(i=0; isspace(zIn[i]); i++){} if( zIn[i]=='-' ){ p->sign = 1; @@ -2768,7 +3044,7 @@ static Decimal *decimal_new( } if( iExp>0 ){ p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_no_mem; + if( p->a==0 ) goto new_from_text_failed; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; } @@ -2787,7 +3063,7 @@ static Decimal *decimal_new( } if( iExp>0 ){ p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_no_mem; + if( p->a==0 ) goto new_from_text_failed; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); p->nDigit += iExp; @@ -2796,7 +3072,76 @@ static Decimal *decimal_new( } return p; -new_no_mem: +new_from_text_failed: + if( p ){ + if( p->a ) sqlite3_free(p->a); + sqlite3_free(p); + } + return 0; +} + +/* Forward reference */ +static Decimal *decimalFromDouble(double); + +/* +** Allocate a new Decimal object from an sqlite3_value. Return a pointer +** to the new object, or NULL if there is an error. If the pCtx argument +** is not NULL, then errors are reported on it as well. +** +** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted +** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length +** 8 bytes, the resulting double value is expanded into its decimal equivalent. +** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, +** then NULL is returned. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, /* Report error here, if not null */ + sqlite3_value *pIn, /* Construct the decimal object from this */ + int bTextOnly /* Always interpret pIn as text if true */ +){ + Decimal *p = 0; + int eType = sqlite3_value_type(pIn); + if( bTextOnly && (eType==SQLITE_FLOAT || eType==SQLITE_BLOB) ){ + eType = SQLITE_TEXT; + } + switch( eType ){ + case SQLITE_TEXT: + case SQLITE_INTEGER: { + const char *zIn = (const char*)sqlite3_value_text(pIn); + int n = sqlite3_value_bytes(pIn); + p = decimalNewFromText(zIn, n); + if( p==0 ) goto new_failed; + break; + } + + case SQLITE_FLOAT: { + p = decimalFromDouble(sqlite3_value_double(pIn)); + break; + } + + case SQLITE_BLOB: { + const unsigned char *x; + unsigned int i; + sqlite3_uint64 v = 0; + double r; + + if( sqlite3_value_bytes(pIn)!=sizeof(r) ) break; + x = sqlite3_value_blob(pIn); + for(i=0; ioom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + for(nZero=0; nZeroa[nZero]==0; nZero++){} + nFrac = p->nFrac + (nDigit - p->nDigit); + nDigit -= nZero; + z = sqlite3_malloc( nDigit+20 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( nDigit==0 ){ + zero = 0; + a = &zero; + nDigit = 1; + nFrac = 0; + }else{ + a = &p->a[nZero]; + } + if( p->sign && nDigit>0 ){ + z[0] = '-'; + }else{ + z[0] = '+'; + } + z[1] = a[0]+'0'; + z[2] = '.'; + if( nDigit==1 ){ + z[3] = '0'; + i = 4; + }else{ + for(i=1; iisNull ) goto cmp_done; - pB = decimal_new(context, argv[1], 0, 0); + pB = decimal_new(context, argv[1], 1); if( pB==0 || pB->isNull ) goto cmp_done; rc = decimal_cmp(pA, pB); if( rc<0 ) rc = -1; @@ -2963,7 +3353,7 @@ static void decimal_expand(Decimal *p, int nDigit, int nFrac){ } /* -** Add the value pB into pA. +** Add the value pB into pA. A := A + B. ** ** Both pA and pB might become denormalized by this routine. */ @@ -3032,6 +3422,172 @@ static void decimal_add(Decimal *pA, Decimal *pB){ } } +/* +** Multiply A by B. A := A * B +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMul(Decimal *pA, Decimal *pB){ + signed char *acc = 0; + int i, j, k; + int minFrac; + + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + if( acc==0 ){ + pA->oom = 1; + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + +mul_end: + sqlite3_free(acc); +} + +/* +** Create a new Decimal object that contains an integer power of 2. +*/ +static Decimal *decimalPow2(int N){ + Decimal *pA = 0; /* The result to be returned */ + Decimal *pX = 0; /* Multiplier */ + if( N<-20000 || N>20000 ) goto pow2_fault; + pA = decimalNewFromText("1.0", 3); + if( pA==0 || pA->oom ) goto pow2_fault; + if( N==0 ) return pA; + if( N>0 ){ + pX = decimalNewFromText("2.0", 3); + }else{ + N = -N; + pX = decimalNewFromText("0.5", 3); + } + if( pX==0 || pX->oom ) goto pow2_fault; + while( 1 /* Exit by break */ ){ + if( N & 1 ){ + decimalMul(pA, pX); + if( pA->oom ) goto pow2_fault; + } + N >>= 1; + if( N==0 ) break; + decimalMul(pX, pX); + } + decimal_free(pX); + return pA; + +pow2_fault: + decimal_free(pA); + decimal_free(pX); + return 0; +} + +/* +** Use an IEEE754 binary64 ("double") to generate a new Decimal object. +*/ +static Decimal *decimalFromDouble(double r){ + sqlite3_int64 m, a; + int e; + int isNeg; + Decimal *pA; + Decimal *pX; + char zNum[100]; + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 ){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + e = e - 1075; + if( e>971 ){ + return 0; /* A NaN or an Infinity */ + } + } + + /* At this point m is the integer significand and e is the exponent */ + sqlite3_snprintf(sizeof(zNum), zNum, "%lld", m); + pA = decimalNewFromText(zNum, (int)strlen(zNum)); + pX = decimalPow2(e); + decimalMul(pA, pX); + decimal_free(pX); + return pA; +} + +/* +** SQL Function: decimal(X) +** OR: decimal_exp(X) +** +** Convert input X into decimal and then back into text. +** +** If X is originally a float, then a full decimal expansion of that floating +** point value is done. Or if X is an 8-byte blob, it is interpreted +** as a float and similarly expanded. +** +** The decimal_exp(X) function returns the result in exponential notation. +** decimal(X) returns a complete decimal, without the e+NNN at the end. +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0); + UNUSED_PARAMETER(argc); + if( p ){ + if( sqlite3_user_data(context)!=0 ){ + decimal_result_sci(context, p); + }else{ + decimal_result(context, p); + } + decimal_free(p); + } +} + /* ** Compare text in decimal order. */ @@ -3042,8 +3598,8 @@ static int decimalCollFunc( ){ const unsigned char *zA = (const unsigned char*)pKey1; const unsigned char *zB = (const unsigned char*)pKey2; - Decimal *pA = decimal_new(0, 0, nKey1, zA); - Decimal *pB = decimal_new(0, 0, nKey2, zB); + Decimal *pA = decimalNewFromText((const char*)zA, nKey1); + Decimal *pB = decimalNewFromText((const char*)zB, nKey2); int rc; UNUSED_PARAMETER(notUsed); if( pA==0 || pB==0 ){ @@ -3068,8 +3624,8 @@ static void decimalAddFunc( int argc, sqlite3_value **argv ){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); decimal_add(pA, pB); decimal_result(context, pA); @@ -3081,8 +3637,8 @@ static void decimalSubFunc( int argc, sqlite3_value **argv ){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); if( pB ){ pB->sign = !pB->sign; @@ -3120,7 +3676,7 @@ static void decimalSumStep( p->nFrac = 0; } if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pArg = decimal_new(context, argv[0], 0, 0); + pArg = decimal_new(context, argv[0], 1); decimal_add(p, pArg); decimal_free(pArg); } @@ -3135,7 +3691,7 @@ static void decimalSumInverse( p = sqlite3_aggregate_context(context, sizeof(*p)); if( p==0 ) return; if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pArg = decimal_new(context, argv[0], 0, 0); + pArg = decimal_new(context, argv[0], 1); if( pArg ) pArg->sign = !pArg->sign; decimal_add(p, pArg); decimal_free(pArg); @@ -3156,66 +3712,49 @@ static void decimalSumFinalize(sqlite3_context *context){ ** SQL Function: decimal_mul(X, Y) ** ** Return the product of X and Y. -** -** All significant digits after the decimal point are retained. -** Trailing zeros after the decimal point are omitted as long as -** the number of digits after the decimal point is no less than -** either the number of digits in either input. */ static void decimalMulFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); - signed char *acc = 0; - int i, j, k; - int minFrac; + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); if( pA==0 || pA->oom || pA->isNull || pB==0 || pB->oom || pB->isNull ){ goto mul_end; } - acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); - if( acc==0 ){ - sqlite3_result_error_nomem(context); + decimalMul(pA, pB); + if( pA->oom ){ goto mul_end; } - memset(acc, 0, pA->nDigit + pB->nDigit + 2); - minFrac = pA->nFrac; - if( pB->nFracnFrac; - for(i=pA->nDigit-1; i>=0; i--){ - signed char f = pA->a[i]; - int carry = 0, x; - for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ - x = acc[k] + f*pB->a[j] + carry; - acc[k] = x%10; - carry = x/10; - } - x = acc[k] + carry; - acc[k] = x%10; - acc[k-1] += x/10; - } - sqlite3_free(pA->a); - pA->a = acc; - acc = 0; - pA->nDigit += pB->nDigit + 2; - pA->nFrac += pB->nFrac; - pA->sign ^= pB->sign; - while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ - pA->nFrac--; - pA->nDigit--; - } decimal_result(context, pA); mul_end: - sqlite3_free(acc); decimal_free(pA); decimal_free(pB); } +/* +** SQL Function: decimal_pow2(N) +** +** Return the N-th power of 2. N must be an integer. +*/ +static void decimalPow2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); + decimal_result_sci(context, pA); + decimal_free(pA); + } +} + #ifdef _WIN32 #endif @@ -3228,13 +3767,16 @@ int sqlite3_decimal_init( static const struct { const char *zFuncName; int nArg; + int iArg; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { - { "decimal", 1, decimalFunc }, - { "decimal_cmp", 2, decimalCmpFunc }, - { "decimal_add", 2, decimalAddFunc }, - { "decimal_sub", 2, decimalSubFunc }, - { "decimal_mul", 2, decimalMulFunc }, + { "decimal", 1, 0, decimalFunc }, + { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_cmp", 2, 0, decimalCmpFunc }, + { "decimal_add", 2, 0, decimalAddFunc }, + { "decimal_sub", 2, 0, decimalSubFunc }, + { "decimal_mul", 2, 0, decimalMulFunc }, + { "decimal_pow2", 1, 0, decimalPow2Func }, }; unsigned int i; (void)pzErrMsg; /* Unused parameter */ @@ -3244,7 +3786,7 @@ int sqlite3_decimal_init( for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){ rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg, SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, - 0, aFunc[i].xFunc, 0, 0); + aFunc[i].iArg ? db : 0, aFunc[i].xFunc, 0, 0); } if( rc==SQLITE_OK ){ rc = sqlite3_create_window_function(db, "decimal_sum", 1, @@ -4272,6 +4814,37 @@ static void ieee754func_to_blob( } } +/* +** SQL Function: ieee754_inc(r,N) +** +** Move the floating point value r by N quantums and return the new +** values. +** +** Behind the scenes: this routine merely casts r into a 64-bit unsigned +** integer, adds N, then casts the value back into float. +** +** Example: To find the smallest positive number: +** +** SELECT ieee754_inc(0.0,+1); +*/ +static void ieee754inc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double r; + sqlite3_int64 N; + sqlite3_uint64 m1, m2; + double r2; + UNUSED_PARAMETER(argc); + r = sqlite3_value_double(argv[0]); + N = sqlite3_value_int64(argv[1]); + memcpy(&m1, &r, 8); + m2 = m1 + N; + memcpy(&r2, &m2, 8); + sqlite3_result_double(context, r2); +} + #ifdef _WIN32 @@ -4293,7 +4866,7 @@ int sqlite3_ieee_init( { "ieee754_exponent", 1, 2, ieee754func }, { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, - + { "ieee754_inc", 2, 0, ieee754inc }, }; unsigned int i; int rc = SQLITE_OK; @@ -4642,6 +5215,10 @@ static int seriesColumn( return SQLITE_OK; } +#ifndef LARGEST_UINT64 +#define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) +#endif + /* ** Return the rowid for the current row, logically equivalent to n+1 where ** "n" is the ascending integer in the aforesaid production definition. @@ -4649,7 +5226,7 @@ static int seriesColumn( static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; sqlite3_uint64 n = pCur->ss.uSeqIndexNow; - *pRowid = (sqlite3_int64)((n<0xffffffffffffffff)? n+1 : 0); + *pRowid = (sqlite3_int64)((n #include #include +#include #include @@ -10194,7 +10775,8 @@ static int zipfileRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollback */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); @@ -10563,7 +11145,7 @@ void sqlite3_expert_destroy(sqlite3expert*); #endif /* !defined(SQLITE_AMALGAMATION) */ -#ifndef SQLITE_OMIT_VIRTUALTABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE /* typedef sqlite3_int64 i64; */ /* typedef sqlite3_uint64 u64; */ @@ -11193,6 +11775,7 @@ static int idxRegisterVtab(sqlite3expert *p){ 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0, /* xIntegrity */ }; return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p); @@ -12349,6 +12932,88 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ return rc; } +/* +** Define and possibly pretend to use a useless collation sequence. +** This pretense allows expert to accept SQL using custom collations. +*/ +int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){ + (void)up1; + (void)up2; + (void)up3; + (void)up4; + (void)up5; + assert(0); /* VDBE should never be run. */ + return 0; +} +/* And a callback to register above upon actual need */ +void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){ + (void)up1; + sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0); +} + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) +/* +** dummy functions for no-op implementation of UDFs during expert's work +*/ +void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){ + (void)up1; + (void)up2; + (void)up3; + assert(0); /* VDBE should never be run. */ +} +void dummyUDFvalue(sqlite3_context *up1){ + (void)up1; + assert(0); /* VDBE should never be run. */ +} + +/* +** Register UDFs from user database with another. +*/ +int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){ + sqlite3_stmt *pStmt; + int rc = sqlite3_prepare_v2(dbSrc, + "SELECT name,type,enc,narg,flags " + "FROM pragma_function_list() " + "WHERE builtin==0", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + int nargs = sqlite3_column_int(pStmt,3); + int flags = sqlite3_column_int(pStmt,4); + const char *name = (char*)sqlite3_column_text(pStmt,0); + const char *type = (char*)sqlite3_column_text(pStmt,1); + const char *enc = (char*)sqlite3_column_text(pStmt,2); + if( name==0 || type==0 || enc==0 ){ + /* no-op. Only happens on OOM */ + }else{ + int ienc = SQLITE_UTF8; + int rcf = SQLITE_ERROR; + if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE; + else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE; + ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY)); + if( strcmp(type,"w")==0 ){ + rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0, + dummyUDF,dummyUDFvalue,0,0,0); + }else if( strcmp(type,"a")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + 0,dummyUDF,dummyUDFvalue); + }else if( strcmp(type,"s")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + dummyUDF,0,0); + } + if( rcf!=SQLITE_OK ){ + rc = rcf; + break; + } + } + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + return rc; +} +#endif + /* ** Allocate a new sqlite3expert object. */ @@ -12375,7 +13040,21 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0); } } - + + /* Allow custom collations to be dealt with through prepare. */ + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS); + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS); + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + /* Register UDFs from database [db] with [dbm] and [dbv]. */ + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbm); + } + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbv); + } +#endif /* Copy the entire schema of database [db] into [dbm]. */ if( rc==SQLITE_OK ){ @@ -12451,6 +13130,10 @@ int sqlite3_expert_sql( while( rc==SQLITE_OK && zStmt && zStmt[0] ){ sqlite3_stmt *pStmt = 0; + /* Ensure that the provided statement compiles against user's DB. */ + rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt); + if( rc!=SQLITE_OK ) break; + sqlite3_finalize(pStmt); rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt); if( rc==SQLITE_OK ){ if( pStmt ){ @@ -12915,13 +13598,12 @@ int sqlite3_recover_finish(sqlite3_recover*); */ #if !defined(SQLITEINT_H) -/* #include "sqlite3ext.h" */ +/* #include "sqlite3.h" */ /* typedef unsigned char u8; */ /* typedef unsigned int u32; */ #endif -SQLITE_EXTENSION_INIT1 #include #include @@ -13506,8 +14188,14 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){ bNextPage = 1; }else{ + int szField = 0; pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); - pCsr->pPtr += dbdataValueBytes(iType); + szField = dbdataValueBytes(iType); + if( (pCsr->nRec - (pCsr->pPtr - pCsr->pRec))pPtr = &pCsr->pRec[pCsr->nRec]; + }else{ + pCsr->pPtr += szField; + } } } } @@ -13770,7 +14458,8 @@ static int sqlite3DbdataRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); @@ -13780,15 +14469,11 @@ static int sqlite3DbdataRegister(sqlite3 *db){ return rc; } -#ifdef _WIN32 - -#endif int sqlite3_dbdata_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ - SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; return sqlite3DbdataRegister(db); } @@ -15902,7 +16587,7 @@ static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ if( iFree>(n-4) ) return 0; iNext = recoverGetU16(&a[iFree]); nByte = recoverGetU16(&a[iFree+2]); - if( iFree+nByte>n ) return 0; + if( iFree+nByte>n || nByte<4 ) return 0; if( iNext && iNextArraySize(aExplainWidth) ){ - nArg = ArraySize(aExplainWidth); + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4, 6, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = ArraySize(aExplainWidth); + int iIndent = 1; + + if( p->cMode==MODE_ScanExp ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = ArraySize(aScanExpWidth); + iIndent = 3; } + if( nArg>nWidth ) nArg = nWidth; + + /* If this is the first row seen, print out the headers */ if( p->cnt++==0 ){ for(i=0; iout, w, azCol[i]); + utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]); fputs(i==nArg-1 ? "\n" : " ", p->out); } for(i=0; iout, w); + print_dashes(p->out, aWidth[i]); fputs(i==nArg-1 ? "\n" : " ", p->out); } } + + /* If there is no data, exit early. */ if( azArg==0 ) break; + for(i=0; iw ){ - w = strlenChar(azArg[i]); + if( zVal && strlenChar(zVal)>w ){ + w = strlenChar(zVal); + zSep = " "; } - if( i==1 && p->aiIndent && p->pStmt ){ + if( i==iIndent && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); + fputs(i==nArg-1 ? "\n" : zSep, p->out); } break; } @@ -18347,7 +19053,7 @@ static int run_table_dump_query( */ static char *save_err_msg( sqlite3 *db, /* Database to query */ - const char *zPhase, /* When the error occcurs */ + const char *zPhase, /* When the error occurs */ int rc, /* Error code returned from API */ const char *zSql /* SQL string, or NULL */ ){ @@ -18607,17 +19313,11 @@ static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ } #endif -/* -** Display scan stats. -*/ -static void display_scanstats( +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void display_explain_scanstats( sqlite3 *db, /* Database to query */ ShellState *pArg /* Pointer to ShellState */ ){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else static const int f = SQLITE_SCANSTAT_COMPLEX; sqlite3_stmt *p = pArg->pStmt; int ii = 0; @@ -18631,7 +19331,7 @@ static void display_scanstats( if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ break; } - n = strlen(z) + scanStatsHeight(p, ii)*3; + n = (int)strlen(z) + scanStatsHeight(p, ii)*3; if( n>nWidth ) nWidth = n; } nWidth += 4; @@ -18643,12 +19343,12 @@ static void display_scanstats( i64 nCycle = 0; int iId = 0; int iPid = 0; - const char *z = 0; + const char *zo = 0; const char *zName = 0; char *zText = 0; double rEst = 0.0; - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ break; } sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); @@ -18659,7 +19359,7 @@ static void display_scanstats( sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); - zText = sqlite3_mprintf("%s", z); + zText = sqlite3_mprintf("%s", zo); if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ char *z = 0; if( nCycle>=0 && nTotal>0 ){ @@ -18689,8 +19389,9 @@ static void display_scanstats( } eqp_render(pArg, nTotal); -#endif } +#endif + /* ** Parameter azArray points to a zero-terminated array of strings. zStr @@ -18728,8 +19429,6 @@ static int str_in_array(const char *zStr, const char **azArray){ ** and "Goto" by 2 spaces. */ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - const char *zSql; /* The text of the SQL statement */ - const char *z; /* Used to check if this is an EXPLAIN */ int *abYield = 0; /* True if op is an OP_Yield */ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ @@ -18740,65 +19439,45 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; - /* Try to figure out if this is really an EXPLAIN statement. If this - ** cannot be verified, return early. */ - if( sqlite3_column_count(pSql)!=8 ){ - p->cMode = p->mode; - return; - } - zSql = sqlite3_sql(pSql); - if( zSql==0 ) return; - for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); - if( sqlite3_strnicmp(z, "explain", 7) ){ - p->cMode = p->mode; - return; - } + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(pSql)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ int i; int iAddr = sqlite3_column_int(pSql, 0); const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - - /* Set p2 to the P2 field of the current opcode. Then, assuming that - ** p2 is an instruction address, set variable p2op to the index of that - ** instruction in the aiIndent[] array. p2 and p2op may be different if - ** the current instruction is part of a sub-program generated by an - ** SQL trigger or foreign key. */ + int p1 = sqlite3_column_int(pSql, 2); int p2 = sqlite3_column_int(pSql, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ int p2op = (p2 + (iOp-iAddr)); /* Grow the p->aiIndent array as required */ if( iOp>=nAlloc ){ - if( iOp==0 ){ - /* Do further verfication that this is explain output. Abort if - ** it is not */ - static const char *explainCols[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" }; - int jj; - for(jj=0; jjcMode = p->mode; - sqlite3_reset(pSql); - return; - } - } - } nAlloc += 100; p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); shell_check_oom(p->aiIndent); abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); shell_check_oom(abYield); } + abYield[iOp] = str_in_array(zOp, azYield); p->aiIndent[iOp] = 0; p->nIndent = iOp+1; - if( str_in_array(zOp, azNext) && p2op>0 ){ for(i=p2op; iaiIndent[i] += 2; } - if( str_in_array(zOp, azGoto) && p2opnIndent - && (abYield[p2op] || sqlite3_column_int(pSql, 2)) - ){ + if( str_in_array(zOp, azGoto) && p2opaiIndent[i] += 2; } } @@ -18818,6 +19497,48 @@ static void explain_data_delete(ShellState *p){ p->iIndent = 0; } +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles" + " FROM bytecode(?)"; + + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + + pArg->cnt = 0; + pArg->cMode = MODE_ScanExp; + explain_data_prepare(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } +#endif +} + /* ** Disable and restore .wheretrace and .treetrace/.selecttrace settings. */ @@ -19113,7 +19834,7 @@ static char *quoted_column(sqlite3_stmt *pStmt, int i){ */ static void exec_prepared_stmt_columnar( ShellState *p, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statment to run */ + sqlite3_stmt *pStmt /* Statement to run */ ){ sqlite3_int64 nRow = 0; int nColumn = 0; @@ -19339,7 +20060,7 @@ static void exec_prepared_stmt_columnar( */ static void exec_prepared_stmt( ShellState *pArg, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statment to run */ + sqlite3_stmt *pStmt /* Statement to run */ ){ int rc; sqlite3_uint64 nRow = 0; @@ -19598,7 +20319,7 @@ static int shell_exec( if( zStmtSql==0 ) zStmtSql = ""; while( IsSpace(zStmtSql[0]) ) zStmtSql++; - /* save off the prepared statment handle and reset row count */ + /* save off the prepared statement handle and reset row count */ if( pArg ){ pArg->pStmt = pStmt; pArg->cnt = 0; @@ -19607,16 +20328,15 @@ static int shell_exec( /* Show the EXPLAIN QUERY PLAN if .eqp is on */ if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ sqlite3_stmt *pExplain; - char *zEQP; int triggerEQP = 0; disable_debug_trace_modes(); sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); if( pArg->autoEQP>=AUTOEQP_trigger ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); } - zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); - shell_check_oom(zEQP); - rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + pExplain = pStmt; + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 2); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); @@ -19628,36 +20348,31 @@ static int shell_exec( } eqp_render(pArg, 0); } - sqlite3_finalize(pExplain); - sqlite3_free(zEQP); if( pArg->autoEQP>=AUTOEQP_full ){ /* Also do an EXPLAIN for ".eqp full" mode */ - zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql); - shell_check_oom(zEQP); - rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 1); if( rc==SQLITE_OK ){ pArg->cMode = MODE_Explain; + assert( sqlite3_stmt_isexplain(pExplain)==1 ); explain_data_prepare(pArg, pExplain); exec_prepared_stmt(pArg, pExplain); explain_data_delete(pArg); } - sqlite3_finalize(pExplain); - sqlite3_free(zEQP); } if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); - /* Reprepare pStmt before reactiving trace modes */ - sqlite3_finalize(pStmt); - sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( pArg ) pArg->pStmt = pStmt; } + sqlite3_reset(pStmt); + sqlite3_stmt_explain(pStmt, 0); restore_debug_trace_modes(); } if( pArg ){ + int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1); pArg->cMode = pArg->mode; if( pArg->autoExplain ){ - if( sqlite3_stmt_isexplain(pStmt)==1 ){ + if( bIsExplain ){ pArg->cMode = MODE_Explain; } if( sqlite3_stmt_isexplain(pStmt)==2 ){ @@ -19667,7 +20382,7 @@ static int shell_exec( /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ - if( pArg->cMode==MODE_Explain ){ + if( pArg->cMode==MODE_Explain && bIsExplain ){ explain_data_prepare(pArg, pStmt); } } @@ -19783,7 +20498,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){ */ if( preserveRowid && isIPK ){ /* If a single PRIMARY KEY column with type INTEGER was seen, then it - ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** might be an alias for the ROWID. But it might also be a WITHOUT ROWID ** table or a INTEGER PRIMARY KEY DESC column, neither of which are ** ROWID aliases. To distinguish these cases, check to see if ** there is a "pk" entry in "PRAGMA index_list". There will be @@ -20044,7 +20759,6 @@ static const char *(azHelp[]) = { " --async Write to FILE without journal and fsync()", #endif ".bail on|off Stop after hitting an error. Default OFF", - ".binary on|off Turn binary output on or off. Default OFF", #ifndef SQLITE_SHELL_FIDDLE ".cd DIRECTORY Change the working directory to DIRECTORY", #endif @@ -20054,6 +20768,9 @@ static const char *(azHelp[]) = { ".clone NEWDB Clone data into NEWDB from the existing database", #endif ".connection [close] [#] Open or close an auxiliary database connection", +#if defined(_WIN32) || defined(WIN32) + ".crnl on|off Translate \\n to \\r\\n. Default ON", +#endif ".databases List names and files of attached databases", ".dbconfig ?op? ?val? List or change sqlite3_db_config() options", #if SQLITE_SHELL_HAVE_RECOVER @@ -20760,7 +21477,7 @@ static void open_db(ShellState *p, int openFlags){ /* Let custom-included extensions get their ..._init() called. * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) - * inititialization routine to be called. + * initialization routine to be called. */ { int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); @@ -20778,6 +21495,12 @@ static void open_db(ShellState *p, int openFlags){ } #endif + sqlite3_create_function(p->db, "strtod", 1, SQLITE_UTF8, 0, + shellStrtod, 0, 0); + sqlite3_create_function(p->db, "dtostr", 1, SQLITE_UTF8, 0, + shellDtostr, 0, 0); + sqlite3_create_function(p->db, "dtostr", 2, SQLITE_UTF8, 0, + shellDtostr, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, @@ -20837,7 +21560,7 @@ static void open_db(ShellState *p, int openFlags){ } /* -** Attempt to close the databaes connection. Report errors. +** Attempt to close the database connection. Report errors. */ void close_db(sqlite3 *db){ int rc = sqlite3_close(db); @@ -21331,7 +22054,7 @@ static void tryToCloneData( if( rc ){ utf8_printf(stderr, "Error %d: %s on [%s]\n", sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), - zQuery); + zInsert); goto end_data_xfer; } for(k=0; k<2; k++){ @@ -23284,7 +24007,6 @@ static int do_meta_command(char *zLine, ShellState *p){ azArg[nArg++] = &zLine[h]; while( zLine[h] && !IsSpace(zLine[h]) ){ h++; } if( zLine[h] ) zLine[h++] = 0; - resolve_backslashes(azArg[nArg-1]); } } azArg[nArg] = 0; @@ -23403,6 +24125,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + /* Undocumented. Legacy only. See "crnl" below */ if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){ if( nArg==2 ){ if( booleanValue(azArg[1]) ){ @@ -23411,6 +24134,8 @@ static int do_meta_command(char *zLine, ShellState *p){ setTextMode(p->out, 1); } }else{ + raw_printf(stderr, "The \".binary\" command is deprecated." + " Use \".crnl\" instead.\n"); raw_printf(stderr, "Usage: .binary on|off\n"); rc = 1; } @@ -23538,6 +24263,22 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='c' && n==4 && cli_strncmp(azArg[0], "crnl", n)==0 ){ + if( nArg==2 ){ + if( booleanValue(azArg[1]) ){ + setTextMode(p->out, 1); + }else{ + setBinaryMode(p->out, 1); + } + }else{ +#if !defined(_WIN32) && !defined(WIN32) + raw_printf(stderr, "The \".crnl\" is a no-op on non-Windows machines.\n"); +#endif + raw_printf(stderr, "Usage: .crnl on|off\n"); + rc = 1; + } + }else + if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ char **azName = 0; int nName = 0; @@ -24004,8 +24745,10 @@ static int do_meta_command(char *zLine, ShellState *p){ "SELECT rowid FROM sqlite_schema" " WHERE name GLOB 'sqlite_stat[134]'", -1, &pStmt, 0); - doStats = sqlite3_step(pStmt)==SQLITE_ROW; - sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + doStats = sqlite3_step(pStmt)==SQLITE_ROW; + sqlite3_finalize(pStmt); + } } if( doStats==0 ){ raw_printf(p->out, "/* No STAT tables available */\n"); @@ -24294,6 +25037,14 @@ static int do_meta_command(char *zLine, ShellState *p){ ** the remaining columns. */ if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + /* + ** For CSV mode, per RFC 4180, accept EOF in lieu of final + ** record terminator but only for last field of multi-field row. + ** (If there are too few fields, it's not valid CSV anyway.) + */ + if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ + z = ""; + } sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); if( iscanstatsOn = 3; + }else if( cli_strcmp(azArg[1], "est")==0 ){ p->scanstatsOn = 2; }else{ @@ -25847,7 +26601,8 @@ static int do_meta_command(char *zLine, ShellState *p){ "with tabcols as materialized(\n" "select tname, cname\n" "from (" - " select ss.tname as tname, ti.name as cname\n" + " select printf('\"%%w\"',ss.tname) as tname," + " printf('\"%%w\"',ti.name) as cname\n" " from (%z) ss\n inner join pragma_table_info(tname) ti))\n" "select 'SELECT total(bad_text_count) AS bad_text_count\n" "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n" @@ -26119,7 +26874,7 @@ static int do_meta_command(char *zLine, ShellState *p){ static const struct { const char *zCtrlName; /* Name of a test-control option */ int ctrlCode; /* Integer code for that option */ - int unSafe; /* Not valid for --safe mode */ + int unSafe; /* Not valid unless --unsafe-testing */ const char *zUsage; /* Usage notes */ } aCtrl[] = { {"always", SQLITE_TESTCTRL_ALWAYS, 1, "BOOLEAN" }, @@ -26129,6 +26884,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"byteorder", SQLITE_TESTCTRL_BYTEORDER, 0, "" }, {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN" }, /*{"fault_install", SQLITE_TESTCTRL_FAULT_INSTALL, 1,"" },*/ + {"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" }, {"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"}, {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" }, {"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" }, @@ -26144,6 +26900,7 @@ static int do_meta_command(char *zLine, ShellState *p){ {"seek_count", SQLITE_TESTCTRL_SEEK_COUNT, 0, "" }, {"sorter_mmap", SQLITE_TESTCTRL_SORTER_MMAP, 0, "NMAX" }, {"tune", SQLITE_TESTCTRL_TUNE, 1, "ID VALUE" }, + {"uselongdouble", SQLITE_TESTCTRL_USELONGDOUBLE,0,"?BOOLEAN|\"default\"?"}, }; int testctrl = -1; int iCtrl = -1; @@ -26152,12 +26909,6 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, n2; const char *zCmd = 0; - if( !ShellHasFlag(p,SHFLG_TestingMode) ){ - utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n", - "testctrl"); - rc = 1; - goto meta_command_exit; - } open_db(p, 0); zCmd = nArg>=2 ? azArg[1] : "help"; @@ -26171,6 +26922,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( cli_strcmp(zCmd,"help")==0 ){ utf8_printf(p->out, "Available test-controls:\n"); for(i=0; iout, " .testctrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } @@ -26182,6 +26934,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ** of the option name, or a numerical value. */ n2 = strlen30(zCmd); for(i=0; ibSafeMode ){ - utf8_printf(stderr, - "line %d: \".testctrl %s\" may not be used in safe mode\n", - p->lineno, aCtrl[iCtrl].zCtrlName); - exit(1); }else{ switch(testctrl){ /* sqlite3_test_control(int, db, int) */ case SQLITE_TESTCTRL_OPTIMIZATIONS: + case SQLITE_TESTCTRL_FK_NO_ACTION: if( nArg==3 ){ unsigned int opt = (unsigned int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); @@ -26274,6 +27023,21 @@ static int do_meta_command(char *zLine, ShellState *p){ } break; + /* sqlite3_test_control(int, int) */ + case SQLITE_TESTCTRL_USELONGDOUBLE: { + int opt = -1; + if( nArg==3 ){ + if( cli_strcmp(azArg[2],"default")==0 ){ + opt = 2; + }else{ + opt = booleanValue(azArg[2]); + } + } + rc2 = sqlite3_test_control(testctrl, opt); + isOk = 1; + break; + } + /* sqlite3_test_control(sqlite3*) */ case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: rc2 = sqlite3_test_control(testctrl, p->db); @@ -26511,6 +27275,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif /* SQLITE_USER_AUTHENTICATION */ if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ + char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit"; utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB @@ -26521,11 +27286,11 @@ static int do_meta_command(char *zLine, ShellState *p){ #if defined(__clang__) && defined(__clang_major__) utf8_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." CTIMEOPT_VAL(__clang_minor__) "." - CTIMEOPT_VAL(__clang_patchlevel__) "\n"); + CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); #elif defined(_MSC_VER) - utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) "\n"); + utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); #elif defined(__GNUC__) && defined(__VERSION__) - utf8_printf(p->out, "gcc-" __VERSION__ "\n"); + utf8_printf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); #endif }else @@ -27143,10 +27908,14 @@ static const char zOptions[] = " -multiplex enable the multiplexor VFS\n" #endif " -newline SEP set output row separator. Default: '\\n'\n" +#if SHELL_WIN_UTF8_OPT + " -no-utf8 do not try to set up UTF-8 output (for legacy)\n" +#endif " -nofollow refuse to open symbolic links to database files\n" " -nonce STRING set the safe-mode escape nonce\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" + " -pcachetrace trace all page cache operations\n" " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -safe enable safe-mode\n" @@ -27158,7 +27927,7 @@ static const char zOptions[] = " -table set output mode to 'table'\n" " -tabs set output mode to 'tabs'\n" " -unsafe-testing allow unsafe commands and modes for testing\n" -#if SHELL_WIN_UTF8_OPT +#if SHELL_WIN_UTF8_OPT && 0 /* Option is accepted, but is now the default. */ " -utf8 setup interactive console code page for UTF-8\n" #endif " -version show SQLite version\n" @@ -27309,7 +28078,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdout_is_console = isatty(1); #endif #if SHELL_WIN_UTF8_OPT - atexit(console_restore); /* Needs revision for CLI as library call */ + probe_console(); /* Check for console I/O and UTF-8 capability. */ + if( !mbcs_opted ) atexit(console_restore); #endif atexit(sayAbnormalExit); #ifdef SQLITE_DEBUG @@ -27396,8 +28166,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, - ** the size of the alternative malloc heap, - ** and the first command to execute. + ** the size of the alternative malloc heap, options affecting commands + ** or SQL run from the command line, and the first command to execute. */ #ifndef SQLITE_SHELL_FIDDLE verify_uninitialized(); @@ -27409,7 +28179,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( data.aAuxDb->zDbFilename==0 ){ data.aAuxDb->zDbFilename = z; }else{ - /* Excesss arguments are interpreted as SQL (or dot-commands) and + /* Excess arguments are interpreted as SQL (or dot-commands) and ** mean that nothing is read from stdin */ readStdin = 0; nCmd++; @@ -27431,12 +28201,26 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ (void)cmdline_option_value(argc, argv, ++i); }else if( cli_strcmp(z,"-init")==0 ){ zInitFile = cmdline_option_value(argc, argv, ++i); + }else if( cli_strcmp(z,"-interactive")==0 ){ + /* Need to check for interactive override here to so that it can + ** affect console setup (for Windows only) and testing thereof. + */ + stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ /* Need to check for batch mode here to so we can avoid printing ** informational messages (like from process_sqliterc) before ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; + }else if( cli_strcmp(z,"-utf8")==0 ){ +#if SHELL_WIN_UTF8_OPT + /* Option accepted, but is ignored except for this diagnostic. */ + if( mbcs_opted ) fprintf(stderr, "Cannot do UTF-8 at this console.\n"); +#endif /* SHELL_WIN_UTF8_OPT */ + }else if( cli_strcmp(z,"-no-utf8")==0 ){ +#if SHELL_WIN_UTF8_OPT + mbcs_opted = 1; +#endif /* SHELL_WIN_UTF8_OPT */ }else if( cli_strcmp(z,"-heap")==0 ){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) const char *zSize; @@ -27494,7 +28278,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif #ifdef SQLITE_ENABLE_MULTIPLEX }else if( cli_strcmp(z,"-multiplex")==0 ){ - extern int sqlite3_multiple_initialize(const char*,int); + extern int sqlite3_multiplex_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif }else if( cli_strcmp(z,"-mmap")==0 ){ @@ -27533,11 +28317,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( cli_strcmp(z, "-memtrace")==0 ){ sqlite3MemTraceActivate(stderr); + }else if( cli_strcmp(z, "-pcachetrace")==0 ){ + sqlite3PcacheTraceActivate(stderr); }else if( cli_strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( cli_strcmp(z,"-nonce")==0 ){ free(data.zNonce); - data.zNonce = strdup(argv[++i]); + data.zNonce = strdup(cmdline_option_value(argc, argv, ++i)); }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ ShellSetFlag(&data,SHFLG_TestingMode); }else if( cli_strcmp(z,"-safe")==0 ){ @@ -27573,6 +28359,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ exit(1); } } +#if SHELL_WIN_UTF8_OPT + /* Get indicated Windows console setup done before running invocation commands. */ + if( in_console || out_console ){ + console_prepare_utf8(); + } + if( !in_console ){ + setBinaryMode(stdin, 0); + } +#endif if( data.pAuxDb->zDbFilename==0 ){ #ifndef SQLITE_OMIT_MEMORYDB @@ -27696,16 +28491,17 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-bail")==0 ){ /* No-op. The bail_on_error flag should already be set. */ }else if( cli_strcmp(z,"-version")==0 ){ - printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid()); + printf("%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(), + 8*(int)sizeof(char*)); return 0; }else if( cli_strcmp(z,"-interactive")==0 ){ - stdin_is_interactive = 1; + /* already handled */ }else if( cli_strcmp(z,"-batch")==0 ){ - stdin_is_interactive = 0; + /* already handled */ }else if( cli_strcmp(z,"-utf8")==0 ){ -#if SHELL_WIN_UTF8_OPT - console_utf8 = 1; -#endif /* SHELL_WIN_UTF8_OPT */ + /* already handled */ + }else if( cli_strcmp(z,"-no-utf8")==0 ){ + /* already handled */ }else if( cli_strcmp(z,"-heap")==0 ){ i++; }else if( cli_strcmp(z,"-pagecache")==0 ){ @@ -27720,6 +28516,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ i++; }else if( cli_strcmp(z,"-memtrace")==0 ){ i++; + }else if( cli_strcmp(z,"-pcachetrace")==0 ){ + i++; #ifdef SQLITE_ENABLE_SORTER_REFERENCES }else if( cli_strcmp(z,"-sorterref")==0 ){ i++; @@ -27785,14 +28583,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } data.cMode = data.mode; } -#if SHELL_WIN_UTF8_OPT - if( console_utf8 && stdin_is_interactive ){ - console_prepare(); - }else{ - setBinaryMode(stdin, 0); - console_utf8 = 0; - } -#endif if( !readStdin ){ /* Run all arguments that do not begin with '-' as if they were separate @@ -27828,11 +28618,20 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( stdin_is_interactive ){ char *zHome; char *zHistory; + const char *zCharset = ""; int nHistory; +#if SHELL_WIN_UTF8_OPT + switch( console_utf8_in+2*console_utf8_out ){ + default: case 0: break; + case 1: zCharset = " (utf8 in)"; break; + case 2: zCharset = " (utf8 out)"; break; + case 3: zCharset = " (utf8 I/O)"; break; + } +#endif printf( - "SQLite version %s %.19s\n" /*extra-version-info*/ + "SQLite version %s %.19s%s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", - sqlite3_libversion(), sqlite3_sourceid() + sqlite3_libversion(), sqlite3_sourceid(), zCharset ); if( warnInmemoryDb ){ printf("Connected to a "); diff --git a/core/src/sqlite/sqlite3.c b/core/src/sqlite/sqlite3.c index dd3b5c575..8f9309a8b 100644 --- a/core/src/sqlite/sqlite3.c +++ b/core/src/sqlite/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.42.0. By combining all the individual C code files into this +** version 3.44.0. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -16,6 +16,9 @@ ** if you want a wrapper to interface SQLite with your choice of programming ** language. The code for the "sqlite3" command-line shell is also in a ** separate file. This file contains only code for the core SQLite library. +** +** The content in this amalgamation comes from Fossil check-in +** 17129ba1ff7f0daf37100ee82d507aef7827. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -50,11 +53,11 @@ ** used on lines of code that actually ** implement parts of coverage testing. ** -** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false +** OPTIMIZATION-IF-TRUE - This branch is allowed to always be false ** and the correct answer is still obtained, ** though perhaps more slowly. ** -** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true +** OPTIMIZATION-IF-FALSE - This branch is allowed to always be true ** and the correct answer is still obtained, ** though perhaps more slowly. ** @@ -456,9 +459,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.42.0" -#define SQLITE_VERSION_NUMBER 3042000 -#define SQLITE_SOURCE_ID "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0" +#define SQLITE_VERSION "3.44.0" +#define SQLITE_VERSION_NUMBER 3044000 +#define SQLITE_SOURCE_ID "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -838,6 +841,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -1500,7 +1504,7 @@ struct sqlite3_io_methods { ** by clients within the current process, only within other processes. ** **
  • [[SQLITE_FCNTL_CKSM_FILE]] -** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use interally by the +** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the ** [checksum VFS shim] only. ** **
  • [[SQLITE_FCNTL_RESET_CACHE]] @@ -2436,7 +2440,7 @@ struct sqlite3_mem_methods { ** is stored in each sorted record and the required column values loaded ** from the database as records are returned in sorted order. The default ** value for this option is to never use this optimization. Specifying a -** negative value for this option restores the default behaviour. +** negative value for this option restores the default behavior. ** This option is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. ** @@ -2611,7 +2615,7 @@ struct sqlite3_mem_methods { ** database handle, SQLite checks if this will mean that there are now no ** connections at all to the database. If so, it performs a checkpoint ** operation before closing the connection. This option may be used to -** override this behaviour. The first parameter passed to this operation +** override this behavior. The first parameter passed to this operation ** is an integer - positive to disable checkpoints-on-close, or zero (the ** default) to enable them, and negative to leave the setting unchanged. ** The second parameter is a pointer to an integer @@ -2764,7 +2768,7 @@ struct sqlite3_mem_methods { ** the [VACUUM] command will fail with an obscure error when attempting to ** process a table with generated columns and a descending index. This is ** not considered a bug since SQLite versions 3.3.0 and earlier do not support -** either generated columns or decending indexes. +** either generated columns or descending indexes. ** ** ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] @@ -3045,6 +3049,7 @@ SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); ** ** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether ** or not an interrupt is currently in effect for [database connection] D. +** It returns 1 if an interrupt is currently in effect, or 0 otherwise. */ SQLITE_API void sqlite3_interrupt(sqlite3*); SQLITE_API int sqlite3_is_interrupted(sqlite3*); @@ -3698,8 +3703,10 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, ** M argument should be the bitwise OR-ed combination of ** zero or more [SQLITE_TRACE] constants. ** -** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides -** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P) +** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or +** sqlite3_trace_v2(D,M,X,P) for the [database connection] D. Each +** database connection may have at most one trace callback. ** ** ^The X callback is invoked whenever any of the events identified by ** mask M occur. ^The integer return value from the callback is currently @@ -4068,7 +4075,7 @@ SQLITE_API int sqlite3_open_v2( ** as F) must be one of: **
      **
    • A database filename pointer created by the SQLite core and -** passed into the xOpen() method of a VFS implemention, or +** passed into the xOpen() method of a VFS implementation, or **
    • A filename obtained from [sqlite3_db_filename()], or **
    • A new filename constructed using [sqlite3_create_filename()]. **
    @@ -4181,7 +4188,7 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); /* ** CAPI3REF: Create and Destroy VFS Filenames ** -** These interfces are provided for use by [VFS shim] implementations and +** These interfaces are provided for use by [VFS shim] implementations and ** are not useful outside of that context. ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of @@ -4261,6 +4268,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language ** text that describes the error, as either UTF-8 or UTF-16 respectively. +** (See how SQLite handles [invalid UTF] for exceptions to this rule.) ** ^(Memory to hold the error message string is managed internally. ** The application does not need to worry about freeing the result. ** However, the error string might be overwritten or deallocated by @@ -4728,6 +4736,41 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN +** setting for [prepared statement] S. If E is zero, then S becomes +** a normal prepared statement. If E is 1, then S behaves as if +** its SQL text began with "[EXPLAIN]". If E is 2, then S behaves as if +** its SQL text began with "[EXPLAIN QUERY PLAN]". +** +** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared. +** SQLite tries to avoid a reprepare, but a reprepare might be necessary +** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode. +** +** Because of the potential need to reprepare, a call to +** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be +** reprepared because it was created using [sqlite3_prepare()] instead of +** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and +** hence has no saved SQL text with which to reprepare. +** +** Changing the explain setting for a prepared statement does not change +** the original SQL text for the statement. Hence, if the SQL text originally +** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0) +** is called to convert the statement into an ordinary statement, the EXPLAIN +** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S) +** output, even though the statement now acts like a normal SQL statement. +** +** This routine returns SQLITE_OK if the explain mode is successfully +** changed, or an error code if the explain mode could not be changed. +** The explain mode cannot be changed while a statement is active. +** Hence, it is good practice to call [sqlite3_reset(S)] +** immediately prior to calling sqlite3_stmt_explain(S,E). +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt @@ -4891,7 +4934,7 @@ typedef struct sqlite3_context sqlite3_context; ** with it may be passed. ^It is called to dispose of the BLOB or string even ** if the call to the bind API fails, except the destructor is not called if ** the third parameter is a NULL pointer or the fourth parameter is negative. -** ^ (2) The special constant, [SQLITE_STATIC], may be passsed to indicate that +** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that ** the application remains responsible for disposing of the object. ^In this ** case, the object and the provided pointer to it must remain valid until ** either the prepared statement is finalized or the same SQL parameter is @@ -5570,20 +5613,33 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S ** back to the beginning of its program. ** -** ^If the most recent call to [sqlite3_step(S)] for the -** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], -** or if [sqlite3_step(S)] has never before been called on S, -** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** ^The return code from [sqlite3_reset(S)] indicates whether or not +** the previous evaluation of prepared statement S completed successfully. +** ^If [sqlite3_step(S)] has never before been called on S or if +** [sqlite3_step(S)] has not been called since the previous call +** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return +** [SQLITE_OK]. ** ** ^If the most recent call to [sqlite3_step(S)] for the ** [prepared statement] S indicated an error, then ** [sqlite3_reset(S)] returns an appropriate [error code]. +** ^The [sqlite3_reset(S)] interface might also return an [error code] +** if there were no prior errors but the process of resetting +** the prepared statement caused a new error. ^For example, if an +** [INSERT] statement with a [RETURNING] clause is only stepped one time, +** that one call to [sqlite3_step(S)] might return SQLITE_ROW but +** the overall statement might still fail and the [sqlite3_reset(S)] call +** might return SQLITE_BUSY if locking constraints prevent the +** database change from committing. Therefore, it is important that +** applications check the return code from [sqlite3_reset(S)] even if +** no prior call to [sqlite3_step(S)] indicated a problem. ** ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} @@ -5794,7 +5850,7 @@ SQLITE_API int sqlite3_create_window_function( ** [application-defined SQL function] ** that has side-effects or that could potentially leak sensitive information. ** This will prevent attacks in which an application is tricked -** into using a database file that has had its schema surreptiously +** into using a database file that has had its schema surreptitiously ** modified to invoke the application-defined function in ways that are ** harmful. **

    @@ -6138,32 +6194,32 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to -** associate metadata with argument values. If the same value is passed to -** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated metadata may be preserved. An example -** of where this might be useful is in a regular-expression matching -** function. The compiled version of the regular expression can be stored as -** metadata associated with the pattern string. +** associate auxiliary data with argument values. If the same argument +** value is passed to multiple invocations of the same SQL function during +** query execution, under some circumstances the associated auxiliary data +** might be preserved. An example of where this might be useful is in a +** regular-expression matching function. The compiled version of the regular +** expression can be stored as auxiliary data associated with the pattern string. ** Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data ** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument ** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no metadata +** function argument. ^If there is no auxiliary data ** associated with the function argument, the sqlite3_get_auxdata(C,N) interface ** returns a NULL pointer. ** -** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th -** argument of the application-defined function. ^Subsequent +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the +** N-th argument of the application-defined function. ^Subsequent ** calls to sqlite3_get_auxdata(C,N) return P from the most recent -** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or -** NULL if the metadata has been discarded. +** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or +** NULL if the auxiliary data has been discarded. ** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, ** SQLite will invoke the destructor function X with parameter P exactly -** once, when the metadata is discarded. -** SQLite is free to discard the metadata at any time, including:

      +** once, when the auxiliary data is discarded. +** SQLite is free to discard the auxiliary data at any time, including:
        **
      • ^(when the corresponding function parameter changes)^, or **
      • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the ** SQL statement)^, or @@ -6179,7 +6235,7 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** function implementation should not make any use of P after ** sqlite3_set_auxdata() has been called. ** -** ^(In practice, metadata is preserved between function calls for +** ^(In practice, auxiliary data is preserved between function calls for ** function parameters that are compile-time constants, including literal ** values and [parameters] and expressions composed from the same.)^ ** @@ -6189,10 +6245,67 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** ** These routines must be called from the same thread in which ** the SQL function is running. +** +** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()]. */ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +/* +** CAPI3REF: Database Connection Client Data +** METHOD: sqlite3 +** +** These functions are used to associate one or more named pointers +** with a [database connection]. +** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P +** to be attached to [database connection] D using name N. Subsequent +** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P +** or a NULL pointer if there were no prior calls to +** sqlite3_set_clientdata() with the same values of D and N. +** Names are compared using strcmp() and are thus case sensitive. +** +** If P and X are both non-NULL, then the destructor X is invoked with +** argument P on the first of the following occurrences: +**
          +**
        • An out-of-memory error occurs during the call to +** sqlite3_set_clientdata() which attempts to register pointer P. +**
        • A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made +** with the same D and N parameters. +**
        • The database connection closes. SQLite does not make any guarantees +** about the order in which destructors are called, only that all +** destructors will be called exactly once at some point during the +** database connection closing process. +**
        +** +** SQLite does not do anything with client data other than invoke +** destructors on the client data at the appropriate time. The intended +** use for client data is to provide a mechanism for wrapper libraries +** to store additional information about an SQLite database connection. +** +** There is no limit (other than available memory) on the number of different +** client data pointers (with different names) that can be attached to a +** single database connection. However, the implementation is optimized +** for the case of having only one or two different client data names. +** Applications and wrapper libraries are discouraged from using more than +** one client data name each. +** +** There is no way to enumerate the client data pointers +** associated with a database connection. The N parameter can be thought +** of as a secret key such that only code that knows the secret key is able +** to access the associated data. +** +** Security Warning: These interfaces should not be exposed in scripting +** languages or in other circumstances where it might be possible for an +** an attacker to invoke them. Any agent that can invoke these interfaces +** can probably also take control of the process. +** +** Database connection client data is only available for SQLite +** version 3.44.0 ([dateof:3.44.0]) and later. +** +** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()]. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3*,const char*); +SQLITE_API int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*)); /* ** CAPI3REF: Constants Defining Special Destructor Behavior @@ -6825,7 +6938,7 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); /* -** CAPI3REF: Allowed return values from [sqlite3_txn_state()] +** CAPI3REF: Allowed return values from sqlite3_txn_state() ** KEYWORDS: {transaction state} ** ** These constants define the current transaction state of a database file. @@ -6957,7 +7070,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^Each call to the sqlite3_autovacuum_pages() interface overrides all ** previous invocations for that database connection. ^If the callback ** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer, -** then the autovacuum steps callback is cancelled. The return value +** then the autovacuum steps callback is canceled. The return value ** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might ** be some other error code if something goes wrong. The current ** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other @@ -7476,6 +7589,10 @@ struct sqlite3_module { /* The methods above are in versions 1 and 2 of the sqlite_module object. ** Those below are for version 3 and greater. */ int (*xShadowName)(const char*); + /* The methods above are in versions 1 through 3 of the sqlite_module object. + ** Those below are for version 4 and greater. */ + int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema, + const char *zTabName, int mFlags, char **pzErr); }; /* @@ -7963,7 +8080,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); ** code is returned and the transaction rolled back. ** ** Calling this function with an argument that is not a NULL pointer or an -** open blob handle results in undefined behaviour. ^Calling this routine +** open blob handle results in undefined behavior. ^Calling this routine ** with a null pointer (such as would be returned by a failed call to ** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function ** is passed a valid open blob handle, the values returned by the @@ -8443,6 +8560,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 #define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_FK_NO_ACTION 7 #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 @@ -8471,7 +8589,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TRACEFLAGS 31 #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 -#define SQLITE_TESTCTRL_LAST 33 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 +#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -9927,7 +10046,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_DIRECTONLY]]
        SQLITE_VTAB_DIRECTONLY
        **
        Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** prohibits that virtual table from being used from within triggers and ** views. **
        @@ -10117,7 +10236,7 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*); ** communicated to the xBestIndex method as a ** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^ If xBestIndex wants to use ** this constraint, it must set the corresponding -** aConstraintUsage[].argvIndex to a postive integer. ^(Then, under +** aConstraintUsage[].argvIndex to a positive integer. ^(Then, under ** the usual mode of handling IN operators, SQLite generates [bytecode] ** that invokes the [xFilter|xFilter() method] once for each value ** on the right-hand side of the IN operator.)^ Thus the virtual table @@ -10546,7 +10665,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** When the [sqlite3_blob_write()] API is used to update a blob column, ** the pre-update hook is invoked with SQLITE_DELETE. This is because the ** in this case the new values are not available. In this case, when a -** callback made with op==SQLITE_DELETE is actuall a write using the +** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the ** pre-update hook is being invoked for some other reason, including a @@ -10807,6 +10926,13 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const c ** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy ** of the database exists. ** +** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set, +** the returned buffer content will remain accessible and unchanged +** until either the next write operation on the connection or when +** the connection is closed, and applications must not modify the +** buffer. If the bit had been clear, the returned buffer will not +** be accessed by SQLite after the call. +** ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory ** allocation error occurs. @@ -10855,6 +10981,9 @@ SQLITE_API unsigned char *sqlite3_serialize( ** SQLite will try to increase the buffer size using sqlite3_realloc64() ** if writes on the database cause it to grow larger than M bytes. ** +** Applications must not modify the buffer P or invalidate it before +** the database connection D is closed. +** ** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the ** database is currently in a read transaction or is involved in a backup ** operation. @@ -10863,6 +10992,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** +** The deserialized database should not be in [WAL mode]. If the database +** is in WAL mode, then any attempt to use the database file will result +** in an [SQLITE_CANTOPEN] error. The application can set the +** [file format version numbers] (bytes 18 and 19) of the input database P +** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the +** database file into rollback mode and work around this limitation. +** ** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the ** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then ** [sqlite3_free()] is invoked on argument P prior to returning. @@ -11935,6 +12071,18 @@ SQLITE_API int sqlite3changeset_concat( ); +/* +** CAPI3REF: Upgrade the Schema of a Changeset/Patchset +*/ +SQLITE_API int sqlite3changeset_upgrade( + sqlite3 *db, + const char *zDb, + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + + + /* ** CAPI3REF: Changegroup Handle ** @@ -11981,6 +12129,38 @@ typedef struct sqlite3_changegroup sqlite3_changegroup; */ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); +/* +** CAPI3REF: Add a Schema to a Changegroup +** METHOD: sqlite3_changegroup_schema +** +** This method may be used to optionally enforce the rule that the changesets +** added to the changegroup handle must match the schema of database zDb +** ("main", "temp", or the name of an attached database). If +** sqlite3changegroup_add() is called to add a changeset that is not compatible +** with the configured schema, SQLITE_SCHEMA is returned and the changegroup +** object is left in an undefined state. +** +** A changeset schema is considered compatible with the database schema in +** the same way as for sqlite3changeset_apply(). Specifically, for each +** table in the changeset, there exists a database table with: +** +**
          +**
        • The name identified by the changeset, and +**
        • at least as many columns as recorded in the changeset, and +**
        • the primary key columns in the same position as recorded in +** the changeset. +**
        +** +** The output of the changegroup object always has the same schema as the +** database nominated using this function. In cases where changesets passed +** to sqlite3changegroup_add() have fewer columns than the corresponding table +** in the database schema, these are filled in using the default column +** values from the database schema. This makes it possible to combined +** changesets that have different numbers of columns for a single table +** within a changegroup, provided that they are otherwise compatible. +*/ +SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb); + /* ** CAPI3REF: Add A Changeset To A Changegroup ** METHOD: sqlite3_changegroup @@ -12049,13 +12229,18 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); ** If the new changeset contains changes to a table that is already present ** in the changegroup, then the number of columns and the position of the ** primary key columns for the table must be consistent. If this is not the -** case, this function fails with SQLITE_SCHEMA. If the input changeset -** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is -** returned. Or, if an out-of-memory condition occurs during processing, this -** function returns SQLITE_NOMEM. In all cases, if an error occurs the state -** of the final contents of the changegroup is undefined. +** case, this function fails with SQLITE_SCHEMA. Except, if the changegroup +** object has been configured with a database schema using the +** sqlite3changegroup_schema() API, then it is possible to combine changesets +** with different numbers of columns for a single table, provided that +** they are otherwise compatible. +** +** If the input changeset appears to be corrupt and the corruption is +** detected, SQLITE_CORRUPT is returned. Or, if an out-of-memory condition +** occurs during processing, this function returns SQLITE_NOMEM. ** -** If no error occurs, SQLITE_OK is returned. +** In all cases, if an error occurs the state of the final contents of the +** changegroup is undefined. If no error occurs, SQLITE_OK is returned. */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); @@ -12320,10 +12505,17 @@ SQLITE_API int sqlite3changeset_apply_v2( **
      • an insert change if all fields of the conflicting row match ** the row being inserted. **
      +** +**
      SQLITE_CHANGESETAPPLY_FKNOACTION
      +** If this flag it set, then all foreign key constraints in the target +** database behave as if they were declared with "ON UPDATE NO ACTION ON +** DELETE NO ACTION", even if they are actually CASCADE, RESTRICT, SET NULL +** or SET DEFAULT. */ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 #define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 +#define SQLITE_CHANGESETAPPLY_FKNOACTION 0x0008 /* ** CAPI3REF: Constants Passed To The Conflict Handler @@ -13064,7 +13256,7 @@ struct Fts5PhraseIter { ** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 2 */ void *(*xUserData)(Fts5Context*); @@ -13293,8 +13485,8 @@ struct Fts5ExtensionApi { ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; @@ -13342,7 +13534,7 @@ struct fts5_api { int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -13351,7 +13543,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -13359,7 +13551,7 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); @@ -13470,7 +13662,7 @@ struct fts5_api { ** level of recursion for each term. A stack overflow can result ** if the number of terms is too large. In practice, most SQL ** never has more than 3 or 4 terms. Use a value of 0 to disable -** any limit on the number of terms in a compount SELECT. +** any limit on the number of terms in a compound SELECT. */ #ifndef SQLITE_MAX_COMPOUND_SELECT # define SQLITE_MAX_COMPOUND_SELECT 500 @@ -13714,6 +13906,16 @@ struct fts5_api { # endif #endif +/* +** Enable SQLITE_USE_SEH by default on MSVC builds. Only omit +** SEH support if the -DSQLITE_OMIT_SEH option is given. +*/ +#if defined(_MSC_VER) && !defined(SQLITE_OMIT_SEH) +# define SQLITE_USE_SEH 1 +#else +# undef SQLITE_USE_SEH +#endif + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -14573,8 +14775,31 @@ typedef INT16_TYPE LogEst; ** the end of buffer S. This macro returns true if P points to something ** contained within the buffer S. */ -#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) +#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) +/* +** P is one byte past the end of a large buffer. Return true if a span of bytes +** between S..E crosses the end of that buffer. In other words, return true +** if the sub-buffer S..E-1 overflows the buffer whose last byte is P-1. +** +** S is the start of the span. E is one byte past the end of end of span. +** +** P +** |-----------------| FALSE +** |-------| +** S E +** +** P +** |-----------------| +** |-------| TRUE +** S E +** +** P +** |-----------------| +** |-------| FALSE +** S E +*/ +#define SQLITE_OVERFLOW(P,S,E) (((uptr)(S)<(uptr)(P))&&((uptr)(E)>(uptr)(P))) /* ** Macros to determine whether the machine is big or little endian, @@ -14584,16 +14809,33 @@ typedef INT16_TYPE LogEst; ** using C-preprocessor macros. If that is unsuccessful, or if ** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined ** at run-time. +** +** If you are building SQLite on some obscure platform for which the +** following ifdef magic does not work, you can always include either: +** +** -DSQLITE_BYTEORDER=1234 +** +** or +** +** -DSQLITE_BYTEORDER=4321 +** +** to cause the build to work for little-endian or big-endian processors, +** respectively. */ -#ifndef SQLITE_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ +#ifndef SQLITE_BYTEORDER /* Replicate changes at tag-20230904a */ +# if defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define SQLITE_BYTEORDER 4321 +# elif defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define SQLITE_BYTEORDER 1234 +# elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1 +# define SQLITE_BYTEORDER 4321 +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) -# define SQLITE_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) || \ - defined(__ARMEB__) || defined(__AARCH64EB__) -# define SQLITE_BYTEORDER 4321 +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 # else # define SQLITE_BYTEORDER 0 # endif @@ -14808,7 +15050,7 @@ struct BusyHandler { /* ** Name of table that holds the database schema. ** -** The PREFERRED names are used whereever possible. But LEGACY is also +** The PREFERRED names are used wherever possible. But LEGACY is also ** used for backwards compatibility. ** ** 1. Queries can use either the PREFERRED or the LEGACY names @@ -14917,11 +15159,13 @@ typedef struct Column Column; typedef struct Cte Cte; typedef struct CteUse CteUse; typedef struct Db Db; +typedef struct DbClientData DbClientData; typedef struct DbFixer DbFixer; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct FKey FKey; +typedef struct FpDecode FpDecode; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; @@ -14940,6 +15184,7 @@ typedef struct Parse Parse; typedef struct ParseCleanup ParseCleanup; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; +typedef struct RCStr RCStr; typedef struct RenameToken RenameToken; typedef struct Returning Returning; typedef struct RowSet RowSet; @@ -15577,6 +15822,10 @@ SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); # define enable_simulated_io_errors() #endif +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) +SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager*); +#endif + #endif /* SQLITE_PAGER_H */ /************** End of pager.h ***********************************************/ @@ -15906,9 +16155,7 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int flags); SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor*); -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor*); -#endif SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); @@ -16353,19 +16600,20 @@ typedef struct VdbeOpList VdbeOpList; #define OP_VCreate 171 #define OP_VDestroy 172 #define OP_VOpen 173 -#define OP_VInitIn 174 /* synopsis: r[P2]=ValueList(P1,P3) */ -#define OP_VColumn 175 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 176 -#define OP_Pagecount 177 -#define OP_MaxPgcnt 178 -#define OP_ClrSubtype 179 /* synopsis: r[P1].subtype = 0 */ -#define OP_FilterAdd 180 /* synopsis: filter(P1) += key(P3@P4) */ -#define OP_Trace 181 -#define OP_CursorHint 182 -#define OP_ReleaseReg 183 /* synopsis: release r[P1@P2] mask P3 */ -#define OP_Noop 184 -#define OP_Explain 185 -#define OP_Abortable 186 +#define OP_VCheck 174 +#define OP_VInitIn 175 /* synopsis: r[P2]=ValueList(P1,P3) */ +#define OP_VColumn 176 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 177 +#define OP_Pagecount 178 +#define OP_MaxPgcnt 179 +#define OP_ClrSubtype 180 /* synopsis: r[P1].subtype = 0 */ +#define OP_FilterAdd 181 /* synopsis: filter(P1) += key(P3@P4) */ +#define OP_Trace 182 +#define OP_CursorHint 183 +#define OP_ReleaseReg 184 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 185 +#define OP_Explain 186 +#define OP_Abortable 187 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -16383,7 +16631,7 @@ typedef struct VdbeOpList VdbeOpList; /* 8 */ 0x01, 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01,\ /* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x49, 0x49, 0x49,\ /* 24 */ 0x49, 0x01, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,\ -/* 32 */ 0x41, 0x01, 0x01, 0x01, 0x41, 0x01, 0x41, 0x41,\ +/* 32 */ 0x41, 0x01, 0x41, 0x41, 0x41, 0x01, 0x41, 0x41,\ /* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ /* 48 */ 0x01, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ /* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x41,\ @@ -16395,14 +16643,14 @@ typedef struct VdbeOpList VdbeOpList; /* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ /* 112 */ 0x40, 0x00, 0x12, 0x40, 0x40, 0x10, 0x40, 0x00,\ /* 120 */ 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10, 0x10,\ -/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50,\ +/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x50,\ /* 136 */ 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50, 0x40,\ /* 144 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ /* 152 */ 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00, 0x04,\ /* 160 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x50, 0x40,\ -/* 176 */ 0x00, 0x10, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00,\ -/* 184 */ 0x00, 0x00, 0x00,} +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10, 0x50,\ +/* 176 */ 0x40, 0x00, 0x10, 0x10, 0x02, 0x00, 0x00, 0x00,\ +/* 184 */ 0x00, 0x00, 0x00, 0x00,} /* The resolve3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -16577,7 +16825,7 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** The VdbeCoverage macros are used to set a coverage testing point ** for VDBE branch instructions. The coverage testing points are line ** numbers in the sqlite3.c source file. VDBE branch coverage testing -** only works with an amalagmation build. That's ok since a VDBE branch +** only works with an amalgamation build. That's ok since a VDBE branch ** coverage build designed for testing the test suite only. No application ** should ever ship with VDBE branch coverage measuring turned on. ** @@ -16595,7 +16843,7 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** // NULL option is not possible ** ** VdbeCoverageEqNe(v) // Previous OP_Jump is only interested -** // in distingishing equal and not-equal. +** // in distinguishing equal and not-equal. ** ** Every VDBE branch operation must be tagged with one of the macros above. ** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and @@ -16605,7 +16853,7 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** During testing, the test application will invoke ** sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE,...) to set a callback ** routine that is invoked as each bytecode branch is taken. The callback -** contains the sqlite3.c source line number ov the VdbeCoverage macro and +** contains the sqlite3.c source line number of the VdbeCoverage macro and ** flags to indicate whether or not the branch was taken. The test application ** is responsible for keeping track of this and reporting byte-code branches ** that are never taken. @@ -16944,7 +17192,7 @@ SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); /* ** Default synchronous levels. ** -** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ +** Note that (for historical reasons) the PAGER_SYNCHRONOUS_* macros differ ** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1. ** ** PAGER_SYNCHRONOUS DEFAULT_SYNCHRONOUS @@ -16983,7 +17231,7 @@ struct Db { ** An instance of the following structure stores a database schema. ** ** Most Schema objects are associated with a Btree. The exception is -** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. +** the Schema for the TEMP database (sqlite3.aDb[1]) which is free-standing. ** In shared cache mode, a single Schema object can be shared by multiple ** Btrees that refer to the same underlying BtShared object. ** @@ -17094,7 +17342,7 @@ struct Lookaside { LookasideSlot *pInit; /* List of buffers not previously used */ LookasideSlot *pFree; /* List of available buffers */ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE - LookasideSlot *pSmallInit; /* List of small buffers not prediously used */ + LookasideSlot *pSmallInit; /* List of small buffers not previously used */ LookasideSlot *pSmallFree; /* List of available small buffers */ void *pMiddle; /* First byte past end of full-size buffers and ** the first byte of LOOKASIDE_SMALL buffers */ @@ -17111,7 +17359,7 @@ struct LookasideSlot { #define EnableLookaside db->lookaside.bDisable--;\ db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue -/* Size of the smaller allocations in two-size lookside */ +/* Size of the smaller allocations in two-size lookaside */ #ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE # define LOOKASIDE_SMALL 0 #else @@ -17311,6 +17559,7 @@ struct sqlite3 { i64 nDeferredCons; /* Net deferred constraints this transaction. */ i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ + DbClientData *pDbData; /* sqlite3_set_clientdata() content */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MAIN ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -17393,6 +17642,7 @@ struct sqlite3 { /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ #define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_FkNoAction HI(0x00008) /* Treat all FK as NO ACTION */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG @@ -17450,6 +17700,7 @@ struct sqlite3 { #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ #define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ +#define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -17532,6 +17783,7 @@ struct FuncDestructor { ** SQLITE_FUNC_ANYORDER == NC_OrderAgg == SF_OrderByReqd ** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG ** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG +** SQLITE_FUNC_BYTELEN == OPFLAG_BYTELENARG ** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API ** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API ** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS -- opposite meanings!!! @@ -17539,7 +17791,7 @@ struct FuncDestructor { ** ** Note that even though SQLITE_FUNC_UNSAFE and SQLITE_INNOCUOUS have the ** same bit value, their meanings are inverted. SQLITE_FUNC_UNSAFE is -** used internally and if set means tha the function has side effects. +** used internally and if set means that the function has side effects. ** SQLITE_INNOCUOUS is used by application code and means "not unsafe". ** See multiple instances of tag-20230109-1. */ @@ -17550,6 +17802,7 @@ struct FuncDestructor { #define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ #define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ #define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_BYTELEN 0x00c0 /* Built-in octet_length() function */ #define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ /* 0x0200 -- available for reuse */ #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ @@ -18129,7 +18382,7 @@ struct FKey { ** foreign key. ** ** The OE_Default value is a place holder that means to use whatever -** conflict resolution algorthm is required from context. +** conflict resolution algorithm is required from context. ** ** The following symbolic values are used to record which type ** of conflict resolution action to take. @@ -18405,6 +18658,9 @@ struct AggInfo { FuncDef *pFunc; /* The aggregate function implementation */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ int iDistAddr; /* Address of OP_OpenEphemeral */ + int iOBTab; /* Ephemeral table to implement ORDER BY */ + u8 bOBPayload; /* iOBTab has payload columns separate from key */ + u8 bOBUnique; /* Enforce uniqueness on iOBTab keys */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ @@ -18543,7 +18799,7 @@ struct Expr { ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old ** EP_Unlikely: 134217728 times likelihood - ** TK_IN: ephemerial table holding RHS + ** TK_IN: ephemeral table holding RHS ** TK_SELECT_COLUMN: Number of columns on the LHS ** TK_SELECT: 1st register of result vector */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. @@ -18589,7 +18845,7 @@ struct Expr { #define EP_Reduced 0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ #define EP_Win 0x008000 /* Contains window functions */ #define EP_TokenOnly 0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ - /* 0x020000 // Available for reuse */ +#define EP_FullSize 0x020000 /* Expr structure must remain full sized */ #define EP_IfNullRow 0x040000 /* The TK_IF_NULL_ROW opcode */ #define EP_Unlikely 0x080000 /* unlikely() or likelihood() function */ #define EP_ConstFunc 0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ @@ -18619,12 +18875,15 @@ struct Expr { #define ExprClearProperty(E,P) (E)->flags&=~(P) #define ExprAlwaysTrue(E) (((E)->flags&(EP_OuterON|EP_IsTrue))==EP_IsTrue) #define ExprAlwaysFalse(E) (((E)->flags&(EP_OuterON|EP_IsFalse))==EP_IsFalse) +#define ExprIsFullSize(E) (((E)->flags&(EP_Reduced|EP_TokenOnly))==0) /* Macros used to ensure that the correct members of unions are accessed ** in Expr. */ #define ExprUseUToken(E) (((E)->flags&EP_IntValue)==0) #define ExprUseUValue(E) (((E)->flags&EP_IntValue)!=0) +#define ExprUseWOfst(E) (((E)->flags&(EP_InnerON|EP_OuterON))==0) +#define ExprUseWJoin(E) (((E)->flags&(EP_InnerON|EP_OuterON))!=0) #define ExprUseXList(E) (((E)->flags&EP_xIsSelect)==0) #define ExprUseXSelect(E) (((E)->flags&EP_xIsSelect)!=0) #define ExprUseYTab(E) (((E)->flags&(EP_WinFunc|EP_Subrtn))==0) @@ -18734,6 +18993,7 @@ struct ExprList { #define ENAME_NAME 0 /* The AS clause of a result set */ #define ENAME_SPAN 1 /* Complete text of the result set expression */ #define ENAME_TAB 2 /* "DB.TABLE.NAME" for the result set */ +#define ENAME_ROWID 3 /* "DB.TABLE._rowid_" for * expansion of rowid */ /* ** An instance of this structure can hold a simple list of identifiers, @@ -18813,7 +19073,7 @@ struct SrcItem { unsigned notCte :1; /* This item may not match a CTE */ unsigned isUsing :1; /* u3.pUsing is valid */ unsigned isOn :1; /* u3.pOn was once valid and non-NULL */ - unsigned isSynthUsing :1; /* u3.pUsing is synthensized from NATURAL */ + unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ @@ -19342,6 +19602,7 @@ struct Parse { int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ + IndexedExpr *pIdxPartExpr; /* Exprs constrained by index WHERE clauses */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ @@ -19349,6 +19610,9 @@ struct Parse { int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ +#endif #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -19362,12 +19626,9 @@ struct Parse { int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ Returning *pReturning; /* The RETURNING clause */ } u1; - u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ -#endif + LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ @@ -19491,6 +19752,7 @@ struct AuthContext { #define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ +#define OPFLAG_BYTELENARG 0xc0 /* OP_Column only for octet_length() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ #define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ #define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ @@ -19612,6 +19874,7 @@ struct Returning { int iRetCur; /* Transient table holding RETURNING results */ int nRetCol; /* Number of in pReturnEL after expansion */ int iRetReg; /* Register array for holding a row of RETURNING */ + char zName[40]; /* Name of trigger: "sqlite_returning_%p" */ }; /* @@ -19633,6 +19896,25 @@ struct sqlite3_str { #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) +/* +** The following object is the header for an "RCStr" or "reference-counted +** string". An RCStr is passed around and used like any other char* +** that has been dynamically allocated. The important interface +** differences: +** +** 1. RCStr strings are reference counted. They are deallocated +** when the reference count reaches zero. +** +** 2. Use sqlite3RCStrUnref() to free an RCStr string rather than +** sqlite3_free() +** +** 3. Make a (read-only) copy of a read-only RCStr string using +** sqlite3RCStrRef(). +*/ +struct RCStr { + u64 nRCRef; /* Number of references */ + /* Total structure size should be a multiple of 8 bytes for alignment */ +}; /* ** A pointer to this structure is used to communicate information @@ -19659,7 +19941,7 @@ typedef struct { /* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled ** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning ** parameters are for temporary use during development, to help find -** optimial values for parameters in the query planner. The should not +** optimal values for parameters in the query planner. The should not ** be used on trunk check-ins. They are a temporary mechanism available ** for transient development builds only. ** @@ -19685,6 +19967,7 @@ struct Sqlite3Config { u8 bUseCis; /* Use covering indices for full-scans */ u8 bSmallMalloc; /* Avoid large memory allocations if true */ u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ + u8 bUseLongDouble; /* Make use of long double */ int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ @@ -19771,6 +20054,7 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ int walkerDepth; /* Number of subqueries */ u16 eCode; /* A small processing code */ + u16 mWFlags; /* Use-dependent flags */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int n; /* A counter */ @@ -19810,6 +20094,7 @@ struct DbFixer { /* Forward declarations */ SQLITE_PRIVATE int sqlite3WalkExpr(Walker*, Expr*); +SQLITE_PRIVATE int sqlite3WalkExprNN(Walker*, Expr*); SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); @@ -19890,6 +20175,16 @@ struct CteUse { }; +/* Client data associated with sqlite3_set_clientdata() and +** sqlite3_get_clientdata(). +*/ +struct DbClientData { + DbClientData *pNext; /* Next in a linked list */ + void *pData; /* The data */ + void (*xDestructor)(void*); /* Destructor. Might be NULL */ + char zName[1]; /* Name of this client data. MUST BE LAST */ +}; + #ifdef SQLITE_DEBUG /* ** An instance of the TreeView object is used for printing the content of @@ -20191,6 +20486,20 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; +/* +** An instance of this object receives the decoding of a floating point +** value into an approximate decimal representation. +*/ +struct FpDecode { + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ + int n; /* Significant digits in the decode */ + int iDP; /* Location of the decimal point */ + char *z; /* Start of significant digits */ + char zBuf[24]; /* Storage for significant digits */ +}; + +SQLITE_PRIVATE void sqlite3FpDecode(FpDecode*,double,int,int); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) @@ -20280,6 +20589,8 @@ SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, const Token*, int); +SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy(Parse*,Expr*,ExprList*); +SQLITE_PRIVATE void sqlite3ExprOrderByAggregateError(Parse*,Expr*); SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); @@ -20481,7 +20792,7 @@ SQLITE_PRIVATE int sqlite3ExprCompare(const Parse*,const Expr*,const Expr*, int) SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr*,Expr*,int); SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList*,const ExprList*, int); SQLITE_PRIVATE int sqlite3ExprImpliesExpr(const Parse*,const Expr*,const Expr*, int); -SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr*,int); +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr*,int,int); SQLITE_PRIVATE void sqlite3AggInfoPersistWalkerInit(Walker*,Parse*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); @@ -20516,6 +20827,7 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); +SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab); SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); @@ -20630,6 +20942,7 @@ SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); + SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); SQLITE_PRIVATE i64 sqlite3RealToI64(double); SQLITE_PRIVATE int sqlite3Int64ToText(i64,char*); @@ -20734,6 +21047,7 @@ SQLITE_PRIVATE void sqlite3FileSuffix3(const char*, char*); SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); @@ -20785,7 +21099,8 @@ SQLITE_PRIVATE int sqlite3MatchEName( const struct ExprList_item*, const char*, const char*, - const char* + const char*, + int* ); SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr*); SQLITE_PRIVATE u8 sqlite3StrIHash(const char*); @@ -20841,6 +21156,11 @@ SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); +SQLITE_PRIVATE char *sqlite3RCStrRef(char*); +SQLITE_PRIVATE void sqlite3RCStrUnref(void*); +SQLITE_PRIVATE char *sqlite3RCStrNew(u64); +SQLITE_PRIVATE char *sqlite3RCStrResize(char*,u64); + SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, i64); SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); @@ -21092,6 +21412,7 @@ SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse*, int); #define sqlite3SelectExprHeight(x) 0 #define sqlite3ExprCheckHeight(x,y) #endif +SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr*,int); SQLITE_PRIVATE u32 sqlite3Get4byte(const u8*); SQLITE_PRIVATE void sqlite3Put4byte(u8*, u32); @@ -21377,9 +21698,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC "4_BYTE_ALIGNED_MALLOC", #endif -#ifdef SQLITE_64BIT_STATS - "64BIT_STATS", -#endif #ifdef SQLITE_ALLOW_COVERING_INDEX_SCAN # if SQLITE_ALLOW_COVERING_INDEX_SCAN != 1 "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), @@ -21675,6 +21993,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS "EXPLAIN_ESTIMATED_ROWS", #endif +#ifdef SQLITE_EXTRA_AUTOEXT + "EXTRA_AUTOEXT=" CTIMEOPT_VAL(SQLITE_EXTRA_AUTOEXT), +#endif #ifdef SQLITE_EXTRA_IFNULLROW "EXTRA_IFNULLROW", #endif @@ -21716,6 +22037,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_INTEGRITY_CHECK_ERROR_MAX "INTEGRITY_CHECK_ERROR_MAX=" CTIMEOPT_VAL(SQLITE_INTEGRITY_CHECK_ERROR_MAX), #endif +#ifdef SQLITE_LEGACY_JSON_VALID + "LEGACY_JSON_VALID", +#endif #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS "LIKE_DOESNT_MATCH_BLOBS", #endif @@ -21953,6 +22277,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS "OMIT_SCHEMA_VERSION_PRAGMAS", #endif +#ifdef SQLITE_OMIT_SEH + "OMIT_SEH", +#endif #ifdef SQLITE_OMIT_SHARED_CACHE "OMIT_SHARED_CACHE", #endif @@ -22350,6 +22677,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0, /* bSmallMalloc */ 1, /* bExtraSchemaChecks */ + sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */ 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ @@ -22579,6 +22907,9 @@ typedef struct VdbeSorter VdbeSorter; /* Elements of the linked list at Vdbe.pAuxData */ typedef struct AuxData AuxData; +/* A cache of large TEXT or BLOB values in a VdbeCursor */ +typedef struct VdbeTxtBlbCache VdbeTxtBlbCache; + /* Types of VDBE cursors */ #define CURTYPE_BTREE 0 #define CURTYPE_SORTER 1 @@ -22610,6 +22941,7 @@ struct VdbeCursor { Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ Bool noReuse:1; /* OpenEphemeral may not reuse this cursor */ + Bool colCache:1; /* pCache pointer is initialized and non-NULL */ u16 seekHit; /* See the OP_SeekHit and OP_IfNoHope opcodes */ union { /* pBtx for isEphermeral. pAltMap otherwise */ Btree *pBtx; /* Separate file holding temporary table */ @@ -22650,6 +22982,7 @@ struct VdbeCursor { #ifdef SQLITE_ENABLE_COLUMN_USED_MASK u64 maskUsed; /* Mask of columns used by this cursor */ #endif + VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for @@ -22662,12 +22995,25 @@ struct VdbeCursor { #define IsNullCursor(P) \ ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0) - /* ** A value for VdbeCursor.cacheStatus that means the cache is always invalid. */ #define CACHE_STALE 0 +/* +** Large TEXT or BLOB values can be slow to load, so we want to avoid +** loading them more than once. For that reason, large TEXT and BLOB values +** can be stored in a cache defined by this object, and attached to the +** VdbeCursor using the pCache field. +*/ +struct VdbeTxtBlbCache { + char *pCValue; /* A RCStr buffer to hold the value */ + i64 iOffset; /* File offset of the row being cached */ + int iCol; /* Column for which the cache is valid */ + u32 cacheStatus; /* Vdbe.cacheCtr value */ + u32 colCacheCtr; /* Column cache counter */ +}; + /* ** When a sub-program is executed (OP_Program), a structure of this type ** is allocated to store the current value of the program counter, as @@ -22988,16 +23334,18 @@ struct Vdbe { u32 nWrite; /* Number of write operations that have occurred */ #endif u16 nResColumn; /* Number of columns in one row of the result set */ + u16 nResAlloc; /* Column slots allocated to aColName[] */ u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 eVdbeState; /* On of the VDBE_*_STATE values */ bft expired:2; /* 1: recompile VM immediately 2: when convenient */ - bft explain:2; /* True if EXPLAIN present on SQL command */ + bft explain:2; /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */ bft changeCntOn:1; /* True to update the change-counter */ bft usesStmtJournal:1; /* True if uses a statement journal */ bft readOnly:1; /* True for statements that do not write */ bft bIsReader:1; /* True for statements that read */ + bft haveEqpOps:1; /* Bytecode supports EXPLAIN QUERY PLAN */ yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ yDbMask lockMask; /* Subset of btreeMask that requires a lock */ u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */ @@ -23044,7 +23392,7 @@ struct PreUpdate { i64 iKey1; /* First key value passed to hook */ i64 iKey2; /* Second key value passed to hook */ Mem *aNew; /* Array of new.* values */ - Table *pTab; /* Schema object being upated */ + Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ }; @@ -23134,6 +23482,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetZeroBlob(Mem*,int); SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); #endif SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); @@ -23730,8 +24079,8 @@ struct DateTime { */ static int getDigits(const char *zDate, const char *zFormat, ...){ /* The aMx[] array translates the 3rd character of each format - ** spec into a max size: a b c d e f */ - static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 14712 }; va_list ap; int cnt = 0; char nextC; @@ -24072,17 +24421,14 @@ static void computeYMD(DateTime *p){ ** Compute the Hour, Minute, and Seconds from the julian day number. */ static void computeHMS(DateTime *p){ - int s; + int day_ms, day_min; /* milliseconds, minutes into the day */ if( p->validHMS ) return; computeJD(p); - s = (int)((p->iJD + 43200000) % 86400000); - p->s = s/1000.0; - s = (int)p->s; - p->s -= s; - p->h = s/3600; - s -= p->h*3600; - p->m = s/60; - p->s += s - p->m*60; + day_ms = (int)((p->iJD + 43200000) % 86400000); + p->s = (day_ms % 60000)/1000.0; + day_min = day_ms/60000; + p->m = day_min % 60; + p->h = day_min / 60; p->rawS = 0; p->validHMS = 1; } @@ -24261,6 +24607,25 @@ static const struct { { 4, "year", 14713.0, 31536000.0 }, }; +/* +** If the DateTime p is raw number, try to figure out if it is +** a julian day number of a unix timestamp. Set the p value +** appropriately. +*/ +static void autoAdjustDate(DateTime *p){ + if( !p->rawS || p->validJD ){ + p->rawS = 0; + }else if( p->s>=-21086676*(i64)10000 /* -4713-11-24 12:00:00 */ + && p->s<=(25340230*(i64)10000)+799 /* 9999-12-31 23:59:59 */ + ){ + double r = p->s*1000.0 + 210866760000000.0; + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + } +} + /* ** Process a modifier to a date-time stamp. The modifiers are ** as follows: @@ -24304,19 +24669,8 @@ static int parseModifier( */ if( sqlite3_stricmp(z, "auto")==0 ){ if( idx>1 ) return 1; /* IMP: R-33611-57934 */ - if( !p->rawS || p->validJD ){ - rc = 0; - p->rawS = 0; - }else if( p->s>=-21086676*(i64)10000 /* -4713-11-24 12:00:00 */ - && p->s<=(25340230*(i64)10000)+799 /* 9999-12-31 23:59:59 */ - ){ - r = p->s*1000.0 + 210866760000000.0; - clearYMD_HMS_TZ(p); - p->iJD = (sqlite3_int64)(r + 0.5); - p->validJD = 1; - p->rawS = 0; - rc = 0; - } + autoAdjustDate(p); + rc = 0; } break; } @@ -24482,18 +24836,73 @@ static int parseModifier( case '9': { double rRounder; int i; - for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + int Y,M,D,h,m,x; + const char *z2 = z; + char z0 = z[0]; + for(n=1; z[n]; n++){ + if( z[n]==':' ) break; + if( sqlite3Isspace(z[n]) ) break; + if( z[n]=='-' ){ + if( n==5 && getDigits(&z[1], "40f", &Y)==1 ) break; + if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; + } + } if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ - rc = 1; + assert( rc==1 ); break; } - if( z[n]==':' ){ + if( z[n]=='-' ){ + /* A modifier of the form (+|-)YYYY-MM-DD adds or subtracts the + ** specified number of years, months, and days. MM is limited to + ** the range 0-11 and DD is limited to 0-30. + */ + if( z0!='+' && z0!='-' ) break; /* Must start with +/- */ + if( n==5 ){ + if( getDigits(&z[1], "40f-20a-20d", &Y, &M, &D)!=3 ) break; + }else{ + assert( n==6 ); + if( getDigits(&z[1], "50f-20a-20d", &Y, &M, &D)!=3 ) break; + z++; + } + if( M>=12 ) break; /* M range 0..11 */ + if( D>=31 ) break; /* D range 0..30 */ + computeYMD_HMS(p); + p->validJD = 0; + if( z0=='-' ){ + p->Y -= Y; + p->M -= M; + D = -D; + }else{ + p->Y += Y; + p->M += M; + } + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + computeJD(p); + p->validHMS = 0; + p->validYMD = 0; + p->iJD += (i64)D*86400000; + if( z[11]==0 ){ + rc = 0; + break; + } + if( sqlite3Isspace(z[11]) + && getDigits(&z[12], "20c:20e", &h, &m)==2 + ){ + z2 = &z[12]; + n = 2; + }else{ + break; + } + } + if( z2[n]==':' ){ /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the ** specified number of hours, minutes, seconds, and fractional seconds ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be ** omitted. */ - const char *z2 = z; + DateTime tx; sqlite3_int64 day; if( !sqlite3Isdigit(*z2) ) z2++; @@ -24503,7 +24912,7 @@ static int parseModifier( tx.iJD -= 43200000; day = tx.iJD/86400000; tx.iJD -= day*86400000; - if( z[0]=='-' ) tx.iJD = -tx.iJD; + if( z0=='-' ) tx.iJD = -tx.iJD; computeJD(p); clearYMD_HMS_TZ(p); p->iJD += tx.iJD; @@ -24519,7 +24928,7 @@ static int parseModifier( if( n>10 || n<3 ) break; if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); - rc = 1; + assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; for(i=0; iM += (int)r; @@ -24687,7 +25095,7 @@ static void datetimeFunc( zBuf[16] = '0' + (x.m)%10; zBuf[17] = ':'; if( x.useSubsec ){ - s = (int)1000.0*x.s; + s = (int)(1000.0*x.s + 0.5); zBuf[18] = '0' + (s/10000)%10; zBuf[19] = '0' + (s/1000)%10; zBuf[20] = '.'; @@ -24734,7 +25142,7 @@ static void timeFunc( zBuf[4] = '0' + (x.m)%10; zBuf[5] = ':'; if( x.useSubsec ){ - s = (int)1000.0*x.s; + s = (int)(1000.0*x.s + 0.5); zBuf[6] = '0' + (s/10000)%10; zBuf[7] = '0' + (s/1000)%10; zBuf[8] = '.'; @@ -24805,7 +25213,7 @@ static void dateFunc( ** %M minute 00-59 ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 sunday==0 +** %w day of week 0-6 Sunday==0 ** %W week of year 00-53 ** %Y year 0000-9999 ** %% % @@ -24831,13 +25239,16 @@ static void strftimeFunc( computeJD(&x); computeYMD_HMS(&x); for(i=j=0; zFmt[i]; i++){ + char cf; if( zFmt[i]!='%' ) continue; if( j12 ) h -= 12; + if( h==0 ) h = 12; + sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); break; } case 'W': /* Fall thru */ @@ -24859,7 +25283,7 @@ static void strftimeFunc( y.D = 1; computeJD(&y); nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( zFmt[i]=='W' ){ + if( cf=='W' ){ int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ wd = (int)(((x.iJD+43200000)/86400000)%7); sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); @@ -24880,6 +25304,19 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d",x.m); break; } + case 'p': /* Fall thru */ + case 'P': { + if( x.h>=12 ){ + sqlite3_str_append(&sRes, cf=='p' ? "PM" : "pm", 2); + }else{ + sqlite3_str_append(&sRes, cf=='p' ? "AM" : "am", 2); + } + break; + } + case 'R': { + sqlite3_str_appendf(&sRes, "%02d:%02d", x.h, x.m); + break; + } case 's': { if( x.useSubsec ){ sqlite3_str_appendf(&sRes,"%.3f", @@ -24894,9 +25331,15 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d",(int)x.s); break; } + case 'T': { + sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); + break; + } + case 'u': /* Fall thru */ case 'w': { - sqlite3_str_appendchar(&sRes, 1, - (char)(((x.iJD+129600000)/86400000) % 7) + '0'); + char c = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + if( c=='0' && cf=='u' ) c = '7'; + sqlite3_str_appendchar(&sRes, 1, c); break; } case 'Y': { @@ -24945,6 +25388,117 @@ static void cdateFunc( dateFunc(context, 0, 0); } +/* +** timediff(DATE1, DATE2) +** +** Return the amount of time that must be added to DATE2 in order to +** convert it into DATE2. The time difference format is: +** +** +YYYY-MM-DD HH:MM:SS.SSS +** +** The initial "+" becomes "-" if DATE1 occurs before DATE2. For +** date/time values A and B, the following invariant should hold: +** +** datetime(A) == (datetime(B, timediff(A,B)) +** +** Both DATE arguments must be either a julian day number, or an +** ISO-8601 string. The unix timestamps are not supported by this +** routine. +*/ +static void timediffFunc( + sqlite3_context *context, + int NotUsed1, + sqlite3_value **argv +){ + char sign; + int Y, M; + DateTime d1, d2; + sqlite3_str sRes; + UNUSED_PARAMETER(NotUsed1); + if( isDate(context, 1, &argv[0], &d1) ) return; + if( isDate(context, 1, &argv[1], &d2) ) return; + computeYMD_HMS(&d1); + computeYMD_HMS(&d2); + if( d1.iJD>=d2.iJD ){ + sign = '+'; + Y = d1.Y - d2.Y; + if( Y ){ + d2.Y = d1.Y; + d2.validJD = 0; + computeJD(&d2); + } + M = d1.M - d2.M; + if( M<0 ){ + Y--; + M += 12; + } + if( M!=0 ){ + d2.M = d1.M; + d2.validJD = 0; + computeJD(&d2); + } + while( d1.iJDd2.iJD ){ + M--; + if( M<0 ){ + M = 11; + Y--; + } + d2.M++; + if( d2.M>12 ){ + d2.M = 1; + d2.Y++; + } + d2.validJD = 0; + computeJD(&d2); + } + d1.iJD = d2.iJD - d1.iJD; + d1.iJD += (u64)1486995408 * (u64)100000; + } + d1.validYMD = 0; + d1.validHMS = 0; + d1.validTZ = 0; + computeYMD_HMS(&d1); + sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); + sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", + sign, Y, M, d1.D-1, d1.h, d1.m, d1.s); + sqlite3ResultStrAccum(context, &sRes); +} + + /* ** current_timestamp() ** @@ -25019,6 +25573,7 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ PURE_DATE(time, -1, 0, 0, timeFunc ), PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), + PURE_DATE(timediff, 2, 0, 0, timediffFunc ), DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), @@ -25172,7 +25727,7 @@ SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite ** is using a regular VFS, it is called after the corresponding ** transaction has been committed. Injecting a fault at this point - ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM + ** confuses the test scripts - the COMMIT command returns SQLITE_NOMEM ** but the transaction is committed anyway. ** ** The core must call OsFileControl() though, not OsFileControlHint(), @@ -25793,7 +26348,7 @@ static void *sqlite3MemMalloc(int nByte){ ** or sqlite3MemRealloc(). ** ** For this low-level routine, we already know that pPrior!=0 since -** cases where pPrior==0 will have been intecepted and dealt with +** cases where pPrior==0 will have been intercepted and dealt with ** by higher-level routines. */ static void sqlite3MemFree(void *pPrior){ @@ -25881,7 +26436,7 @@ static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } len = sizeof(cpuCount); - /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + /* One usually wants to use hw.activecpu for MT decisions, but not here */ sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); if( cpuCount>1 ){ /* defer MT decisions to system malloc */ @@ -27873,7 +28428,7 @@ static void checkMutexFree(sqlite3_mutex *p){ assert( SQLITE_MUTEX_FAST<2 ); assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( ((CheckMutex*)p)->iType<2 ) #endif { @@ -28348,7 +28903,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ /* ** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields -** are necessary under two condidtions: (1) Debug builds and (2) using +** are necessary under two conditions: (1) Debug builds and (2) using ** home-grown mutexes. Encapsulate these conditions into a single #define. */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) @@ -28545,7 +29100,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ */ static void pthreadMutexFree(sqlite3_mutex *p){ assert( p->nRef==0 ); -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) #endif { @@ -28849,7 +29404,7 @@ struct sqlite3_mutex { CRITICAL_SECTION mutex; /* Mutex controlling the lock */ int id; /* Mutex type */ #ifdef SQLITE_DEBUG - volatile int nRef; /* Number of enterances */ + volatile int nRef; /* Number of entrances */ volatile DWORD owner; /* Thread holding this mutex */ volatile LONG trace; /* True to trace changes */ #endif @@ -30109,7 +30664,7 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ if( db->mallocFailed || rc ){ return apiHandleError(db, rc); } - return rc & db->errMask; + return 0; } /************** End of malloc.c **********************************************/ @@ -30221,57 +30776,6 @@ static const et_info fmtinfo[] = { ** %!S Like %S but prefer the zName over the zAlias */ -/* Floating point constants used for rounding */ -static const double arRound[] = { - 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, - 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, -}; - -/* -** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point -** conversions will work. -*/ -#ifndef SQLITE_OMIT_FLOATING_POINT -/* -** "*val" is a double such that 0.1 <= *val < 10.0 -** Return the ascii code for the leading digit of *val, then -** multiply "*val" by 10.0 to renormalize. -** -** Example: -** input: *val = 3.14159 -** output: *val = 1.4159 function return = '3' -** -** The counter *cnt is incremented each time. After counter exceeds -** 16 (the number of significant digits in a 64-bit float) '0' is -** always returned. -*/ -static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ - int digit; - LONGDOUBLE_TYPE d; - if( (*cnt)<=0 ) return '0'; - (*cnt)--; - digit = (int)*val; - d = digit; - digit += '0'; - *val = (*val - d)*10.0; - return (char)digit; -} -#endif /* SQLITE_OMIT_FLOATING_POINT */ - -#ifndef SQLITE_OMIT_FLOATING_POINT -/* -** "*val" is a u64. *msd is a divisor used to extract the -** most significant digit of *val. Extract that most significant -** digit and return it. -*/ -static char et_getdigit_int(u64 *val, u64 *msd){ - u64 x = (*val)/(*msd); - *val -= x*(*msd); - if( *msd>=10 ) *msd /= 10; - return '0' + (char)(x & 15); -} -#endif /* SQLITE_OMIT_FLOATING_POINT */ - /* ** Set the StrAccum object to an error mode. */ @@ -30363,20 +30867,15 @@ SQLITE_API void sqlite3_str_vappendf( u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ - LONGDOUBLE_TYPE realvalue; /* Value for real types */ - sqlite_uint64 msd; /* Divisor to get most-significant-digit - ** of longvalue */ + double realvalue; /* Value for real types */ const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ char *zExtra = 0; /* Malloced memory used by some conversion */ -#ifndef SQLITE_OMIT_FLOATING_POINT - int exp, e2; /* exponent of real numbers */ - int nsd; /* Number of significant digits returned */ - double rounder; /* Used for rounding floating point values */ + int exp, e2; /* exponent of real numbers */ etByte flag_dp; /* True if decimal point should be shown */ etByte flag_rtz; /* True if trailing zeros should be removed */ -#endif + PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ @@ -30651,94 +31150,61 @@ SQLITE_API void sqlite3_str_vappendf( break; case etFLOAT: case etEXP: - case etGENERIC: + case etGENERIC: { + FpDecode s; + int iRound; + int j; + if( bArgList ){ realvalue = getDoubleArg(pArgList); }else{ realvalue = va_arg(ap,double); } -#ifdef SQLITE_OMIT_FLOATING_POINT - length = 0; -#else if( precision<0 ) precision = 6; /* Set default precision */ #ifdef SQLITE_FP_PRECISION_LIMIT if( precision>SQLITE_FP_PRECISION_LIMIT ){ precision = SQLITE_FP_PRECISION_LIMIT; } #endif - if( realvalue<0.0 ){ - realvalue = -realvalue; - prefix = '-'; + if( xtype==etFLOAT ){ + iRound = -precision; + }else if( xtype==etGENERIC ){ + iRound = precision; }else{ - prefix = flag_prefix; + iRound = precision+1; } - exp = 0; - if( xtype==etGENERIC && precision>0 ) precision--; - testcase( precision>0xfff ); - if( realvalue<1.0e+16 - && realvalue==(LONGDOUBLE_TYPE)(longvalue = (u64)realvalue) - ){ - /* Number is a pure integer that can be represented as u64 */ - for(msd=1; msd*10<=longvalue; msd *= 10, exp++){} - if( exp>precision && xtype!=etFLOAT ){ - u64 rnd = msd/2; - int kk = precision; - while( kk-- > 0 ){ rnd /= 10; } - longvalue += rnd; - } - }else{ - msd = 0; - longvalue = 0; /* To prevent a compiler warning */ - idx = precision & 0xfff; - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } - if( xtype==etFLOAT ){ - double rx = (double)realvalue; - sqlite3_uint64 u; - int ex; - memcpy(&u, &rx, sizeof(u)); - ex = -1023 + (int)((u>>52)&0x7ff); - if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; - realvalue += rounder; - } - if( sqlite3IsNaN((double)realvalue) ){ - if( flag_zeropad ){ - bufpt = "null"; - length = 4; + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); + if( s.isSpecial ){ + if( s.isSpecial==2 ){ + bufpt = flag_zeropad ? "null" : "NaN"; + length = sqlite3Strlen30(bufpt); + break; + }else if( flag_zeropad ){ + s.z[0] = '9'; + s.iDP = 1000; + s.n = 1; + }else{ + memcpy(buf, "-Inf", 5); + bufpt = buf; + if( s.sign=='-' ){ + /* no-op */ + }else if( flag_prefix ){ + buf[0] = flag_prefix; }else{ - bufpt = "NaN"; - length = 3; + bufpt++; } + length = sqlite3Strlen30(bufpt); break; } - - /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - if( ALWAYS(realvalue>0.0) ){ - LONGDOUBLE_TYPE scale = 1.0; - while( realvalue>=1e100*scale && exp<=350){ scale*=1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale*=1e10; exp+=10; } - while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } - realvalue /= scale; - while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } - if( exp>350 ){ - if( flag_zeropad ){ - realvalue = 9.0; - exp = 999; - }else{ - bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); - break; - } - } - if( xtype!=etFLOAT ){ - realvalue += rounder; - if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } - } - } } + if( s.sign=='-' ){ + prefix = '-'; + }else{ + prefix = flag_prefix; + } + + exp = s.iDP-1; + if( xtype==etGENERIC && precision>0 ) precision--; /* ** If the field type is etGENERIC, then convert to either etEXP @@ -30758,9 +31224,8 @@ SQLITE_API void sqlite3_str_vappendf( if( xtype==etEXP ){ e2 = 0; }else{ - e2 = exp; + e2 = s.iDP - 1; } - nsd = 16 + flag_altform2*10; bufpt = buf; { i64 szBufNeeded; /* Size of a temporary buffer needed */ @@ -30778,16 +31243,12 @@ SQLITE_API void sqlite3_str_vappendf( *(bufpt++) = prefix; } /* Digits prior to the decimal point */ + j = 0; if( e2<0 ){ *(bufpt++) = '0'; - }else if( msd>0 ){ - for(; e2>=0; e2--){ - *(bufpt++) = et_getdigit_int(&longvalue,&msd); - if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; - } }else{ for(; e2>=0; e2--){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + *(bufpt++) = j1 ) *(bufpt++) = ','; } } @@ -30797,19 +31258,12 @@ SQLITE_API void sqlite3_str_vappendf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0; precision--, e2++){ - assert( precision>0 ); + for(e2++; e2<0 && precision>0; precision--, e2++){ *(bufpt++) = '0'; } /* Significant digits after the decimal point */ - if( msd>0 ){ - while( (precision--)>0 ){ - *(bufpt++) = et_getdigit_int(&longvalue,&msd); - } - }else{ - while( (precision--)>0 ){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); - } + while( (precision--)>0 ){ + *(bufpt++) = jcharset]; if( exp<0 ){ *(bufpt++) = '-'; exp = -exp; @@ -30858,8 +31313,8 @@ SQLITE_API void sqlite3_str_vappendf( while( nPad-- ) bufpt[i++] = '0'; length = width; } -#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ break; + } case etSIZE: if( !bArgList ){ *(va_arg(ap,int*)) = pAccum->nChar; @@ -31583,6 +32038,75 @@ SQLITE_API void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ va_end(ap); } + +/***************************************************************************** +** Reference counted string storage +*****************************************************************************/ + +/* +** Increase the reference count of the string by one. +** +** The input parameter is returned. +*/ +SQLITE_PRIVATE char *sqlite3RCStrRef(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + p->nRCRef++; + return z; +} + +/* +** Decrease the reference count by one. Free the string when the +** reference count reaches zero. +*/ +SQLITE_PRIVATE void sqlite3RCStrUnref(void *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + if( p->nRCRef>=2 ){ + p->nRCRef--; + }else{ + sqlite3_free(p); + } +} + +/* +** Create a new string that is capable of holding N bytes of text, not counting +** the zero byte at the end. The string is uninitialized. +** +** The reference count is initially 1. Call sqlite3RCStrUnref() to free the +** newly allocated string. +** +** This routine returns 0 on an OOM. +*/ +SQLITE_PRIVATE char *sqlite3RCStrNew(u64 N){ + RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); + if( p==0 ) return 0; + p->nRCRef = 1; + return (char*)&p[1]; +} + +/* +** Change the size of the string so that it is able to hold N bytes. +** The string might be reallocated, so return the new allocation. +*/ +SQLITE_PRIVATE char *sqlite3RCStrResize(char *z, u64 N){ + RCStr *p = (RCStr*)z; + RCStr *pNew; + assert( p!=0 ); + p--; + assert( p->nRCRef==1 ); + pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); + if( pNew==0 ){ + sqlite3_free(p); + return 0; + }else{ + return (char*)&pNew[1]; + } +} + /************** End of printf.c **********************************************/ /************** Begin file treeview.c ****************************************/ /* @@ -31999,6 +32523,7 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u sqlite3TreeViewItem(pView, "FILTER", 1); sqlite3TreeViewExpr(pView, pWin->pFilter, 0); sqlite3TreeViewPop(&pView); + if( pWin->eFrmType==TK_FILTER ) return; } sqlite3TreeViewPush(&pView, more); if( pWin->zName ){ @@ -32008,7 +32533,7 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u } if( pWin->zBase ) nElement++; if( pWin->pOrderBy ) nElement++; - if( pWin->eFrmType ) nElement++; + if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ) nElement++; if( pWin->eExclude ) nElement++; if( pWin->zBase ){ sqlite3TreeViewPush(&pView, (--nElement)>0); @@ -32021,7 +32546,7 @@ SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u if( pWin->pOrderBy ){ sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY"); } - if( pWin->eFrmType ){ + if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ){ char zBuf[30]; const char *zFrmType = "ROWS"; if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE"; @@ -32230,7 +32755,8 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m }; assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); assert( pExpr->pRight ); - assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE ); + assert( sqlite3ExprSkipCollateAndLikely(pExpr->pRight)->op + == TK_TRUEFALSE ); x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); zUniOp = azOp[x]; break; @@ -32268,7 +32794,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m assert( ExprUseXList(pExpr) ); pFarg = pExpr->x.pList; #ifndef SQLITE_OMIT_WINDOWFUNC - pWin = ExprHasProperty(pExpr, EP_WinFunc) ? pExpr->y.pWin : 0; + pWin = IsWindowFunc(pExpr) ? pExpr->y.pWin : 0; #else pWin = 0; #endif @@ -32294,7 +32820,13 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs); } if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0); + sqlite3TreeViewExprList(pView, pFarg, pWin!=0 || pExpr->pLeft, 0); + if( pExpr->pLeft ){ + Expr *pOB = pExpr->pLeft; + assert( pOB->op==TK_ORDER ); + assert( ExprUseXList(pOB) ); + sqlite3TreeViewExprList(pView, pOB->x.pList, pWin!=0, "ORDERBY"); + } } #ifndef SQLITE_OMIT_WINDOWFUNC if( pWin ){ @@ -32303,6 +32835,10 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m #endif break; } + case TK_ORDER: { + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, "ORDERBY"); + break; + } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: { assert( ExprUseXSelect(pExpr) ); @@ -33889,7 +34425,7 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ /* ** Calls to sqlite3FaultSim() are used to simulate a failure during testing, ** or to bypass normal error detection during testing in order to let -** execute proceed futher downstream. +** execute proceed further downstream. ** ** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0). The ** sqlite3FaultSim() function only returns non-zero during testing. @@ -34006,6 +34542,23 @@ SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3 *db){ */ SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ if( rc==SQLITE_IOERR_NOMEM ) return; +#ifdef SQLITE_USE_SEH + if( rc==SQLITE_IOERR_IN_PAGE ){ + int ii; + int iErr; + sqlite3BtreeEnterAll(db); + for(ii=0; iinDb; ii++){ + if( db->aDb[ii].pBt ){ + iErr = sqlite3PagerWalSystemErrno(sqlite3BtreePager(db->aDb[ii].pBt)); + if( iErr ){ + db->iSysErrno = iErr; + } + } + } + sqlite3BtreeLeaveAll(db); + return; + } +#endif rc &= 0xff; if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ db->iSysErrno = sqlite3OsGetLastError(db->pVfs); @@ -34050,12 +34603,16 @@ SQLITE_PRIVATE void sqlite3ProgressCheck(Parse *p){ p->rc = SQLITE_INTERRUPT; } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( db->xProgress && (++p->nProgressSteps)>=db->nProgressOps ){ - if( db->xProgress(db->pProgressArg) ){ - p->nErr++; - p->rc = SQLITE_INTERRUPT; + if( db->xProgress ){ + if( p->rc==SQLITE_INTERRUPT ){ + p->nProgressSteps = 0; + }else if( (++p->nProgressSteps)>=db->nProgressOps ){ + if( db->xProgress(db->pProgressArg) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } + p->nProgressSteps = 0; } - p->nProgressSteps = 0; } #endif } @@ -34251,43 +34808,40 @@ SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ return h; } -/* -** Compute 10 to the E-th power. Examples: E==1 results in 10. -** E==2 results in 100. E==50 results in 1.0e50. +/* Double-Double multiplication. (x[0],x[1]) *= (y,yy) ** -** This routine only works for values of E between 1 and 341. +** Reference: +** T. J. Dekker, "A Floating-Point Technique for Extending the +** Available Precision". 1971-07-26. */ -static LONGDOUBLE_TYPE sqlite3Pow10(int E){ -#if defined(_MSC_VER) - static const LONGDOUBLE_TYPE x[] = { - 1.0e+001L, - 1.0e+002L, - 1.0e+004L, - 1.0e+008L, - 1.0e+016L, - 1.0e+032L, - 1.0e+064L, - 1.0e+128L, - 1.0e+256L - }; - LONGDOUBLE_TYPE r = 1.0; - int i; - assert( E>=0 && E<=307 ); - for(i=0; E!=0; i++, E >>=1){ - if( E & 1 ) r *= x[i]; - } - return r; -#else - LONGDOUBLE_TYPE x = 10.0; - LONGDOUBLE_TYPE r = 1.0; - while(1){ - if( E & 1 ) r *= x; - E >>= 1; - if( E==0 ) break; - x *= x; - } - return r; -#endif +static void dekkerMul2(volatile double *x, double y, double yy){ + /* + ** The "volatile" keywords on parameter x[] and on local variables + ** below are needed force intermediate results to be truncated to + ** binary64 rather than be carried around in an extended-precision + ** format. The truncation is necessary for the Dekker algorithm to + ** work. Intel x86 floating point might omit the truncation without + ** the use of volatile. + */ + volatile double tx, ty, p, q, c, cc; + double hx, hy; + u64 m; + memcpy(&m, (void*)&x[0], 8); + m &= 0xfffffffffc000000LL; + memcpy(&hx, &m, 8); + tx = x[0] - hx; + memcpy(&m, &y, 8); + m &= 0xfffffffffc000000LL; + memcpy(&hy, &m, 8); + ty = y - hy; + p = hx*hy; + q = hx*ty + tx*hy; + c = p+q; + cc = p - c + q + tx*ty; + cc = x[0]*yy + x[1]*y + cc; + x[0] = c + cc; + x[1] = c - x[0]; + x[1] += cc; } /* @@ -34328,12 +34882,11 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en const char *zEnd; /* sign * significand * (10 ^ (esign * exponent)) */ int sign = 1; /* sign of significand */ - i64 s = 0; /* significand */ + u64 s = 0; /* significand */ int d = 0; /* adjust exponent for shifting decimal point */ int esign = 1; /* sign of exponent */ int e = 0; /* exponent */ int eValid = 1; /* True exponent is either not used or is well-formed */ - double result; int nDigit = 0; /* Number of digits processed */ int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ @@ -34373,7 +34926,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en while( z=((LARGEST_INT64-9)/10) ){ + if( s>=((LARGEST_UINT64-9)/10) ){ /* skip non-significant significand digits ** (increase exponent by d to shift decimal left) */ while( z0 ){ /*OPTIMIZATION-IF-TRUE*/ - if( esign>0 ){ - if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ - s *= 10; - }else{ - if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ - s /= 10; - } - e--; - } + /* adjust exponent by d, and update sign */ + e = (e*esign) + d; - /* adjust the sign of significand */ - s = sign<0 ? -s : s; + /* Try to adjust the exponent to make it smaller */ + while( e>0 && s<(LARGEST_UINT64/10) ){ + s *= 10; + e--; + } + while( e<0 && (s%10)==0 ){ + s /= 10; + e++; + } - if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ - result = (double)s; + if( e==0 ){ + *pResult = s; + }else if( sqlite3Config.bUseLongDouble ){ + LONGDOUBLE_TYPE r = (LONGDOUBLE_TYPE)s; + if( e>0 ){ + while( e>=100 ){ e-=100; r *= 1.0e+100L; } + while( e>=10 ){ e-=10; r *= 1.0e+10L; } + while( e>=1 ){ e-=1; r *= 1.0e+01L; } }else{ - /* attempt to handle extremely small/large numbers better */ - if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ - if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ - LONGDOUBLE_TYPE scale = sqlite3Pow10(e-308); - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else{ assert( e>=342 ); - if( esign<0 ){ - result = 0.0*s; - }else{ + while( e<=-100 ){ e+=100; r *= 1.0e-100L; } + while( e<=-10 ){ e+=10; r *= 1.0e-10L; } + while( e<=-1 ){ e+=1; r *= 1.0e-01L; } + } + assert( r>=0.0 ); + if( r>+1.7976931348623157081452742373e+308L ){ #ifdef INFINITY - result = INFINITY*s; + *pResult = +INFINITY; #else - result = 1e308*1e308*s; /* Infinity */ + *pResult = 1.0e308*10.0; #endif - } - } - }else{ - LONGDOUBLE_TYPE scale = sqlite3Pow10(e); - if( esign<0 ){ - result = s / scale; - }else{ - result = s * scale; - } + }else{ + *pResult = (double)r; + } + }else{ + double rr[2]; + u64 s2; + rr[0] = (double)s; + s2 = (u64)rr[0]; + rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); + if( e>0 ){ + while( e>=100 ){ + e -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( e>=10 ){ + e -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( e>=1 ){ + e -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } + }else{ + while( e<=-100 ){ + e += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( e<=-10 ){ + e += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( e<=-1 ){ + e += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); } } + *pResult = rr[0]+rr[1]; + if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; } + if( sign<0 ) *pResult = -*pResult; + assert( !sqlite3IsNaN(*pResult) ); - /* store the result */ - *pResult = result; - - /* return true if number and no extra non-whitespace chracters after */ +atof_return: + /* return true if number and no extra non-whitespace characters after */ if( z==zEnd && nDigit>0 && eValid && eType>0 ){ return eType; }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ @@ -34636,7 +35199,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc /* This test and assignment is needed only to suppress UB warnings ** from clang and -fsanitize=undefined. This test and assignment make ** the code a little larger and slower, and no harm comes from omitting - ** them, but we must appaise the undefined-behavior pharisees. */ + ** them, but we must appease the undefined-behavior pharisees. */ *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; }else if( neg ){ *pNum = -(i64)u; @@ -34714,7 +35277,9 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { - return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789")); + if( z[n] ) n++; + return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8); } } @@ -34793,6 +35358,153 @@ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ return x; } +/* +** Decode a floating-point value into an approximate decimal +** representation. +** +** Round the decimal representation to n significant digits if +** n is positive. Or round to -n signficant digits after the +** decimal point if n is negative. No rounding is performed if +** n is zero. +** +** The significant digits of the decimal representation are +** stored in p->z[] which is a often (but not always) a pointer +** into the middle of p->zBuf[]. There are p->n significant digits. +** The p->z[] array is *not* zero-terminated. +*/ +SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ + int i; + u64 v; + int e, exp = 0; + p->isSpecial = 0; + p->z = p->zBuf; + + /* Convert negative numbers to positive. Deal with Infinity, 0.0, and + ** NaN. */ + if( r<0.0 ){ + p->sign = '-'; + r = -r; + }else if( r==0.0 ){ + p->sign = '+'; + p->n = 1; + p->iDP = 1; + p->z = "0"; + return; + }else{ + p->sign = '+'; + } + memcpy(&v,&r,8); + e = v>>52; + if( (e&0x7ff)==0x7ff ){ + p->isSpecial = 1 + (v!=0x7ff0000000000000LL); + p->n = 0; + p->iDP = 0; + return; + } + + /* Multiply r by powers of ten until it lands somewhere in between + ** 1.0e+19 and 1.0e+17. + */ + if( sqlite3Config.bUseLongDouble ){ + LONGDOUBLE_TYPE rr = r; + if( rr>=1.0e+19 ){ + while( rr>=1.0e+119L ){ exp+=100; rr *= 1.0e-100L; } + while( rr>=1.0e+29L ){ exp+=10; rr *= 1.0e-10L; } + while( rr>=1.0e+19L ){ exp++; rr *= 1.0e-1L; } + }else{ + while( rr<1.0e-97L ){ exp-=100; rr *= 1.0e+100L; } + while( rr<1.0e+07L ){ exp-=10; rr *= 1.0e+10L; } + while( rr<1.0e+17L ){ exp--; rr *= 1.0e+1L; } + } + v = (u64)rr; + }else{ + /* If high-precision floating point is not available using "long double", + ** then use Dekker-style double-double computation to increase the + ** precision. + ** + ** The error terms on constants like 1.0e+100 computed using the + ** decimal extension, for example as follows: + ** + ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); + */ + double rr[2]; + rr[0] = r; + rr[1] = 0.0; + if( rr[0]>9.223372036854774784e+18 ){ + while( rr[0]>9.223372036854774784e+118 ){ + exp += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( rr[0]>9.223372036854774784e+28 ){ + exp += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( rr[0]>9.223372036854774784e+18 ){ + exp += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); + } + }else{ + while( rr[0]<9.223372036854774784e-83 ){ + exp -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( rr[0]<9.223372036854774784e+07 ){ + exp -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( rr[0]<9.22337203685477478e+17 ){ + exp -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } + } + v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; + } + + + /* Extract significant digits. */ + i = sizeof(p->zBuf)-1; + assert( v>0 ); + while( v ){ p->zBuf[i--] = (v%10) + '0'; v /= 10; } + assert( i>=0 && izBuf)-1 ); + p->n = sizeof(p->zBuf) - 1 - i; + assert( p->n>0 ); + assert( p->nzBuf) ); + p->iDP = p->n + exp; + if( iRound<0 ){ + iRound = p->iDP - iRound; + if( iRound==0 && p->zBuf[i+1]>='5' ){ + iRound = 1; + p->zBuf[i--] = '0'; + p->n++; + p->iDP++; + } + } + if( iRound>0 && (iRoundn || p->n>mxRound) ){ + char *z = &p->zBuf[i+1]; + if( iRound>mxRound ) iRound = mxRound; + p->n = iRound; + if( z[iRound]>='5' ){ + int j = iRound-1; + while( 1 /*exit-by-break*/ ){ + z[j]++; + if( z[j]<='9' ) break; + z[j] = '0'; + if( j==0 ){ + p->z[i--] = '1'; + p->n++; + p->iDP++; + break; + }else{ + j--; + } + } + } + } + p->z = &p->zBuf[i+1]; + assert( i+p->n < sizeof(p->zBuf) ); + while( ALWAYS(p->n>0) && p->z[p->n-1]=='0' ){ p->n--; } +} + /* ** Try to convert z into an unsigned 32-bit integer. Return true on ** success and false if there is an error. @@ -35056,121 +35768,32 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ ** this function assumes the single-byte case has already been handled. */ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ - u32 a,b; + u64 v64; + u8 n; - /* The 1-byte case. Overwhelmingly the most common. Handled inline - ** by the getVarin32() macro */ - a = *p; - /* a: p0 (unmasked) */ -#ifndef getVarint32 - if (!(a&0x80)) - { - /* Values between 0 and 127 */ - *v = a; - return 1; - } -#endif + /* Assume that the single-byte case has already been handled by + ** the getVarint32() macro */ + assert( (p[0] & 0x80)!=0 ); - /* The 2-byte case */ - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 128 and 16383 */ - a &= 0x7f; - a = a<<7; - *v = a | b; + if( (p[1] & 0x80)==0 ){ + /* This is the two-byte case */ + *v = ((p[0]&0x7f)<<7) | p[1]; return 2; } - - /* The 3-byte case */ - p++; - a = a<<14; - a |= *p; - /* a: p0<<14 | p2 (unmasked) */ - if (!(a&0x80)) - { - /* Values between 16384 and 2097151 */ - a &= (0x7f<<14)|(0x7f); - b &= 0x7f; - b = b<<7; - *v = a | b; + if( (p[2] & 0x80)==0 ){ + /* This is the three-byte case */ + *v = ((p[0]&0x7f)<<14) | ((p[1]&0x7f)<<7) | p[2]; return 3; } - - /* A 32-bit varint is used to store size information in btrees. - ** Objects are rarely larger than 2MiB limit of a 3-byte varint. - ** A 3-byte varint is sufficient, for example, to record the size - ** of a 1048569-byte BLOB or string. - ** - ** We only unroll the first 1-, 2-, and 3- byte cases. The very - ** rare larger cases can be handled by the slower 64-bit varint - ** routine. - */ -#if 1 - { - u64 v64; - u8 n; - - n = sqlite3GetVarint(p-2, &v64); - assert( n>3 && n<=9 ); - if( (v64 & SQLITE_MAX_U32)!=v64 ){ - *v = 0xffffffff; - }else{ - *v = (u32)v64; - } - return n; - } - -#else - /* For following code (kept for historical record only) shows an - ** unrolling for the 3- and 4-byte varint cases. This code is - ** slightly faster, but it is also larger and much harder to test. - */ - p++; - b = b<<14; - b |= *p; - /* b: p1<<14 | p3 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 2097152 and 268435455 */ - b &= (0x7f<<14)|(0x7f); - a &= (0x7f<<14)|(0x7f); - a = a<<7; - *v = a | b; - return 4; - } - - p++; - a = a<<14; - a |= *p; - /* a: p0<<28 | p2<<14 | p4 (unmasked) */ - if (!(a&0x80)) - { - /* Values between 268435456 and 34359738367 */ - a &= SLOT_4_2_0; - b &= SLOT_4_2_0; - b = b<<7; - *v = a | b; - return 5; - } - - /* We can only reach this point when reading a corrupt database - ** file. In that case we are not in any hurry. Use the (relatively - ** slow) general-purpose sqlite3GetVarint() routine to extract the - ** value. */ - { - u64 v64; - u8 n; - - p -= 4; - n = sqlite3GetVarint(p, &v64); - assert( n>5 && n<=9 ); + /* four or more bytes */ + n = sqlite3GetVarint(p, &v64); + assert( n>3 && n<=9 ); + if( (v64 & SQLITE_MAX_U32)!=v64 ){ + *v = 0xffffffff; + }else{ *v = (u32)v64; - return n; } -#endif + return n; } /* @@ -35321,7 +35944,7 @@ SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ } /* -** Attempt to add, substract, or multiply the 64-bit signed value iB against +** Attempt to add, subtract, or multiply the 64-bit signed value iB against ** the other 64-bit signed integer at *pA and store the result in *pA. ** Return 0 on success. Or if the operation would have resulted in an ** overflow, leave *pA unchanged and return 1. @@ -35634,7 +36257,7 @@ SQLITE_PRIVATE int sqlite3VListNameToNum(VList *pIn, const char *zName, int nNam #define SQLITE_HWTIME_H /* -** The following routine only works on pentium-class (or newer) processors. +** The following routine only works on Pentium-class (or newer) processors. ** It uses the RDTSC opcode to read the cycle count value out of the ** processor and returns that value. This can be used for high-res ** profiling. @@ -35806,7 +36429,7 @@ static void insertElement( } -/* Resize the hash table so that it cantains "new_size" buckets. +/* Resize the hash table so that it contains "new_size" buckets. ** ** The hash table might fail to resize if sqlite3_malloc() fails or ** if the new size is the same as the prior size. @@ -36166,19 +36789,20 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 171 */ "VCreate" OpHelp(""), /* 172 */ "VDestroy" OpHelp(""), /* 173 */ "VOpen" OpHelp(""), - /* 174 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), - /* 175 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 176 */ "VRename" OpHelp(""), - /* 177 */ "Pagecount" OpHelp(""), - /* 178 */ "MaxPgcnt" OpHelp(""), - /* 179 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), - /* 180 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), - /* 181 */ "Trace" OpHelp(""), - /* 182 */ "CursorHint" OpHelp(""), - /* 183 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), - /* 184 */ "Noop" OpHelp(""), - /* 185 */ "Explain" OpHelp(""), - /* 186 */ "Abortable" OpHelp(""), + /* 174 */ "VCheck" OpHelp(""), + /* 175 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), + /* 176 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 177 */ "VRename" OpHelp(""), + /* 178 */ "Pagecount" OpHelp(""), + /* 179 */ "MaxPgcnt" OpHelp(""), + /* 180 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), + /* 181 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), + /* 182 */ "Trace" OpHelp(""), + /* 183 */ "CursorHint" OpHelp(""), + /* 184 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 185 */ "Noop" OpHelp(""), + /* 186 */ "Explain" OpHelp(""), + /* 187 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -37192,7 +37816,7 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void){ ** This source file is organized into divisions where the logic for various ** subfunctions is contained within the appropriate division. PLEASE ** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed -** in the correct division and should be clearly labeled. +** in the correct division and should be clearly labelled. ** ** The layout of divisions is as follows: ** @@ -37779,7 +38403,7 @@ static int robustFchown(int fd, uid_t uid, gid_t gid){ /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** "unix" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -38301,7 +38925,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** If you close a file descriptor that points to a file that has locks, ** all locks on that file that are owned by the current process are ** released. To work around this problem, each unixInodeInfo object -** maintains a count of the number of pending locks on tha inode. +** maintains a count of the number of pending locks on the inode. ** When an attempt is made to close an unixFile, if there are ** other unixFile open on the same inode that are holding locks, the call ** to close() the file descriptor is deferred until all of the locks clear. @@ -38315,7 +38939,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** not posix compliant. Under LinuxThreads, a lock created by thread ** A cannot be modified or overridden by a different thread B. ** Only thread A can modify the lock. Locking behavior is correct -** if the appliation uses the newer Native Posix Thread Library (NPTL) +** if the application uses the newer Native Posix Thread Library (NPTL) ** on linux - with NPTL a lock created by thread A can override locks ** in thread B. But there is no way to know at compile-time which ** threading library is being used. So there is no way to know at @@ -38517,7 +39141,7 @@ static void storeLastErrno(unixFile *pFile, int error){ } /* -** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. +** Close all file descriptors accumulated in the unixInodeInfo->pUnused list. */ static void closePendingFds(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; @@ -38880,7 +39504,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** slightly in order to be compatible with Windows95 systems simultaneously ** accessing the same database file, in case that is ever required. ** - ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** Symbols defined in os.h identify the 'pending byte' and the 'reserved ** byte', each single bytes at well known offsets, and the 'shared byte ** range', a range of 510 bytes at a well known offset. ** @@ -38888,7 +39512,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** byte'. If this is successful, 'shared byte range' is read-locked ** and the lock on the 'pending byte' released. (Legacy note: When ** SQLite was first developed, Windows95 systems were still very common, - ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** and Windows95 lacks a shared-lock capability. So on Windows95, a ** single randomly selected by from the 'shared byte range' is locked. ** Windows95 is now pretty much extinct, but this work-around for the ** lack of shared-locks on Windows95 lives on, for backwards @@ -38909,7 +39533,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** obtaining a write-lock on the 'pending byte'. This ensures that no new ** SHARED locks can be obtained, but existing SHARED locks are allowed to ** persist. If the call to this function fails to obtain the EXCLUSIVE - ** lock in this case, it holds the PENDING lock intead. The client may + ** lock in this case, it holds the PENDING lock instead. The client may ** then re-attempt the EXCLUSIVE lock later on, after existing SHARED ** locks have cleared. */ @@ -38937,7 +39561,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* Make sure the locking sequence is correct. ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); @@ -40155,7 +40779,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ /* Make sure the locking sequence is correct ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); @@ -40271,7 +40895,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + pInode->sharedByte, 1, 0)) ){ int failed2 = SQLITE_OK; - /* now attemmpt to get the exclusive lock range */ + /* now attempt to get the exclusive lock range */ failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 1); if( failed && (failed2 = afpSetLock(context->dbPath, pFile, @@ -40320,9 +40944,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unixInodeInfo *pInode; afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; int skipShared = 0; -#ifdef SQLITE_TEST - int h = pFile->h; -#endif assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, @@ -40338,9 +40959,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); #ifdef SQLITE_DEBUG /* When reducing a lock such that other processes can start @@ -40389,9 +41007,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; pInode->nShared--; if( pInode->nShared==0 ){ - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); if( !skipShared ){ rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); } @@ -40566,7 +41181,7 @@ static int unixRead( #endif #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -40718,7 +41333,7 @@ static int unixWrite( #endif #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -40840,7 +41455,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a ** no-op. But go ahead and call fstat() to validate the file ** descriptor as we need a method to provoke a failure during - ** coverate testing. + ** coverage testing. */ #ifdef SQLITE_NO_SYNC { @@ -43885,12 +44500,17 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** than the argument. */ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ -#if OS_VXWORKS || _POSIX_C_SOURCE >= 199309L +#if !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP+0 struct timespec sp; - sp.tv_sec = microseconds / 1000000; sp.tv_nsec = (microseconds % 1000000) * 1000; + + /* Almost all modern unix systems support nanosleep(). But if you are + ** compiling for one of the rare exceptions, you can use + ** -DHAVE_NANOSLEEP=0 (perhaps in conjuction with -DHAVE_USLEEP if + ** usleep() is available) in order to bypass the use of nanosleep() */ nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); return microseconds; #elif defined(HAVE_USLEEP) && HAVE_USLEEP @@ -46480,7 +47100,7 @@ static struct win_syscall { /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** "win32" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -48060,7 +48680,7 @@ static int winRead( pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -48138,7 +48758,7 @@ static int winWrite( pFile->h, pBuf, amt, offset, pFile->locktype)); #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -48248,7 +48868,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ ** all references to memory-mapped content are closed. That is doable, ** but involves adding a few branches in the common write code path which ** could slow down normal operations slightly. Hence, we have decided for - ** now to simply make trancations a no-op if there are pending reads. We + ** now to simply make transactions a no-op if there are pending reads. We ** can maybe revisit this decision in the future. */ return SQLITE_OK; @@ -48307,7 +48927,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ #ifdef SQLITE_TEST /* ** Count the number of fullsyncs and normal syncs. This is used to test -** that syncs and fullsyncs are occuring at the right times. +** that syncs and fullsyncs are occurring at the right times. */ SQLITE_API int sqlite3_sync_count = 0; SQLITE_API int sqlite3_fullsync_count = 0; @@ -48664,7 +49284,7 @@ static int winLock(sqlite3_file *id, int locktype){ */ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); - res = winUnlockReadLock(pFile); + (void)winUnlockReadLock(pFile); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ @@ -50068,6 +50688,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789"; size_t i, j; + DWORD pid; int nPre = sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX); int nMax, nBuf, nDir, nLen; char *zBuf; @@ -50280,7 +50901,10 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ j = sqlite3Strlen30(zBuf); sqlite3_randomness(15, &zBuf[j]); + pid = osGetCurrentProcessId(); for(i=0; i<15; i++, j++){ + zBuf[j] += pid & 0xff; + pid >>= 8; zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; } zBuf[j] = 0; @@ -52645,7 +53269,7 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ h = BITVEC_HASH(i++); /* if there wasn't a hash collision, and this doesn't */ /* completely fill the hash, then just add it without */ - /* worring about sub-dividing and re-hashing. */ + /* worrying about sub-dividing and re-hashing. */ if( !p->u.aHash[h] ){ if (p->nSet<(BITVEC_NINT-1)) { goto bitvec_set_end; @@ -52978,7 +53602,7 @@ struct PCache { ** Return 1 if pPg is on the dirty list for pCache. Return 0 if not. ** This routine runs inside of assert() statements only. */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ PgHdr *p; for(p=pCache->pDirty; p; p=p->pDirtyNext){ @@ -52986,6 +53610,16 @@ static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ } return 0; } +static int pageNotOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 0; + } + return 1; +} +#else +# define pageOnDirtyList(A,B) 1 +# define pageNotOnDirtyList(A,B) 1 #endif /* @@ -53006,7 +53640,7 @@ SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ assert( pCache!=0 ); /* Every page has an associated PCache */ if( pPg->flags & PGHDR_CLEAN ){ assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ - assert( !pageOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirty list */ + assert( pageNotOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirtylist */ }else{ assert( (pPg->flags & PGHDR_DIRTY)!=0 );/* If not CLEAN must be DIRTY */ assert( pPg->pDirtyNext==0 || pPg->pDirtyNext->pDirtyPrev==pPg ); @@ -53142,7 +53776,7 @@ static int numberOfCachePages(PCache *p){ return p->szCache; }else{ i64 n; - /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + /* IMPLEMENTATION-OF: R-59858-46238 If the argument N is negative, then the ** number of cache pages is adjusted to be a number of pages that would ** use approximately abs(N*1024) bytes of memory based on the current ** page size. */ @@ -53630,7 +54264,7 @@ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ } /* -** Sort the list of pages in accending order by pgno. Pages are +** Sort the list of pages in ascending order by pgno. Pages are ** connected by pDirty pointers. The pDirtyPrev pointers are ** corrupted by this sort. ** @@ -53870,7 +54504,7 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** If N is positive, then N pages worth of memory are allocated using a single ** sqlite3Malloc() call and that memory is used for the first N pages allocated. ** Or if N is negative, then -1024*N bytes of memory are allocated and used -** for as many pages as can be accomodated. +** for as many pages as can be accommodated. ** ** Only one of (2) or (3) can be used. Once the memory available to (2) or ** (3) is exhausted, subsequent allocations fail over to the general-purpose @@ -53904,7 +54538,7 @@ typedef struct PGroup PGroup; ** in memory directly after the associated page data, if the database is ** corrupt, code at the b-tree layer may overread the page buffer and ** read part of this structure before the corruption is detected. This -** can cause a valgrind error if the unitialized gap is accessed. Using u16 +** can cause a valgrind error if the uninitialized gap is accessed. Using u16 ** ensures there is no such gap, and therefore no bytes of uninitialized ** memory in the structure. ** @@ -55124,7 +55758,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** The TEST primitive includes a "batch" number. The TEST primitive ** will only see elements that were inserted before the last change ** in the batch number. In other words, if an INSERT occurs between -** two TESTs where the TESTs have the same batch nubmer, then the +** two TESTs where the TESTs have the same batch number, then the ** value added by the INSERT will not be visible to the second TEST. ** The initial batch number is zero, so if the very first TEST contains ** a non-zero batch number, it will see all prior INSERTs. @@ -55656,6 +56290,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 # define sqlite3WalFile(x) 0 +# undef SQLITE_USE_SEH #else #define WAL_SAVEPOINT_NDATA 4 @@ -55762,6 +56397,10 @@ SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock); SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db); #endif +#ifdef SQLITE_USE_SEH +SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal*); +#endif + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* SQLITE_WAL_H */ @@ -56047,7 +56686,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** outstanding transactions have been abandoned, the pager is able to ** transition back to OPEN state, discarding the contents of the ** page-cache and any other in-memory state at the same time. Everything -** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** is reloaded from disk (and, if necessary, hot-journal rollback performed) ** when a read-transaction is next opened on the pager (transitioning ** the pager into READER state). At that point the system has recovered ** from the error. @@ -57238,9 +57877,32 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } + + /* The random check-hash initializer */ - sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + } +#ifdef SQLITE_DEBUG + else{ + /* The Pager.cksumInit variable is usually randomized above to protect + ** against there being existing records in the journal file. This is + ** dangerous, as following a crash they may be mistaken for records + ** written by the current transaction and rolled back into the database + ** file, causing corruption. The following assert statements verify + ** that this is not required in "journal_mode=memory" mode, as in that + ** case the journal file is always 0 bytes in size at this point. + ** It is advantageous to avoid the sqlite3_randomness() call if possible + ** as it takes the global PRNG mutex. */ + i64 sz = 0; + sqlite3OsFileSize(pPager->jfd, &sz); + assert( sz==0 ); + assert( pPager->journalOff==journalHdrOffset(pPager) ); + assert( sqlite3JournalIsInMemory(pPager->jfd) ); + } +#endif put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); /* The assumed sector size for this process */ @@ -57420,7 +58082,7 @@ static int readJournalHdr( ** + 4 bytes: super-journal name checksum. ** + 8 bytes: aJournalMagic[]. ** -** The super-journal page checksum is the sum of the bytes in thesuper-journal +** The super-journal page checksum is the sum of the bytes in the super-journal ** name, where each byte is interpreted as a signed 8-bit integer. ** ** If zSuper is a NULL pointer (occurs for a single database transaction), @@ -57473,7 +58135,7 @@ static int writeSuperJournal(Pager *pPager, const char *zSuper){ } pPager->journalOff += (nSuper+20); - /* If the pager is in peristent-journal mode, then the physical + /* If the pager is in persistent-journal mode, then the physical ** journal-file may extend past the end of the super-journal name ** and 8 bytes of magic data just written to the file. This is ** dangerous because the code to rollback a hot-journal file @@ -57643,7 +58305,7 @@ static void pager_unlock(Pager *pPager){ /* ** This function is called whenever an IOERR or FULL error that requires -** the pager to transition into the ERROR state may ahve occurred. +** the pager to transition into the ERROR state may have occurred. ** The first argument is a pointer to the pager structure, the second ** the error-code about to be returned by a pager API function. The ** value returned is a copy of the second argument to this function. @@ -57884,6 +58546,9 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ return (rc==SQLITE_OK?rc2:rc); } +/* Forward reference */ +static int pager_playback(Pager *pPager, int isHot); + /* ** Execute a rollback if a transaction is active and unlock the ** database file. @@ -57912,13 +58577,28 @@ static void pagerUnlockAndRollback(Pager *pPager){ assert( pPager->eState==PAGER_READER ); pager_end_transaction(pPager, 0, 0); } + }else if( pPager->eState==PAGER_ERROR + && pPager->journalMode==PAGER_JOURNALMODE_MEMORY + && isOpen(pPager->jfd) + ){ + /* Special case for a ROLLBACK due to I/O error with an in-memory + ** journal: We have to rollback immediately, before the journal is + ** closed, because once it is closed, all content is forgotten. */ + int errCode = pPager->errCode; + u8 eLock = pPager->eLock; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + pPager->eLock = EXCLUSIVE_LOCK; + pager_playback(pPager, 1); + pPager->errCode = errCode; + pPager->eLock = eLock; } pager_unlock(pPager); } /* ** Parameter aData must point to a buffer of pPager->pageSize bytes -** of data. Compute and return a checksum based ont the contents of the +** of data. Compute and return a checksum based on the contents of the ** page of data and the current value of pPager->cksumInit. ** ** This is not a real checksum. It is really just the sum of the @@ -58884,7 +59564,7 @@ static int pagerWalFrames( assert( pPager->pWal ); assert( pList ); #ifdef SQLITE_DEBUG - /* Verify that the page list is in accending order */ + /* Verify that the page list is in ascending order */ for(p=pList; p && p->pDirty; p=p->pDirty){ assert( p->pgno < p->pDirty->pgno ); } @@ -59015,7 +59695,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ #ifndef SQLITE_OMIT_WAL /* ** Check if the *-wal file that corresponds to the database opened by pPager -** exists if the database is not empy, or verify that the *-wal file does +** exists if the database is not empty, or verify that the *-wal file does ** not exist (by deleting it) if the database file is empty. ** ** If the database is not empty and the *-wal file exists, open the pager @@ -60425,11 +61105,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int rc = SQLITE_OK; /* Return code */ int tempFile = 0; /* True for temp files (incl. in-memory files) */ int memDb = 0; /* True if this is an in-memory file */ -#ifndef SQLITE_OMIT_DESERIALIZE int memJM = 0; /* Memory journal mode */ -#else -# define memJM 0 -#endif int readOnly = 0; /* True if this is a read-only file */ int journalFileSize; /* Bytes to allocate for each journal fd */ char *zPathname = 0; /* Full path to database file */ @@ -60548,12 +61224,13 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** specific formatting and order of the various filenames, so if the format ** changes here, be sure to change it there as well. */ + assert( SQLITE_PTRSIZE==sizeof(Pager*) ); pPtr = (u8 *)sqlite3MallocZero( ROUND8(sizeof(*pPager)) + /* Pager structure */ ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ journalFileSize * 2 + /* The two journal files */ - sizeof(pPager) + /* Space to hold a pointer */ + SQLITE_PTRSIZE + /* Space to hold a pointer */ 4 + /* Database prefix */ nPathname + 1 + /* database filename */ nUriByte + /* query parameters */ @@ -60574,7 +61251,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); - memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); + memcpy(pPtr, &pPager, SQLITE_PTRSIZE); pPtr += SQLITE_PTRSIZE; /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ pPtr += 4; /* Skip zero prefix */ @@ -60628,9 +61305,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); assert( !memDb ); -#ifndef SQLITE_OMIT_DESERIALIZE pPager->memVfs = memJM = (fout&SQLITE_OPEN_MEMORY)!=0; -#endif readOnly = (fout&SQLITE_OPEN_READONLY)!=0; /* If the file was successfully opened for read/write access, @@ -60767,7 +61442,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* ** Return the sqlite3_file for the main database given the name -** of the corresonding WAL or Journal name as passed into +** of the corresponding WAL or Journal name as passed into ** xOpen. */ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){ @@ -61409,8 +62084,20 @@ SQLITE_PRIVATE int sqlite3PagerGet( DbPage **ppPage, /* Write a pointer to the page here */ int flags /* PAGER_GET_XXX flags */ ){ - /* printf("PAGE %u\n", pgno); fflush(stdout); */ +#if 0 /* Trace page fetch by setting to 1 */ + int rc; + printf("PAGE %u\n", pgno); + fflush(stdout); + rc = pPager->xGet(pPager, pgno, ppPage, flags); + if( rc ){ + printf("PAGE %u failed with 0x%02x\n", pgno, rc); + fflush(stdout); + } + return rc; +#else + /* Normal, high-speed version of sqlite3PagerGet() */ return pPager->xGet(pPager, pgno, ppPage, flags); +#endif } /* @@ -62286,6 +62973,13 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); if( rc==SQLITE_OK ){ rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbFileSize ){ + char *pTmp = pPager->pTmpSpace; + int szPage = (int)pPager->pageSize; + memset(pTmp, 0, szPage); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, + ((i64)pPager->dbSize*pPager->pageSize)-szPage); + } if( rc==SQLITE_OK ){ rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); } @@ -63052,7 +63746,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; - /* When transistioning from TRUNCATE or PERSIST to any other journal + /* When transitioning from TRUNCATE or PERSIST to any other journal ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ @@ -63097,7 +63791,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } assert( state==pPager->eState ); } - }else if( eMode==PAGER_JOURNALMODE_OFF ){ + }else if( eMode==PAGER_JOURNALMODE_OFF || eMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3OsClose(pPager->jfd); } } @@ -63480,6 +64174,12 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ } #endif +#ifdef SQLITE_USE_SEH +SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager *pPager){ + return sqlite3WalSystemErrno(pPager->pWal); +} +#endif + #endif /* SQLITE_OMIT_DISKIO */ /************** End of pager.c ***********************************************/ @@ -63770,7 +64470,7 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; ** ** Technically, the various VFSes are free to implement these locks however ** they see fit. However, compatibility is encouraged so that VFSes can -** interoperate. The standard implemention used on both unix and windows +** interoperate. The standard implementation used on both unix and windows ** is for the index number to indicate a byte offset into the ** WalCkptInfo.aLock[] array in the wal-index header. In other words, all ** locks are on the shm file. The WALINDEX_LOCK_OFFSET constant (which @@ -63846,7 +64546,7 @@ struct WalIndexHdr { ** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) ** for any aReadMark[] means that entry is unused. aReadMark[0] is ** a special case; its value is never used and it exists as a place-holder -** to avoid having to offset aReadMark[] indexs by one. Readers holding +** to avoid having to offset aReadMark[] indexes by one. Readers holding ** WAL_READ_LOCK(0) always ignore the entire WAL and read all content ** directly from the database. ** @@ -64014,7 +64714,15 @@ struct Wal { u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ +#ifdef SQLITE_USE_SEH + u32 lockMask; /* Mask of locks held */ + void *pFree; /* Pointer to sqlite3_free() if exception thrown */ + u32 *pWiValue; /* Value to write into apWiData[iWiPg] */ + int iWiPg; /* Write pWiValue into apWiData[iWiPg] */ + int iSysErrno; /* System error code following exception */ +#endif #ifdef SQLITE_DEBUG + int nSehTry; /* Number of nested SEH_TRY{} blocks */ u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT @@ -64096,6 +64804,113 @@ struct WalIterator { sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ ) +/* +** Structured Exception Handling (SEH) is a Windows-specific technique +** for catching exceptions raised while accessing memory-mapped files. +** +** The -DSQLITE_USE_SEH compile-time option means to use SEH to catch and +** deal with system-level errors that arise during WAL -shm file processing. +** Without this compile-time option, any system-level faults that appear +** while accessing the memory-mapped -shm file will cause a process-wide +** signal to be deliver, which will more than likely cause the entire +** process to exit. +*/ +#ifdef SQLITE_USE_SEH +#include + +/* Beginning of a block of code in which an exception might occur */ +# define SEH_TRY __try { \ + assert( walAssertLockmask(pWal) && pWal->nSehTry==0 ); \ + VVA_ONLY(pWal->nSehTry++); + +/* The end of a block of code in which an exception might occur */ +# define SEH_EXCEPT(X) \ + VVA_ONLY(pWal->nSehTry--); \ + assert( pWal->nSehTry==0 ); \ + } __except( sehExceptionFilter(pWal, GetExceptionCode(), GetExceptionInformation() ) ){ X } + +/* Simulate a memory-mapping fault in the -shm file for testing purposes */ +# define SEH_INJECT_FAULT sehInjectFault(pWal) + +/* +** The second argument is the return value of GetExceptionCode() for the +** current exception. Return EXCEPTION_EXECUTE_HANDLER if the exception code +** indicates that the exception may have been caused by accessing the *-shm +** file mapping. Or EXCEPTION_CONTINUE_SEARCH otherwise. +*/ +static int sehExceptionFilter(Wal *pWal, int eCode, EXCEPTION_POINTERS *p){ + VVA_ONLY(pWal->nSehTry--); + if( eCode==EXCEPTION_IN_PAGE_ERROR ){ + if( p && p->ExceptionRecord && p->ExceptionRecord->NumberParameters>=3 ){ + /* From MSDN: For this type of exception, the first element of the + ** ExceptionInformation[] array is a read-write flag - 0 if the exception + ** was thrown while reading, 1 if while writing. The second element is + ** the virtual address being accessed. The "third array element specifies + ** the underlying NTSTATUS code that resulted in the exception". */ + pWal->iSysErrno = (int)p->ExceptionRecord->ExceptionInformation[2]; + } + return EXCEPTION_EXECUTE_HANDLER; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +/* +** If one is configured, invoke the xTestCallback callback with 650 as +** the argument. If it returns true, throw the same exception that is +** thrown by the system if the *-shm file mapping is accessed after it +** has been invalidated. +*/ +static void sehInjectFault(Wal *pWal){ + int res; + assert( pWal->nSehTry>0 ); + + res = sqlite3FaultSim(650); + if( res!=0 ){ + ULONG_PTR aArg[3]; + aArg[0] = 0; + aArg[1] = 0; + aArg[2] = (ULONG_PTR)res; + RaiseException(EXCEPTION_IN_PAGE_ERROR, 0, 3, (const ULONG_PTR*)aArg); + } +} + +/* +** There are two ways to use this macro. To set a pointer to be freed +** if an exception is thrown: +** +** SEH_FREE_ON_ERROR(0, pPtr); +** +** and to cancel the same: +** +** SEH_FREE_ON_ERROR(pPtr, 0); +** +** In the first case, there must not already be a pointer registered to +** be freed. In the second case, pPtr must be the registered pointer. +*/ +#define SEH_FREE_ON_ERROR(X,Y) \ + assert( (X==0 || Y==0) && pWal->pFree==X ); pWal->pFree = Y + +/* +** There are two ways to use this macro. To arrange for pWal->apWiData[iPg] +** to be set to pValue if an exception is thrown: +** +** SEH_SET_ON_ERROR(iPg, pValue); +** +** and to cancel the same: +** +** SEH_SET_ON_ERROR(0, 0); +*/ +#define SEH_SET_ON_ERROR(X,Y) pWal->iWiPg = X; pWal->pWiValue = Y + +#else +# define SEH_TRY VVA_ONLY(pWal->nSehTry++); +# define SEH_EXCEPT(X) VVA_ONLY(pWal->nSehTry--); assert( pWal->nSehTry==0 ); +# define SEH_INJECT_FAULT assert( pWal->nSehTry>0 ); +# define SEH_FREE_ON_ERROR(X,Y) +# define SEH_SET_ON_ERROR(X,Y) +#endif /* ifdef SQLITE_USE_SEH */ + + /* ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are @@ -64168,6 +64983,7 @@ static int walIndexPage( int iPage, /* The page we seek */ volatile u32 **ppPage /* Write the page pointer here */ ){ + SEH_INJECT_FAULT; if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){ return walIndexPageRealloc(pWal, iPage, ppPage); } @@ -64179,6 +64995,7 @@ static int walIndexPage( */ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ assert( pWal->nWiData>0 && pWal->apWiData[0] ); + SEH_INJECT_FAULT; return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); } @@ -64187,6 +65004,7 @@ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ */ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ assert( pWal->nWiData>0 && pWal->apWiData[0] ); + SEH_INJECT_FAULT; return (volatile WalIndexHdr*)pWal->apWiData[0]; } @@ -64376,7 +65194,7 @@ static int walDecodeFrame( return 0; } - /* A frame is only valid if the page number is creater than zero. + /* A frame is only valid if the page number is greater than zero. */ pgno = sqlite3Get4byte(&aFrame[0]); if( pgno==0 ){ @@ -64384,7 +65202,7 @@ static int walDecodeFrame( } /* A frame is only valid if a checksum of the WAL header, - ** all prior frams, the first 16 bytes of this frame-header, + ** all prior frames, the first 16 bytes of this frame-header, ** and the frame-data matches the checksum in the last 8 ** bytes of this frame-header. */ @@ -64444,12 +65262,18 @@ static int walLockShared(Wal *pWal, int lockIdx){ WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, walLockName(lockIdx), rc ? "failed" : "ok")); VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) +#ifdef SQLITE_USE_SEH + if( rc==SQLITE_OK ) pWal->lockMask |= (1 << lockIdx); +#endif return rc; } static void walUnlockShared(Wal *pWal, int lockIdx){ if( pWal->exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); +#ifdef SQLITE_USE_SEH + pWal->lockMask &= ~(1 << lockIdx); +#endif WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); } static int walLockExclusive(Wal *pWal, int lockIdx, int n){ @@ -64460,12 +65284,20 @@ static int walLockExclusive(Wal *pWal, int lockIdx, int n){ WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, walLockName(lockIdx), n, rc ? "failed" : "ok")); VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) +#ifdef SQLITE_USE_SEH + if( rc==SQLITE_OK ){ + pWal->lockMask |= (((1<exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); +#ifdef SQLITE_USE_SEH + pWal->lockMask &= ~(((1<apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; } @@ -64816,6 +65649,7 @@ static int walIndexRecover(Wal *pWal){ /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); + SEH_FREE_ON_ERROR(0, aFrame); if( !aFrame ){ rc = SQLITE_NOMEM_BKPT; goto recovery_error; @@ -64834,6 +65668,7 @@ static int walIndexRecover(Wal *pWal){ rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); assert( aShare!=0 || rc!=SQLITE_OK ); if( aShare==0 ) break; + SEH_SET_ON_ERROR(iPg, aShare); pWal->apWiData[iPg] = aPrivate; for(iFrame=iFirst; iFrame<=iLast; iFrame++){ @@ -64861,6 +65696,7 @@ static int walIndexRecover(Wal *pWal){ } } pWal->apWiData[iPg] = aShare; + SEH_SET_ON_ERROR(0,0); nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); nHdr32 = nHdr / sizeof(u32); #ifndef SQLITE_SAFER_WALINDEX_RECOVERY @@ -64891,9 +65727,11 @@ static int walIndexRecover(Wal *pWal){ } } #endif + SEH_INJECT_FAULT; if( iFrame<=iLast ) break; } + SEH_FREE_ON_ERROR(aFrame, 0); sqlite3_free(aFrame); } @@ -64921,6 +65759,7 @@ static int walIndexRecover(Wal *pWal){ }else{ pInfo->aReadMark[i] = READMARK_NOT_USED; } + SEH_INJECT_FAULT; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc!=SQLITE_BUSY ){ goto recovery_error; @@ -65078,7 +65917,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( } /* -** Change the size to which the WAL file is trucated on each reset. +** Change the size to which the WAL file is truncated on each reset. */ SQLITE_PRIVATE void sqlite3WalLimit(Wal *pWal, i64 iLimit){ if( pWal ) pWal->mxWalSize = iLimit; @@ -65304,23 +66143,16 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3_malloc64(nByte); + p = (WalIterator *)sqlite3_malloc64(nByte + + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) + ); if( !p ){ return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; - - /* Allocate temporary space used by the merge-sort routine. This block - ** of memory will be freed before this function returns. - */ - aTmp = (ht_slot *)sqlite3_malloc64( - sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); - if( !aTmp ){ - rc = SQLITE_NOMEM_BKPT; - } - + aTmp = (ht_slot*)&(((u8*)p)[nByte]); + SEH_FREE_ON_ERROR(0, p); for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && iaSegment[i].aPgno = (u32 *)sLoc.aPgno; } } - sqlite3_free(aTmp); - if( rc!=SQLITE_OK ){ + SEH_FREE_ON_ERROR(p, 0); walIteratorFree(p); p = 0; } @@ -65576,13 +66407,13 @@ static int walCheckpoint( mxSafeFrame = pWal->hdr.mxFrame; mxPage = pWal->hdr.nPage; for(i=1; iaReadMark+i); + u32 y = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT; if( mxSafeFrame>y ){ assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); - AtomicStore(pInfo->aReadMark+i, iMark); + AtomicStore(pInfo->aReadMark+i, iMark); SEH_INJECT_FAULT; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; @@ -65603,8 +66434,7 @@ static int walCheckpoint( && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK ){ u32 nBackfill = pInfo->nBackfill; - - pInfo->nBackfillAttempted = mxSafeFrame; + pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT; /* Sync the WAL to disk */ rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); @@ -65635,6 +66465,7 @@ static int walCheckpoint( while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); + SEH_INJECT_FAULT; if( AtomicLoad(&db->u1.isInterrupted) ){ rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; break; @@ -65664,7 +66495,7 @@ static int walCheckpoint( } } if( rc==SQLITE_OK ){ - AtomicStore(&pInfo->nBackfill, mxSafeFrame); + AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT; } } @@ -65686,6 +66517,7 @@ static int walCheckpoint( */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); + SEH_INJECT_FAULT; if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ @@ -65717,6 +66549,7 @@ static int walCheckpoint( } walcheckpoint_out: + SEH_FREE_ON_ERROR(pIter, 0); walIteratorFree(pIter); return rc; } @@ -65739,6 +66572,93 @@ static void walLimitSize(Wal *pWal, i64 nMax){ } } +#ifdef SQLITE_USE_SEH +/* +** This is the "standard" exception handler used in a few places to handle +** an exception thrown by reading from the *-shm mapping after it has become +** invalid in SQLITE_USE_SEH builds. It is used as follows: +** +** SEH_TRY { ... } +** SEH_EXCEPT( rc = walHandleException(pWal); ) +** +** This function does three things: +** +** 1) Determines the locks that should be held, based on the contents of +** the Wal.readLock, Wal.writeLock and Wal.ckptLock variables. All other +** held locks are assumed to be transient locks that would have been +** released had the exception not been thrown and are dropped. +** +** 2) Frees the pointer at Wal.pFree, if any, using sqlite3_free(). +** +** 3) Set pWal->apWiData[pWal->iWiPg] to pWal->pWiValue if not NULL +** +** 4) Returns SQLITE_IOERR. +*/ +static int walHandleException(Wal *pWal){ + if( pWal->exclusiveMode==0 ){ + static const int S = 1; + static const int E = (1<lockMask & ~( + (pWal->readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) + | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) + | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) + ); + for(ii=0; iipFree); + pWal->pFree = 0; + if( pWal->pWiValue ){ + pWal->apWiData[pWal->iWiPg] = pWal->pWiValue; + pWal->pWiValue = 0; + } + return SQLITE_IOERR_IN_PAGE; +} + +/* +** Assert that the Wal.lockMask mask, which indicates the locks held +** by the connenction, is consistent with the Wal.readLock, Wal.writeLock +** and Wal.ckptLock variables. To be used as: +** +** assert( walAssertLockmask(pWal) ); +*/ +static int walAssertLockmask(Wal *pWal){ + if( pWal->exclusiveMode==0 ){ + static const int S = 1; + static const int E = (1<readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) + | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) + | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) +#ifdef SQLITE_ENABLE_SNAPSHOT + | (pWal->pSnapshot ? (pWal->lockMask & (1 << WAL_CKPT_LOCK)) : 0) +#endif + ); + assert( mExpect==pWal->lockMask ); + } + return 1; +} + +/* +** Return and zero the "system error" field set when an +** EXCEPTION_IN_PAGE_ERROR exception is caught. +*/ +SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal *pWal){ + int iRet = 0; + if( pWal ){ + iRet = pWal->iSysErrno; + pWal->iSysErrno = 0; + } + return iRet; +} + +#else +# define walAssertLockmask(x) 1 +#endif /* ifdef SQLITE_USE_SEH */ + /* ** Close a connection to a log file. */ @@ -65753,6 +66673,8 @@ SQLITE_PRIVATE int sqlite3WalClose( if( pWal ){ int isDelete = 0; /* True to unlink wal and wal-index files */ + assert( walAssertLockmask(pWal) ); + /* If an EXCLUSIVE lock can be obtained on the database file (using the ** ordinary, rollback-mode locking methods, this guarantees that the ** connection associated with this log file is the only connection to @@ -65777,7 +66699,7 @@ SQLITE_PRIVATE int sqlite3WalClose( ); if( bPersist!=1 ){ /* Try to delete the WAL file if the checkpoint completed and - ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** fsynced (rc==SQLITE_OK) and if we are not in persistent-wal ** mode (!bPersist) */ isDelete = 1; }else if( pWal->mxWalSize>=0 ){ @@ -65844,7 +66766,7 @@ static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ ** give false-positive warnings about these accesses because the tools do not ** account for the double-read and the memory barrier. The use of mutexes ** here would be problematic as the memory being accessed is potentially - ** shared among multiple processes and not all mutex implementions work + ** shared among multiple processes and not all mutex implementations work ** reliably in that environment. */ aHdr = walIndexHdr(pWal); @@ -66295,6 +67217,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ assert( pWal->nWiData>0 ); assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); + SEH_INJECT_FAULT; if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0) @@ -66344,7 +67267,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } #endif for(i=1; iaReadMark+i); + u32 thisMark = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT; if( mxReadMark<=thisMark && thisMark<=mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; @@ -66410,7 +67333,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** we can guarantee that the checkpointer that set nBackfill could not ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ - pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; + pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT; walShmBarrier(pWal); if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) @@ -66425,6 +67348,54 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } #ifdef SQLITE_ENABLE_SNAPSHOT +/* +** This function does the work of sqlite3WalSnapshotRecover(). +*/ +static int walSnapshotRecover( + Wal *pWal, /* WAL handle */ + void *pBuf1, /* Temp buffer pWal->szPage bytes in size */ + void *pBuf2 /* Temp buffer pWal->szPage bytes in size */ +){ + int szPage = (int)pWal->szPage; + int rc; + i64 szDb; /* Size of db file in bytes */ + + rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + u32 i = pInfo->nBackfillAttempted; + for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ + WalHashLoc sLoc; /* Hash table location */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ + i64 iWalOff; /* Offset of wal file entry */ + + rc = walHashGet(pWal, walFramePage(i), &sLoc); + if( rc!=SQLITE_OK ) break; + assert( i - sLoc.iZero - 1 >=0 ); + pgno = sLoc.aPgno[i-sLoc.iZero-1]; + iDbOff = (i64)(pgno-1) * szPage; + + if( iDbOff+szPage<=szDb ){ + iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; + rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); + + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); + } + + if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ + break; + } + } + + pInfo->nBackfillAttempted = i-1; + } + } + + return rc; +} + /* ** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted ** variable so that older snapshots can be accessed. To do this, loop @@ -66450,50 +67421,21 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ assert( pWal->readLock>=0 ); rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc==SQLITE_OK ){ - volatile WalCkptInfo *pInfo = walCkptInfo(pWal); - int szPage = (int)pWal->szPage; - i64 szDb; /* Size of db file in bytes */ - - rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); - if( rc==SQLITE_OK ){ - void *pBuf1 = sqlite3_malloc(szPage); - void *pBuf2 = sqlite3_malloc(szPage); - if( pBuf1==0 || pBuf2==0 ){ - rc = SQLITE_NOMEM; - }else{ - u32 i = pInfo->nBackfillAttempted; - for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ - WalHashLoc sLoc; /* Hash table location */ - u32 pgno; /* Page number in db file */ - i64 iDbOff; /* Offset of db file entry */ - i64 iWalOff; /* Offset of wal file entry */ - - rc = walHashGet(pWal, walFramePage(i), &sLoc); - if( rc!=SQLITE_OK ) break; - assert( i - sLoc.iZero - 1 >=0 ); - pgno = sLoc.aPgno[i-sLoc.iZero-1]; - iDbOff = (i64)(pgno-1) * szPage; - - if( iDbOff+szPage<=szDb ){ - iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; - rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); - - if( rc==SQLITE_OK ){ - rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); - } - - if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ - break; - } - } - - pInfo->nBackfillAttempted = i-1; - } + void *pBuf1 = sqlite3_malloc(pWal->szPage); + void *pBuf2 = sqlite3_malloc(pWal->szPage); + if( pBuf1==0 || pBuf2==0 ){ + rc = SQLITE_NOMEM; + }else{ + pWal->ckptLock = 1; + SEH_TRY { + rc = walSnapshotRecover(pWal, pBuf1, pBuf2); } - - sqlite3_free(pBuf1); - sqlite3_free(pBuf2); + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + pWal->ckptLock = 0; } + + sqlite3_free(pBuf1); + sqlite3_free(pBuf2); walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); } @@ -66502,28 +67444,20 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ #endif /* SQLITE_ENABLE_SNAPSHOT */ /* -** Begin a read transaction on the database. -** -** This routine used to be called sqlite3OpenSnapshot() and with good reason: -** it takes a snapshot of the state of the WAL and wal-index for the current -** instant in time. The current thread will continue to use this snapshot. -** Other threads might append new content to the WAL and wal-index but -** that extra content is ignored by the current thread. -** -** If the database contents have changes since the previous read -** transaction, then *pChanged is set to 1 before returning. The -** Pager layer will use this to know that its cache is stale and -** needs to be flushed. +** This function does the work of sqlite3WalBeginReadTransaction() (see +** below). That function simply calls this one inside an SEH_TRY{...} block. */ -SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ +static int walBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ #ifdef SQLITE_ENABLE_SNAPSHOT + int ckptLock = 0; int bChanged = 0; WalIndexHdr *pSnapshot = pWal->pSnapshot; #endif assert( pWal->ckptLock==0 ); + assert( pWal->nSehTry>0 ); #ifdef SQLITE_ENABLE_SNAPSHOT if( pSnapshot ){ @@ -66546,7 +67480,7 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ if( rc!=SQLITE_OK ){ return rc; } - pWal->ckptLock = 1; + ckptLock = 1; } #endif @@ -66610,15 +67544,37 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ } /* Release the shared CKPT lock obtained above. */ - if( pWal->ckptLock ){ + if( ckptLock ){ assert( pSnapshot ); walUnlockShared(pWal, WAL_CKPT_LOCK); - pWal->ckptLock = 0; } #endif return rc; } +/* +** Begin a read transaction on the database. +** +** This routine used to be called sqlite3OpenSnapshot() and with good reason: +** it takes a snapshot of the state of the WAL and wal-index for the current +** instant in time. The current thread will continue to use this snapshot. +** Other threads might append new content to the WAL and wal-index but +** that extra content is ignored by the current thread. +** +** If the database contents have changes since the previous read +** transaction, then *pChanged is set to 1 before returning. The +** Pager layer will use this to know that its cache is stale and +** needs to be flushed. +*/ +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ + int rc; + SEH_TRY { + rc = walBeginReadTransaction(pWal, pChanged); + } + SEH_EXCEPT( rc = walHandleException(pWal); ) + return rc; +} + /* ** Finish with a read transaction. All this does is release the ** read-lock. @@ -66639,7 +67595,7 @@ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ ** Return SQLITE_OK if successful, or an error code if an error occurs. If an ** error does occur, the final value of *piRead is undefined. */ -SQLITE_PRIVATE int sqlite3WalFindFrame( +static int walFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ u32 *piRead /* OUT: Frame number (or zero) */ @@ -66702,6 +67658,7 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( } nCollide = HASHTABLE_NSLOT; iKey = walHash(pgno); + SEH_INJECT_FAULT; while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ u32 iFrame = iH + sLoc.iZero; if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){ @@ -66738,6 +67695,30 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( return SQLITE_OK; } +/* +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. +** +** The difference between this function and walFindFrame() is that this +** function wraps walFindFrame() in an SEH_TRY{...} block. +*/ +SQLITE_PRIVATE int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ +){ + int rc; + SEH_TRY { + rc = walFindFrame(pWal, pgno, piRead); + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + return rc; +} + /* ** Read the contents of frame iRead from the wal file into buffer pOut ** (which is nOut bytes in size). Return SQLITE_OK if successful, or an @@ -66819,12 +67800,17 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** time the read transaction on this connection was started, then ** the write is disallowed. */ - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + SEH_TRY { + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + + if( rc!=SQLITE_OK ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; - rc = SQLITE_BUSY_SNAPSHOT; } - return rc; } @@ -66860,30 +67846,33 @@ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *p Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; - /* Restore the clients cache of the wal-index header to the state it - ** was in before the client began writing to the database. - */ - memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); - - for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; - iFrame++ - ){ - /* This call cannot fail. Unless the page for which the page number - ** is passed as the second argument is (a) in the cache and - ** (b) has an outstanding reference, then xUndo is either a no-op - ** (if (a) is false) or simply expels the page from the cache (if (b) - ** is false). - ** - ** If the upper layer is doing a rollback, it is guaranteed that there - ** are no outstanding references to any page other than page 1. And - ** page 1 is never written to the log until the transaction is - ** committed. As a result, the call to xUndo may not fail. + SEH_TRY { + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. */ - assert( walFramePgno(pWal, iFrame)!=1 ); - rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + iFrame++ + ){ + /* This call cannot fail. Unless the page for which the page number + ** is passed as the second argument is (a) in the cache and + ** (b) has an outstanding reference, then xUndo is either a no-op + ** (if (a) is false) or simply expels the page from the cache (if (b) + ** is false). + ** + ** If the upper layer is doing a rollback, it is guaranteed that there + ** are no outstanding references to any page other than page 1. And + ** page 1 is never written to the log until the transaction is + ** committed. As a result, the call to xUndo may not fail. + */ + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + } + if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) } return rc; } @@ -66927,7 +67916,10 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ pWal->hdr.mxFrame = aWalData[0]; pWal->hdr.aFrameCksum[0] = aWalData[1]; pWal->hdr.aFrameCksum[1] = aWalData[2]; - walCleanupHash(pWal); + SEH_TRY { + walCleanupHash(pWal); + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) } return rc; @@ -67108,7 +68100,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). */ -SQLITE_PRIVATE int sqlite3WalFrames( +static int walFrames( Wal *pWal, /* Wal handle to write to */ int szPage, /* Database page-size in bytes */ PgHdr *pList, /* List of dirty pages to write */ @@ -67219,7 +68211,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; - VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + VVA_ONLY(rc =) walFindFrame(pWal, p->pgno, &iWrite); assert( rc==SQLITE_OK || iWrite==0 ); if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; @@ -67338,6 +68330,29 @@ SQLITE_PRIVATE int sqlite3WalFrames( return rc; } +/* +** Write a set of frames to the log. The caller must hold the write-lock +** on the log file (obtained using sqlite3WalBeginWriteTransaction()). +** +** The difference between this function and walFrames() is that this +** function wraps walFrames() in an SEH_TRY{...} block. +*/ +SQLITE_PRIVATE int sqlite3WalFrames( + Wal *pWal, /* Wal handle to write to */ + int szPage, /* Database page-size in bytes */ + PgHdr *pList, /* List of dirty pages to write */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; + SEH_TRY { + rc = walFrames(pWal, szPage, pList, nTruncate, isCommit, sync_flags); + } + SEH_EXCEPT( rc = walHandleException(pWal); ) + return rc; +} + /* ** This routine is called to implement sqlite3_wal_checkpoint() and ** related interfaces. @@ -67417,30 +68432,33 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* Read the wal-index header. */ - if( rc==SQLITE_OK ){ - walDisableBlocking(pWal); - rc = walIndexReadHdr(pWal, &isChanged); - (void)walEnableBlocking(pWal); - if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ - sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + SEH_TRY { + if( rc==SQLITE_OK ){ + walDisableBlocking(pWal); + rc = walIndexReadHdr(pWal, &isChanged); + (void)walEnableBlocking(pWal); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } - } - - /* Copy data from the log to the database file. */ - if( rc==SQLITE_OK ){ - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); - } + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); + } - /* If no error occurred, set the output variables. */ - if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ - if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; - if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + /* If no error occurred, set the output variables. */ + if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + SEH_INJECT_FAULT; + if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + } } } + SEH_EXCEPT( rc = walHandleException(pWal); ) if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was @@ -67517,7 +68535,9 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ ** locks are taken in this case). Nor should the pager attempt to ** upgrade to exclusive-mode following such an error. */ +#ifndef SQLITE_USE_SEH assert( pWal->readLock>=0 || pWal->lockError ); +#endif assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ @@ -67618,16 +68638,19 @@ SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ */ SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){ int rc; - rc = walLockShared(pWal, WAL_CKPT_LOCK); - if( rc==SQLITE_OK ){ - WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; - if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) - || pNew->mxFramenBackfillAttempted - ){ - rc = SQLITE_ERROR_SNAPSHOT; - walUnlockShared(pWal, WAL_CKPT_LOCK); + SEH_TRY { + rc = walLockShared(pWal, WAL_CKPT_LOCK); + if( rc==SQLITE_OK ){ + WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; + if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + || pNew->mxFramenBackfillAttempted + ){ + rc = SQLITE_ERROR_SNAPSHOT; + walUnlockShared(pWal, WAL_CKPT_LOCK); + } } } + SEH_EXCEPT( rc = walHandleException(pWal); ) return rc; } @@ -67866,7 +68889,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** 0x81 0x00 becomes 0x00000080 ** 0x82 0x00 becomes 0x00000100 ** 0x80 0x7f becomes 0x0000007f -** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x91 0xd1 0xac 0x78 becomes 0x12345678 ** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 ** ** Variable length integers are used for rowids and to hold the number of @@ -67949,7 +68972,7 @@ typedef struct CellInfo CellInfo; ** page that has been loaded into memory. The information in this object ** is derived from the raw on-disk page content. ** -** As each database page is loaded into memory, the pager allocats an +** As each database page is loaded into memory, the pager allocates an ** instance of this object and zeros the first 8 bytes. (This is the ** "extra" information associated with each page of the pager.) ** @@ -68381,7 +69404,7 @@ struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ u8 *aPgRef; /* 1 bit per page in the db (see above) */ - Pgno nPage; /* Number of pages in the database */ + Pgno nCkPage; /* Pages in the database. 0 for partial check */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ @@ -68405,7 +69428,7 @@ struct IntegrityCk { /* ** get2byteAligned(), unlike get2byte(), requires that its argument point to a -** two-byte aligned address. get2bytea() is only used for accessing the +** two-byte aligned address. get2byteAligned() is only used for accessing the ** cell addresses in a btree header. */ #if SQLITE_BYTEORDER==4321 @@ -68582,7 +69605,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ ** ** There is a corresponding leave-all procedures. ** -** Enter the mutexes in accending order by BtShared pointer address +** Enter the mutexes in ascending order by BtShared pointer address ** to avoid the possibility of deadlock when two threads with ** two or more btrees in common both try to lock all their btrees ** at the same instant. @@ -70249,7 +71272,7 @@ static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ pPage->xParseCell(pPage, pCell, &info); if( info.nLocalaDataEnd, pCell, pCell+info.nLocal) ){ + if( SQLITE_OVERFLOW(pSrc->aDataEnd, pCell, pCell+info.nLocal) ){ testcase( pSrc!=pPage ); *pRC = SQLITE_CORRUPT_BKPT; return; @@ -70350,7 +71373,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ iCellStart = get2byte(&data[hdr+5]); if( nCell>0 ){ temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); + memcpy(temp, data, usableSize); src = temp; for(i=0; iiPage. -** -** The page is fetched as read-write unless pCur is not NULL and is -** a read-only cursor. -** -** If an error occurs, then *ppPage is undefined. It -** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - BtCursor *pCur, /* Cursor to receive the page, or NULL */ int bReadOnly /* True for a read-only page */ ){ int rc; DbPage *pDbPage; + MemPage *pPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pCur==0 || ppPage==&pCur->pPage ); - assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); - assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ - rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error1; + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; } rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ - goto getAndInitPage_error1; + *ppPage = 0; + return rc; } - *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - if( (*ppPage)->isInit==0 ){ + pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( pPage->isInit==0 ){ btreePageFromDbPage(pDbPage, pgno, pBt); - rc = btreeInitPage(*ppPage); + rc = btreeInitPage(pPage); if( rc!=SQLITE_OK ){ - goto getAndInitPage_error2; + releasePage(pPage); + *ppPage = 0; + return rc; } } - assert( (*ppPage)->pgno==pgno || CORRUPT_DB ); - assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); - - /* If obtaining a child page for a cursor, we must verify that the page is - ** compatible with the root page. */ - if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ - rc = SQLITE_CORRUPT_PGNO(pgno); - goto getAndInitPage_error2; - } + assert( pPage->pgno==pgno || CORRUPT_DB ); + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); + *ppPage = pPage; return SQLITE_OK; - -getAndInitPage_error2: - releasePage(*ppPage); -getAndInitPage_error1: - if( pCur ){ - pCur->iPage--; - pCur->pPage = pCur->apPage[pCur->iPage]; - } - testcase( pgno==0 ); - assert( pgno!=0 || rc!=SQLITE_OK ); - return rc; } /* @@ -71177,7 +72173,7 @@ static void pageReinit(DbPage *pData){ ** call to btreeInitPage() will likely return SQLITE_CORRUPT. ** But no harm is done by this. And it is very important that ** btreeInitPage() be called on every btree page so we make - ** the call for every page that comes in for re-initing. */ + ** the call for every page that comes in for re-initializing. */ btreeInitPage(pPage); } } @@ -71356,6 +72352,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( assert( sizeof(u16)==2 ); assert( sizeof(Pgno)==4 ); + /* Suppress false-positive compiler warning from PVS-Studio */ + memset(&zDbHeader[16], 0, 8); + pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -71572,7 +72571,7 @@ static SQLITE_NOINLINE int allocateTempSpace(BtShared *pBt){ ** can mean that fillInCell() only initializes the first 2 or 3 ** bytes of pTmpSpace, but that the first 4 bytes are copied from ** it into a database page. This is not actually a problem, but it - ** does cause a valgrind error when the 1 or 2 bytes of unitialized + ** does cause a valgrind error when the 1 or 2 bytes of uninitialized ** data is passed to system call write(). So to avoid this error, ** zero the first 4 bytes of temp space here. ** @@ -71807,7 +72806,7 @@ SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ /* ** Return the number of bytes of space at the end of every page that -** are intentually left unused. This is the "reserved" space that is +** are intentionally left unused. This is the "reserved" space that is ** sometimes used by extensions. ** ** The value returned is the larger of the current reserve size and @@ -72054,7 +73053,6 @@ static int lockBtree(BtShared *pBt){ ){ goto page1_init_failed; } - pBt->btsFlags |= BTS_PAGESIZE_FIXED; assert( (pageSize & 7)==0 ); /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte ** integer at offset 20 is the number of bytes of space at the end of @@ -72074,6 +73072,7 @@ static int lockBtree(BtShared *pBt){ releasePageOne(pPage1); pBt->usableSize = usableSize; pBt->pageSize = pageSize; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); @@ -72093,6 +73092,7 @@ static int lockBtree(BtShared *pBt){ if( usableSize<480 ){ goto page1_init_failed; } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; pBt->pageSize = pageSize; pBt->usableSize = usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -72271,7 +73271,11 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** when A already has a read lock, we encourage A to give up and let B ** proceed. */ -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ +static SQLITE_NOINLINE int btreeBeginTrans( + Btree *p, /* The btree in which to start the transaction */ + int wrflag, /* True to start a write transaction */ + int *pSchemaVersion /* Put schema version number here, if not NULL */ +){ BtShared *pBt = p->pBt; Pager *pPager = pBt->pPager; int rc = SQLITE_OK; @@ -72443,6 +73447,28 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVers sqlite3BtreeLeave(p); return rc; } +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ + BtShared *pBt; + if( p->sharable + || p->inTrans==TRANS_NONE + || (p->inTrans==TRANS_READ && wrflag!=0) + ){ + return btreeBeginTrans(p,wrflag,pSchemaVersion); + } + pBt = p->pBt; + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + return sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + }else{ + return SQLITE_OK; + } +} #ifndef SQLITE_OMIT_AUTOVACUUM @@ -73538,7 +74564,6 @@ SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor *pCur){ pCur->curFlags &= ~BTCF_Pinned; } -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC /* ** Return the offset into the database file for the start of the ** payload to which the cursor is pointing. @@ -73550,7 +74575,6 @@ SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor *pCur){ return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + (i64)(pCur->info.pPayload - pCur->pPage->aData); } -#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ /* ** Return the number of bytes of payload for the entry that pCur is @@ -73576,7 +74600,7 @@ SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor *pCur){ ** routine always returns 2147483647 (which is the largest record ** that SQLite can handle) or more. But returning a smaller value might ** prevent large memory allocations when trying to interpret a -** corrupt datrabase. +** corrupt database. ** ** The current implementation merely returns the size of the underlying ** database file. @@ -74038,6 +75062,7 @@ SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ + int rc; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageapPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; pCur->iPage++; - return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur, - pCur->curPagerFlags); + rc = getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur->curPagerFlags); + assert( pCur->pPage!=0 || rc!=SQLITE_OK ); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(newPgno); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + } + return rc; } #ifdef SQLITE_DEBUG @@ -74159,7 +75194,7 @@ static int moveToRoot(BtCursor *pCur){ sqlite3BtreeClearCursor(pCur); } rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage, - 0, pCur->curPagerFlags); + pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -74271,7 +75306,7 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ *pRes = 0; rc = moveToLeftmost(pCur); }else if( rc==SQLITE_EMPTY ){ - assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + assert( pCur->pgnoRoot==0 || (pCur->pPage!=0 && pCur->pPage->nCell==0) ); *pRes = 1; rc = SQLITE_OK; } @@ -74376,7 +75411,7 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( /* If the requested key is one more than the previous key, then ** try to get there using sqlite3BtreeNext() rather than a full ** binary search. This is an optimization only. The correct answer - ** is still obtained without this case, only a little more slowely */ + ** is still obtained without this case, only a little more slowly. */ if( pCur->info.nKey+1==intKey ){ *pRes = 0; rc = sqlite3BtreeNext(pCur, 0); @@ -74772,10 +75807,36 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( }else{ chldPg = get4byte(findCell(pPage, lwr)); } - pCur->ix = (u16)lwr; - rc = moveToChild(pCur, chldPg); - if( rc ) break; - } + + /* This block is similar to an in-lined version of: + ** + ** pCur->ix = (u16)lwr; + ** rc = moveToChild(pCur, chldPg); + ** if( rc ) break; + */ + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + pCur->aiIdx[pCur->iPage] = (u16)lwr; + pCur->apPage[pCur->iPage] = pCur->pPage; + pCur->ix = 0; + pCur->iPage++; + rc = getAndInitPage(pCur->pBt, chldPg, &pCur->pPage, pCur->curPagerFlags); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(chldPg); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + break; + } + /* + ***** End of in-lined moveToChild() call */ + } moveto_index_finish: pCur->info.nSize = 0; assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); @@ -75559,7 +76620,7 @@ static SQLITE_NOINLINE int clearCellOverflow( /* Call xParseCell to compute the size of a cell. If the cell contains ** overflow, then invoke cellClearOverflow to clear out that overflow. -** STore the result code (SQLITE_OK or some error code) in rc. +** Store the result code (SQLITE_OK or some error code) in rc. ** ** Implemented as macro to force inlining for performance. */ @@ -76170,12 +77231,13 @@ static int rebuildPage( int k; /* Current slot in pCArray->apEnd[] */ u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ + assert( nCell>0 ); assert( i(u32)usableSize) ){ j = 0; } + if( j>(u32)usableSize ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); - for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kixNx[k]<=i; k++){} pSrcEnd = pCArray->apEnd[k]; pData = pEnd; @@ -76238,7 +77300,7 @@ static int rebuildPage( ** Finally, argument pBegin points to the byte immediately following the ** end of the space required by this page for the cell-pointer area (for ** all cells - not just those inserted by the current call). If the content -** area must be extended to before this point in order to accomodate all +** area must be extended to before this point in order to accommodate all ** cells in apCell[], then the cells do not fit and non-zero is returned. */ static int pageInsertArray( @@ -76258,7 +77320,7 @@ static int pageInsertArray( u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ if( iEnd<=iFirst ) return 0; - for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kixNx[k]<=i ; k++){} pEnd = pCArray->apEnd[k]; while( 1 /*Exit by break*/ ){ int sz, rc; @@ -76476,6 +77538,7 @@ static int editPage( return SQLITE_OK; editpage_fail: /* Unable to edit this page. Rebuild it from scratch instead. */ + if( nNew<1 ) return SQLITE_CORRUPT_BKPT; populateCellCache(pCArray, iNew, nNew); return rebuildPage(pCArray, iNew, nNew, pPg); } @@ -76553,7 +77616,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ ** with entries for the new page, and any pointer from the ** cell on the page to an overflow page. If either of these ** operations fails, the return code is set, but the contents - ** of the parent page are still manipulated by thh code below. + ** of the parent page are still manipulated by the code below. ** That is Ok, at this point the parent page is guaranteed to ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. @@ -76829,7 +77892,7 @@ static int balance_nonroot( pgno = get4byte(pRight); while( 1 ){ if( rc==SQLITE_OK ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); } if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); @@ -77143,7 +78206,7 @@ static int balance_nonroot( } } - /* Sanity check: For a non-corrupt database file one of the follwing + /* Sanity check: For a non-corrupt database file one of the following ** must be true: ** (1) We found one or more cells (cntNew[0])>0), or ** (2) pPage is a virtual root page. A virtual root page is when @@ -77368,9 +78431,9 @@ static int balance_nonroot( iOvflSpace += sz; assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace <= (int)pBt->pageSize ); - for(k=0; b.ixNx[k]<=j && ALWAYS(k=0 && iPg=1 || i>=0 ); + assert( iPg=0 /* On the upwards pass, or... */ || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ @@ -77760,7 +78825,7 @@ static int btreeOverwriteContent( ){ int nData = pX->nData - iOffset; if( nData<=0 ){ - /* Overwritting with zeros */ + /* Overwriting with zeros */ int i; for(i=0; ipData to write */ @@ -78543,7 +79608,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ MemPage *pRoot; Pgno pgnoRoot; int rc; - int ptfFlags; /* Page-type flage for the root page of new table */ + int ptfFlags; /* Page-type flags for the root page of new table */ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -78712,7 +79777,7 @@ static int clearDatabasePage( if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; if( (pBt->openFlags & BTREE_SINGLE)==0 && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) @@ -79133,7 +80198,8 @@ static void checkAppendMsg( ** corresponds to page iPg is already set. */ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); } @@ -79141,7 +80207,8 @@ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. */ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); } @@ -79155,7 +80222,7 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Also check that the page number is in bounds. */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ - if( iPage>pCheck->nPage || iPage==0 ){ + if( iPage>pCheck->nCkPage || iPage==0 ){ checkAppendMsg(pCheck, "invalid page number %u", iPage); return 1; } @@ -79378,10 +80445,11 @@ static int checkTreePage( if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; pCheck->zPfx = "Tree %u page %u: "; - pCheck->v0 = pCheck->v1 = iPage; + pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); + if( rc==SQLITE_IOERR_NOMEM ) pCheck->rc = SQLITE_NOMEM; goto end_of_check; } @@ -79652,15 +80720,15 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; - sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.nCkPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; - if( sCheck.nPage==0 ){ + if( sCheck.nCkPage==0 ){ goto integrity_ck_cleanup; } - sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + sCheck.aPgRef = sqlite3MallocZero((sCheck.nCkPage / 8)+ 1); if( !sCheck.aPgRef ){ checkOom(&sCheck); goto integrity_ck_cleanup; @@ -79672,7 +80740,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( } i = PENDING_BYTE_PAGE(pBt); - if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + if( i<=sCheck.nCkPage ) setPageReferenced(&sCheck, i); /* Check the integrity of the freelist */ @@ -79715,6 +80783,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif + sCheck.v0 = aRoot[i]; checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); } pBt->db->flags = savedDbFlags; @@ -79722,7 +80791,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( /* Make sure every page in the file is referenced */ if( !bPartial ){ - for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ + for(i=1; i<=sCheck.nCkPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ checkAppendMsg(&sCheck, "Page %u: never used", i); @@ -81141,6 +82210,40 @@ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ return SQLITE_OK; } +/* +** If pMem is already a string, detect if it is a zero-terminated +** string, or make it into one if possible, and mark it as such. +** +** This is an optimization. Correct operation continues even if +** this routine is a no-op. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ + if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ + /* pMem must be a string, and it cannot be an ephemeral or static string */ + return; + } + if( pMem->enc!=SQLITE_UTF8 ) return; + if( NEVER(pMem->z==0) ) return; + if( pMem->flags & MEM_Dyn ){ + if( pMem->xDel==sqlite3_free + && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) + ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } + if( pMem->xDel==sqlite3RCStrUnref ){ + /* Blindly assume that all RCStr objects are zero-terminated */ + pMem->flags |= MEM_Term; + return; + } + }else if( pMem->szMalloc >= pMem->n+1 ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } +} + /* ** It is already known that pMem contains an unterminated string. ** Add the zero terminator. @@ -81402,36 +82505,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemReleaseMalloc(Mem *p){ if( p->szMalloc ) vdbeMemClear(p); } -/* -** Convert a 64-bit IEEE double into a 64-bit signed integer. -** If the double is out of range of a 64-bit signed integer then -** return the closest available 64-bit signed integer. -*/ -static SQLITE_NOINLINE i64 doubleToInt64(double r){ -#ifdef SQLITE_OMIT_FLOATING_POINT - /* When floating-point is omitted, double and int64 are the same thing */ - return r; -#else - /* - ** Many compilers we encounter do not define constants for the - ** minimum and maximum 64-bit integers, or they define them - ** inconsistently. And many do not understand the "LL" notation. - ** So we define our own static constants here using nothing - ** larger than a 32-bit integer constant. - */ - static const i64 maxInt = LARGEST_INT64; - static const i64 minInt = SMALLEST_INT64; - - if( r<=(double)minInt ){ - return minInt; - }else if( r>=(double)maxInt ){ - return maxInt; - }else{ - return (i64)r; - } -#endif -} - /* ** Return some kind of integer value which is the best we can do ** at representing the value that *pMem describes as an integer. @@ -81458,7 +82531,7 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem *pMem){ testcase( flags & MEM_IntReal ); return pMem->u.i; }else if( flags & MEM_Real ){ - return doubleToInt64(pMem->u.r); + return sqlite3RealToI64(pMem->u.r); }else if( (flags & (MEM_Str|MEM_Blob))!=0 && pMem->z!=0 ){ return memIntValue(pMem); }else{ @@ -81520,7 +82593,7 @@ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ if( pMem->flags & MEM_IntReal ){ MemSetTypeFlag(pMem, MEM_Int); }else{ - i64 ix = doubleToInt64(pMem->u.r); + i64 ix = sqlite3RealToI64(pMem->u.r); /* Only mark the value as an integer if ** @@ -81588,8 +82661,8 @@ SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ ** from UBSAN. */ SQLITE_PRIVATE i64 sqlite3RealToI64(double r){ - if( r<=(double)SMALLEST_INT64 ) return SMALLEST_INT64; - if( r>=(double)LARGEST_INT64) return LARGEST_INT64; + if( r<-9223372036854774784.0 ) return SMALLEST_INT64; + if( r>+9223372036854774784.0 ) return LARGEST_INT64; return (i64)r; } @@ -81660,6 +82733,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ break; } default: { + int rc; assert( aff==SQLITE_AFF_TEXT ); assert( MEM_Str==(MEM_Blob>>3) ); pMem->flags |= (pMem->flags&MEM_Blob)>>3; @@ -81667,7 +82741,9 @@ SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; - return sqlite3VdbeChangeEncoding(pMem, encoding); + rc = sqlite3VdbeChangeEncoding(pMem, encoding); + if( rc ) return rc; + sqlite3VdbeMemZeroTerminateIfAble(pMem); } } return SQLITE_OK; @@ -82191,6 +83267,24 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } +/* Return true if sqlit3_value object pVal is a string or blob value +** that uses the destructor specified in the second argument. +** +** TODO: Maybe someday promote this interface into a published API so +** that third-party extensions can get access to it? +*/ +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ + if( ALWAYS(pVal!=0) + && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0) + && (pVal->flags & MEM_Dyn)!=0 + && pVal->xDel==xFree + ){ + return 1; + }else{ + return 0; + } +} + /* ** Create a new sqlite3_value object. */ @@ -82258,6 +83352,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ } pRec->nField = p->iVal+1; + sqlite3VdbeMemSetNull(&pRec->aMem[p->iVal]); return &pRec->aMem[p->iVal]; } #else @@ -82517,6 +83612,7 @@ static int valueFromExpr( if( pVal ){ pVal->flags = MEM_Int; pVal->u.i = pExpr->u.zToken[4]==0; + sqlite3ValueApplyAffinity(pVal, affinity, enc); } } @@ -83046,6 +84142,35 @@ static void test_addop_breakpoint(int pc, Op *pOp){ } #endif +/* +** Slow paths for sqlite3VdbeAddOp3() and sqlite3VdbeAddOp4Int() for the +** unusual case when we need to increase the size of the Vdbe.aOp[] array +** before adding the new opcode. +*/ +static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ + assert( p->nOpAlloc<=p->nOp ); + if( growOpArray(p, 1) ) return 1; + assert( p->nOpAlloc>p->nOp ); + return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +} +static SQLITE_NOINLINE int addOp4IntSlow( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + if( p->db->mallocFailed==0 ){ + VdbeOp *pOp = &p->aOp[addr]; + pOp->p4type = P4_INT32; + pOp->p4.i = p4; + } + return addr; +} + + /* ** Add a new instruction to the list of instructions current in the ** VDBE. Return the address of the new instruction. @@ -83056,17 +84181,16 @@ static void test_addop_breakpoint(int pc, Op *pOp){ ** ** op The opcode for this instruction ** -** p1, p2, p3 Operands -** -** Use the sqlite3VdbeResolveLabel() function to fix an address and -** the sqlite3VdbeChangeP4() function to change the value of the P4 -** operand. +** p1, p2, p3, p4 Operands */ -static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ - assert( p->nOpAlloc<=p->nOp ); - if( growOpArray(p, 1) ) return 1; - assert( p->nOpAlloc>p->nOp ); - return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ + return sqlite3VdbeAddOp3(p, op, 0, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ + return sqlite3VdbeAddOp3(p, op, p1, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ + return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ int i; @@ -83089,6 +84213,9 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; + + /* Replicate this logic in sqlite3VdbeAddOp4Int() + ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS pOp->zComment = 0; #endif @@ -83105,16 +84232,59 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ #ifdef SQLITE_VDBE_COVERAGE pOp->iSrcLine = 0; #endif + /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ** Replicate in sqlite3VdbeAddOp4Int() */ + return i; } -SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ - return sqlite3VdbeAddOp3(p, op, 0, 0, 0); -} -SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ - return sqlite3VdbeAddOp3(p, op, p1, 0, 0); -} -SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ - return sqlite3VdbeAddOp3(p, op, p1, p2, 0); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int i; + VdbeOp *pOp; + + i = p->nOp; + if( p->nOpAlloc<=i ){ + return addOp4IntSlow(p, op, p1, p2, p3, p4); + } + p->nOp++; + pOp = &p->aOp[i]; + assert( pOp!=0 ); + pOp->opcode = (u8)op; + pOp->p5 = 0; + pOp->p1 = p1; + pOp->p2 = p2; + pOp->p3 = p3; + pOp->p4.i = p4; + pOp->p4type = P4_INT32; + + /* Replicate this logic in sqlite3VdbeAddOp3() + ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOp->zComment = 0; +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + test_addop_breakpoint(i, &p->aOp[i]); + } +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOp->iSrcLine = 0; +#endif + /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ** Replicate in sqlite3VdbeAddOp3() */ + + return i; } /* Generate code for an unconditional jump to instruction iDest @@ -83292,7 +84462,7 @@ SQLITE_PRIVATE int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, if( bPush){ pParse->addrExplain = iThis; } - sqlite3VdbeScanStatus(v, iThis, 0, 0, 0, 0); + sqlite3VdbeScanStatus(v, iThis, -1, -1, 0, 0); } return addr; } @@ -83322,26 +84492,6 @@ SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere, sqlite3MayAbort(p->pParse); } -/* -** Add an opcode that includes the p4 value as an integer. -*/ -SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( - Vdbe *p, /* Add the opcode to this VM */ - int op, /* The new opcode */ - int p1, /* The P1 operand */ - int p2, /* The P2 operand */ - int p3, /* The P3 operand */ - int p4 /* The P4 operand as an integer */ -){ - int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); - if( p->db->mallocFailed==0 ){ - VdbeOp *pOp = &p->aOp[addr]; - pOp->p4type = P4_INT32; - pOp->p4.i = p4; - } - return addr; -} - /* Insert the end of a co-routine */ SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ @@ -83654,7 +84804,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; assert( p->aOp[0].opcode==OP_Init ); - while( 1 /* Loop termates when it reaches the OP_Init opcode */ ){ + while( 1 /* Loop terminates when it reaches the OP_Init opcode */ ){ /* Only JUMP opcodes and the short list of special opcodes in the switch ** below need to be considered. The mkopcodeh.tcl generator script groups ** all these opcodes together near the front of the opcode list. Skip @@ -83776,6 +84926,10 @@ SQLITE_PRIVATE void sqlite3VdbeNoJumpsOutsideSubrtn( int iDest = pOp->p2; /* Jump destination */ if( iDest==0 ) continue; if( pOp->opcode==OP_Gosub ) continue; + if( pOp->p3==20230325 && pOp->opcode==OP_NotNull ){ + /* This is a deliberately taken illegal branch. tag-20230325-2 */ + continue; + } if( iDest<0 ){ int j = ADDR(iDest); assert( j>=0 ); @@ -84024,8 +85178,8 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters( pScan = 0; } if( pScan ){ - pScan->addrLoop = addrLoop; - pScan->addrVisit = addrVisit; + if( addrLoop>0 ) pScan->addrLoop = addrLoop; + if( addrVisit>0 ) pScan->addrVisit = addrVisit; } } } @@ -84108,7 +85262,7 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ /* ** If the input FuncDef structure is ephemeral, then free it. If -** the FuncDef is not ephermal, then do nothing. +** the FuncDef is not ephemeral, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ assert( db!=0 ); @@ -84272,7 +85426,6 @@ SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters( } #endif /* SQLITE_DEBUG */ - /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -85193,7 +86346,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); - p->nResColumn = 4; + assert( p->nResColumn==4 ); }else{ sqlite3VdbeMemSetInt64(pMem+0, i); sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), @@ -85212,7 +86365,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( sqlite3VdbeMemSetNull(pMem+7); #endif sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); - p->nResColumn = 8; + assert( p->nResColumn==8 ); } p->pResultRow = pMem; if( db->mallocFailed ){ @@ -85426,26 +86579,9 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); if( pParse->explain ){ - static const char * const azColName[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "id", "parent", "notused", "detail" - }; - int iFirst, mx, i; if( nMem<10 ) nMem = 10; p->explain = pParse->explain; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(p, 4); - iFirst = 8; - mx = 12; - }else{ - sqlite3VdbeSetNumCols(p, 8); - iFirst = 0; - mx = 8; - } - for(i=iFirst; inResColumn = 12 - 4*p->explain; } p->expired = 0; @@ -85497,7 +86633,23 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx); } +static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){ + VdbeTxtBlbCache *pCache = pCx->pCache; + assert( pCx->colCache ); + pCx->colCache = 0; + pCx->pCache = 0; + if( pCache->pCValue ){ + sqlite3RCStrUnref(pCache->pCValue); + pCache->pCValue = 0; + } + sqlite3DbFree(p->db, pCache); + sqlite3VdbeFreeCursorNN(p, pCx); +} SQLITE_PRIVATE void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){ + if( pCx->colCache ){ + freeCursorWithCache(p, pCx); + return; + } switch( pCx->eCurType ){ case CURTYPE_SORTER: { sqlite3VdbeSorterClose(p->db, pCx); @@ -85598,12 +86750,12 @@ SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ int n; sqlite3 *db = p->db; - if( p->nResColumn ){ - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + if( p->nResAlloc ){ + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); sqlite3DbFree(db, p->aColName); } n = nResColumn*COLNAME_N; - p->nResColumn = (u16)nResColumn; + p->nResColumn = p->nResAlloc = (u16)nResColumn; p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); if( p->aColName==0 ) return; initMemArray(p->aColName, n, db, MEM_Null); @@ -85628,14 +86780,14 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( ){ int rc; Mem *pColName; - assert( idxnResColumn ); + assert( idxnResAlloc ); assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE_DYNAMIC ); return SQLITE_NOMEM_BKPT; } assert( p->aColName!=0 ); - pColName = &(p->aColName[idx+var*p->nResColumn]); + pColName = &(p->aColName[idx+var*p->nResAlloc]); rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; @@ -86148,6 +87300,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3VdbeLeave(p); return SQLITE_BUSY; }else if( rc!=SQLITE_OK ){ + sqlite3SystemError(db, rc); p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -86459,7 +87612,7 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ assert( db!=0 ); assert( p->db==0 || p->db==db ); if( p->aColName ){ - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); sqlite3DbNNFreeNN(db, p->aColName); } for(pSub=p->pProgram; pSub; pSub=pNext){ @@ -87059,6 +88212,15 @@ static int vdbeRecordCompareDebug( if( d1+(u64)serial_type1+2>(u64)nKey1 && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1 ){ + if( serial_type1>=1 + && serial_type1<=7 + && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)<=(u64)nKey1+8 + && CORRUPT_DB + ){ + return 1; /* corrupt record not detected by + ** sqlite3VdbeRecordCompareWithSkip(). Return true + ** to avoid firing the assert() */ + } break; } @@ -87227,20 +88389,33 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem return n1 - n2; } +/* The following two functions are used only within testcase() to prove +** test coverage. These functions do no exist for production builds. +** We must use separate SQLITE_NOINLINE functions here, since otherwise +** optimizer code movement causes gcov to become very confused. +*/ +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) +static int SQLITE_NOINLINE doubleLt(double a, double b){ return a8 ){ + if( sqlite3IsNaN(r) ){ + /* SQLite considers NaN to be a NULL. And all integer values are greater + ** than NULL */ + return 1; + } + if( sqlite3Config.bUseLongDouble ){ LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i; testcase( xr ); testcase( x==r ); - if( xr ) return +1; /*NO_TEST*/ /* work around bugs in gcov */ - return 0; /*NO_TEST*/ /* work around bugs in gcov */ + return (xr); }else{ i64 y; double s; @@ -87250,9 +88425,10 @@ SQLITE_PRIVATE int sqlite3IntFloatCompare(i64 i, double r){ if( iy ) return +1; s = (double)i; - if( sr ) return +1; - return 0; + testcase( doubleLt(s,r) ); + testcase( doubleLt(r,s) ); + testcase( doubleEq(r,s) ); + return (sr); } } @@ -87502,7 +88678,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* Serial types 12 or greater are strings and blobs (greater than ** numbers). Types 10 and 11 are currently "reserved for future ** use", so it doesn't really matter what the results of comparing - ** them to numberic values are. */ + ** them to numeric values are. */ rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; @@ -88620,7 +89796,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value *pOld){ ** is too big or if an OOM occurs. ** ** The invokeValueDestructor(P,X) routine invokes destructor function X() -** on value P is not going to be used and need to be destroyed. +** on value P if P is not going to be used and need to be destroyed. */ static void setResultStrOrError( sqlite3_context *pCtx, /* Function context */ @@ -88650,7 +89826,7 @@ static void setResultStrOrError( static int invokeValueDestructor( const void *p, /* Value to destroy */ void (*xDel)(void*), /* The destructor */ - sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ + sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if not NULL */ ){ assert( xDel!=SQLITE_DYNAMIC ); if( xDel==0 ){ @@ -88660,7 +89836,14 @@ static int invokeValueDestructor( }else{ xDel((void*)p); } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx!=0 ){ + sqlite3_result_error_toobig(pCtx); + } +#else + assert( pCtx!=0 ); sqlite3_result_error_toobig(pCtx); +#endif return SQLITE_TOOBIG; } SQLITE_API void sqlite3_result_blob( @@ -88669,6 +89852,12 @@ SQLITE_API void sqlite3_result_blob( int n, void (*xDel)(void *) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 || n<0 ){ + invokeValueDestructor(z, xDel, pCtx); + return; + } +#endif assert( n>=0 ); assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); @@ -88679,8 +89868,14 @@ SQLITE_API void sqlite3_result_blob64( sqlite3_uint64 n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ @@ -88688,30 +89883,48 @@ SQLITE_API void sqlite3_result_blob64( } } SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } @@ -88721,14 +89934,25 @@ SQLITE_API void sqlite3_result_pointer( const char *zPType, void (*xDestructor)(void*) ){ - Mem *pOut = pCtx->pOut; + Mem *pOut; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(pPtr, xDestructor, 0); + return; + } +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); sqlite3VdbeMemRelease(pOut); pOut->flags = MEM_Null; sqlite3VdbeMemSetPointer(pOut, pPtr, zPType, xDestructor); } SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ - Mem *pOut = pCtx->pOut; + Mem *pOut; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); pOut->eSubtype = eSubtype & 0xff; pOut->flags |= MEM_Subtype; @@ -88739,6 +89963,12 @@ SQLITE_API void sqlite3_result_text( int n, void (*xDel)(void *) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } @@ -88749,6 +89979,12 @@ SQLITE_API void sqlite3_result_text64( void (*xDel)(void *), unsigned char enc ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); if( enc!=SQLITE_UTF8 ){ @@ -88759,6 +89995,7 @@ SQLITE_API void sqlite3_result_text64( (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); + sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut); } } #ifndef SQLITE_OMIT_UTF16 @@ -88791,7 +90028,16 @@ SQLITE_API void sqlite3_result_text16le( } #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ - Mem *pOut = pCtx->pOut; + Mem *pOut; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; + if( pValue==0 ){ + sqlite3_result_null(pCtx); + return; + } +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemCopy(pOut, pValue); sqlite3VdbeChangeEncoding(pOut, pCtx->enc); @@ -88803,7 +90049,12 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ sqlite3_result_zeroblob64(pCtx, n>0 ? n : 0); } SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ - Mem *pOut = pCtx->pOut; + Mem *pOut; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return SQLITE_MISUSE_BKPT; +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(pCtx); @@ -88817,6 +90068,9 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ #endif } SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif pCtx->isError = errCode ? errCode : -1; #ifdef SQLITE_DEBUG if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode; @@ -88829,6 +90083,9 @@ SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ /* Force an SQLITE_TOOBIG error. */ SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, @@ -88837,6 +90094,9 @@ SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ /* An SQLITE_NOMEM error. */ SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); pCtx->isError = SQLITE_NOMEM_BKPT; @@ -89089,7 +90349,11 @@ SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ ** pointer to it. */ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#else assert( p && p->pFunc ); +#endif return p->pFunc->pUserData; } @@ -89104,7 +90368,11 @@ SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ ** application defined function. */ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#else assert( p && p->pOut ); +#endif return p->pOut->db; } @@ -89123,7 +90391,11 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ ** value, as a signal to the xUpdate routine that the column is unchanged. */ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#else assert( p ); +#endif return sqlite3_value_nochange(p->pOut); } @@ -89131,7 +90403,7 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ ** The destructor function for a ValueList object. This needs to be ** a separate function, unknowable to the application, to ensure that ** calls to sqlite3_vtab_in_first()/sqlite3_vtab_in_next() that are not -** preceeded by activation of IN processing via sqlite3_vtab_int() do not +** preceded by activation of IN processing via sqlite3_vtab_int() do not ** try to access a fake ValueList object inserted by a hostile extension. */ SQLITE_PRIVATE void sqlite3VdbeValueListFree(void *pToDelete){ @@ -89151,7 +90423,7 @@ static int valueFromValueList( ValueList *pRhs; *ppOut = 0; - if( pVal==0 ) return SQLITE_MISUSE; + if( pVal==0 ) return SQLITE_MISUSE_BKPT; if( (pVal->flags & MEM_Dyn)==0 || pVal->xDel!=sqlite3VdbeValueListFree ){ return SQLITE_ERROR; }else{ @@ -89282,6 +90554,9 @@ SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return 0; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); #if SQLITE_ENABLE_STAT4 if( pCtx->pVdbe==0 ) return 0; @@ -89314,8 +90589,12 @@ SQLITE_API void sqlite3_set_auxdata( void (*xDelete)(void*) ){ AuxData *pAuxData; - Vdbe *pVdbe = pCtx->pVdbe; + Vdbe *pVdbe; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif + pVdbe= pCtx->pVdbe; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); #ifdef SQLITE_ENABLE_STAT4 if( pVdbe==0 ) goto failed; @@ -89371,7 +90650,8 @@ SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - return pVm ? pVm->nResColumn : 0; + if( pVm==0 ) return 0; + return pVm->nResColumn; } /* @@ -89460,7 +90740,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ ** sqlite3_column_real() ** sqlite3_column_bytes() ** sqlite3_column_bytes16() -** sqiite3_column_blob() +** sqlite3_column_blob() */ static void columnMallocFailure(sqlite3_stmt *pStmt) { @@ -89544,6 +90824,32 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ return iType; } +/* +** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN. +*/ +static const char * const azExplainColNames8[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", /* EXPLAIN */ + "id", "parent", "notused", "detail" /* EQP */ +}; +static const u16 azExplainColNames16data[] = { + /* 0 */ 'a', 'd', 'd', 'r', 0, + /* 5 */ 'o', 'p', 'c', 'o', 'd', 'e', 0, + /* 12 */ 'p', '1', 0, + /* 15 */ 'p', '2', 0, + /* 18 */ 'p', '3', 0, + /* 21 */ 'p', '4', 0, + /* 24 */ 'p', '5', 0, + /* 27 */ 'c', 'o', 'm', 'm', 'e', 'n', 't', 0, + /* 35 */ 'i', 'd', 0, + /* 38 */ 'p', 'a', 'r', 'e', 'n', 't', 0, + /* 45 */ 'n', 'o', 't', 'u', 's', 'e', 'd', 0, + /* 53 */ 'd', 'e', 't', 'a', 'i', 'l', 0 +}; +static const u8 iExplainColNames16[] = { + 0, 5, 12, 15, 18, 21, 24, 27, + 35, 38, 45, 53 +}; + /* ** Convert the N-th element of pStmt->pColName[] into a string using ** xFunc() then return that string. If N is out of range, return 0. @@ -89576,15 +90882,29 @@ static const void *columnName( return 0; } #endif + if( N<0 ) return 0; ret = 0; p = (Vdbe *)pStmt; db = p->db; assert( db!=0 ); - n = sqlite3_column_count(pStmt); - if( N=0 ){ + sqlite3_mutex_enter(db->mutex); + + if( p->explain ){ + if( useType>0 ) goto columnName_end; + n = p->explain==1 ? 8 : 4; + if( N>=n ) goto columnName_end; + if( useUtf16 ){ + int i = iExplainColNames16[N + 8*p->explain - 8]; + ret = (void*)&azExplainColNames16data[i]; + }else{ + ret = (void*)azExplainColNames8[N + 8*p->explain - 8]; + } + goto columnName_end; + } + n = p->nResColumn; + if( NmallocFailed; N += useType*n; - sqlite3_mutex_enter(db->mutex); #ifndef SQLITE_OMIT_UTF16 if( useUtf16 ){ ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); @@ -89601,8 +90921,9 @@ static const void *columnName( sqlite3OomClear(db); ret = 0; } - sqlite3_mutex_leave(db->mutex); } +columnName_end: + sqlite3_mutex_leave(db->mutex); return ret; } @@ -89695,7 +91016,7 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ /* ** Unbind the value bound to variable i in virtual machine p. This is the ** the same as binding a NULL value to the column. If the "i" parameter is -** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK. +** out of range, then SQLITE_RANGE is returned. Otherwise SQLITE_OK. ** ** A successful evaluation of this routine acquires the mutex on p. ** the mutex is released if any kind of error occurs. @@ -89710,7 +91031,7 @@ static int vdbeUnbind(Vdbe *p, unsigned int i){ } sqlite3_mutex_enter(p->db->mutex); if( p->eVdbeState!=VDBE_READY_STATE ){ - sqlite3Error(p->db, SQLITE_MISUSE); + sqlite3Error(p->db, SQLITE_MISUSE_BKPT); sqlite3_mutex_leave(p->db->mutex); sqlite3_log(SQLITE_MISUSE, "bind on a busy prepared statement: [%s]", p->zSql); @@ -89939,6 +91260,9 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ int rc; Vdbe *p = (Vdbe *)pStmt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->db->mutex); if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ rc = SQLITE_TOOBIG; @@ -90059,6 +91383,42 @@ SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->explain : 0; } +/* +** Set the explain mode for a statement. +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){ + Vdbe *v = (Vdbe*)pStmt; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(v->db->mutex); + if( ((int)v->explain)==eMode ){ + rc = SQLITE_OK; + }else if( eMode<0 || eMode>2 ){ + rc = SQLITE_ERROR; + }else if( (v->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ + rc = SQLITE_ERROR; + }else if( v->eVdbeState!=VDBE_READY_STATE ){ + rc = SQLITE_BUSY; + }else if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){ + /* No reprepare necessary */ + v->explain = eMode; + rc = SQLITE_OK; + }else{ + v->explain = eMode; + rc = sqlite3Reprepare(v); + v->haveEqpOps = eMode==2; + } + if( v->explain ){ + v->nResColumn = 12 - 4*v->explain; + }else{ + v->nResColumn = v->nResAlloc; + } + sqlite3_mutex_leave(v->db->mutex); + return rc; +} + /* ** Return true if the prepared statement is in need of being reset. */ @@ -90198,10 +91558,16 @@ static UnpackedRecord *vdbeUnpackRecord( ** a field of the row currently being updated or deleted. */ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; Mem *pMem; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 || ppValue==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + p = db->pPreUpdate; /* Test that this call is being made from within an SQLITE_DELETE or ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */ if( !p || p->op==SQLITE_INSERT ){ @@ -90262,7 +91628,12 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa ** the number of columns in the row being updated, deleted or inserted. */ SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif return (p ? p->keyinfo.nKeyField : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -90280,7 +91651,12 @@ SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ ** or SET DEFAULT action is considered a trigger. */ SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif return (p ? p->v->nFrame : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -90291,7 +91667,12 @@ SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ ** only. */ SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif return (p ? p->iBlobWrite : -1); } #endif @@ -90302,10 +91683,16 @@ SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *db){ ** a field of the row currently being updated or inserted. */ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; + PreUpdate *p; int rc = SQLITE_OK; Mem *pMem; +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 || ppValue==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + p = db->pPreUpdate; if( !p || p->op==SQLITE_DELETE ){ rc = SQLITE_MISUSE_BKPT; goto preupdate_new_out; @@ -90384,11 +91771,20 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; - VdbeOp *aOp = p->aOp; - int nOp = p->nOp; + VdbeOp *aOp; + int nOp; ScanStatus *pScan = 0; int idx; +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 || pOut==0 + || iScanStatusOpSQLITE_SCANSTAT_NCYCLE ){ + return 1; + } +#endif + aOp = p->aOp; + nOp = p->nOp; if( p->pFrame ){ VdbeFrame *pFrame; for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); @@ -90535,7 +91931,7 @@ SQLITE_API int sqlite3_stmt_scanstatus( SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; int ii; - for(ii=0; iinOp; ii++){ + for(ii=0; p!=0 && iinOp; ii++){ Op *pOp = &p->aOp[ii]; pOp->nExec = 0; pOp->nCycle = 0; @@ -91298,6 +92694,9 @@ SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); } sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); + if( f & MEM_Term ){ + sqlite3_str_appendf(pStr, "(0-term)"); + } } } #endif @@ -91434,6 +92833,93 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ return h; } + +/* +** For OP_Column, factor out the case where content is loaded from +** overflow pages, so that the code to implement this case is separate +** the common case where all content fits on the page. Factoring out +** the code reduces register pressure and helps the common case +** to run faster. +*/ +static SQLITE_NOINLINE int vdbeColumnFromOverflow( + VdbeCursor *pC, /* The BTree cursor from which we are reading */ + int iCol, /* The column to read */ + int t, /* The serial-type code for the column value */ + i64 iOffset, /* Offset to the start of the content value */ + u32 cacheStatus, /* Current Vdbe.cacheCtr value */ + u32 colCacheCtr, /* Current value of the column cache counter */ + Mem *pDest /* Store the value into this register. */ +){ + int rc; + sqlite3 *db = pDest->db; + int encoding = pDest->enc; + int len = sqlite3VdbeSerialTypeLen(t); + assert( pC->eCurType==CURTYPE_BTREE ); + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG; + if( len > 4000 && pC->pKeyInfo==0 ){ + /* Cache large column values that are on overflow pages using + ** an RCStr (reference counted string) so that if they are reloaded, + ** that do not have to be copied a second time. The overhead of + ** creating and managing the cache is such that this is only + ** profitable for larger TEXT and BLOB values. + ** + ** Only do this on table-btrees so that writes to index-btrees do not + ** need to clear the cache. This buys performance in the common case + ** in exchange for generality. + */ + VdbeTxtBlbCache *pCache; + char *pBuf; + if( pC->colCache==0 ){ + pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) ); + if( pC->pCache==0 ) return SQLITE_NOMEM; + pC->colCache = 1; + } + pCache = pC->pCache; + if( pCache->pCValue==0 + || pCache->iCol!=iCol + || pCache->cacheStatus!=cacheStatus + || pCache->colCacheCtr!=colCacheCtr + || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor) + ){ + if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue); + pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 ); + if( pBuf==0 ) return SQLITE_NOMEM; + rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf); + if( rc ) return rc; + pBuf[len] = 0; + pBuf[len+1] = 0; + pBuf[len+2] = 0; + pCache->iCol = iCol; + pCache->cacheStatus = cacheStatus; + pCache->colCacheCtr = colCacheCtr; + pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor); + }else{ + pBuf = pCache->pCValue; + } + assert( t>=12 ); + sqlite3RCStrRef(pBuf); + if( t&1 ){ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding, + sqlite3RCStrUnref); + pDest->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0, + sqlite3RCStrUnref); + } + }else{ + rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest); + if( rc ) return rc; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ + pDest->z[len] = 0; + pDest->flags |= MEM_Term; + } + } + pDest->flags &= ~MEM_Ephem; + return rc; +} + + /* ** Return the symbolic name for the data type of a pMem */ @@ -91476,6 +92962,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ + u32 colCacheCtr = 0; /* Column cache counter */ #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) u64 *pnCycle = 0; int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; @@ -91671,8 +93158,8 @@ SQLITE_PRIVATE int sqlite3VdbeExec( case OP_Goto: { /* jump */ #ifdef SQLITE_DEBUG - /* In debuggging mode, when the p5 flags is set on an OP_Goto, that - ** means we should really jump back to the preceeding OP_ReleaseReg + /* In debugging mode, when the p5 flags is set on an OP_Goto, that + ** means we should really jump back to the preceding OP_ReleaseReg ** instruction. */ if( pOp->p5 ){ assert( pOp->p2 < (int)(pOp - aOp) ); @@ -91880,7 +93367,7 @@ case OP_HaltIfNull: { /* in3 */ ** P5 is a value between 0 and 4, inclusive, that modifies the P4 string. ** ** 0: (no change) -** 1: NOT NULL contraint failed: P4 +** 1: NOT NULL constraint failed: P4 ** 2: UNIQUE constraint failed: P4 ** 3: CHECK constraint failed: P4 ** 4: FOREIGN KEY constraint failed: P4 @@ -93011,10 +94498,10 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** opcodes are allowed to occur between this instruction and the previous ** OP_Lt or OP_Gt. ** -** If result of an OP_Eq comparison on the same two operands as the -** prior OP_Lt or OP_Gt would have been true, then jump to P2. -** If the result of an OP_Eq comparison on the two previous -** operands would have been false or NULL, then fall through. +** If the result of an OP_Eq comparison on the same two operands as +** the prior OP_Lt or OP_Gt would have been true, then jump to P2. If +** the result of an OP_Eq comparison on the two previous operands +** would have been false or NULL, then fall through. */ case OP_ElseEq: { /* same as TK_ESCAPE, jump */ @@ -93444,7 +94931,7 @@ case OP_IsType: { /* jump */ /* Opcode: ZeroOrNull P1 P2 P3 * * ** Synopsis: r[P2] = 0 OR NULL ** -** If all both registers P1 and P3 are NOT NULL, then store a zero in +** If both registers P1 and P3 are NOT NULL, then store a zero in ** register P2. If either registers P1 or P3 are NULL then put ** a NULL in register P2. */ @@ -93798,11 +95285,16 @@ case OP_Column: { /* ncycle */ pDest->flags = aFlag[t&1]; } }else{ + u8 p5; pDest->enc = encoding; + assert( pDest->db==db ); /* This branch happens only when content is on overflow pages */ - if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 - && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) - || (len = sqlite3VdbeSerialTypeLen(t))==0 + if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0 + && (p5==OPFLAG_TYPEOFARG + || (t>=12 && ((t&1)==0 || p5==OPFLAG_BYTELENARG)) + ) + ) + || sqlite3VdbeSerialTypeLen(t)==0 ){ /* Content is irrelevant for ** 1. the typeof() function, @@ -93819,11 +95311,13 @@ case OP_Column: { /* ncycle */ */ sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest); }else{ - if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big; - rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); - pDest->flags &= ~MEM_Ephem; + rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2], + p->cacheCtr, colCacheCtr, pDest); + if( rc ){ + if( rc==SQLITE_NOMEM ) goto no_mem; + if( rc==SQLITE_TOOBIG ) goto too_big; + goto abort_due_to_error; + } } } @@ -94285,7 +95779,6 @@ case OP_MakeRecord: { /* NULL value. No change in zPayload */ }else{ u64 v; - u32 i; if( serial_type==7 ){ assert( sizeof(v)==sizeof(pRec->u.r) ); memcpy(&v, &pRec->u.r, sizeof(v)); @@ -94293,12 +95786,17 @@ case OP_MakeRecord: { }else{ v = pRec->u.i; } - len = i = sqlite3SmallTypeSizes[serial_type]; - assert( i>0 ); - while( 1 /*exit-by-break*/ ){ - zPayload[--i] = (u8)(v&0xFF); - if( i==0 ) break; - v >>= 8; + len = sqlite3SmallTypeSizes[serial_type]; + assert( len>=1 && len<=8 && len!=5 && len!=7 ); + switch( len ){ + default: zPayload[7] = (u8)(v&0xff); v >>= 8; + zPayload[6] = (u8)(v&0xff); v >>= 8; + case 6: zPayload[5] = (u8)(v&0xff); v >>= 8; + zPayload[4] = (u8)(v&0xff); v >>= 8; + case 4: zPayload[3] = (u8)(v&0xff); v >>= 8; + case 3: zPayload[2] = (u8)(v&0xff); v >>= 8; + case 2: zPayload[1] = (u8)(v&0xff); v >>= 8; + case 1: zPayload[0] = (u8)(v&0xff); } zPayload += len; } @@ -95107,7 +96605,7 @@ case OP_OpenEphemeral: { /* ncycle */ } pCx = p->apCsr[pOp->p1]; if( pCx && !pCx->noReuse && ALWAYS(pOp->p2<=pCx->nField) ){ - /* If the ephermeral table is already open and has no duplicates from + /* If the ephemeral table is already open and has no duplicates from ** OP_OpenDup, then erase all existing content so that the table is ** empty again, rather than creating a new table. */ assert( pCx->isEphemeral ); @@ -95598,7 +97096,7 @@ case OP_SeekGT: { /* jump, in3, group, ncycle */ ** row. If This.P5 is false (0) then a jump is made to SeekGE.P2. If ** This.P5 is true (non-zero) then a jump is made to This.P2. The P5==0 ** case occurs when there are no inequality constraints to the right of -** the IN constraing. The jump to SeekGE.P2 ends the loop. The P5!=0 case +** the IN constraint. The jump to SeekGE.P2 ends the loop. The P5!=0 case ** occurs when there are inequality constraints to the right of the IN ** operator. In that case, the This.P2 will point either directly to or ** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for @@ -95606,7 +97104,7 @@ case OP_SeekGT: { /* jump, in3, group, ncycle */ ** ** Possible outcomes from this opcode:
        ** -**
      1. If the cursor is initally not pointed to any valid row, then +**
      2. If the cursor is initially not pointed to any valid row, then ** fall through into the subsequent OP_SeekGE opcode. ** **
      3. If the cursor is left pointing to a row that is before the target @@ -95838,13 +97336,13 @@ case OP_IfNotOpen: { /* jump */ ** operands to OP_NotFound and OP_IdxGT. ** ** This opcode is an optimization attempt only. If this opcode always -** falls through, the correct answer is still obtained, but extra works +** falls through, the correct answer is still obtained, but extra work ** is performed. ** ** A value of N in the seekHit flag of cursor P1 means that there exists ** a key P3:N that will match some record in the index. We want to know ** if it is possible for a record P3:P4 to match some record in the -** index. If it is not possible, we can skips some work. So if seekHit +** index. If it is not possible, we can skip some work. So if seekHit ** is less than P4, attempt to find out if a match is possible by running ** OP_NotFound. ** @@ -96356,6 +97854,7 @@ case OP_Insert: { ); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; + colCacheCtr++; /* Invoke the update-hook if required. */ if( rc ) goto abort_due_to_error; @@ -96409,13 +97908,18 @@ case OP_RowCell: { ** left in an undefined state. ** ** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this -** delete one of several associated with deleting a table row and all its -** associated index entries. Exactly one of those deletes is the "primary" -** delete. The others are all on OPFLAG_FORDELETE cursors or else are -** marked with the AUXDELETE flag. +** delete is one of several associated with deleting a table row and +** all its associated index entries. Exactly one of those deletes is +** the "primary" delete. The others are all on OPFLAG_FORDELETE +** cursors or else are marked with the AUXDELETE flag. +** +** If the OPFLAG_NCHANGE (0x01) flag of P2 (NB: P2 not P5) is set, then +** the row change count is incremented (otherwise not). ** -** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row -** change count is incremented (otherwise not). +** If the OPFLAG_ISNOOP (0x40) flag of P2 (not P5!) is set, then the +** pre-update-hook for deletes is run, but the btree is otherwise unchanged. +** This happens when the OP_Delete is to be shortly followed by an OP_Insert +** with the same key, causing the btree entry to be overwritten. ** ** P1 must not be pseudo-table. It has to be a real table with ** multiple rows. @@ -96516,6 +98020,7 @@ case OP_Delete: { rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; + colCacheCtr++; pC->seekResult = 0; if( rc ) goto abort_due_to_error; @@ -96583,13 +98088,13 @@ case OP_SorterCompare: { ** Write into register P2 the current sorter data for sorter cursor P1. ** Then clear the column header cache on cursor P3. ** -** This opcode is normally use to move a record out of the sorter and into +** This opcode is normally used to move a record out of the sorter and into ** a register that is the source for a pseudo-table cursor created using ** OpenPseudo. That pseudo-table cursor is the one that is identified by ** parameter P3. Clearing the P3 column cache as part of this opcode saves ** us from having to issue a separate NullRow instruction to clear that cache. */ -case OP_SorterData: { +case OP_SorterData: { /* ncycle */ VdbeCursor *pC; pOut = &aMem[pOp->p2]; @@ -96864,8 +98369,8 @@ case OP_IfSmaller: { /* jump */ ** regression tests can determine whether or not the optimizer is ** correctly optimizing out sorts. */ -case OP_SorterSort: /* jump */ -case OP_Sort: { /* jump */ +case OP_SorterSort: /* jump ncycle */ +case OP_Sort: { /* jump ncycle */ #ifdef SQLITE_TEST sqlite3_sort_count++; sqlite3_search_count--; @@ -97392,7 +98897,7 @@ case OP_IdxGE: { /* jump, ncycle */ ** file is given by P1. ** ** The table being destroyed is in the main database file if P3==0. If -** P3==1 then the table to be clear is in the auxiliary database file +** P3==1 then the table to be destroyed is in the auxiliary database file ** that is used to store tables create using CREATE TEMPORARY TABLE. ** ** If AUTOVACUUM is enabled then it is possible that another root page @@ -97452,8 +98957,8 @@ case OP_Destroy: { /* out2 */ ** in the database file is given by P1. But, unlike Destroy, do not ** remove the table or index from the database file. ** -** The table being clear is in the main database file if P2==0. If -** P2==1 then the table to be clear is in the auxiliary database file +** The table being cleared is in the main database file if P2==0. If +** P2==1 then the table to be cleared is in the auxiliary database file ** that is used to store tables create using CREATE TEMPORARY TABLE. ** ** If the P3 value is non-zero, then the row change count is incremented @@ -97539,13 +99044,41 @@ case OP_CreateBtree: { /* out2 */ /* Opcode: SqlExec * * * P4 * ** ** Run the SQL statement or statements specified in the P4 string. +** Disable Auth and Trace callbacks while those statements are running if +** P1 is true. */ case OP_SqlExec: { + char *zErr; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth; +#endif + u8 mTrace; + sqlite3VdbeIncrWriteCounter(p, 0); db->nSqlExec++; - rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + zErr = 0; +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; +#endif + mTrace = db->mTrace; + if( pOp->p1 ){ +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + db->mTrace = 0; + } + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); db->nSqlExec--; - if( rc ) goto abort_due_to_error; +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + db->mTrace = mTrace; + if( zErr || rc ){ + sqlite3VdbeError(p, "%s", zErr); + sqlite3_free(zErr); + if( rc==SQLITE_NOMEM ) goto no_mem; + goto abort_due_to_error; + } break; } @@ -98279,7 +99812,7 @@ case OP_AggStep1: { /* If this function is inside of a trigger, the register array in aMem[] ** might change from one evaluation to the next. The next block of code ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ + ** reinitializes the relevant parts of the sqlite3_context object */ if( pCtx->pMem != pMem ){ pCtx->pMem = pMem; for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; @@ -98766,6 +100299,53 @@ case OP_VOpen: { /* ncycle */ } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VCheck P1 P2 P3 P4 * +** +** P4 is a pointer to a Table object that is a virtual table in schema P1 +** that supports the xIntegrity() method. This opcode runs the xIntegrity() +** method for that virtual table, using P3 as the integer argument. If +** an error is reported back, the table name is prepended to the error +** message and that message is stored in P2. If no errors are seen, +** register P2 is set to NULL. +*/ +case OP_VCheck: { /* out2 */ + Table *pTab; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + char *zErr = 0; + + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetNull(pOut); /* Innocent until proven guilty */ + assert( pOp->p4type==P4_TABLE ); + pTab = pOp->p4.pTab; + assert( pTab!=0 ); + assert( IsVirtual(pTab) ); + assert( pTab->u.vtab.p!=0 ); + pVtab = pTab->u.vtab.p->pVtab; + assert( pVtab!=0 ); + pModule = pVtab->pModule; + assert( pModule!=0 ); + assert( pModule->iVersion>=4 ); + assert( pModule->xIntegrity!=0 ); + pTab->nTabRef++; + sqlite3VtabLock(pTab->u.vtab.p); + assert( pOp->p1>=0 && pOp->p1nDb ); + rc = pModule->xIntegrity(pVtab, db->aDb[pOp->p1].zDbSName, pTab->zName, + pOp->p3, &zErr); + sqlite3VtabUnlock(pTab->u.vtab.p); + sqlite3DeleteTable(db, pTab); + if( rc ){ + sqlite3_free(zErr); + goto abort_due_to_error; + } + if( zErr ){ + sqlite3VdbeMemSetStr(pOut, zErr, -1, SQLITE_UTF8, sqlite3_free); + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VInitIn P1 P2 P3 * * ** Synopsis: r[P2]=ValueList(P1,P3) @@ -99157,7 +100737,7 @@ case OP_MaxPgcnt: { /* out2 */ ** This opcode works exactly like OP_Function. The only difference is in ** its name. This opcode is used in places where the function must be ** purely non-deterministic. Some built-in date/time functions can be -** either determinitic of non-deterministic, depending on their arguments. +** either deterministic of non-deterministic, depending on their arguments. ** When those function are used in a non-deterministic way, they will check ** to see if they were called using OP_PureFunc instead of OP_Function, and ** if they were, they throw an error. @@ -99175,7 +100755,7 @@ case OP_Function: { /* group */ /* If this function is inside of a trigger, the register array in aMem[] ** might change from one evaluation to the next. The next block of code ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ + ** reinitializes the relevant parts of the sqlite3_context object */ pOut = &aMem[pOp->p3]; if( pCtx->pOut != pOut ){ pCtx->pVdbe = p; @@ -99251,7 +100831,7 @@ case OP_FilterAdd: { printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n)); } #endif - h %= pIn1->n; + h %= (pIn1->n*8); pIn1->z[h/8] |= 1<<(h&7); break; } @@ -99287,7 +100867,7 @@ case OP_Filter: { /* jump */ printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n)); } #endif - h %= pIn1->n; + h %= (pIn1->n*8); if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){ VdbeBranchTaken(1, 2); p->aCounter[SQLITE_STMTSTATUS_FILTER_HIT]++; @@ -99539,7 +101119,7 @@ default: { /* This is really OP_Noop, OP_Explain */ } if( opProperty==0xff ){ /* Never happens. This code exists to avoid a harmless linkage - ** warning aboud sqlite3VdbeRegisterDump() being defined but not + ** warning about sqlite3VdbeRegisterDump() being defined but not ** used. */ sqlite3VdbeRegisterDump(p); } @@ -99712,8 +101292,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ /* Set the value of register r[1] in the SQL statement to integer iRow. ** This is done directly as a performance optimization */ - v->aMem[1].flags = MEM_Int; - v->aMem[1].u.i = iRow; + sqlite3VdbeMemSetInt64(&v->aMem[1], iRow); /* If the statement has been run before (and is paused at the OP_ResultRow) ** then back it up to the point where it does the OP_NotExists. This could @@ -99796,7 +101375,7 @@ SQLITE_API int sqlite3_blob_open( #endif *ppBlob = 0; #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zTable==0 ){ + if( !sqlite3SafetyCheckOk(db) || zTable==0 || zColumn==0 ){ return SQLITE_MISUSE_BKPT; } #endif @@ -100257,7 +101836,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** The threshold for the amount of main memory to use before flushing ** records to a PMA is roughly the same as the limit configured for the ** page-cache of the main database. Specifically, the threshold is set to -** the value returned by "PRAGMA main.page_size" multipled by +** the value returned by "PRAGMA main.page_size" multiplied by ** that returned by "PRAGMA main.cache_size", in bytes. ** ** If the sorter is running in single-threaded mode, then all PMAs generated @@ -100280,7 +101859,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ** ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the ** sorter is running in single-threaded mode, then these PMAs are merged -** incrementally as keys are retreived from the sorter by the VDBE. The +** incrementally as keys are retrieved from the sorter by the VDBE. The ** MergeEngine object, described in further detail below, performs this ** merge. ** @@ -100358,7 +101937,7 @@ struct SorterFile { struct SorterList { SorterRecord *pList; /* Linked list of records */ u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ - int szPMA; /* Size of pList as PMA in bytes */ + i64 szPMA; /* Size of pList as PMA in bytes */ }; /* @@ -100443,7 +102022,7 @@ struct MergeEngine { ** ** Essentially, this structure contains all those fields of the VdbeSorter ** structure for which each thread requires a separate instance. For example, -** each thread requries its own UnpackedRecord object to unpack records in +** each thread requeries its own UnpackedRecord object to unpack records in ** as part of comparison operations. ** ** Before a background thread is launched, variable bDone is set to 0. Then, @@ -100467,10 +102046,10 @@ typedef int (*SorterCompare)(SortSubtask*,int*,const void*,int,const void*,int); struct SortSubtask { SQLiteThread *pThread; /* Background thread, if any */ int bDone; /* Set if thread is finished but not joined */ + int nPMA; /* Number of PMAs currently in file */ VdbeSorter *pSorter; /* Sorter that owns this sub-task */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ SorterList list; /* List for thread to write to a PMA */ - int nPMA; /* Number of PMAs currently in file */ SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ @@ -100515,7 +102094,7 @@ struct VdbeSorter { ** PMA, in sorted order. The next key to be read is cached in nKey/aKey. ** aKey might point into aMap or into aBuffer. If neither of those locations ** contain a contiguous representation of the key, then aAlloc is allocated -** and the key is copied into aAlloc and aKey is made to poitn to aAlloc. +** and the key is copied into aAlloc and aKey is made to point to aAlloc. ** ** pFd==0 at EOF. */ @@ -101886,7 +103465,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ ** the background thread from a sub-tasks previous turn is still running, ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, ** fall back to using the final sub-task. The first (pSorter->nTask-1) - ** sub-tasks are prefered as they use background threads - the final + ** sub-tasks are preferred as they use background threads - the final ** sub-task uses the main thread. */ for(i=0; iiPrev + i + 1) % nWorker; @@ -101944,8 +103523,8 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ int bFlush; /* True to flush contents of memory to PMA */ - int nReq; /* Bytes of memory required */ - int nPMA; /* Bytes of PMA space required */ + i64 nReq; /* Bytes of memory required */ + i64 nPMA; /* Bytes of PMA space required */ int t; /* serial type of first record field */ assert( pCsr->eCurType==CURTYPE_SORTER ); @@ -102370,7 +103949,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); - /* Set up the required files for pIncr. A multi-theaded IncrMerge object + /* Set up the required files for pIncr. A multi-threaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ @@ -103010,6 +104589,8 @@ static int bytecodevtabConnect( "p5 INT," "comment TEXT," "subprog TEXT," + "nexec INT," + "ncycle INT," "stmt HIDDEN" ");", @@ -103172,7 +104753,7 @@ static int bytecodevtabColumn( } } } - i += 10; + i += 20; } } switch( i ){ @@ -103222,16 +104803,31 @@ static int bytecodevtabColumn( } break; } - case 10: /* tables_used.type */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + case 9: /* nexec */ + sqlite3_result_int(ctx, pOp->nExec); + break; + case 10: /* ncycle */ + sqlite3_result_int(ctx, pOp->nCycle); + break; +#else + case 9: /* nexec */ + case 10: /* ncycle */ + sqlite3_result_int(ctx, 0); + break; +#endif + + case 20: /* tables_used.type */ sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); break; - case 11: /* tables_used.schema */ + case 21: /* tables_used.schema */ sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); break; - case 12: /* tables_used.name */ + case 22: /* tables_used.name */ sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); break; - case 13: /* tables_used.wr */ + case 23: /* tables_used.wr */ sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); break; } @@ -103305,7 +104901,7 @@ static int bytecodevtabBestIndex( int rc = SQLITE_CONSTRAINT; struct sqlite3_index_constraint *p; bytecodevtab *pVTab = (bytecodevtab*)tab; - int iBaseCol = pVTab->bTablesUsed ? 4 : 8; + int iBaseCol = pVTab->bTablesUsed ? 4 : 10; pIdxInfo->estimatedCost = (double)100; pIdxInfo->estimatedRows = 100; pIdxInfo->idxNum = 0; @@ -103352,7 +104948,8 @@ static sqlite3_module bytecodevtabModule = { /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, - /* xShadowName */ 0 + /* xShadowName */ 0, + /* xIntegrity */ 0 }; @@ -103876,7 +105473,7 @@ static int walkWindowList(Walker *pWalker, Window *pList, int bOneOnly){ ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3WalkExprNN(Walker *pWalker, Expr *pExpr){ int rc; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); @@ -103885,7 +105482,9 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ if( rc ) return rc & WRC_Abort; if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ assert( pExpr->x.pList==0 || pExpr->pRight==0 ); - if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + if( pExpr->pLeft && sqlite3WalkExprNN(pWalker, pExpr->pLeft) ){ + return WRC_Abort; + } if( pExpr->pRight ){ assert( !ExprHasProperty(pExpr, EP_WinFunc) ); pExpr = pExpr->pRight; @@ -103909,7 +105508,7 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ - return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; + return pExpr ? sqlite3WalkExprNN(pWalker,pExpr) : WRC_Continue; } /* @@ -104035,7 +105634,7 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ } /* Increase the walkerDepth when entering a subquery, and -** descrease when leaving the subquery. +** decrease when leaving the subquery. */ SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker *pWalker, Select *pSelect){ UNUSED_PARAMETER(pSelect); @@ -104179,21 +105778,36 @@ static void resolveAlias( } /* -** Subqueries stores the original database, table and column names for their -** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". -** Check to see if the zSpan given to this routine matches the zDb, zTab, -** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will -** match anything. +** Subqueries store the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN", +** and mark the expression-list item by setting ExprList.a[].fg.eEName +** to ENAME_TAB. +** +** Check to see if the zSpan/eEName of the expression-list item passed to this +** routine matches the zDb, zTab, and zCol. If any of zDb, zTab, and zCol are +** NULL then those fields will match anything. Return true if there is a match, +** or false otherwise. +** +** SF_NestedFrom subqueries also store an entry for the implicit rowid (or +** _rowid_, or oid) column by setting ExprList.a[].fg.eEName to ENAME_ROWID, +** and setting zSpan to "DATABASE.TABLE.". This type of pItem +** argument matches if zCol is a rowid alias. If it is not NULL, (*pbRowid) +** is set to 1 if there is this kind of match. */ SQLITE_PRIVATE int sqlite3MatchEName( const struct ExprList_item *pItem, const char *zCol, const char *zTab, - const char *zDb + const char *zDb, + int *pbRowid ){ int n; const char *zSpan; - if( pItem->fg.eEName!=ENAME_TAB ) return 0; + int eEName = pItem->fg.eEName; + if( eEName!=ENAME_TAB && (eEName!=ENAME_ROWID || NEVER(pbRowid==0)) ){ + return 0; + } + assert( pbRowid==0 || *pbRowid==0 ); zSpan = pItem->zEName; for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ @@ -104205,9 +105819,11 @@ SQLITE_PRIVATE int sqlite3MatchEName( return 0; } zSpan += n+1; - if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ - return 0; + if( zCol ){ + if( eEName==ENAME_TAB && sqlite3StrICmp(zSpan, zCol)!=0 ) return 0; + if( eEName==ENAME_ROWID && sqlite3IsRowid(zCol)==0 ) return 0; } + if( eEName==ENAME_ROWID ) *pbRowid = 1; return 1; } @@ -104340,7 +105956,7 @@ static int lookupName( ){ int i, j; /* Loop counters */ int cnt = 0; /* Number of matching column names */ - int cntTab = 0; /* Number of matching table names */ + int cntTab = 0; /* Number of potential "rowid" matches */ int nSubquery = 0; /* How many levels of subquery */ sqlite3 *db = pParse->db; /* The database connection */ SrcItem *pItem; /* Use for looping over pSrcList items */ @@ -104417,39 +106033,49 @@ static int lookupName( assert( pEList!=0 ); assert( pEList->nExpr==pTab->nCol ); for(j=0; jnExpr; j++){ - if( !sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb) ){ + int bRowid = 0; /* True if possible rowid match */ + if( !sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb, &bRowid) ){ continue; } - if( cnt>0 ){ - if( pItem->fg.isUsing==0 - || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 - ){ - /* Two or more tables have the same column name which is - ** not joined by USING. This is an error. Signal as much - ** by clearing pFJMatch and letting cnt go above 1. */ - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else - if( (pItem->fg.jointype & JT_RIGHT)==0 ){ - /* An INNER or LEFT JOIN. Use the left-most table */ - continue; - }else - if( (pItem->fg.jointype & JT_LEFT)==0 ){ - /* A RIGHT JOIN. Use the right-most table */ - cnt = 0; - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else{ - /* For a FULL JOIN, we must construct a coalesce() func */ - extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + if( bRowid==0 ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + } } + cnt++; + hit = 1; + }else if( cnt>0 ){ + /* This is a potential rowid match, but there has already been + ** a real match found. So this can be ignored. */ + continue; } - cnt++; - cntTab = 2; + cntTab++; pMatch = pItem; pExpr->iColumn = j; pEList->a[j].fg.bUsed = 1; - hit = 1; + + /* rowid cannot be part of a USING clause - assert() this. */ + assert( bRowid==0 || pEList->a[j].fg.bUsingTerm==0 ); if( pEList->a[j].fg.bUsingTerm ) break; } if( hit || zTab==0 ) continue; @@ -104644,10 +106270,10 @@ static int lookupName( && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) - && ALWAYS(VisibleRowid(pMatch->pTab)) + && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) ){ cnt = 1; - pExpr->iColumn = -1; + if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -105100,6 +106726,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); #endif assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); + assert( pExpr->pLeft==0 || pExpr->pLeft->op==TK_ORDER ); zId = pExpr->u.zToken; pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ @@ -105241,6 +106868,10 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pNC->nNcErr++; } #endif + else if( is_agg==0 && pExpr->pLeft ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + pNC->nNcErr++; + } if( is_agg ){ /* Window functions may not be arguments of aggregate functions. ** Or arguments of other window functions. But aggregate functions @@ -105259,6 +106890,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pWin ){ Select *pSel = pNC->pWinSelect; @@ -105769,7 +107405,7 @@ static int resolveOrderGroupBy( } for(j=0; jpEList->nExpr; j++){ if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ - /* Since this expresion is being changed into a reference + /* Since this expression is being changed into a reference ** to an identical expression in the result set, remove all Window ** objects belonging to the expression from the Select.pWin list. */ windowRemoveExprFromSelect(pSelect, pE); @@ -105822,10 +107458,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ while( p ){ assert( (p->selFlags & SF_Expanded)!=0 ); assert( (p->selFlags & SF_Resolved)==0 ); - assert( db->suppressErr==0 ); /* SF_Resolved not set if errors suppressed */ p->selFlags |= SF_Resolved; - /* Resolve the expressions in the LIMIT and OFFSET clauses. These ** are not allowed to refer to any names, so pass an empty NameContext. */ @@ -106092,7 +107726,8 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( return SQLITE_ERROR; } #endif - sqlite3WalkExpr(&w, pExpr); + assert( pExpr!=0 ); + sqlite3WalkExprNN(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif @@ -106134,7 +107769,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( return WRC_Abort; } #endif - sqlite3WalkExpr(&w, pExpr); + sqlite3WalkExprNN(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif @@ -106156,7 +107791,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( /* ** Resolve all names in all expressions of a SELECT and in all -** decendents of the SELECT, including compounds off of p->pPrior, +** descendants of the SELECT, including compounds off of p->pPrior, ** subqueries in expressions, and subqueries used as FROM clause ** terms. ** @@ -106306,6 +107941,7 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ if( op==TK_SELECT_COLUMN ){ assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); assert( pExpr->iColumn < pExpr->iTable ); + assert( pExpr->iColumn >= 0 ); assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); return sqlite3ExprAffinity( pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr @@ -106542,7 +108178,7 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ /* ** Return the collation sequence for the expression pExpr. If ** there is no defined collating sequence, return a pointer to the -** defautl collation sequence. +** default collation sequence. ** ** See also: sqlite3ExprCollSeq() ** @@ -106672,7 +108308,7 @@ SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq( return pColl; } -/* Expresssion p is a comparison operator. Return a collation sequence +/* Expression p is a comparison operator. Return a collation sequence ** appropriate for the comparison operator. ** ** This is normally just a wrapper around sqlite3BinaryCompareCollSeq(). @@ -106829,6 +108465,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprForVectorField( */ pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0); if( pRet ){ + ExprSetProperty(pRet, EP_FullSize); pRet->iTable = nField; pRet->iColumn = iField; pRet->pLeft = pVector; @@ -107128,6 +108765,15 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ #define exprSetHeight(y) #endif /* SQLITE_MAX_EXPR_DEPTH>0 */ +/* +** Set the error offset for an Expr node, if possible. +*/ +SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ + if( pExpr==0 ) return; + if( NEVER(ExprUseWJoin(pExpr)) ) return; + pExpr->w.iOfst = iOfst; +} + /* ** This routine is the core allocator for Expr nodes. ** @@ -107410,6 +109056,69 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction( return pNew; } +/* +** Report an error when attempting to use an ORDER BY clause within +** the arguments of a non-aggregate function. +*/ +SQLITE_PRIVATE void sqlite3ExprOrderByAggregateError(Parse *pParse, Expr *p){ + sqlite3ErrorMsg(pParse, + "ORDER BY may not be used with non-aggregate %#T()", p + ); +} + +/* +** Attach an ORDER BY clause to a function call. +** +** functionname( arguments ORDER BY sortlist ) +** \_____________________/ \______/ +** pExpr pOrderBy +** +** The ORDER BY clause is inserted into a new Expr node of type TK_ORDER +** and added to the Expr.pLeft field of the parent TK_FUNCTION node. +*/ +SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The function call to which ORDER BY is to be added */ + ExprList *pOrderBy /* The ORDER BY clause to add */ +){ + Expr *pOB; + sqlite3 *db = pParse->db; + if( NEVER(pOrderBy==0) ){ + assert( db->mallocFailed ); + return; + } + if( pExpr==0 ){ + assert( db->mallocFailed ); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + assert( pExpr->op==TK_FUNCTION ); + assert( pExpr->pLeft==0 ); + assert( ExprUseXList(pExpr) ); + if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){ + /* Ignore ORDER BY on zero-argument aggregates */ + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprListDelete, + pOrderBy); + return; + } + if( IsWindowFunc(pExpr) ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + + pOB = sqlite3ExprAlloc(db, TK_ORDER, 0, 0); + if( pOB==0 ){ + sqlite3ExprListDelete(db, pOrderBy); + return; + } + pOB->x.pList = pOrderBy; + assert( ExprUseXList(pOB) ); + pExpr->pLeft = pOB; + ExprSetProperty(pOB, EP_FullSize); +} + /* ** Check to see if a function is usable according to current access ** rules: @@ -107588,7 +109297,7 @@ SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ /* ** Arrange to cause pExpr to be deleted when the pParse is deleted. ** This is similar to sqlite3ExprDelete() except that the delete is -** deferred untilthe pParse is deleted. +** deferred until the pParse is deleted. ** ** The pExpr might be deleted immediately on an OOM error. ** @@ -107663,11 +109372,7 @@ static int dupedExprStructSize(const Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags || p->op==TK_SELECT_COLUMN -#ifndef SQLITE_OMIT_WINDOWFUNC - || ExprHasProperty(p, EP_WinFunc) -#endif - ){ + if( 0==flags || ExprHasProperty(p, EP_FullSize) ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -107698,56 +109403,93 @@ static int dupedExprNodeSize(const Expr *p, int flags){ /* ** Return the number of bytes required to create a duplicate of the -** expression passed as the first argument. The second argument is a -** mask containing EXPRDUP_XXX flags. +** expression passed as the first argument. ** ** The value returned includes space to create a copy of the Expr struct ** itself and the buffer referred to by Expr.u.zToken, if any. ** -** If the EXPRDUP_REDUCE flag is set, then the return value includes -** space to duplicate all Expr nodes in the tree formed by Expr.pLeft -** and Expr.pRight variables (but not for any structures pointed to or -** descended from the Expr.x.pList or Expr.x.pSelect variables). +** The return value includes space to duplicate all Expr nodes in the +** tree formed by Expr.pLeft and Expr.pRight, but not any other +** substructure such as Expr.x.pList, Expr.x.pSelect, and Expr.y.pWin. */ -static int dupedExprSize(const Expr *p, int flags){ - int nByte = 0; - if( p ){ - nByte = dupedExprNodeSize(p, flags); - if( flags&EXPRDUP_REDUCE ){ - nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); - } - } +static int dupedExprSize(const Expr *p){ + int nByte; + assert( p!=0 ); + nByte = dupedExprNodeSize(p, EXPRDUP_REDUCE); + if( p->pLeft ) nByte += dupedExprSize(p->pLeft); + if( p->pRight ) nByte += dupedExprSize(p->pRight); + assert( nByte==ROUND8(nByte) ); return nByte; } /* -** This function is similar to sqlite3ExprDup(), except that if pzBuffer -** is not NULL then *pzBuffer is assumed to point to a buffer large enough -** to store the copy of expression p, the copies of p->u.zToken -** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte past the -** portion of the buffer copied into by this function. +** An EdupBuf is a memory allocation used to stored multiple Expr objects +** together with their Expr.zToken content. This is used to help implement +** compression while doing sqlite3ExprDup(). The top-level Expr does the +** allocation for itself and many of its decendents, then passes an instance +** of the structure down into exprDup() so that they decendents can have +** access to that memory. */ -static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ +typedef struct EdupBuf EdupBuf; +struct EdupBuf { + u8 *zAlloc; /* Memory space available for storage */ +#ifdef SQLITE_DEBUG + u8 *zEnd; /* First byte past the end of memory */ +#endif +}; + +/* +** This function is similar to sqlite3ExprDup(), except that if pEdupBuf +** is not NULL then it points to memory that can be used to store a copy +** of the input Expr p together with its p->u.zToken (if any). pEdupBuf +** is updated with the new buffer tail prior to returning. +*/ +static Expr *exprDup( + sqlite3 *db, /* Database connection (for memory allocation) */ + const Expr *p, /* Expr tree to be duplicated */ + int dupFlags, /* EXPRDUP_REDUCE for compression. 0 if not */ + EdupBuf *pEdupBuf /* Preallocated storage space, or NULL */ +){ Expr *pNew; /* Value to return */ - u8 *zAlloc; /* Memory space from which to build Expr object */ + EdupBuf sEdupBuf; /* Memory space from which to build Expr object */ u32 staticFlag; /* EP_Static if space not obtained from malloc */ + int nToken = -1; /* Space needed for p->u.zToken. -1 means unknown */ assert( db!=0 ); assert( p ); assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); - assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pEdupBuf==0 || dupFlags==EXPRDUP_REDUCE ); /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; + if( pEdupBuf ){ + sEdupBuf.zAlloc = pEdupBuf->zAlloc; +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = pEdupBuf->zEnd; +#endif staticFlag = EP_Static; - assert( zAlloc!=0 ); + assert( sEdupBuf.zAlloc!=0 ); + assert( dupFlags==EXPRDUP_REDUCE ); }else{ - zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + int nAlloc; + if( dupFlags ){ + nAlloc = dupedExprSize(p); + }else if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30NN(p->u.zToken)+1; + nAlloc = ROUND8(EXPR_FULLSIZE + nToken); + }else{ + nToken = 0; + nAlloc = ROUND8(EXPR_FULLSIZE); + } + assert( nAlloc==ROUND8(nAlloc) ); + sEdupBuf.zAlloc = sqlite3DbMallocRawNN(db, nAlloc); +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = sEdupBuf.zAlloc ? sEdupBuf.zAlloc+nAlloc : 0; +#endif + staticFlag = 0; } - pNew = (Expr *)zAlloc; + pNew = (Expr *)sEdupBuf.zAlloc; + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); if( pNew ){ /* Set nNewSize to the size allocated for the structure pointed to @@ -107756,22 +109498,27 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ ** by the copy of the p->u.zToken string (if any). */ const unsigned nStructSize = dupedExprStructSize(p, dupFlags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; - }else{ - nToken = 0; + int nNewSize = nStructSize & 0xfff; + if( nToken<0 ){ + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } } if( dupFlags ){ + assert( (int)(sEdupBuf.zEnd - sEdupBuf.zAlloc) >= nNewSize+nToken ); assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); + memcpy(sEdupBuf.zAlloc, p, nNewSize); }else{ u32 nSize = (u32)exprStructSize(p); - memcpy(zAlloc, p, nSize); + assert( (int)(sEdupBuf.zEnd - sEdupBuf.zAlloc) >= + (int)EXPR_FULLSIZE+nToken ); + memcpy(sEdupBuf.zAlloc, p, nSize); if( nSizeu.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + assert( nToken>=0 ); + if( nToken>0 ){ + char *zToken = pNew->u.zToken = (char*)&sEdupBuf.zAlloc[nNewSize]; memcpy(zToken, p->u.zToken, nToken); + nNewSize += nToken; } + sEdupBuf.zAlloc += ROUND8(nNewSize); + + if( ((p->flags|pNew->flags)&(EP_TokenOnly|EP_Leaf))==0 ){ - if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ if( ExprUseXSelect(p) ){ pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, + p->op!=TK_ORDER ? dupFlags : 0); } - } - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly|EP_WinFunc) ){ - zAlloc += dupedExprNodeSize(p, dupFlags); - if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ - pNew->pLeft = p->pLeft ? - exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; - pNew->pRight = p->pRight ? - exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; - } #ifndef SQLITE_OMIT_WINDOWFUNC if( ExprHasProperty(p, EP_WinFunc) ){ pNew->y.pWin = sqlite3WindowDup(db, pNew, p->y.pWin); assert( ExprHasProperty(pNew, EP_WinFunc) ); } #endif /* SQLITE_OMIT_WINDOWFUNC */ - if( pzBuffer ){ - *pzBuffer = zAlloc; - } - }else{ - if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ - if( pNew->op==TK_SELECT_COLUMN ){ + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( dupFlags ){ + if( p->op==TK_SELECT_COLUMN ){ pNew->pLeft = p->pLeft; - assert( p->pRight==0 || p->pRight==p->pLeft - || ExprHasProperty(p->pLeft, EP_Subquery) ); + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); + }else{ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &sEdupBuf) : 0; + } + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &sEdupBuf) : 0; + }else{ + if( p->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); }else{ pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); } @@ -107829,6 +109582,8 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ } } } + if( pEdupBuf ) memcpy(pEdupBuf, &sEdupBuf, sizeof(sEdupBuf)); + assert( sEdupBuf.zAlloc <= sEdupBuf.zEnd ); return pNew; } @@ -108093,11 +109848,7 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *p, int flags) ** initially NULL, then create a new expression list. ** ** The pList argument must be either NULL or a pointer to an ExprList -** obtained from a prior call to sqlite3ExprListAppend(). This routine -** may not be used with an ExprList obtained from sqlite3ExprListDup(). -** Reason: This routine assumes that the number of slots in pList->a[] -** is a power of two. That is true for sqlite3ExprListAppend() returns -** but is not necessarily true from the return value of sqlite3ExprListDup(). +** obtained from a prior call to sqlite3ExprListAppend(). ** ** If a memory allocation error occurs, the entire list is freed and ** NULL is returned. If non-NULL is returned, then it is guaranteed @@ -108430,7 +110181,7 @@ SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ ** and 0 if it is FALSE. */ SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr *pExpr){ - pExpr = sqlite3ExprSkipCollate((Expr*)pExpr); + pExpr = sqlite3ExprSkipCollateAndLikely((Expr*)pExpr); assert( pExpr->op==TK_TRUEFALSE ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 @@ -108923,6 +110674,27 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ return 0; } +/* +** Return a pointer to a buffer containing a usable rowid alias for table +** pTab. An alias is usable if there is not an explicit user-defined column +** of the same name. +*/ +SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab){ + const char *azOpt[] = {"_ROWID_", "ROWID", "OID"}; + int ii; + assert( VisibleRowid(pTab) ); + for(ii=0; iinCol; iCol++){ + if( sqlite3_stricmp(azOpt[ii], pTab->aCol[iCol].zCnName)==0 ) break; + } + if( iCol==pTab->nCol ){ + return azOpt[ii]; + } + } + return 0; +} + /* ** pX is the RHS of an IN operator. If pX is a SELECT statement ** that can be simplified to a direct table access, then return @@ -109023,7 +110795,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. +** populated ephemeral table. ** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be ** implemented as a sequence of comparisons. ** @@ -109036,7 +110808,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** an ephemeral table might need to be generated from the RHS and then ** pX->iTable made to point to the ephemeral table instead of an ** existing table. In this case, the creation and initialization of the -** ephmeral table might be put inside of a subroutine, the EP_Subrtn flag +** ephemeral table might be put inside of a subroutine, the EP_Subrtn flag ** will be set on pX and the pX->y.sub fields will be set to show where ** the subroutine is coded. ** @@ -109048,12 +110820,12 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table will be created unless the selected columns are guaranteed +** An ephemeral table will be created unless the selected columns are guaranteed ** to be unique - either because it is an INTEGER PRIMARY KEY or due to ** a UNIQUE constraint or index. ** ** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used -** for fast set membership tests) then an epheremal table must +** for fast set membership tests) then an ephemeral table must ** be used unless is a single INTEGER PRIMARY KEY column or an ** index can be found with the specified as its left-most. ** @@ -109386,7 +111158,7 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ ** x IN (SELECT a FROM b) -- IN operator with subquery on the right ** ** The pExpr parameter is the IN operator. The cursor number for the -** constructed ephermeral table is returned. The first time the ephemeral +** constructed ephemeral table is returned. The first time the ephemeral ** table is computed, the cursor number is also stored in pExpr->iTable, ** however the cursor number returned might not be the same, as it might ** have been duplicated using OP_OpenDup. @@ -110201,10 +111973,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( u8 p5 /* P5 value for OP_Column + FLAGS */ ){ assert( pParse->pVdbe!=0 ); + assert( (p5 & (OPFLAG_NOCHNG|OPFLAG_TYPEOFARG|OPFLAG_LENGTHARG))==p5 ); + assert( IsVirtual(pTab) || (p5 & OPFLAG_NOCHNG)==0 ); sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Column ) pOp->p5 = p5; + if( pOp->opcode==OP_VColumn ) pOp->p5 = (p5 & OPFLAG_NOCHNG); } return iReg; } @@ -110233,7 +112008,7 @@ static void exprToRegister(Expr *pExpr, int iReg){ /* ** Evaluate an expression (either a vector or a scalar expression) and store -** the result in continguous temporary registers. Return the index of +** the result in contiguous temporary registers. Return the index of ** the first register used to store the result. ** ** If the returned result register is a temporary scalar, then also write @@ -110273,7 +112048,7 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ */ static void setDoNotMergeFlagOnCopy(Vdbe *v){ if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){ - sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ + sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergeable */ } } @@ -110363,13 +112138,13 @@ static int exprCodeInlineFunction( } case INLINEFUNC_implies_nonnull_row: { - /* REsult of sqlite3ExprImpliesNonNullRow() */ + /* Result of sqlite3ExprImpliesNonNullRow() */ Expr *pA1; assert( nFarg==2 ); pA1 = pFarg->a[1].pExpr; if( pA1->op==TK_COLUMN ){ sqlite3VdbeAddOp2(v, OP_Integer, - sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable), + sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable,1), target); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); @@ -110457,6 +112232,41 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( } +/* +** Expresion pExpr is guaranteed to be a TK_COLUMN or equivalent. This +** function checks the Parse.pIdxPartExpr list to see if this column +** can be replaced with a constant value. If so, it generates code to +** put the constant value in a register (ideally, but not necessarily, +** register iTarget) and returns the register number. +** +** Or, if the TK_COLUMN cannot be replaced by a constant, zero is +** returned. +*/ +static int exprPartidxExprLookup(Parse *pParse, Expr *pExpr, int iTarget){ + IndexedExpr *p; + for(p=pParse->pIdxPartExpr; p; p=p->pIENext){ + if( pExpr->iColumn==p->iIdxCol && pExpr->iTable==p->iDataCur ){ + Vdbe *v = pParse->pVdbe; + int addr = 0; + int ret; + + if( p->bMaybeNullRow ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNullRow, p->iIdxCur); + } + ret = sqlite3ExprCodeTarget(pParse, p->pExpr, iTarget); + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Affinity, ret, 1, 0, + (const char*)&p->aff, 1); + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP3(v, addr, ret); + } + return ret; + } + } + return 0; +} + + /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -110493,6 +112303,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; } + assert( op!=TK_ORDER ); switch( op ){ case TK_AGG_COLUMN: { AggInfo *pAggInfo = pExpr->pAggInfo; @@ -110506,7 +112317,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #ifdef SQLITE_VDBE_COVERAGE /* Verify that the OP_Null above is exercised by tests ** tag-20230325-2 */ - sqlite3VdbeAddOp2(v, OP_NotNull, target, 1); + sqlite3VdbeAddOp3(v, OP_NotNull, target, 1, 20230325); VdbeCoverageNeverTaken(v); #endif break; @@ -110545,7 +112356,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( ExprHasProperty(pExpr, EP_FixedCol) ){ /* This COLUMN expression is really a constant due to WHERE clause ** constraints, and that constant is coded by the pExpr->pLeft - ** expresssion. However, make sure the constant has the correct + ** expression. However, make sure the constant has the correct ** datatype by applying the Affinity of the table column to the ** constant. */ @@ -110614,6 +112425,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) iTab = pParse->iSelfTab - 1; } } + else if( pParse->pIdxPartExpr + && 0!=(r1 = exprPartidxExprLookup(pParse, pExpr, target)) + ){ + return r1; + } assert( ExprUseYTab(pExpr) ); assert( pExpr->y.pTab!=0 ); iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, @@ -110871,7 +112687,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ErrorMsg(pParse, "unknown function: %#T()", pExpr); break; } - if( pDef->funcFlags & SQLITE_FUNC_INLINE ){ + if( (pDef->funcFlags & SQLITE_FUNC_INLINE)!=0 && ALWAYS(pFarg!=0) ){ assert( (pDef->funcFlags & SQLITE_FUNC_UNSAFE)==0 ); assert( (pDef->funcFlags & SQLITE_FUNC_DIRECT)==0 ); return exprCodeInlineFunction(pParse, pFarg, @@ -110897,10 +112713,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) r1 = sqlite3GetTempRange(pParse, nFarg); } - /* For length() and typeof() functions with a column argument, + /* For length() and typeof() and octet_length() functions, ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG - ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data - ** loading. + ** or OPFLAG_TYPEOFARG or OPFLAG_BYTELENARG respectively, to avoid + ** unnecessary data loading. */ if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ u8 exprOp; @@ -110910,14 +112726,16 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); - testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); - pFarg->a[0].pExpr->op2 = - pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + assert( SQLITE_FUNC_BYTELEN==OPFLAG_BYTELENARG ); + assert( (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG)==OPFLAG_BYTELENARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_LENGTHARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_TYPEOFARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG); + pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG; } } - sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, - SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR); }else{ r1 = 0; } @@ -111272,9 +113090,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** once. If no functions are involved, then factor the code out and put it at ** the end of the prepared statement in the initialization section. ** -** If regDest>=0 then the result is always stored in that register and the +** If regDest>0 then the result is always stored in that register and the ** result is not reusable. If regDest<0 then this routine is free to -** store the value whereever it wants. The register where the expression +** store the value wherever it wants. The register where the expression ** is stored is returned. When regDest<0, two identical expressions might ** code to the same register, if they do not contain function calls and hence ** are factored out into the initialization section at the end of the @@ -111287,6 +113105,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce( ){ ExprList *p; assert( ConstFactorOk(pParse) ); + assert( regDest!=0 ); p = pParse->pConstExpr; if( regDest<0 && p ){ struct ExprList_item *pItem; @@ -112192,7 +114011,7 @@ static int exprImpliesNotNull( ** pE1: x!=123 pE2: x IS NOT NULL Result: true ** pE1: x!=?1 pE2: x IS NOT NULL Result: true ** pE1: x IS NULL pE2: x IS NOT NULL Result: false -** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** pE1: x IS ?2 pE2: x IS NOT NULL Result: false ** ** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has ** Expr.iTable<0 then assume a table number given by iTab. @@ -112229,11 +114048,29 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr( return 0; } +/* This is a helper function to impliesNotNullRow(). In this routine, +** set pWalker->eCode to one only if *both* of the input expressions +** separately have the implies-not-null-row property. +*/ +static void bothImplyNotNullRow(Walker *pWalker, Expr *pE1, Expr *pE2){ + if( pWalker->eCode==0 ){ + sqlite3WalkExpr(pWalker, pE1); + if( pWalker->eCode ){ + pWalker->eCode = 0; + sqlite3WalkExpr(pWalker, pE2); + } + } +} + /* ** This is the Expr node callback for sqlite3ExprImpliesNonNullRow(). ** If the expression node requires that the table at pWalker->iCur ** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. ** +** pWalker->mWFlags is non-zero if this inquiry is being undertaking on +** behalf of a RIGHT JOIN (or FULL JOIN). That makes a difference when +** evaluating terms in the ON clause of an inner join. +** ** This routine controls an optimization. False positives (setting ** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives ** (never setting pWalker->eCode) is a harmless missed optimization. @@ -112242,28 +114079,33 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); if( ExprHasProperty(pExpr, EP_OuterON) ) return WRC_Prune; + if( ExprHasProperty(pExpr, EP_InnerON) && pWalker->mWFlags ){ + /* If iCur is used in an inner-join ON clause to the left of a + ** RIGHT JOIN, that does *not* mean that the table must be non-null. + ** But it is difficult to check for that condition precisely. + ** To keep things simple, any use of iCur from any inner-join is + ** ignored while attempting to simplify a RIGHT JOIN. */ + return WRC_Prune; + } switch( pExpr->op ){ case TK_ISNOT: case TK_ISNULL: case TK_NOTNULL: case TK_IS: - case TK_OR: case TK_VECTOR: - case TK_CASE: - case TK_IN: case TK_FUNCTION: case TK_TRUTH: + case TK_CASE: testcase( pExpr->op==TK_ISNOT ); testcase( pExpr->op==TK_ISNULL ); testcase( pExpr->op==TK_NOTNULL ); testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_OR ); testcase( pExpr->op==TK_VECTOR ); - testcase( pExpr->op==TK_CASE ); - testcase( pExpr->op==TK_IN ); testcase( pExpr->op==TK_FUNCTION ); testcase( pExpr->op==TK_TRUTH ); + testcase( pExpr->op==TK_CASE ); return WRC_Prune; + case TK_COLUMN: if( pWalker->u.iCur==pExpr->iTable ){ pWalker->eCode = 1; @@ -112271,21 +114113,38 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ } return WRC_Prune; + case TK_OR: case TK_AND: - if( pWalker->eCode==0 ){ + /* Both sides of an AND or OR must separately imply non-null-row. + ** Consider these cases: + ** 1. NOT (x AND y) + ** 2. x OR y + ** If only one of x or y is non-null-row, then the overall expression + ** can be true if the other arm is false (case 1) or true (case 2). + */ + testcase( pExpr->op==TK_OR ); + testcase( pExpr->op==TK_AND ); + bothImplyNotNullRow(pWalker, pExpr->pLeft, pExpr->pRight); + return WRC_Prune; + + case TK_IN: + /* Beware of "x NOT IN ()" and "x NOT IN (SELECT 1 WHERE false)", + ** both of which can be true. But apart from these cases, if + ** the left-hand side of the IN is NULL then the IN itself will be + ** NULL. */ + if( ExprUseXList(pExpr) && ALWAYS(pExpr->x.pList->nExpr>0) ){ sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( pWalker->eCode ){ - pWalker->eCode = 0; - sqlite3WalkExpr(pWalker, pExpr->pRight); - } } return WRC_Prune; case TK_BETWEEN: - if( sqlite3WalkExpr(pWalker, pExpr->pLeft)==WRC_Abort ){ - assert( pWalker->eCode ); - return WRC_Abort; - } + /* In "x NOT BETWEEN y AND z" either x must be non-null-row or else + ** both y and z must be non-null row */ + assert( ExprUseXList(pExpr) ); + assert( pExpr->x.pList->nExpr==2 ); + sqlite3WalkExpr(pWalker, pExpr->pLeft); + bothImplyNotNullRow(pWalker, pExpr->x.pList->a[0].pExpr, + pExpr->x.pList->a[1].pExpr); return WRC_Prune; /* Virtual tables are allowed to use constraints like x=NULL. So @@ -112347,7 +114206,7 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ ** be non-NULL, then the LEFT JOIN can be safely converted into an ** ordinary join. */ -SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab, int isRJ){ Walker w; p = sqlite3ExprSkipCollateAndLikely(p); if( p==0 ) return 0; @@ -112355,7 +114214,7 @@ SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ p = p->pLeft; }else{ while( p->op==TK_AND ){ - if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab) ) return 1; + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab, isRJ) ) return 1; p = p->pRight; } } @@ -112363,6 +114222,7 @@ SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ w.xSelectCallback = 0; w.xSelectCallback2 = 0; w.eCode = 0; + w.mWFlags = isRJ!=0; w.u.iCur = iTab; sqlite3WalkExpr(&w, p); return w.eCode; @@ -112423,7 +114283,7 @@ SQLITE_PRIVATE int sqlite3ExprCoveredByIndex( } -/* Structure used to pass information throught the Walker in order to +/* Structure used to pass information throughout the Walker in order to ** implement sqlite3ReferencesSrcList(). */ struct RefSrcList { @@ -112530,6 +114390,12 @@ SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList assert( pExpr->op==TK_AGG_FUNCTION ); assert( ExprUseXList(pExpr) ); sqlite3WalkExprList(&w, pExpr->x.pList); + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + assert( pExpr->pLeft->x.pList!=0 ); + sqlite3WalkExprList(&w, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC if( ExprHasProperty(pExpr, EP_WinFunc) ){ sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); @@ -112639,7 +114505,7 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ ** Return the index in aCol[] of the entry that describes that column. ** ** If no prior entry is found, create a new one and return -1. The -** new column will have an idex of pAggInfo->nColumn-1. +** new column will have an index of pAggInfo->nColumn-1. */ static void findOrCreateAggInfoColumn( Parse *pParse, /* Parsing context */ @@ -112652,6 +114518,7 @@ static void findOrCreateAggInfoColumn( assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ + if( pCol->pCExpr==pExpr ) return; if( pCol->iTable==pExpr->iTable && pCol->iColumn==pExpr->iColumn && pExpr->op!=TK_IF_NULL_ROW @@ -112793,14 +114660,42 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); i = addAggInfoFunc(pParse->db, pAggInfo); if( i>=0 ){ + int nArg; assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; assert( ExprUseUToken(pExpr) ); + nArg = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, - pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); - if( pExpr->flags & EP_Distinct ){ + pExpr->u.zToken, nArg, enc, 0); + assert( pItem->bOBUnique==0 ); + if( pExpr->pLeft + && (pItem->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)==0 + ){ + /* The NEEDCOLL test above causes any ORDER BY clause on + ** aggregate min() or max() to be ignored. */ + ExprList *pOBList; + assert( nArg>0 ); + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + pItem->iOBTab = pParse->nTab++; + pOBList = pExpr->pLeft->x.pList; + assert( pOBList->nExpr>0 ); + assert( pItem->bOBUnique==0 ); + if( pOBList->nExpr==1 + && nArg==1 + && sqlite3ExprCompare(0,pOBList->a[0].pExpr, + pExpr->x.pList->a[0].pExpr,0)==0 + ){ + pItem->bOBPayload = 0; + pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct); + }else{ + pItem->bOBPayload = 1; + } + }else{ + pItem->iOBTab = -1; + } + if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){ pItem->iDistinct = pParse->nTab++; }else{ pItem->iDistinct = -1; @@ -113436,14 +115331,19 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ /* Verify that constraints are still satisfied */ if( pNew->pCheck!=0 || (pCol->notNull && (pCol->colFlags & COLFLAG_GENERATED)!=0) + || (pTab->tabFlags & TF_Strict)!=0 ){ sqlite3NestedParse(pParse, "SELECT CASE WHEN quick_check GLOB 'CHECK*'" " THEN raise(ABORT,'CHECK constraint failed')" + " WHEN quick_check GLOB 'non-* value in*'" + " THEN raise(ABORT,'type mismatch on DEFAULT')" " ELSE raise(ABORT,'NOT NULL constraint failed')" " END" " FROM pragma_quick_check(%Q,%Q)" - " WHERE quick_check GLOB 'CHECK*' OR quick_check GLOB 'NULL*'", + " WHERE quick_check GLOB 'CHECK*'" + " OR quick_check GLOB 'NULL*'" + " OR quick_check GLOB 'non-* value in*'", zTab, zDb ); } @@ -113532,7 +115432,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->u.tab.pDfltList = sqlite3ExprListDup(db, pTab->u.tab.pDfltList, 0); pNew->pSchema = db->aDb[iDb].pSchema; pNew->u.tab.addColOffset = pTab->u.tab.addColOffset; - pNew->nTabRef = 1; + assert( pNew->nTabRef==1 ); exit_begin_add_column: sqlite3SrcListDelete(db, pSrc); @@ -114037,7 +115937,7 @@ static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ } /* -** An error occured while parsing or otherwise processing a database +** An error occurred while parsing or otherwise processing a database ** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an ** ALTER TABLE RENAME COLUMN program. The error message emitted by the ** sub-routine is currently stored in pParse->zErrMsg. This function @@ -117143,14 +119043,15 @@ static int loadStatTbl( decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); - /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + /* Take a copy of the sample. Add 8 extra 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the ** sqlite3VdbeRecordCompare() may read up to two varints past the ** end of the allocated buffer before it realizes it is dealing with - ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a corrupt record. Or it might try to read a large integer from the + ** buffer. In any case, eight 0x00 bytes prevents this from causing ** a buffer overread. */ pSample->n = sqlite3_column_bytes(pStmt, 4); - pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 8); if( pSample->p==0 ){ sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; @@ -118108,7 +120009,7 @@ SQLITE_PRIVATE int sqlite3AuthCheck( sqlite3 *db = pParse->db; int rc; - /* Don't do any authorization checks if the database is initialising + /* Don't do any authorization checks if the database is initializing ** or if the parser is being invoked from within sqlite3_declare_vtab. */ assert( !IN_RENAME_OBJECT || db->xAuth==0 ); @@ -118409,29 +120310,26 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ pParse->nVtabLock = 0; #endif +#ifndef SQLITE_OMIT_SHARED_CACHE /* Once all the cookies have been verified and transactions opened, ** obtain the required table-locks. This is a no-op unless the ** shared-cache feature is enabled. */ - codeTableLocks(pParse); + if( pParse->nTableLock ) codeTableLocks(pParse); +#endif /* Initialize any AUTOINCREMENT data structures required. */ - sqlite3AutoincrementBegin(pParse); + if( pParse->pAinc ) sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops. - ** - ** The pConstExpr list might also contain expressions that we simply - ** want to keep around until the Parse object is deleted. Such - ** expressions have iConstExprReg==0. Do not generate code for - ** those expressions, of course. + /* Code constant expressions that were factored out of inner loops. */ if( pParse->pConstExpr ){ ExprList *pEL = pParse->pConstExpr; pParse->okConstFactor = 0; for(i=0; inExpr; i++){ - int iReg = pEL->a[i].u.iConstExprReg; - sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); + assert( pEL->a[i].u.iConstExprReg>0 ); + sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); } } @@ -118930,7 +120828,7 @@ SQLITE_PRIVATE void sqlite3ColumnSetColl( } /* -** Return the collating squence name for a column +** Return the collating sequence name for a column */ SQLITE_PRIVATE const char *sqlite3ColumnColl(Column *pCol){ const char *z; @@ -119585,20 +121483,13 @@ SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ } #endif -/* -** Name of the special TEMP trigger used to implement RETURNING. The -** name begins with "sqlite_" so that it is guaranteed not to collide -** with any application-generated triggers. -*/ -#define RETURNING_TRIGGER_NAME "sqlite_returning" - /* ** Clean up the data structures associated with the RETURNING clause. */ static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){ Hash *pHash; pHash = &(db->aDb[1].pSchema->trigHash); - sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, 0); + sqlite3HashInsert(pHash, pRet->zName, 0); sqlite3ExprListDelete(db, pRet->pReturnEL); sqlite3DbFree(db, pRet); } @@ -119641,7 +121532,9 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet); testcase( pParse->earlyCleanup ); if( db->mallocFailed ) return; - pRet->retTrig.zName = RETURNING_TRIGGER_NAME; + sqlite3_snprintf(sizeof(pRet->zName), pRet->zName, + "sqlite_returning_%p", pParse); + pRet->retTrig.zName = pRet->zName; pRet->retTrig.op = TK_RETURNING; pRet->retTrig.tr_tm = TRIGGER_AFTER; pRet->retTrig.bReturning = 1; @@ -119652,9 +121545,9 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pRet->retTStep.pTrig = &pRet->retTrig; pRet->retTStep.pExprList = pList; pHash = &(db->aDb[1].pSchema->trigHash); - assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 + assert( sqlite3HashFind(pHash, pRet->zName)==0 || pParse->nErr || pParse->ifNotExists ); - if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) + if( sqlite3HashInsert(pHash, pRet->zName, &pRet->retTrig) ==&pRet->retTrig ){ sqlite3OomFault(db); } @@ -119688,7 +121581,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ } if( !IN_RENAME_OBJECT ) sqlite3DequoteToken(&sName); - /* Because keywords GENERATE ALWAYS can be converted into indentifiers + /* Because keywords GENERATE ALWAYS can be converted into identifiers ** by the parser, we can sometimes end up with a typename that ends ** with "generated always". Check for this case and omit the surplus ** text. */ @@ -119909,7 +121802,7 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The parsed expression of the default value */ const char *zStart, /* Start of the default value text */ - const char *zEnd /* First character past end of defaut value text */ + const char *zEnd /* First character past end of default value text */ ){ Table *p; Column *pCol; @@ -120257,7 +122150,7 @@ static int identLength(const char *z){ ** to the specified offset in the buffer and updates *pIdx to refer ** to the first byte after the last byte written before returning. ** -** If the string zSignedIdent consists entirely of alpha-numeric +** If the string zSignedIdent consists entirely of alphanumeric ** characters, does not begin with a digit and is not an SQL keyword, ** then it is copied to the output buffer exactly as it is. Otherwise, ** it is quoted using double-quotes. @@ -120409,7 +122302,7 @@ static void estimateIndexWidth(Index *pIdx){ for(i=0; inColumn; i++){ i16 x = pIdx->aiColumn[i]; assert( xpTable->nCol ); - wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + wIndex += x<0 ? 1 : aCol[x].szEst; } pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } @@ -121098,6 +122991,17 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Reparse everything to update our internal data structures */ sqlite3VdbeAddParseSchemaOp(v, iDb, sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName),0); + + /* Test for cycles in generated columns and illegal expressions + ** in CHECK constraints and in DEFAULT clauses. */ + if( p->tabFlags & TF_HasGenerated ){ + sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", + db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); + } + sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3MPrintf(db, "PRAGMA \"%w\".integrity_check(%Q)", + db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } /* Add the table to the in-memory representation of the database. @@ -122147,7 +124051,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( #ifndef SQLITE_OMIT_TEMPDB /* If the index name was unqualified, check if the table ** is a temp table. If so, set the database to 1. Do not do this - ** if initialising a database schema. + ** if initializing a database schema. */ if( !db->init.busy ){ pTab = sqlite3SrcListLookup(pParse, pTblName); @@ -123804,7 +125708,7 @@ SQLITE_PRIVATE void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ /* ** This routine is invoked once per CTE by the parser while parsing a -** WITH clause. The CTE described by teh third argument is added to +** WITH clause. The CTE described by the third argument is added to ** the WITH clause of the second argument. If the second argument is ** NULL, then a new WITH argument is created. */ @@ -124446,8 +126350,9 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc>=1 ); pTab = sqlite3LocateTableItem(pParse, 0, pItem); - sqlite3DeleteTable(pParse->db, pItem->pTab); + if( pItem->pTab ) sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; + pItem->fg.notCte = 1; if( pTab ){ pTab->nTabRef++; if( pItem->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pItem) ){ @@ -124600,7 +126505,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( sqlite3 *db = pParse->db; Expr *pLhs = NULL; /* LHS of IN(SELECT...) operator */ Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ - ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + ExprList *pEList = NULL; /* Expression list containing only pSelectRowid*/ SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ Select *pSelect = NULL; /* Complete SELECT tree */ Table *pTab; @@ -124638,14 +126543,20 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( ); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol>=1 ); if( pPk->nKeyCol==1 ){ - const char *zName = pTab->aCol[pPk->aiColumn[0]].zCnName; + const char *zName; + assert( pPk->aiColumn[0]>=0 && pPk->aiColumn[0]nCol ); + zName = pTab->aCol[pPk->aiColumn[0]].zCnName; pLhs = sqlite3Expr(db, TK_ID, zName); pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName)); }else{ int i; for(i=0; inKeyCol; i++){ - Expr *p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); + Expr *p; + assert( pPk->aiColumn[i]>=0 && pPk->aiColumn[i]nCol ); + p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); pEList = sqlite3ExprListAppend(pParse, pEList, p); } pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); @@ -124674,7 +126585,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( pOrderBy,0,pLimit ); - /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + /* now generate the new WHERE rowid IN clause for the DELETE/UPDATE */ pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0); sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; @@ -124903,7 +126814,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; - nPk = 1; + assert( nPk==1 ); iRowSet = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); }else{ @@ -124931,7 +126842,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( pWInfo==0 ) goto delete_from_cleanup; eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); - assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF + || OptimizationDisabled(db, SQLITE_OnePass) ); if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse); if( sqlite3WhereUsesDeferredSeek(pWInfo) ){ sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur); @@ -125268,9 +127180,11 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); /* Invoke AFTER DELETE trigger programs. */ - sqlite3CodeRowTrigger(pParse, pTrigger, - TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel - ); + if( pTrigger ){ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); + } /* Jump here if the row had already been deleted before any BEFORE ** trigger programs were invoked. Or if a trigger program throws a @@ -125583,6 +127497,42 @@ static void lengthFunc( } } +/* +** Implementation of the octet_length() function +*/ +static void bytelengthFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_BLOB: { + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + i64 m = sqlite3_context_db_handle(context)->enc<=SQLITE_UTF8 ? 1 : 2; + sqlite3_result_int64(context, sqlite3_value_bytes(argv[0])*m); + break; + } + case SQLITE_TEXT: { + if( sqlite3_value_encoding(argv[0])<=SQLITE_UTF8 ){ + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + }else{ + sqlite3_result_int(context, sqlite3_value_bytes16(argv[0])); + } + break; + } + default: { + sqlite3_result_null(context); + break; + } + } +} + /* ** Implementation of the abs() function. ** @@ -125859,7 +127809,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ }else if( n==0 ){ r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); }else{ - zBuf = sqlite3_mprintf("%.*f",n,r); + zBuf = sqlite3_mprintf("%!.*f",n,r); if( zBuf==0 ){ sqlite3_result_error_nomem(context); return; @@ -126059,7 +128009,7 @@ struct compareInfo { /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, provde the Utf8Read() +** character is exactly one byte in size. Also, provide the Utf8Read() ** macro for fast reading of the next character in the common case where ** the next character is ASCII. */ @@ -126292,7 +128242,7 @@ SQLITE_API int sqlite3_like_count = 0; /* ** Implementation of the like() SQL function. This function implements -** the build-in LIKE operator. The first argument to the function is the +** the built-in LIKE operator. The first argument to the function is the ** pattern and the second argument is the string. So, the SQL statements: ** ** A LIKE B @@ -126625,6 +128575,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } + *zOut = 0; sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } @@ -126653,7 +128604,8 @@ static void hexFunc( *(z++) = hexdigits[c&0xf]; } *z = 0; - sqlite3_result_text(context, zHex, n*2, sqlite3_free); + sqlite3_result_text64(context, zHex, (u64)(z-zHex), + sqlite3_free, SQLITE_UTF8); } } @@ -126678,7 +128630,7 @@ static int strContainsChar(const u8 *zStr, int nStr, u32 ch){ ** decoded and returned as a blob. ** ** If there is only a single argument, then it must consist only of an -** even number of hexadeximal digits. Otherwise, return NULL. +** even number of hexadecimal digits. Otherwise, return NULL. ** ** Or, if there is a second argument, then any character that appears in ** the second argument is also allowed to appear between pairs of hexadecimal @@ -126947,6 +128899,81 @@ static void trimFunc( sqlite3_result_text(context, (char*)zIn, nIn, SQLITE_TRANSIENT); } +/* The core implementation of the CONCAT(...) and CONCAT_WS(SEP,...) +** functions. +** +** Return a string value that is the concatenation of all non-null +** entries in argv[]. Use zSep as the separator. +*/ +static void concatFuncCore( + sqlite3_context *context, + int argc, + sqlite3_value **argv, + int nSep, + const char *zSep +){ + i64 j, k, n = 0; + int i; + char *z; + for(i=0; i0 ){ + const char *v = (const char*)sqlite3_value_text(argv[i]); + if( v!=0 ){ + if( j>0 && nSep>0 ){ + memcpy(&z[j], zSep, nSep); + j += nSep; + } + memcpy(&z[j], v, k); + j += k; + } + } + } + z[j] = 0; + assert( j<=n ); + sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8); +} + +/* +** The CONCAT(...) function. Generate a string result that is the +** concatentation of all non-null arguments. +*/ +static void concatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + concatFuncCore(context, argc, argv, 0, ""); +} + +/* +** The CONCAT_WS(separator, ...) function. +** +** Generate a string that is the concatenation of 2nd through the Nth +** argument. Use the first argument (which must be non-NULL) as the +** separator. +*/ +static void concatwsFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nSep = sqlite3_value_bytes(argv[0]); + const char *zSep = (const char*)sqlite3_value_text(argv[0]); + if( zSep==0 ) return; + concatFuncCore(context, argc-1, argv+1, nSep, zSep); +} + #ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION /* @@ -127068,13 +129095,68 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ */ typedef struct SumCtx SumCtx; struct SumCtx { - double rSum; /* Floating point sum */ - i64 iSum; /* Integer sum */ + double rSum; /* Running sum as as a double */ + double rErr; /* Error term for Kahan-Babushka-Neumaier summation */ + i64 iSum; /* Running sum as a signed integer */ i64 cnt; /* Number of elements summed */ - u8 overflow; /* True if integer overflow seen */ - u8 approx; /* True if non-integer value was input to the sum */ + u8 approx; /* True if any non-integer value was input to the sum */ + u8 ovrfl; /* Integer overflow seen */ }; +/* +** Do one step of the Kahan-Babushka-Neumaier summation. +** +** https://en.wikipedia.org/wiki/Kahan_summation_algorithm +** +** Variables are marked "volatile" to defeat c89 x86 floating point +** optimizations can mess up this algorithm. +*/ +static void kahanBabuskaNeumaierStep( + volatile SumCtx *pSum, + volatile double r +){ + volatile double s = pSum->rSum; + volatile double t = s + r; + if( fabs(s) > fabs(r) ){ + pSum->rErr += (s - t) + r; + }else{ + pSum->rErr += (r - t) + s; + } + pSum->rSum = t; +} + +/* +** Add a (possibly large) integer to the running sum. +*/ +static void kahanBabuskaNeumaierStepInt64(volatile SumCtx *pSum, i64 iVal){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iBig, iSm; + iSm = iVal % 16384; + iBig = iVal - iSm; + kahanBabuskaNeumaierStep(pSum, iBig); + kahanBabuskaNeumaierStep(pSum, iSm); + }else{ + kahanBabuskaNeumaierStep(pSum, (double)iVal); + } +} + +/* +** Initialize the Kahan-Babaska-Neumaier sum from a 64-bit integer +*/ +static void kahanBabuskaNeumaierInit( + volatile SumCtx *p, + i64 iVal +){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iSm = iVal % 16384; + p->rSum = (double)(iVal - iSm); + p->rErr = (double)iSm; + }else{ + p->rSum = (double)iVal; + p->rErr = 0.0; + } +} + /* ** Routines used to compute the sum, average, and total. ** @@ -127094,15 +129176,29 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ type = sqlite3_value_numeric_type(argv[0]); if( p && type!=SQLITE_NULL ){ p->cnt++; - if( type==SQLITE_INTEGER ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum += v; - if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ - p->approx = p->overflow = 1; + if( p->approx==0 ){ + if( type!=SQLITE_INTEGER ){ + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + }else{ + i64 x = p->iSum; + if( sqlite3AddInt64(&x, sqlite3_value_int64(argv[0]))==0 ){ + p->iSum = x; + }else{ + p->ovrfl = 1; + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + } } }else{ - p->rSum += sqlite3_value_double(argv[0]); - p->approx = 1; + if( type==SQLITE_INTEGER ){ + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + }else{ + p->ovrfl = 0; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + } } } } @@ -127119,13 +129215,18 @@ static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ if( ALWAYS(p) && type!=SQLITE_NULL ){ assert( p->cnt>0 ); p->cnt--; - assert( type==SQLITE_INTEGER || p->approx ); - if( type==SQLITE_INTEGER && p->approx==0 ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum -= v; - p->iSum -= v; + if( !p->approx ){ + p->iSum -= sqlite3_value_int64(argv[0]); + }else if( type==SQLITE_INTEGER ){ + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal!=SMALLEST_INT64 ){ + kahanBabuskaNeumaierStepInt64(p, -iVal); + }else{ + kahanBabuskaNeumaierStepInt64(p, LARGEST_INT64); + kahanBabuskaNeumaierStepInt64(p, 1); + } }else{ - p->rSum -= sqlite3_value_double(argv[0]); + kahanBabuskaNeumaierStep(p, -sqlite3_value_double(argv[0])); } } } @@ -127136,10 +129237,14 @@ static void sumFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - if( p->overflow ){ - sqlite3_result_error(context,"integer overflow",-1); - }else if( p->approx ){ - sqlite3_result_double(context, p->rSum); + if( p->approx ){ + if( p->ovrfl ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( !sqlite3IsNaN(p->rErr) ){ + sqlite3_result_double(context, p->rSum+p->rErr); + }else{ + sqlite3_result_double(context, p->rSum); + } }else{ sqlite3_result_int64(context, p->iSum); } @@ -127149,14 +129254,29 @@ static void avgFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - sqlite3_result_double(context, p->rSum/(double)p->cnt); + double r; + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + sqlite3_result_double(context, r/(double)p->cnt); } } static void totalFinalize(sqlite3_context *context){ SumCtx *p; + double r = 0.0; p = sqlite3_aggregate_context(context, 0); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - sqlite3_result_double(context, p ? p->rSum : (double)0); + if( p ){ + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + } + sqlite3_result_double(context, r); } /* @@ -127275,6 +129395,7 @@ static void minMaxFinalize(sqlite3_context *context){ /* ** group_concat(EXPR, ?SEPARATOR?) +** string_agg(EXPR, SEPARATOR) ** ** The SEPARATOR goes before the EXPR string. This is tragic. The ** groupConcatInverse() implementation would have been easier if the @@ -127378,7 +129499,7 @@ static void groupConcatInverse( if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; pGCC = (GroupConcatCtx*)sqlite3_aggregate_context(context, sizeof(*pGCC)); /* pGCC is always non-NULL since groupConcatStep() will have always - ** run frist to initialize it */ + ** run first to initialize it */ if( ALWAYS(pGCC) ){ int nVS; /* Must call sqlite3_value_text() to convert the argument into text prior @@ -127462,8 +129583,10 @@ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ ** sensitive. */ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + FuncDef *pDef; struct compareInfo *pInfo; int flags; + int nArg; if( caseSensitive ){ pInfo = (struct compareInfo*)&likeInfoAlt; flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; @@ -127471,10 +129594,13 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) pInfo = (struct compareInfo*)&likeInfoNorm; flags = SQLITE_FUNC_LIKE; } - sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); - sqlite3FindFunction(db, "like", 2, SQLITE_UTF8, 0)->funcFlags |= flags; - sqlite3FindFunction(db, "like", 3, SQLITE_UTF8, 0)->funcFlags |= flags; + for(nArg=2; nArg<=3; nArg++){ + sqlite3CreateFunc(db, "like", nArg, SQLITE_UTF8, pInfo, likeFunc, + 0, 0, 0, 0, 0); + pDef = sqlite3FindFunction(db, "like", nArg, SQLITE_UTF8, 0); + pDef->funcFlags |= flags; + pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; + } } /* @@ -127746,6 +129872,37 @@ static void signFunc( sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); } +#ifdef SQLITE_DEBUG +/* +** Implementation of fpdecode(x,y,z) function. +** +** x is a real number that is to be decoded. y is the precision. +** z is the maximum real precision. +*/ +static void fpdecodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FpDecode s; + double x; + int y, z; + char zBuf[100]; + UNUSED_PARAMETER(argc); + assert( argc==3 ); + x = sqlite3_value_double(argv[0]); + y = sqlite3_value_int(argv[1]); + z = sqlite3_value_int(argv[2]); + sqlite3FpDecode(&s, x, y, z); + if( s.isSpecial==2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "NaN"); + }else{ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%c%.*s/%d", s.sign, s.n, s.z, s.iDP); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); +} +#endif /* SQLITE_DEBUG */ + /* ** All of the FuncDef structures in the aBuiltinFunc[] array above ** to the global function hash table. This occurs at start-time (as @@ -127810,12 +129967,16 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(subtype, 1, 0, 0, subtypeFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION2(octet_length, 1, 0, 0, bytelengthFunc,SQLITE_FUNC_BYTELEN), FUNCTION(instr, 2, 0, 0, instrFunc ), FUNCTION(printf, -1, 0, 0, printfFunc ), FUNCTION(format, -1, 0, 0, printfFunc ), FUNCTION(unicode, 1, 0, 0, unicodeFunc ), FUNCTION(char, -1, 0, 0, charFunc ), FUNCTION(abs, 1, 0, 0, absFunc ), +#ifdef SQLITE_DEBUG + FUNCTION(fpdecode, 3, 0, 0, fpdecodeFunc ), +#endif #ifndef SQLITE_OMIT_FLOATING_POINT FUNCTION(round, 1, 0, 0, roundFunc ), FUNCTION(round, 2, 0, 0, roundFunc ), @@ -127825,6 +129986,11 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(hex, 1, 0, 0, hexFunc ), FUNCTION(unhex, 1, 0, 0, unhexFunc ), FUNCTION(unhex, 2, 0, 0, unhexFunc ), + FUNCTION(concat, -1, 0, 0, concatFunc ), + FUNCTION(concat, 0, 0, 0, 0 ), + FUNCTION(concat_ws, -1, 0, 0, concatwsFunc ), + FUNCTION(concat_ws, 0, 0, 0, 0 ), + FUNCTION(concat_ws, 1, 0, 0, 0 ), INLINE_FUNC(ifnull, 2, INLINEFUNC_coalesce, 0 ), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), @@ -127854,6 +130020,8 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + WAGGREGATE(string_agg, 2, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), #ifdef SQLITE_CASE_SENSITIVE_LIKE @@ -128796,6 +130964,7 @@ static int isSetNullAction(Parse *pParse, FKey *pFKey){ if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) ){ + assert( (pTop->db->flags & SQLITE_FkNoAction)==0 ); return 1; } } @@ -128990,6 +131159,8 @@ SQLITE_PRIVATE void sqlite3FkCheck( } if( regOld!=0 ){ int eAction = pFKey->aAction[aChange!=0]; + if( (db->flags & SQLITE_FkNoAction) ) eAction = OE_None; + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); /* If this is a deferred FK constraint, or a CASCADE or SET NULL ** action applies, then any foreign key violations caused by @@ -129105,7 +131276,11 @@ SQLITE_PRIVATE int sqlite3FkRequired( /* Check if any parent key columns are being modified. */ for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ if( fkParentIsModified(pTab, p, aChange, chngRowid) ){ - if( p->aAction[1]!=OE_None ) return 2; + if( (pParse->db->flags & SQLITE_FkNoAction)==0 + && p->aAction[1]!=OE_None + ){ + return 2; + } bHaveFK = 1; } } @@ -129155,6 +131330,7 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; + if( (db->flags & SQLITE_FkNoAction) ) action = OE_None; if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ return 0; } @@ -129386,9 +131562,8 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ if( pFKey->pPrevTo ){ pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ - void *p = (void *)pFKey->pNextTo; - const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); + const char *z = (pFKey->pNextTo ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, pFKey->pNextTo); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; @@ -129451,8 +131626,10 @@ SQLITE_PRIVATE void sqlite3OpenTable( assert( pParse->pVdbe!=0 ); v = pParse->pVdbe; assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); - sqlite3TableLock(pParse, iDb, pTab->tnum, - (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( !pParse->db->noSharedCache ){ + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + } if( HasRowid(pTab) ){ sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); VdbeComment((v, "%s", pTab->zName)); @@ -129581,7 +131758,7 @@ SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ ** For STRICT tables: ** ------------------ ** -** Generate an appropropriate OP_TypeCheck opcode that will verify the +** Generate an appropriate OP_TypeCheck opcode that will verify the ** datatypes against the column definitions in pTab. If iReg==0, that ** means an OP_MakeRecord opcode has already been generated and should be ** the last opcode generated. The new OP_TypeCheck needs to be inserted @@ -130873,7 +133050,7 @@ SQLITE_PRIVATE void sqlite3Insert( /* This is the Walker callback from sqlite3ExprReferencesUpdatedColumn(). * Set bit 0x01 of pWalker->eCode if pWalker->eCode to 0 and if this ** expression node references any of the -** columns that are being modifed by an UPDATE statement. +** columns that are being modified by an UPDATE statement. */ static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_COLUMN ){ @@ -131096,7 +133273,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int *aiChng, /* column i is unchanged if aiChng[i]<0 */ Upsert *pUpsert /* ON CONFLICT clauses, if any. NULL otherwise */ ){ - Vdbe *v; /* VDBE under constrution */ + Vdbe *v; /* VDBE under construction */ Index *pIdx; /* Pointer to one of the indices */ Index *pPk = 0; /* The PRIMARY KEY index for WITHOUT ROWID tables */ sqlite3 *db; /* Database connection */ @@ -131579,7 +133756,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( pIdx; pIdx = indexIteratorNext(&sIdxIter, &ix) ){ - int regIdx; /* Range of registers hold conent for pIdx */ + int regIdx; /* Range of registers holding content for pIdx */ int regR; /* Range of registers holding conflicting PK */ int iThisCur; /* Cursor for this UNIQUE index */ int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ @@ -132074,6 +134251,8 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( assert( op==OP_OpenRead || op==OP_OpenWrite ); assert( op==OP_OpenWrite || p5==0 ); + assert( piDataCur!=0 ); + assert( piIdxCur!=0 ); if( IsVirtual(pTab) ){ /* This routine is a no-op for virtual tables. Leave the output ** variables *piDataCur and *piIdxCur set to illegal cursor numbers @@ -132086,18 +134265,18 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( assert( v!=0 ); if( iBase<0 ) iBase = pParse->nTab; iDataCur = iBase++; - if( piDataCur ) *piDataCur = iDataCur; + *piDataCur = iDataCur; if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); - }else{ + }else if( pParse->db->noSharedCache==0 ){ sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); } - if( piIdxCur ) *piIdxCur = iBase; + *piIdxCur = iBase; for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - if( piDataCur ) *piDataCur = iIdxCur; + *piDataCur = iIdxCur; p5 = 0; } if( aToOpen==0 || aToOpen[i+1] ){ @@ -132395,7 +134574,7 @@ static int xferOptimization( } #endif #ifndef SQLITE_OMIT_FOREIGN_KEY - /* Disallow the transfer optimization if the destination table constains + /* Disallow the transfer optimization if the destination table contains ** any foreign key constraints. This is more restrictive than necessary. ** But the main beneficiary of the transfer optimization is the VACUUM ** command, and the VACUUM command disables foreign key constraints. So @@ -133105,6 +135284,11 @@ struct sqlite3_api_routines { int (*value_encoding)(sqlite3_value*); /* Version 3.41.0 and later */ int (*is_interrupted)(sqlite3*); + /* Version 3.43.0 and later */ + int (*stmt_explain)(sqlite3_stmt*,int); + /* Version 3.44.0 and later */ + void *(*get_clientdata)(sqlite3*,const char*); + int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); }; /* @@ -133433,6 +135617,11 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_value_encoding sqlite3_api->value_encoding /* Version 3.41.0 and later */ #define sqlite3_is_interrupted sqlite3_api->is_interrupted +/* Version 3.43.0 and later */ +#define sqlite3_stmt_explain sqlite3_api->stmt_explain +/* Version 3.44.0 and later */ +#define sqlite3_get_clientdata sqlite3_api->get_clientdata +#define sqlite3_set_clientdata sqlite3_api->set_clientdata #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -133949,7 +136138,12 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.40.0 and later */ sqlite3_value_encoding, /* Version 3.41.0 and later */ - sqlite3_is_interrupted + sqlite3_is_interrupted, + /* Version 3.43.0 and later */ + sqlite3_stmt_explain, + /* Version 3.44.0 and later */ + sqlite3_get_clientdata, + sqlite3_set_clientdata }; /* True if x is the directory separator character @@ -134029,6 +136223,10 @@ static int sqlite3LoadExtension( */ if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; + /* Do not allow sqlite3_load_extension() to link to a copy of the + ** running application, by passing in an empty filename. */ + if( nMsg==0 ) goto extension_not_found; + handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; iimutex); if( onoff ){ db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; @@ -134210,6 +136411,9 @@ SQLITE_API int sqlite3_auto_extension( void (*xInit)(void) ){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return SQLITE_MISUSE_BKPT; +#endif #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ){ @@ -134262,6 +136466,9 @@ SQLITE_API int sqlite3_cancel_auto_extension( int i; int n = 0; wsdAutoextInit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return 0; +#endif sqlite3_mutex_enter(mutex); for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ if( wsdAutoext.aExt[i]==xInit ){ @@ -135861,7 +138068,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** The first form reports the current local setting for the ** page cache spill size. The second form turns cache spill on - ** or off. When turnning cache spill on, the size is set to the + ** or off. When turning cache spill on, the size is set to the ** current cache_size. The third form sets a spill size that ** may be different form the cache size. ** If N is positive then that is the @@ -136131,7 +138338,11 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif if( sqlite3GetBoolean(zRight, 0) ){ - db->flags |= mask; + if( (mask & SQLITE_WriteSchema)==0 + || (db->flags & SQLITE_Defensive)==0 + ){ + db->flags |= mask; + } }else{ db->flags &= ~mask; if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; @@ -136639,9 +138850,9 @@ SQLITE_PRIVATE void sqlite3Pragma( ** The "quick_check" is reduced version of ** integrity_check designed to detect most database corruption ** without the overhead of cross-checking indexes. Quick_check - ** is linear time wherease integrity_check is O(NlogN). + ** is linear time whereas integrity_check is O(NlogN). ** - ** The maximum nubmer of errors is 100 by default. A different default + ** The maximum number of errors is 100 by default. A different default ** can be specified using a numeric parameter N. ** ** Or, the parameter N can be the name of a table. In that case, only @@ -136764,8 +138975,31 @@ SQLITE_PRIVATE void sqlite3Pragma( int r2; /* Previous key for WITHOUT ROWID tables */ int mxCol; /* Maximum non-virtual column number */ - if( !IsOrdinaryTable(pTab) ) continue; if( pObjTab && pObjTab!=pTab ) continue; + if( !IsOrdinaryTable(pTab) ){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_vtab *pVTab; + int a1; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; + } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); +#endif + continue; + } if( isQuick || HasRowid(pTab) ){ pPk = 0; r2 = 0; @@ -137399,7 +139633,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Schema *pSchema; /* The current schema */ Table *pTab; /* A table in the schema */ Index *pIdx; /* An index of the table */ - LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ @@ -137891,7 +140125,8 @@ static const sqlite3_module pragmaVtabModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; /* @@ -138515,8 +140750,6 @@ SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse *pParse){ db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; assert( pParse->db->pParse==pParse ); db->pParse = pParse->pOuterParse; - pParse->db = 0; - pParse->disableLookaside = 0; } /* @@ -138525,7 +140758,7 @@ SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse *pParse){ ** immediately. ** ** Use this mechanism for uncommon cleanups. There is a higher setup -** cost for this mechansim (an extra malloc), so it should not be used +** cost for this mechanism (an extra malloc), so it should not be used ** for common cleanups that happen on most calls. But for less ** common cleanups, we save a single NULL-pointer comparison in ** sqlite3ParseObjectReset(), which reduces the total CPU cycle count. @@ -138617,7 +140850,12 @@ static int sqlite3Prepare( sParse.pOuterParse = db->pParse; db->pParse = &sParse; sParse.db = db; - sParse.pReprepare = pReprepare; + if( pReprepare ){ + sParse.pReprepare = pReprepare; + sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare); + }else{ + assert( sParse.pReprepare==0 ); + } assert( ppStmt && *ppStmt==0 ); if( db->mallocFailed ){ sqlite3ErrorMsg(&sParse, "out of memory"); @@ -139227,7 +141465,7 @@ static Select *findRightmost(Select *p){ ** NATURAL FULL OUTER JT_NATRUAL|JT_LEFT|JT_RIGHT ** ** To preserve historical compatibly, SQLite also accepts a variety -** of other non-standard and in many cases non-sensical join types. +** of other non-standard and in many cases nonsensical join types. ** This routine makes as much sense at it can from the nonsense join ** type and returns a result. Examples of accepted nonsense join types ** include but are not limited to: @@ -139449,6 +141687,7 @@ static void unsetJoinExpr(Expr *p, int iTable, int nullable){ } if( p->op==TK_FUNCTION ){ assert( ExprUseXList(p) ); + assert( p->pLeft==0 ); if( p->x.pList ){ int i; for(i=0; ix.pList->nExpr; i++){ @@ -139498,7 +141737,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON; - /* If this is a NATURAL join, synthesize an approprate USING clause + /* If this is a NATURAL join, synthesize an appropriate USING clause ** to specify which columns should be joined. */ if( pRight->fg.jointype & JT_NATURAL ){ @@ -139714,7 +141953,7 @@ static void pushOntoSorter( ** (3) Some output columns are omitted from the sort record due to ** the SQLITE_ENABLE_SORTER_REFERENCES optimization, or due to the ** SQLITE_ECEL_OMITREF optimization, or due to the - ** SortCtx.pDeferredRowLoad optimiation. In any of these cases + ** SortCtx.pDeferredRowLoad optimization. In any of these cases ** regOrigData is 0 to prevent this routine from trying to copy ** values that might not yet exist. */ @@ -139770,7 +142009,7 @@ static void pushOntoSorter( testcase( pKI->nAllField > pKI->nKeyField+2 ); pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, pKI->nAllField-pKI->nKeyField-1); - pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */ + pOp = 0; /* Ensure pOp not used after sqlite3VdbeAddOp3() */ addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse); @@ -139864,7 +142103,7 @@ static void codeOffset( ** The returned value in this case is a copy of parameter iTab. ** ** WHERE_DISTINCT_ORDERED: -** In this case rows are being delivered sorted order. The ephermal +** In this case rows are being delivered sorted order. The ephemeral ** table is not required. Instead, the current set of values ** is compared against previous row. If they match, the new row ** is not distinct and control jumps to VM address addrRepeat. Otherwise, @@ -140293,6 +142532,16 @@ static void selectInnerLoop( testcase( eDest==SRT_Fifo ); testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); +#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) + /* A destination of SRT_Table and a non-zero iSDParm2 parameter means + ** that this is an "UPDATE ... FROM" on a virtual table or view. In this + ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. + ** This does not affect operation in any way - it just allows MakeRecord + ** to process OPFLAG_NOCHANGE values without an assert() failing. */ + if( eDest==SRT_Table && pDest->iSDParm2 ){ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); + } +#endif #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -141096,13 +143345,6 @@ SQLITE_PRIVATE void sqlite3GenerateColumnNames( int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ -#ifndef SQLITE_OMIT_EXPLAIN - /* If this is an EXPLAIN, skip this step */ - if( pParse->explain ){ - return; - } -#endif - if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; @@ -141289,7 +143531,7 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( ** kind (maybe a parenthesized subquery in the FROM clause of a larger ** query, or a VIEW, or a CTE). This routine computes type information ** for that Table object based on the Select object that implements the -** subquery. For the purposes of this routine, "type infomation" means: +** subquery. For the purposes of this routine, "type information" means: ** ** * The datatype name, as it might appear in a CREATE TABLE statement ** * Which collating sequence to use for the column @@ -141618,7 +143860,7 @@ static void generateWithRecursiveQuery( int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ int eDest = SRT_Fifo; /* How to write to Queue */ - SelectDest destQueue; /* SelectDest targetting the Queue table */ + SelectDest destQueue; /* SelectDest targeting the Queue table */ int i; /* Loop counter */ int rc; /* Result code */ ExprList *pOrderBy; /* The ORDER BY clause */ @@ -142218,7 +144460,7 @@ SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ /* ** Code an output subroutine for a coroutine implementation of a -** SELECT statment. +** SELECT statement. ** ** The data to be output is contained in pIn->iSdst. There are ** pIn->nSdst columns to be output. pDest is where the output should @@ -142440,7 +144682,7 @@ static int generateOutputSubroutine( ** ** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not ** actually called using Gosub and they do not Return. EofA and EofB loop -** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** until all data is exhausted then jump to the "end" label. AltB, AeqB, ** and AgtB jump to either L2 or to one of EofA or EofB. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -142477,7 +144719,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int addr1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargeted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -142846,11 +145088,14 @@ static Expr *substExpr( #endif { Expr *pNew; - int iColumn = pExpr->iColumn; - Expr *pCopy = pSubst->pEList->a[iColumn].pExpr; + int iColumn; + Expr *pCopy; Expr ifNullRow; + iColumn = pExpr->iColumn; + assert( iColumn>=0 ); assert( pSubst->pEList!=0 && iColumnpEList->nExpr ); assert( pExpr->pRight==0 ); + pCopy = pSubst->pEList->a[iColumn].pExpr; if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ @@ -143199,7 +145444,7 @@ static int compoundHasDifferentAffinities(Select *p){ ** (9) If the subquery uses LIMIT then the outer query may not be aggregate. ** ** (**) Restriction (10) was removed from the code on 2005-02-05 but we -** accidently carried the comment forward until 2014-09-15. Original +** accidentally carried the comment forward until 2014-09-15. Original ** constraint: "If the subquery is aggregate then the outer query ** may not use LIMIT." ** @@ -143291,7 +145536,8 @@ static int compoundHasDifferentAffinities(Select *p){ ** (27b) the subquery is a compound query and the RIGHT JOIN occurs ** in any arm of the compound query. (See also (17g).) ** -** (28) The subquery is not a MATERIALIZED CTE. +** (28) The subquery is not a MATERIALIZED CTE. (This is handled +** in the caller before ever reaching this routine.) ** ** ** In this routine, the "p" parameter is a pointer to the outer query. @@ -143401,9 +145647,9 @@ static int flattenSubquery( if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ return 0; /* Restriction (27a) */ } - if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){ - return 0; /* (28) */ - } + + /* Condition (28) is blocked by the caller */ + assert( !pSubitem->fg.isCte || pSubitem->u2.pCteUse->eM10d!=M10d_Yes ); /* Restriction (17): If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries @@ -143473,7 +145719,7 @@ static int flattenSubquery( testcase( i==SQLITE_DENY ); pParse->zAuthContext = zSavedAuthContext; - /* Delete the transient structures associated with thesubquery */ + /* Delete the transient structures associated with the subquery */ pSub1 = pSubitem->pSelect; sqlite3DbFree(db, pSubitem->zDatabase); sqlite3DbFree(db, pSubitem->zName); @@ -143655,7 +145901,7 @@ static int flattenSubquery( ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values ** do not necessarily correspond to columns in SELECT statement pParent, - ** zero them before transfering the ORDER BY clause. + ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this ** function attempts to flatten a compound sub-query into pParent @@ -143715,8 +145961,7 @@ static int flattenSubquery( } } - /* Finially, delete what is left of the subquery and return - ** success. + /* Finally, delete what is left of the subquery and return success. */ sqlite3AggInfoPersistWalkerInit(&w, pParse); sqlite3WalkSelect(&w,pSub1); @@ -143751,7 +145996,7 @@ struct WhereConst { /* ** Add a new entry to the pConst object. Except, do not add duplicate -** pColumn entires. Also, do not add if doing so would not be appropriate. +** pColumn entries. Also, do not add if doing so would not be appropriate. ** ** The caller guarantees the pColumn is a column and pValue is a constant. ** This routine has to do some additional checks before completing the @@ -143937,7 +146182,7 @@ static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ ** SELECT * FROM t1 WHERE a=123 AND b=123; ** ** The two SELECT statements above should return different answers. b=a -** is alway true because the comparison uses numeric affinity, but b=123 +** is always true because the comparison uses numeric affinity, but b=123 ** is false because it uses text affinity and '0123' is not the same as '123'. ** To work around this, the expression tree is not actually changed from ** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol @@ -144021,7 +146266,7 @@ static int propagateConstants( ** At the time this function is called it is guaranteed that ** ** * the sub-query uses only one distinct window frame, and -** * that the window frame has a PARTITION BY clase. +** * that the window frame has a PARTITION BY clause. */ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ assert( pSubq->pWin->pPartition ); @@ -144290,12 +146535,12 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ assert( pItem->pSelect!=0 ); pSub = pItem->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); - if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ - testcase( pSub->selFlags & SF_Distinct ); - testcase( pSub->selFlags & SF_Aggregate ); - return 0; - } for(pX=pSub; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pX->selFlags & SF_Distinct ); + testcase( pX->selFlags & SF_Aggregate ); + return 0; + } if( pX->pPrior && pX->op!=TK_ALL ){ /* This optimization does not work for compound subqueries that ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ @@ -145101,12 +147346,20 @@ static int selectExpander(Walker *pWalker, Select *p){ ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ char *zTName = 0; /* text of name of TABLE */ + int iErrOfst; if( pE->op==TK_DOT ){ + assert( (selFlags & SF_NestedFrom)==0 ); assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; + assert( ExprUseWOfst(pE->pLeft) ); + iErrOfst = pE->pRight->w.iOfst; + }else{ + assert( ExprUseWOfst(pE) ); + iErrOfst = pE->w.iOfst; } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + int nAdd; /* Number of cols including rowid */ Table *pTab = pFrom->pTab; /* Table for this data source */ ExprList *pNestedFrom; /* Result-set of a nested FROM clause */ char *zTabName; /* AS name for this data source */ @@ -145124,6 +147377,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pNestedFrom = pFrom->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); + assert( VisibleRowid(pTab)==0 ); }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; @@ -145141,6 +147395,7 @@ static int selectExpander(Walker *pWalker, Select *p){ for(ii=0; iinId; ii++){ const char *zUName = pUsing->a[ii].zName; pRight = sqlite3Expr(db, TK_ID, zUName); + sqlite3ExprSetErrorOffset(pRight, iErrOfst); pNew = sqlite3ExprListAppend(pParse, pNew, pRight); if( pNew ){ struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; @@ -145153,33 +147408,48 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ pUsing = 0; } - for(j=0; jnCol; j++){ - char *zName = pTab->aCol[j].zCnName; + + nAdd = pTab->nCol + (VisibleRowid(pTab) && (selFlags&SF_NestedFrom)); + for(j=0; ja[j], 0, zTName, 0)==0 - ){ - continue; - } + if( j==pTab->nCol ){ + zName = sqlite3RowidAlias(pTab); + if( zName==0 ) continue; + }else{ + zName = pTab->aCol[j].zCnName; - /* If a column is marked as 'hidden', omit it from the expanded - ** result-set list unless the SELECT has the SF_IncludeHidden - ** bit set. - */ - if( (p->selFlags & SF_IncludeHidden)==0 - && IsHiddenColumn(&pTab->aCol[j]) - ){ - continue; - } - if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 - && zTName==0 - && (selFlags & (SF_NestedFrom))==0 - ){ - continue; + /* If pTab is actually an SF_NestedFrom sub-select, do not + ** expand any ENAME_ROWID columns. */ + if( pNestedFrom && pNestedFrom->a[j].fg.eEName==ENAME_ROWID ){ + continue; + } + + if( zTName + && pNestedFrom + && sqlite3MatchEName(&pNestedFrom->a[j], 0, zTName, 0, 0)==0 + ){ + continue; + } + + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. + */ + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ + continue; + } + if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 + && zTName==0 + && (selFlags & (SF_NestedFrom))==0 + ){ + continue; + } } + assert( zName ); tableSeen = 1; if( i>0 && zTName==0 && (selFlags & SF_NestedFrom)==0 ){ @@ -145213,6 +147483,7 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ pExpr = pRight; } + sqlite3ExprSetErrorOffset(pExpr, iErrOfst); pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); if( pNew==0 ){ break; /* OOM */ @@ -145228,11 +147499,11 @@ static int selectExpander(Walker *pWalker, Select *p){ zSchemaName, zTabName, zName); testcase( pX->zEName==0 ); } - pX->fg.eEName = ENAME_TAB; + pX->fg.eEName = (j==pTab->nCol ? ENAME_ROWID : ENAME_TAB); if( (pFrom->fg.isUsing && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0) || (pUsing && sqlite3IdListIndex(pUsing, zName)>=0) - || (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 + || (jnCol && (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)) ){ pX->fg.bNoExpand = 1; } @@ -145453,8 +147724,14 @@ static void analyzeAggFuncArgs( pNC->ncFlags |= NC_InAggFunc; for(i=0; inFunc; i++){ Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( pExpr->op==TK_FUNCTION || pExpr->op==TK_AGG_FUNCTION ); assert( ExprUseXList(pExpr) ); sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC assert( !IsWindowFunc(pExpr) ); if( ExprHasProperty(pExpr, EP_WinFunc) ){ @@ -145529,7 +147806,7 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ pExpr->op = TK_AGG_COLUMN; pExpr->iTable = pCol->iTable; pExpr->iColumn = pCol->iColumn; - ExprClearProperty(pExpr, EP_Skip|EP_Collate); + ExprClearProperty(pExpr, EP_Skip|EP_Collate|EP_Unlikely); return WRC_Prune; } @@ -145560,7 +147837,7 @@ static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ ** * The aCol[] and aFunc[] arrays may be modified ** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used ** -** After clling this routine: +** After calling this routine: ** ** * The aCol[] and aFunc[] arrays are fixed ** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used @@ -145609,6 +147886,32 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ pFunc->pFunc->zName)); } } + if( pFunc->iOBTab>=0 ){ + ExprList *pOBList; + KeyInfo *pKeyInfo; + int nExtra = 0; + assert( pFunc->pFExpr->pLeft!=0 ); + assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pFunc->pFExpr->pLeft) ); + pOBList = pFunc->pFExpr->pLeft->x.pList; + if( !pFunc->bOBUnique ){ + nExtra++; /* One extra column for the OP_Sequence */ + } + if( pFunc->bOBPayload ){ + /* extra columns for the function arguments */ + assert( ExprUseXList(pFunc->pFExpr) ); + nExtra += pFunc->pFExpr->x.pList->nExpr; + } + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); + if( !pFunc->bOBUnique && pParse->nErr==0 ){ + pKeyInfo->nKeyField++; + } + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iOBTab, pOBList->nExpr+nExtra, 0, + (char*)pKeyInfo, P4_KEYINFO); + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(ORDER BY)", + pFunc->pFunc->zName)); + } } } @@ -145624,13 +147927,46 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); pList = pF->pFExpr->x.pList; + if( pF->iOBTab>=0 ){ + /* For an ORDER BY aggregate, calls to OP_AggStep where deferred and + ** all content was stored in emphermal table pF->iOBTab. Extract that + ** content now (in ORDER BY order) and make all calls to OP_AggStep + ** before doing the OP_AggFinal call. */ + int iTop; /* Start of loop for extracting columns */ + int nArg; /* Number of columns to extract */ + int nKey; /* Key columns to be skipped */ + int regAgg; /* Extract into this array */ + int j; /* Loop counter */ + + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + + if( pF->bOBPayload==0 ){ + nKey = 0; + }else{ + assert( pF->pFExpr->pLeft!=0 ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + assert( pF->pFExpr->pLeft->x.pList!=0 ); + nKey = pF->pFExpr->pLeft->x.pList->nExpr; + if( ALWAYS(!pF->bOBUnique) ) nKey++; + } + iTop = sqlite3VdbeAddOp1(v, OP_Rewind, pF->iOBTab); VdbeCoverage(v); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j); + } + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, iTop); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + } sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } - /* ** Generate code that will update the accumulator memory cells for an ** aggregate based on the current cursor position. @@ -145639,6 +147975,13 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator ** registers if register regAcc contains 0. The caller will take care ** of setting and clearing regAcc. +** +** For an ORDER BY aggregate, the actual accumulator memory cell update +** is deferred until after all input rows have been received, so that they +** can be run in the requested order. In that case, instead of invoking +** OP_AggStep to update the accumulator, just add the arguments that would +** have been passed into OP_AggStep into the sorting ephemeral table +** (along with the appropriate sort key). */ static void updateAccumulator( Parse *pParse, @@ -145660,6 +148003,8 @@ static void updateAccumulator( int nArg; int addrNext = 0; int regAgg; + int regAggSz = 0; + int regDistinct = 0; ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); assert( !IsWindowFunc(pF->pFExpr) ); @@ -145686,9 +148031,44 @@ static void updateAccumulator( addrNext = sqlite3VdbeMakeLabel(pParse); sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); } - if( pList ){ + if( pF->iOBTab>=0 ){ + /* Instead of invoking AggStep, we must push the arguments that would + ** have been passed to AggStep onto the sorting table. */ + int jj; /* Registered used so far in building the record */ + ExprList *pOBList; /* The ORDER BY clause */ + assert( pList!=0 ); + nArg = pList->nExpr; + assert( nArg>0 ); + assert( pF->pFExpr->pLeft!=0 ); + assert( pF->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + pOBList = pF->pFExpr->pLeft->x.pList; + assert( pOBList!=0 ); + assert( pOBList->nExpr>0 ); + regAggSz = pOBList->nExpr; + if( !pF->bOBUnique ){ + regAggSz++; /* One register for OP_Sequence */ + } + if( pF->bOBPayload ){ + regAggSz += nArg; + } + regAggSz++; /* One extra register to hold result of MakeRecord */ + regAgg = sqlite3GetTempRange(pParse, regAggSz); + regDistinct = regAgg; + sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP); + jj = pOBList->nExpr; + if( !pF->bOBUnique ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj); + jj++; + } + if( pF->bOBPayload ){ + regDistinct = regAgg+jj; + sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); + } + }else if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); + regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; @@ -145699,26 +148079,37 @@ static void updateAccumulator( addrNext = sqlite3VdbeMakeLabel(pParse); } pF->iDistinct = codeDistinct(pParse, eDistinctType, - pF->iDistinct, addrNext, pList, regAgg); - } - if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - CollSeq *pColl = 0; - struct ExprList_item *pItem; - int j; - assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ - for(j=0, pItem=pList->a; !pColl && jpExpr); - } - if( !pColl ){ - pColl = pParse->db->pDfltColl; + pF->iDistinct, addrNext, pList, regDistinct); + } + if( pF->iOBTab>=0 ){ + /* Insert a new record into the ORDER BY table */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regAgg, regAggSz-1, + regAgg+regAggSz-1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pF->iOBTab, regAgg+regAggSz-1, + regAgg, regAggSz-1); + sqlite3ReleaseTempRange(pParse, regAgg, regAggSz); + }else{ + /* Invoke the AggStep function */ + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); + } + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, + (char *)pColl, P4_COLLSEQ); } - if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); - sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } @@ -146214,22 +148605,59 @@ SQLITE_PRIVATE int sqlite3Select( ** to a real table */ assert( pTab!=0 ); - /* Convert LEFT JOIN into JOIN if there are terms of the right table - ** of the LEFT JOIN used in the WHERE clause. + /* Try to simplify joins: + ** + ** LEFT JOIN -> JOIN + ** RIGHT JOIN -> JOIN + ** FULL JOIN -> RIGHT JOIN + ** + ** If terms of the i-th table are used in the WHERE clause in such a + ** way that the i-th table cannot be the NULL row of a join, then + ** perform the appropriate simplification. This is called + ** "OUTER JOIN strength reduction" in the SQLite documentation. */ - if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==JT_LEFT - && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) + if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 + && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor, + pItem->fg.jointype & JT_LTORJ) && OptimizationEnabled(db, SQLITE_SimplifyJoin) ){ - TREETRACE(0x1000,pParse,p, - ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); - pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + if( pItem->fg.jointype & JT_LEFT ){ + if( pItem->fg.jointype & JT_RIGHT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to RIGHT-JOIN on term %d\n",i)); + pItem->fg.jointype &= ~JT_LEFT; + }else{ + TREETRACE(0x1000,pParse,p, + ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); + pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + } + } + if( pItem->fg.jointype & JT_LTORJ ){ + for(j=i+1; jnSrc; j++){ + SrcItem *pI2 = &pTabList->a[j]; + if( pI2->fg.jointype & JT_RIGHT ){ + if( pI2->fg.jointype & JT_LEFT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to LEFT-JOIN on term %d\n",j)); + pI2->fg.jointype &= ~JT_RIGHT; + }else{ + TREETRACE(0x1000,pParse,p, + ("RIGHT-JOIN simplifies to JOIN on term %d\n",j)); + pI2->fg.jointype &= ~(JT_RIGHT|JT_OUTER); + } + } + } + for(j=pTabList->nSrc-1; j>=i; j--){ + pTabList->a[j].fg.jointype &= ~JT_LTORJ; + if( pTabList->a[j].fg.jointype & JT_RIGHT ) break; + } + } assert( pItem->iCursor>=0 ); unsetJoinExpr(p->pWhere, pItem->iCursor, pTabList->a[0].fg.jointype & JT_LTORJ); } - /* No futher action if this term of the FROM clause is not a subquery */ + /* No further action if this term of the FROM clause is not a subquery */ if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of @@ -146240,6 +148668,14 @@ SQLITE_PRIVATE int sqlite3Select( goto select_end; } + /* Do not attempt the usual optimizations (flattening and ORDER BY + ** elimination) on a MATERIALIZED common table expression because + ** a MATERIALIZED common table expression is an optimization fence. + */ + if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ){ + continue; + } + /* Do not try to flatten an aggregate subquery. ** ** Flattening an aggregate subquery is only possible if the outer query @@ -146269,6 +148705,8 @@ SQLITE_PRIVATE int sqlite3Select( ** (a) The outer query has a different ORDER BY clause ** (b) The subquery is part of a join ** See forum post 062d576715d277c8 + ** + ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled. */ if( pSub->pOrderBy!=0 && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ @@ -146483,7 +148921,7 @@ SQLITE_PRIVATE int sqlite3Select( }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, - ** the make the pItem->iCursor be a copy of the ephemerial table that + ** the make the pItem->iCursor be a copy of the ephemeral table that ** holds the result of the materialization. */ CteUse *pCteUse = pItem->u2.pCteUse; sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); @@ -146866,7 +149304,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int addr1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparison jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -146957,9 +149395,13 @@ SQLITE_PRIVATE int sqlite3Select( int nCol; int nGroupBy; - explainTempTable(pParse, +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp; /* Address of OP_Explain instruction */ +#endif + ExplainQueryPlan2(addrExp, (pParse, 0, "USE TEMP B-TREE FOR %s", (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? - "DISTINCT" : "GROUP BY"); + "DISTINCT" : "GROUP BY" + )); groupBySort = 1; nGroupBy = pGroupBy->nExpr; @@ -146984,18 +149426,23 @@ SQLITE_PRIVATE int sqlite3Select( } pAggInfo->directMode = 0; regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, 0, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); + sqlite3VdbeScanStatusRange(v, addrExp, sqlite3VdbeCurrentAddr(v)-2, -1); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, sqlite3VdbeCurrentAddr(v), 0); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); pAggInfo->useSortingIdx = 1; + sqlite3VdbeScanStatusRange(v, addrExp, -1, sortPTab); + sqlite3VdbeScanStatusRange(v, addrExp, -1, pAggInfo->sortingIdx); } /* If there are entries in pAgggInfo->aFunc[] that contain subexpressions @@ -147723,6 +150170,10 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables"); goto trigger_orphan_error; } + if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){ + sqlite3ErrorMsg(pParse, "cannot create triggers on shadow tables"); + goto trigger_orphan_error; + } /* Check that the trigger name is not reserved and that no trigger of the ** specified name exists */ @@ -148506,10 +150957,17 @@ static void codeReturningTrigger( SrcList sFrom; assert( v!=0 ); - assert( pParse->bReturning ); + if( !pParse->bReturning ){ + /* This RETURNING trigger must be for a different statement as + ** this statement lacks a RETURNING clause. */ + return; + } assert( db->pParse==pParse ); pReturning = pParse->u1.pReturning; - assert( pTrigger == &(pReturning->retTrig) ); + if( pTrigger != &(pReturning->retTrig) ){ + /* This RETURNING trigger is for a different statement */ + return; + } memset(&sSelect, 0, sizeof(sSelect)); memset(&sFrom, 0, sizeof(sFrom)); sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); @@ -149246,7 +151704,7 @@ static void updateFromSelect( assert( pTabList->nSrc>1 ); if( pSrc ){ - pSrc->a[0].fg.notCte = 1; + assert( pSrc->a[0].fg.notCte ); pSrc->a[0].iCursor = -1; pSrc->a[0].pTab->nTabRef--; pSrc->a[0].pTab = 0; @@ -149763,7 +152221,7 @@ SQLITE_PRIVATE void sqlite3Update( && !hasFK && !chngKey && !bReplace - && (sNC.ncFlags & NC_Subquery)==0 + && (pWhere==0 || !ExprHasProperty(pWhere, EP_Subquery)) ){ flags |= WHERE_ONEPASS_MULTIROW; } @@ -149835,6 +152293,8 @@ SQLITE_PRIVATE void sqlite3Update( if( !isView ){ int addrOnce = 0; + int iNotUsed1 = 0; + int iNotUsed2 = 0; /* Open every index that needs updating. */ if( eOnePass!=ONEPASS_OFF ){ @@ -149846,7 +152306,7 @@ SQLITE_PRIVATE void sqlite3Update( addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, - aToOpen, 0, 0); + aToOpen, &iNotUsed1, &iNotUsed2); if( addrOnce ){ sqlite3VdbeJumpHereOrPopInst(v, addrOnce); } @@ -150137,8 +152597,10 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); } - sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, - TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + if( pTrigger ){ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + } /* Repeat the above with the next record to be updated, until ** all record selected by the WHERE clause have been updated. @@ -150233,7 +152695,7 @@ static void updateVirtualTable( int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ int regArg; /* First register in VUpdate arg array */ int regRec; /* Register in which to assemble record */ - int regRowid; /* Register for ephem table rowid */ + int regRowid; /* Register for ephemeral table rowid */ int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ int eOnePass; /* True to use onepass strategy */ @@ -150277,7 +152739,9 @@ static void updateVirtualTable( sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0) ); }else{ - pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i)); + Expr *pRowExpr = exprRowColumn(pParse, i); + if( pRowExpr ) pRowExpr->op2 = OPFLAG_NOCHNG; + pList = sqlite3ExprListAppend(pParse, pList, pRowExpr); } } @@ -150354,7 +152818,7 @@ static void updateVirtualTable( sqlite3WhereEnd(pWInfo); } - /* Begin scannning through the ephemeral table. */ + /* Begin scanning through the ephemeral table. */ addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); /* Extract arguments from the current row of the ephemeral table and @@ -150562,7 +153026,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( pExpr = &sCol[0]; } for(jj=0; jja[jj].pExpr,pExpr,iCursor)<2 ){ + if( sqlite3ExprCompare(0,pTarget->a[jj].pExpr,pExpr,iCursor)<2 ){ break; /* Column ii of the index matches column jj of target */ } } @@ -150911,7 +153375,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( ** (possibly synchronous) transaction opened on the main database before ** sqlite3BtreeCopyFile() is called. ** - ** An optimisation would be to use a non-journaled pager. + ** An optimization would be to use a non-journaled pager. ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but ** that actually made the VACUUM run slower. Very little journalling ** actually occurs when doing a vacuum since the vacuum_db is initially @@ -151600,7 +154064,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ ** the information we've collected. ** ** The VM register number pParse->regRowid holds the rowid of an - ** entry in the sqlite_schema table tht was created for this vtab + ** entry in the sqlite_schema table that was created for this vtab ** by sqlite3StartTable(). */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -151940,7 +154404,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3_mutex_enter(db->mutex); pCtx = db->pVtabCtx; if( !pCtx || pCtx->bDeclared ){ - sqlite3Error(db, SQLITE_MISUSE); + sqlite3Error(db, SQLITE_MISUSE_BKPT); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } @@ -152344,7 +154808,7 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ ** ** An eponymous virtual table instance is one that is named after its ** module, and more importantly, does not require a CREATE VIRTUAL TABLE -** statement in order to come into existance. Eponymous virtual table +** statement in order to come into existence. Eponymous virtual table ** instances always exist. They cannot be DROP-ed. ** ** Any virtual table module for which xConnect and xCreate are the same @@ -152535,7 +154999,7 @@ typedef struct WhereRightJoin WhereRightJoin; /* ** This object is a header on a block of allocated memory that will be -** automatically freed when its WInfo oject is destructed. +** automatically freed when its WInfo object is destructed. */ struct WhereMemBlock { WhereMemBlock *pNext; /* Next block in the chain */ @@ -152596,7 +155060,7 @@ struct WhereLevel { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ int iBase; /* Base register of multi-key index record */ - int nPrefix; /* Number of prior entires in the key */ + int nPrefix; /* Number of prior entries in the key */ u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ @@ -152846,7 +155310,7 @@ struct WhereClause { int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ int nBase; /* Number of terms through the last non-Virtual */ - WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ + WhereTerm *a; /* Each a[] describes a term of the WHERE clause */ #if defined(SQLITE_SMALL_STACK) WhereTerm aStatic[1]; /* Initial static space for a[] */ #else @@ -153131,7 +155595,7 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ -#define WHERE_VIEWSCAN 0x02000000 /* A full-scan of a VIEW or subquery */ + /* 0x02000000 -- available for reuse */ #define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ @@ -153434,6 +155898,12 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( if( wsFlags & WHERE_INDEXED ){ sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); } + }else{ + int addr = pSrclist->a[pLvl->iFrom].addrFillSub; + VdbeOp *pOp = sqlite3VdbeGetOp(v, addr-1); + assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine ); + assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr ); + sqlite3VdbeScanStatusRange(v, addrExplain, addr, pOp->p2-1); } } } @@ -153931,7 +156401,7 @@ static int codeAllEqualityTerms( /* Figure out how many memory cells we will need then allocate them. */ regBase = pParse->nMem + 1; - nReg = pLoop->u.btree.nEq + nExtraReg; + nReg = nEq + nExtraReg; pParse->nMem += nReg; zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); @@ -153978,9 +156448,6 @@ static int codeAllEqualityTerms( sqlite3VdbeAddOp2(v, OP_Copy, r1, regBase+j); } } - } - for(j=nSkip; jaLTerm[j]; if( pTerm->eOperator & WO_IN ){ if( pTerm->pExpr->flags & EP_xIsSelect ){ /* No affinity ever needs to be (or should be) applied to a value @@ -154123,7 +156590,7 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ ** 2) transform the expression node to a TK_REGISTER node that reads ** from the newly populated register. ** -** Also, if the node is a TK_COLUMN that does access the table idenified +** Also, if the node is a TK_COLUMN that does access the table identified ** by pCCurHint.iTabCur, and an index is being used (which we will ** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into ** an access of the index rather than the original table. @@ -154741,7 +157208,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( }; assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + assert( TK_GE==TK_GT+3 ); /* ... is correct. */ assert( (pStart->wtFlags & TERM_VNULL)==0 ); testcase( pStart->wtFlags & TERM_VIRTUAL ); @@ -155921,7 +158388,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( ** the WHERE clause of SQL statements. ** ** This file was originally part of where.c but was split out to improve -** readability and editabiliity. This file contains utility routines for +** readability and editability. This file contains utility routines for ** analyzing Expr objects in the WHERE clause. */ /* #include "sqliteInt.h" */ @@ -156137,7 +158604,7 @@ static int isLikeOrGlob( ** range search. The third is because the caller assumes that the pattern ** consists of at least one character after all escapes have been ** removed. */ - if( cnt!=0 && 255!=(u8)z[cnt-1] && (cnt>1 || z[0]!=wc[3]) ){ + if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && 255!=(u8)z[cnt-1] ){ Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ @@ -156710,7 +159177,7 @@ static void exprAnalyzeOrTerm( pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the ** chngToIN set but t1 is not. This term will be either preceded - ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** or followed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); @@ -156972,8 +159439,8 @@ static void exprAnalyze( WhereTerm *pTerm; /* The term to be analyzed */ WhereMaskSet *pMaskSet; /* Set of table index masks */ Expr *pExpr; /* The expression to be analyzed */ - Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ - Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask prereqLeft; /* Prerequisites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequisites of pExpr */ Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ @@ -158923,13 +161390,17 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ int iCur; /* Cursor for table getting the filter */ IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ + IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */ saved_pIdxEpr = pParse->pIdxEpr; + saved_pIdxPartExpr = pParse->pIdxPartExpr; pParse->pIdxEpr = 0; + pParse->pIdxPartExpr = 0; assert( pLoop!=0 ); assert( v!=0 ); assert( pLoop->wsFlags & WHERE_BLOOMFILTER ); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ); addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); do{ @@ -159019,6 +161490,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( }while( iLevel < pWInfo->nLevel ); sqlite3VdbeJumpHere(v, addrOnce); pParse->pIdxEpr = saved_pIdxEpr; + pParse->pIdxPartExpr = saved_pIdxPartExpr; } @@ -159534,7 +162006,7 @@ SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCo ** Value pLoop->nOut is currently set to the estimated number of rows ** visited for scanning (a=? AND b=?). This function reduces that estimate ** by some factor to account for the (c BETWEEN ? AND ?) expression based -** on the stat4 data for the index. this scan will be peformed multiple +** on the stat4 data for the index. this scan will be performed multiple ** times (once for each (a,b) combination that matches a=?) is dealt with ** by the caller. ** @@ -160289,7 +162761,7 @@ static WhereLoop **whereLoopFindLesser( ** rSetup. Call this SETUP-INVARIANT */ assert( p->rSetup>=pTemplate->rSetup ); - /* Any loop using an appliation-defined index (or PRIMARY KEY or + /* Any loop using an application-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 @@ -160316,7 +162788,7 @@ static WhereLoop **whereLoopFindLesser( /* If pTemplate is always better than p, then cause p to be overwritten ** with pTemplate. pTemplate is better than p if: - ** (1) pTemplate has no more dependences than p, and + ** (1) pTemplate has no more dependencies than p, and ** (2) pTemplate has an equal or lower cost than p. */ if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ @@ -160434,7 +162906,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ }else{ /* We will be overwriting WhereLoop p[]. But before we do, first ** go through the rest of the list and delete any other entries besides - ** p[] that are also supplated by pTemplate */ + ** p[] that are also supplanted by pTemplate */ WhereLoop **ppTail = &p->pNextLoop; WhereLoop *pToDel; while( *ppTail ){ @@ -160634,7 +163106,7 @@ static int whereRangeVectorLen( } /* -** Adjust the cost C by the costMult facter T. This only occurs if +** Adjust the cost C by the costMult factor T. This only occurs if ** compiled with -DSQLITE_ENABLE_COSTMULT */ #ifdef SQLITE_ENABLE_COSTMULT @@ -160661,7 +163133,7 @@ static int whereLoopAddBtreeIndex( Index *pProbe, /* An index on pSrc */ LogEst nInMul /* log(Number of iterations due to IN) */ ){ - WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection malloc context */ WhereLoop *pNew; /* Template WhereLoop under construction */ @@ -160971,7 +163443,7 @@ static int whereLoopAddBtreeIndex( assert( pSrc->pTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior - ** pages are small. Thuse szIdxRow gives a good estimate of seek cost. + ** pages are small. Thus szIdxRow gives a good estimate of seek cost. ** But the leaf pages are full-size, so pProbe->szIdxRow would badly ** under-estimate the scanning cost. */ rCostIdx = pNew->nOut + 16; @@ -161278,6 +163750,100 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( return rc; } +/* +** This is an sqlite3ParserAddCleanup() callback that is invoked to +** free the Parse->pIdxEpr list when the Parse object is destroyed. +*/ +static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ + IndexedExpr **pp = (IndexedExpr**)pObject; + while( *pp!=0 ){ + IndexedExpr *p = *pp; + *pp = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } +} + +/* +** This function is called for a partial index - one with a WHERE clause - in +** two scenarios. In both cases, it determines whether or not the WHERE +** clause on the index implies that a column of the table may be safely +** replaced by a constant expression. For example, in the following +** SELECT: +** +** CREATE INDEX i1 ON t1(b, c) WHERE a=; +** SELECT a, b, c FROM t1 WHERE a= AND b=?; +** +** The "a" in the select-list may be replaced by , iff: +** +** (a) is a constant expression, and +** (b) The (a=) comparison uses the BINARY collation sequence, and +** (c) Column "a" has an affinity other than NONE or BLOB. +** +** If argument pItem is NULL, then pMask must not be NULL. In this case this +** function is being called as part of determining whether or not pIdx +** is a covering index. This function clears any bits in (*pMask) +** corresponding to columns that may be replaced by constants as described +** above. +** +** Otherwise, if pItem is not NULL, then this function is being called +** as part of coding a loop that uses index pIdx. In this case, add entries +** to the Parse.pIdxPartExpr list for each column that can be replaced +** by a constant. +*/ +static void wherePartIdxExpr( + Parse *pParse, /* Parse context */ + Index *pIdx, /* Partial index being processed */ + Expr *pPart, /* WHERE clause being processed */ + Bitmask *pMask, /* Mask to clear bits in */ + int iIdxCur, /* Cursor number for index */ + SrcItem *pItem /* The FROM clause entry for the table */ +){ + assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 ); + assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) ); + + if( pPart->op==TK_AND ){ + wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem); + pPart = pPart->pLeft; + } + + if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){ + Expr *pLeft = pPart->pLeft; + Expr *pRight = pPart->pRight; + u8 aff; + + if( pLeft->op!=TK_COLUMN ) return; + if( !sqlite3ExprIsConstant(pRight) ) return; + if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; + if( pLeft->iColumn<0 ) return; + aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; + if( aff>=SQLITE_AFF_TEXT ){ + if( pItem ){ + sqlite3 *db = pParse->db; + IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p)); + if( p ){ + int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0; + p->pExpr = sqlite3ExprDup(db, pRight, 0); + p->iDataCur = pItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = pLeft->iColumn; + p->bMaybeNullRow = bNullRow; + p->pIENext = pParse->pIdxPartExpr; + p->aff = aff; + pParse->pIdxPartExpr = p; + if( p->pIENext==0 ){ + void *pArg = (void*)&pParse->pIdxPartExpr; + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); + } + } + }else if( pLeft->iColumn<(BMS-1) ){ + *pMask &= ~((Bitmask)1 << pLeft->iColumn); + } + } + } +} + + /* ** Add all WhereLoop objects for a single table of the join where the table ** is identified by pBuilder->pNew->iTab. That table is guaranteed to be @@ -161316,7 +163882,7 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex( */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mPrereq /* Extra prerequesites for using this table */ + Bitmask mPrereq /* Extra prerequisites for using this table */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ @@ -161481,9 +164047,6 @@ static int whereLoopAddBtree( #else pNew->rRun = rSize + 16; #endif - if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){ - pNew->wsFlags |= WHERE_VIEWSCAN; - } ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); @@ -161496,6 +164059,11 @@ static int whereLoopAddBtree( pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ m = pSrc->colUsed & pProbe->colNotIdxed; + if( pProbe->pPartIdxWhere ){ + wherePartIdxExpr( + pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0 + ); + } pNew->wsFlags = WHERE_INDEXED; if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); @@ -161823,7 +164391,7 @@ static int whereLoopAddVirtualOne( ** ** Return a pointer to the collation name: ** -** 1. If there is an explicit COLLATE operator on the constaint, return it. +** 1. If there is an explicit COLLATE operator on the constraint, return it. ** ** 2. Else, if the column has an alternative collation, return that. ** @@ -161878,7 +164446,7 @@ SQLITE_API int sqlite3_vtab_rhs_value( sqlite3_value *pVal = 0; int rc = SQLITE_OK; if( iCons<0 || iCons>=pIdxInfo->nConstraint ){ - rc = SQLITE_MISUSE; /* EV: R-30545-25046 */ + rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */ }else{ if( pH->aRhs[iCons]==0 ){ WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset]; @@ -162784,7 +165352,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** For joins of 3 or more tables, track the 10 best paths */ mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); - WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n", + nRowEst, pParse->nQueryLoop)); /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this ** case the purpose of this call is to estimate the number of rows returned @@ -162887,7 +165456,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ); } /* TUNING: Add a small extra penalty (3) to sorting as an - ** extra encouragment to the query planner to select a plan + ** extra encouragement to the query planner to select a plan ** where the rows emerge in the correct order without any sorting ** required. */ rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; @@ -162901,13 +165470,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ } - /* TUNING: A full-scan of a VIEW or subquery in the outer loop - ** is not so bad. */ - if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ - rCost += -10; - nOut += -30; - } - /* Check to see if pWLoop should be added to the set of ** mxChoice best-so-far paths. ** @@ -163457,20 +166019,6 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( } } -/* -** This is an sqlite3ParserAddCleanup() callback that is invoked to -** free the Parse->pIdxEpr list when the Parse object is destroyed. -*/ -static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ - Parse *pParse = (Parse*)pObject; - while( pParse->pIdxEpr!=0 ){ - IndexedExpr *p = pParse->pIdxEpr; - pParse->pIdxEpr = p->pIENext; - sqlite3ExprDelete(db, p->pExpr); - sqlite3DbFreeNN(db, p); - } -} - /* ** The index pIdx is used by a query and contains one or more expressions. ** In other words pIdx is an index on an expression. iIdxCur is the cursor @@ -163532,7 +166080,30 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( #endif pParse->pIdxEpr = p; if( p->pIENext==0 ){ - sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); + void *pArg = (void*)&pParse->pIdxEpr; + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); + } + } +} + +/* +** Set the reverse-scan order mask to one for all tables in the query +** with the exception of MATERIALIZED common table expressions that have +** their own internal ORDER BY clauses. +** +** This implements the PRAGMA reverse_unordered_selects=ON setting. +** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER). +*/ +static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ + int ii; + for(ii=0; iipTabList->nSrc; ii++){ + SrcItem *pItem = &pWInfo->pTabList->a[ii]; + if( !pItem->fg.isCte + || pItem->u2.pCteUse->eM10d!=M10d_Yes + || NEVER(pItem->pSelect==0) + || pItem->pSelect->pOrderBy==0 + ){ + pWInfo->revMask |= MASKBIT(ii); } } } @@ -163595,7 +166166,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( ** ** OUTER JOINS ** -** An outer join of tables t1 and t2 is conceptally coded as follows: +** An outer join of tables t1 and t2 is conceptually coded as follows: ** ** foreach row1 in t1 do ** flag = 0 @@ -163750,7 +166321,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** ** The N-th term of the FROM clause is assigned a bitmask of 1<nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } + + /* TUNING: Assume that a DISTINCT clause on a subquery reduces + ** the output size by a factor of 8 (LogEst -30). + */ + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ + WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n", + pWInfo->nRowOut, pWInfo->nRowOut-30)); + pWInfo->nRowOut -= 30; + } + } + assert( pWInfo->pTabList!=0 ); if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ - pWInfo->revMask = ALLBITS; + whereReverseScanOrder(pWInfo); } if( pParse->nErr ){ goto whereBeginError; @@ -164002,6 +166584,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) && !IsVirtual(pTabList->a[0].pTab) && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) + && OptimizationEnabled(db, SQLITE_OnePass) )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ @@ -164110,6 +166693,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); } + if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){ + wherePartIdxExpr( + pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem + ); + } } pLevel->iIdxCur = iIndexCur; assert( pIx!=0 ); @@ -164731,7 +167319,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** ** These are the same built-in window functions supported by Postgres. ** Although the behaviour of aggregate window functions (functions that -** can be used as either aggregates or window funtions) allows them to +** can be used as either aggregates or window functions) allows them to ** be implemented using an API, built-in window functions are much more ** esoteric. Additionally, some window functions (e.g. nth_value()) ** may only be implemented by caching the entire partition in memory. @@ -165261,7 +167849,7 @@ static Window *windowFind(Parse *pParse, Window *pList, const char *zName){ ** is the Window object representing the associated OVER clause. This ** function updates the contents of pWin as follows: ** -** * If the OVER clause refered to a named window (as in "max(x) OVER win"), +** * If the OVER clause referred to a named window (as in "max(x) OVER win"), ** search list pList for a matching WINDOW definition, and update pWin ** accordingly. If no such WINDOW clause can be found, leave an error ** in pParse. @@ -165882,7 +168470,7 @@ SQLITE_PRIVATE Window *sqlite3WindowAssemble( } /* -** Window *pWin has just been created from a WINDOW clause. Tokne pBase +** Window *pWin has just been created from a WINDOW clause. Token pBase ** is the base window. Earlier windows from the same WINDOW clause are ** stored in the linked list starting at pWin->pNextWin. This function ** either updates *pWin according to the base specification, or else @@ -165926,8 +168514,9 @@ SQLITE_PRIVATE void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){ if( p ){ assert( p->op==TK_FUNCTION ); assert( pWin ); + assert( ExprIsFullSize(p) ); p->y.pWin = pWin; - ExprSetProperty(p, EP_WinFunc); + ExprSetProperty(p, EP_WinFunc|EP_FullSize); pWin->pOwner = p; if( (p->flags & EP_Distinct) && pWin->eFrmType!=TK_FILTER ){ sqlite3ErrorMsg(pParse, @@ -166188,7 +168777,7 @@ struct WindowCsrAndReg { ** ** (ORDER BY a, b GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING) ** -** The windows functions implmentation caches the input rows in a temp +** The windows functions implementation caches the input rows in a temp ** table, sorted by "a, b" (it actually populates the cache lazily, and ** aggressively removes rows once they are no longer required, but that's ** a mere detail). It keeps three cursors open on the temp table. One @@ -167197,7 +169786,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){ ** ** For the most part, the patterns above are adapted to support UNBOUNDED by ** assuming that it is equivalent to "infinity PRECEDING/FOLLOWING" and -** CURRENT ROW by assuming that it is equivilent to "0 PRECEDING/FOLLOWING". +** CURRENT ROW by assuming that it is equivalent to "0 PRECEDING/FOLLOWING". ** This is optimized of course - branches that will never be taken and ** conditions that are always true are omitted from the VM code. The only ** exceptional case is: @@ -167476,7 +170065,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( } /* Allocate registers for the array of values from the sub-query, the - ** samve values in record form, and the rowid used to insert said record + ** same values in record form, and the rowid used to insert said record ** into the ephemeral table. */ regNew = pParse->nMem+1; pParse->nMem += nInput; @@ -167717,7 +170306,8 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( /************** End of window.c **********************************************/ /************** Begin file parse.c *******************************************/ /* This file is automatically generated by Lemon from input grammar -** source file "parse.y". */ +** source file "parse.y". +*/ /* ** 2001-09-15 ** @@ -167734,7 +170324,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( ** The canonical source code to this file ("parse.y") is a Lemon grammar ** file that specifies the input grammar and actions to take while parsing. ** That input file is processed by Lemon to generate a C-language -** implementation of a parser for the given grammer. You might be reading +** implementation of a parser for the given grammar. You might be reading ** this comment as part of the translated C-code. Edits should be made ** to the original parse.y sources. */ @@ -168228,18 +170818,18 @@ typedef union { #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 575 -#define YYNRULE 403 +#define YYNSTATE 579 +#define YYNRULE 405 #define YYNRULE_WITH_ACTION 340 #define YYNTOKEN 185 -#define YY_MAX_SHIFT 574 -#define YY_MIN_SHIFTREDUCE 833 -#define YY_MAX_SHIFTREDUCE 1235 -#define YY_ERROR_ACTION 1236 -#define YY_ACCEPT_ACTION 1237 -#define YY_NO_ACTION 1238 -#define YY_MIN_REDUCE 1239 -#define YY_MAX_REDUCE 1641 +#define YY_MAX_SHIFT 578 +#define YY_MIN_SHIFTREDUCE 838 +#define YY_MAX_SHIFTREDUCE 1242 +#define YY_ERROR_ACTION 1243 +#define YY_ACCEPT_ACTION 1244 +#define YY_NO_ACTION 1245 +#define YY_MIN_REDUCE 1246 +#define YY_MAX_REDUCE 1650 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -168306,218 +170896,218 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2096) +#define YY_ACTTAB_COUNT (2100) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 568, 208, 568, 118, 115, 229, 568, 118, 115, 229, - /* 10 */ 568, 1310, 377, 1289, 408, 562, 562, 562, 568, 409, - /* 20 */ 378, 1310, 1272, 41, 41, 41, 41, 208, 1520, 71, - /* 30 */ 71, 969, 419, 41, 41, 491, 303, 279, 303, 970, - /* 40 */ 397, 71, 71, 125, 126, 80, 1212, 1212, 1047, 1050, - /* 50 */ 1037, 1037, 123, 123, 124, 124, 124, 124, 476, 409, - /* 60 */ 1237, 1, 1, 574, 2, 1241, 550, 118, 115, 229, - /* 70 */ 317, 480, 146, 480, 524, 118, 115, 229, 529, 1323, - /* 80 */ 417, 523, 142, 125, 126, 80, 1212, 1212, 1047, 1050, - /* 90 */ 1037, 1037, 123, 123, 124, 124, 124, 124, 118, 115, - /* 100 */ 229, 327, 122, 122, 122, 122, 121, 121, 120, 120, - /* 110 */ 120, 119, 116, 444, 284, 284, 284, 284, 442, 442, - /* 120 */ 442, 1561, 376, 1563, 1188, 375, 1159, 565, 1159, 565, - /* 130 */ 409, 1561, 537, 259, 226, 444, 101, 145, 449, 316, - /* 140 */ 559, 240, 122, 122, 122, 122, 121, 121, 120, 120, - /* 150 */ 120, 119, 116, 444, 125, 126, 80, 1212, 1212, 1047, - /* 160 */ 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, 142, - /* 170 */ 294, 1188, 339, 448, 120, 120, 120, 119, 116, 444, - /* 180 */ 127, 1188, 1189, 1188, 148, 441, 440, 568, 119, 116, - /* 190 */ 444, 124, 124, 124, 124, 117, 122, 122, 122, 122, - /* 200 */ 121, 121, 120, 120, 120, 119, 116, 444, 454, 113, - /* 210 */ 13, 13, 546, 122, 122, 122, 122, 121, 121, 120, - /* 220 */ 120, 120, 119, 116, 444, 422, 316, 559, 1188, 1189, - /* 230 */ 1188, 149, 1220, 409, 1220, 124, 124, 124, 124, 122, - /* 240 */ 122, 122, 122, 121, 121, 120, 120, 120, 119, 116, - /* 250 */ 444, 465, 342, 1034, 1034, 1048, 1051, 125, 126, 80, - /* 260 */ 1212, 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, - /* 270 */ 124, 124, 1275, 522, 222, 1188, 568, 409, 224, 514, - /* 280 */ 175, 82, 83, 122, 122, 122, 122, 121, 121, 120, - /* 290 */ 120, 120, 119, 116, 444, 1005, 16, 16, 1188, 133, - /* 300 */ 133, 125, 126, 80, 1212, 1212, 1047, 1050, 1037, 1037, - /* 310 */ 123, 123, 124, 124, 124, 124, 122, 122, 122, 122, - /* 320 */ 121, 121, 120, 120, 120, 119, 116, 444, 1038, 546, - /* 330 */ 1188, 373, 1188, 1189, 1188, 252, 1429, 399, 504, 501, - /* 340 */ 500, 111, 560, 566, 4, 924, 924, 433, 499, 340, - /* 350 */ 460, 328, 360, 394, 1233, 1188, 1189, 1188, 563, 568, - /* 360 */ 122, 122, 122, 122, 121, 121, 120, 120, 120, 119, - /* 370 */ 116, 444, 284, 284, 369, 1574, 1600, 441, 440, 154, - /* 380 */ 409, 445, 71, 71, 1282, 565, 1217, 1188, 1189, 1188, - /* 390 */ 85, 1219, 271, 557, 543, 515, 1555, 568, 98, 1218, - /* 400 */ 6, 1274, 472, 142, 125, 126, 80, 1212, 1212, 1047, - /* 410 */ 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, 550, - /* 420 */ 13, 13, 1024, 507, 1220, 1188, 1220, 549, 109, 109, - /* 430 */ 222, 568, 1234, 175, 568, 427, 110, 197, 445, 569, - /* 440 */ 445, 430, 1546, 1014, 325, 551, 1188, 270, 287, 368, - /* 450 */ 510, 363, 509, 257, 71, 71, 543, 71, 71, 359, - /* 460 */ 316, 559, 1606, 122, 122, 122, 122, 121, 121, 120, - /* 470 */ 120, 120, 119, 116, 444, 1014, 1014, 1016, 1017, 27, - /* 480 */ 284, 284, 1188, 1189, 1188, 1154, 568, 1605, 409, 899, - /* 490 */ 190, 550, 356, 565, 550, 935, 533, 517, 1154, 516, - /* 500 */ 413, 1154, 552, 1188, 1189, 1188, 568, 544, 1548, 51, - /* 510 */ 51, 214, 125, 126, 80, 1212, 1212, 1047, 1050, 1037, - /* 520 */ 1037, 123, 123, 124, 124, 124, 124, 1188, 474, 135, - /* 530 */ 135, 409, 284, 284, 1484, 505, 121, 121, 120, 120, - /* 540 */ 120, 119, 116, 444, 1005, 565, 518, 217, 541, 1555, - /* 550 */ 316, 559, 142, 6, 532, 125, 126, 80, 1212, 1212, - /* 560 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, - /* 570 */ 1549, 122, 122, 122, 122, 121, 121, 120, 120, 120, - /* 580 */ 119, 116, 444, 485, 1188, 1189, 1188, 482, 281, 1263, - /* 590 */ 955, 252, 1188, 373, 504, 501, 500, 1188, 340, 570, - /* 600 */ 1188, 570, 409, 292, 499, 955, 874, 191, 480, 316, - /* 610 */ 559, 384, 290, 380, 122, 122, 122, 122, 121, 121, - /* 620 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, - /* 630 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, - /* 640 */ 124, 409, 394, 1132, 1188, 867, 100, 284, 284, 1188, - /* 650 */ 1189, 1188, 373, 1089, 1188, 1189, 1188, 1188, 1189, 1188, - /* 660 */ 565, 455, 32, 373, 233, 125, 126, 80, 1212, 1212, - /* 670 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, - /* 680 */ 1428, 957, 568, 228, 956, 122, 122, 122, 122, 121, - /* 690 */ 121, 120, 120, 120, 119, 116, 444, 1154, 228, 1188, - /* 700 */ 157, 1188, 1189, 1188, 1547, 13, 13, 301, 955, 1228, - /* 710 */ 1154, 153, 409, 1154, 373, 1577, 1172, 5, 369, 1574, - /* 720 */ 429, 1234, 3, 955, 122, 122, 122, 122, 121, 121, - /* 730 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, - /* 740 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, - /* 750 */ 124, 409, 208, 567, 1188, 1025, 1188, 1189, 1188, 1188, - /* 760 */ 388, 850, 155, 1546, 286, 402, 1094, 1094, 488, 568, - /* 770 */ 465, 342, 1315, 1315, 1546, 125, 126, 80, 1212, 1212, - /* 780 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, - /* 790 */ 129, 568, 13, 13, 374, 122, 122, 122, 122, 121, - /* 800 */ 121, 120, 120, 120, 119, 116, 444, 302, 568, 453, - /* 810 */ 528, 1188, 1189, 1188, 13, 13, 1188, 1189, 1188, 1293, - /* 820 */ 463, 1263, 409, 1313, 1313, 1546, 1010, 453, 452, 200, - /* 830 */ 299, 71, 71, 1261, 122, 122, 122, 122, 121, 121, - /* 840 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, - /* 850 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, - /* 860 */ 124, 409, 227, 1069, 1154, 284, 284, 419, 312, 278, - /* 870 */ 278, 285, 285, 1415, 406, 405, 382, 1154, 565, 568, - /* 880 */ 1154, 1191, 565, 1594, 565, 125, 126, 80, 1212, 1212, - /* 890 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, - /* 900 */ 453, 1476, 13, 13, 1530, 122, 122, 122, 122, 121, - /* 910 */ 121, 120, 120, 120, 119, 116, 444, 201, 568, 354, - /* 920 */ 1580, 574, 2, 1241, 838, 839, 840, 1556, 317, 1207, - /* 930 */ 146, 6, 409, 255, 254, 253, 206, 1323, 9, 1191, - /* 940 */ 262, 71, 71, 424, 122, 122, 122, 122, 121, 121, - /* 950 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, - /* 960 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, - /* 970 */ 124, 568, 284, 284, 568, 1208, 409, 573, 313, 1241, - /* 980 */ 349, 1292, 352, 419, 317, 565, 146, 491, 525, 1637, - /* 990 */ 395, 371, 491, 1323, 70, 70, 1291, 71, 71, 240, - /* 1000 */ 1321, 104, 80, 1212, 1212, 1047, 1050, 1037, 1037, 123, - /* 1010 */ 123, 124, 124, 124, 124, 122, 122, 122, 122, 121, - /* 1020 */ 121, 120, 120, 120, 119, 116, 444, 1110, 284, 284, - /* 1030 */ 428, 448, 1519, 1208, 439, 284, 284, 1483, 1348, 311, - /* 1040 */ 474, 565, 1111, 969, 491, 491, 217, 1259, 565, 1532, - /* 1050 */ 568, 970, 207, 568, 1024, 240, 383, 1112, 519, 122, - /* 1060 */ 122, 122, 122, 121, 121, 120, 120, 120, 119, 116, - /* 1070 */ 444, 1015, 107, 71, 71, 1014, 13, 13, 910, 568, - /* 1080 */ 1489, 568, 284, 284, 97, 526, 491, 448, 911, 1322, - /* 1090 */ 1318, 545, 409, 284, 284, 565, 151, 209, 1489, 1491, - /* 1100 */ 262, 450, 55, 55, 56, 56, 565, 1014, 1014, 1016, - /* 1110 */ 443, 332, 409, 527, 12, 295, 125, 126, 80, 1212, - /* 1120 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, - /* 1130 */ 124, 347, 409, 862, 1528, 1208, 125, 126, 80, 1212, - /* 1140 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, - /* 1150 */ 124, 1133, 1635, 474, 1635, 371, 125, 114, 80, 1212, - /* 1160 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, - /* 1170 */ 124, 1489, 329, 474, 331, 122, 122, 122, 122, 121, - /* 1180 */ 121, 120, 120, 120, 119, 116, 444, 203, 1415, 568, - /* 1190 */ 1290, 862, 464, 1208, 436, 122, 122, 122, 122, 121, - /* 1200 */ 121, 120, 120, 120, 119, 116, 444, 553, 1133, 1636, - /* 1210 */ 539, 1636, 15, 15, 890, 122, 122, 122, 122, 121, - /* 1220 */ 121, 120, 120, 120, 119, 116, 444, 568, 298, 538, - /* 1230 */ 1131, 1415, 1553, 1554, 1327, 409, 6, 6, 1165, 1264, - /* 1240 */ 415, 320, 284, 284, 1415, 508, 565, 525, 300, 457, - /* 1250 */ 43, 43, 568, 891, 12, 565, 330, 478, 425, 407, - /* 1260 */ 126, 80, 1212, 1212, 1047, 1050, 1037, 1037, 123, 123, - /* 1270 */ 124, 124, 124, 124, 568, 57, 57, 288, 1188, 1415, - /* 1280 */ 496, 458, 392, 392, 391, 273, 389, 1131, 1552, 847, - /* 1290 */ 1165, 407, 6, 568, 321, 1154, 470, 44, 44, 1551, - /* 1300 */ 1110, 426, 234, 6, 323, 256, 540, 256, 1154, 431, - /* 1310 */ 568, 1154, 322, 17, 487, 1111, 58, 58, 122, 122, - /* 1320 */ 122, 122, 121, 121, 120, 120, 120, 119, 116, 444, - /* 1330 */ 1112, 216, 481, 59, 59, 1188, 1189, 1188, 111, 560, - /* 1340 */ 324, 4, 236, 456, 526, 568, 237, 456, 568, 437, - /* 1350 */ 168, 556, 420, 141, 479, 563, 568, 293, 568, 1091, - /* 1360 */ 568, 293, 568, 1091, 531, 568, 870, 8, 60, 60, - /* 1370 */ 235, 61, 61, 568, 414, 568, 414, 568, 445, 62, - /* 1380 */ 62, 45, 45, 46, 46, 47, 47, 199, 49, 49, - /* 1390 */ 557, 568, 359, 568, 100, 486, 50, 50, 63, 63, - /* 1400 */ 64, 64, 561, 415, 535, 410, 568, 1024, 568, 534, - /* 1410 */ 316, 559, 316, 559, 65, 65, 14, 14, 568, 1024, - /* 1420 */ 568, 512, 930, 870, 1015, 109, 109, 929, 1014, 66, - /* 1430 */ 66, 131, 131, 110, 451, 445, 569, 445, 416, 177, - /* 1440 */ 1014, 132, 132, 67, 67, 568, 467, 568, 930, 471, - /* 1450 */ 1360, 283, 226, 929, 315, 1359, 407, 568, 459, 407, - /* 1460 */ 1014, 1014, 1016, 239, 407, 86, 213, 1346, 52, 52, - /* 1470 */ 68, 68, 1014, 1014, 1016, 1017, 27, 1579, 1176, 447, - /* 1480 */ 69, 69, 288, 97, 108, 1535, 106, 392, 392, 391, - /* 1490 */ 273, 389, 568, 877, 847, 881, 568, 111, 560, 466, - /* 1500 */ 4, 568, 152, 30, 38, 568, 1128, 234, 396, 323, - /* 1510 */ 111, 560, 527, 4, 563, 53, 53, 322, 568, 163, - /* 1520 */ 163, 568, 337, 468, 164, 164, 333, 563, 76, 76, - /* 1530 */ 568, 289, 1508, 568, 31, 1507, 568, 445, 338, 483, - /* 1540 */ 100, 54, 54, 344, 72, 72, 296, 236, 1076, 557, - /* 1550 */ 445, 877, 1356, 134, 134, 168, 73, 73, 141, 161, - /* 1560 */ 161, 1568, 557, 535, 568, 319, 568, 348, 536, 1007, - /* 1570 */ 473, 261, 261, 889, 888, 235, 535, 568, 1024, 568, - /* 1580 */ 475, 534, 261, 367, 109, 109, 521, 136, 136, 130, - /* 1590 */ 130, 1024, 110, 366, 445, 569, 445, 109, 109, 1014, - /* 1600 */ 162, 162, 156, 156, 568, 110, 1076, 445, 569, 445, - /* 1610 */ 410, 351, 1014, 568, 353, 316, 559, 568, 343, 568, - /* 1620 */ 100, 497, 357, 258, 100, 896, 897, 140, 140, 355, - /* 1630 */ 1306, 1014, 1014, 1016, 1017, 27, 139, 139, 362, 451, - /* 1640 */ 137, 137, 138, 138, 1014, 1014, 1016, 1017, 27, 1176, - /* 1650 */ 447, 568, 372, 288, 111, 560, 1018, 4, 392, 392, - /* 1660 */ 391, 273, 389, 568, 1137, 847, 568, 1072, 568, 258, - /* 1670 */ 492, 563, 568, 211, 75, 75, 555, 960, 234, 261, - /* 1680 */ 323, 111, 560, 927, 4, 113, 77, 77, 322, 74, - /* 1690 */ 74, 42, 42, 1369, 445, 48, 48, 1414, 563, 972, - /* 1700 */ 973, 1088, 1087, 1088, 1087, 860, 557, 150, 928, 1342, - /* 1710 */ 113, 1354, 554, 1419, 1018, 1271, 1262, 1250, 236, 1249, - /* 1720 */ 1251, 445, 1587, 1339, 308, 276, 168, 309, 11, 141, - /* 1730 */ 393, 310, 232, 557, 1401, 1024, 335, 291, 1396, 219, - /* 1740 */ 336, 109, 109, 934, 297, 1406, 235, 341, 477, 110, - /* 1750 */ 502, 445, 569, 445, 1389, 1405, 1014, 400, 1289, 365, - /* 1760 */ 223, 1480, 1024, 1479, 1351, 1352, 1350, 1349, 109, 109, - /* 1770 */ 204, 1590, 1228, 558, 265, 218, 110, 205, 445, 569, - /* 1780 */ 445, 410, 387, 1014, 1527, 179, 316, 559, 1014, 1014, - /* 1790 */ 1016, 1017, 27, 230, 1525, 1225, 79, 560, 85, 4, - /* 1800 */ 418, 215, 548, 81, 84, 188, 1402, 173, 181, 461, - /* 1810 */ 451, 35, 462, 563, 183, 1014, 1014, 1016, 1017, 27, - /* 1820 */ 184, 1485, 185, 186, 495, 242, 98, 398, 1408, 36, - /* 1830 */ 1407, 484, 91, 469, 401, 1410, 445, 192, 1474, 246, - /* 1840 */ 1496, 490, 346, 277, 248, 196, 493, 511, 557, 350, - /* 1850 */ 1252, 249, 250, 403, 1309, 1308, 111, 560, 432, 4, - /* 1860 */ 1307, 1300, 93, 1604, 881, 1603, 224, 404, 434, 520, - /* 1870 */ 263, 435, 1573, 563, 1279, 1278, 364, 1024, 306, 1277, - /* 1880 */ 264, 1602, 1559, 109, 109, 370, 1299, 307, 1558, 438, - /* 1890 */ 128, 110, 1374, 445, 569, 445, 445, 546, 1014, 10, - /* 1900 */ 1461, 105, 381, 1373, 34, 571, 99, 1332, 557, 314, - /* 1910 */ 1182, 530, 272, 274, 379, 210, 1331, 547, 385, 386, - /* 1920 */ 275, 572, 1247, 1242, 411, 412, 1512, 165, 178, 1513, - /* 1930 */ 1014, 1014, 1016, 1017, 27, 1511, 1510, 1024, 78, 147, - /* 1940 */ 166, 220, 221, 109, 109, 834, 304, 167, 446, 212, - /* 1950 */ 318, 110, 231, 445, 569, 445, 144, 1086, 1014, 1084, - /* 1960 */ 326, 180, 169, 1207, 182, 334, 238, 913, 241, 1100, - /* 1970 */ 187, 170, 171, 421, 87, 88, 423, 189, 89, 90, - /* 1980 */ 172, 1103, 243, 1099, 244, 158, 18, 245, 345, 247, - /* 1990 */ 1014, 1014, 1016, 1017, 27, 261, 1092, 193, 1222, 489, - /* 2000 */ 194, 37, 366, 849, 494, 251, 195, 506, 92, 19, - /* 2010 */ 498, 358, 20, 503, 879, 361, 94, 892, 305, 159, - /* 2020 */ 513, 39, 95, 1170, 160, 1053, 964, 1139, 96, 174, - /* 2030 */ 1138, 225, 280, 282, 198, 958, 113, 1160, 1156, 260, - /* 2040 */ 21, 22, 23, 1158, 1164, 1163, 1144, 24, 33, 25, - /* 2050 */ 202, 542, 26, 100, 1067, 102, 1054, 103, 7, 1052, - /* 2060 */ 1056, 1109, 1057, 1108, 266, 267, 28, 40, 390, 1019, - /* 2070 */ 861, 112, 29, 564, 1178, 1177, 268, 176, 143, 923, - /* 2080 */ 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, - /* 2090 */ 1238, 1238, 1238, 1238, 269, 1595, + /* 0 */ 572, 210, 572, 119, 116, 231, 572, 119, 116, 231, + /* 10 */ 572, 1317, 379, 1296, 410, 566, 566, 566, 572, 411, + /* 20 */ 380, 1317, 1279, 42, 42, 42, 42, 210, 1529, 72, + /* 30 */ 72, 974, 421, 42, 42, 495, 305, 281, 305, 975, + /* 40 */ 399, 72, 72, 126, 127, 81, 1217, 1217, 1054, 1057, + /* 50 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 480, 411, + /* 60 */ 1244, 1, 1, 578, 2, 1248, 554, 119, 116, 231, + /* 70 */ 319, 484, 147, 484, 528, 119, 116, 231, 533, 1330, + /* 80 */ 419, 527, 143, 126, 127, 81, 1217, 1217, 1054, 1057, + /* 90 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 119, 116, + /* 100 */ 231, 329, 123, 123, 123, 123, 122, 122, 121, 121, + /* 110 */ 121, 120, 117, 448, 286, 286, 286, 286, 446, 446, + /* 120 */ 446, 1568, 378, 1570, 1193, 377, 1164, 569, 1164, 569, + /* 130 */ 411, 1568, 541, 261, 228, 448, 102, 146, 453, 318, + /* 140 */ 563, 242, 123, 123, 123, 123, 122, 122, 121, 121, + /* 150 */ 121, 120, 117, 448, 126, 127, 81, 1217, 1217, 1054, + /* 160 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 143, + /* 170 */ 296, 1193, 341, 452, 121, 121, 121, 120, 117, 448, + /* 180 */ 128, 1193, 1194, 1193, 149, 445, 444, 572, 120, 117, + /* 190 */ 448, 125, 125, 125, 125, 118, 123, 123, 123, 123, + /* 200 */ 122, 122, 121, 121, 121, 120, 117, 448, 458, 114, + /* 210 */ 13, 13, 550, 123, 123, 123, 123, 122, 122, 121, + /* 220 */ 121, 121, 120, 117, 448, 424, 318, 563, 1193, 1194, + /* 230 */ 1193, 150, 1225, 411, 1225, 125, 125, 125, 125, 123, + /* 240 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, + /* 250 */ 448, 469, 344, 1041, 1041, 1055, 1058, 126, 127, 81, + /* 260 */ 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, + /* 270 */ 125, 125, 1282, 526, 224, 1193, 572, 411, 226, 519, + /* 280 */ 177, 83, 84, 123, 123, 123, 123, 122, 122, 121, + /* 290 */ 121, 121, 120, 117, 448, 1010, 16, 16, 1193, 134, + /* 300 */ 134, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, + /* 310 */ 124, 124, 125, 125, 125, 125, 123, 123, 123, 123, + /* 320 */ 122, 122, 121, 121, 121, 120, 117, 448, 1045, 550, + /* 330 */ 1193, 375, 1193, 1194, 1193, 254, 1438, 401, 508, 505, + /* 340 */ 504, 112, 564, 570, 4, 929, 929, 435, 503, 342, + /* 350 */ 464, 330, 362, 396, 1238, 1193, 1194, 1193, 567, 572, + /* 360 */ 123, 123, 123, 123, 122, 122, 121, 121, 121, 120, + /* 370 */ 117, 448, 286, 286, 371, 1581, 1607, 445, 444, 155, + /* 380 */ 411, 449, 72, 72, 1289, 569, 1222, 1193, 1194, 1193, + /* 390 */ 86, 1224, 273, 561, 547, 520, 520, 572, 99, 1223, + /* 400 */ 6, 1281, 476, 143, 126, 127, 81, 1217, 1217, 1054, + /* 410 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 554, + /* 420 */ 13, 13, 1031, 511, 1225, 1193, 1225, 553, 110, 110, + /* 430 */ 224, 572, 1239, 177, 572, 429, 111, 199, 449, 573, + /* 440 */ 449, 432, 1555, 1019, 327, 555, 1193, 272, 289, 370, + /* 450 */ 514, 365, 513, 259, 72, 72, 547, 72, 72, 361, + /* 460 */ 318, 563, 1613, 123, 123, 123, 123, 122, 122, 121, + /* 470 */ 121, 121, 120, 117, 448, 1019, 1019, 1021, 1022, 28, + /* 480 */ 286, 286, 1193, 1194, 1193, 1159, 572, 1612, 411, 904, + /* 490 */ 192, 554, 358, 569, 554, 940, 537, 521, 1159, 437, + /* 500 */ 415, 1159, 556, 1193, 1194, 1193, 572, 548, 548, 52, + /* 510 */ 52, 216, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, + /* 520 */ 1044, 124, 124, 125, 125, 125, 125, 1193, 478, 136, + /* 530 */ 136, 411, 286, 286, 1493, 509, 122, 122, 121, 121, + /* 540 */ 121, 120, 117, 448, 1010, 569, 522, 219, 545, 545, + /* 550 */ 318, 563, 143, 6, 536, 126, 127, 81, 1217, 1217, + /* 560 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, + /* 570 */ 1557, 123, 123, 123, 123, 122, 122, 121, 121, 121, + /* 580 */ 120, 117, 448, 489, 1193, 1194, 1193, 486, 283, 1270, + /* 590 */ 960, 254, 1193, 375, 508, 505, 504, 1193, 342, 574, + /* 600 */ 1193, 574, 411, 294, 503, 960, 879, 193, 484, 318, + /* 610 */ 563, 386, 292, 382, 123, 123, 123, 123, 122, 122, + /* 620 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, + /* 630 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, + /* 640 */ 125, 411, 396, 1139, 1193, 872, 101, 286, 286, 1193, + /* 650 */ 1194, 1193, 375, 1096, 1193, 1194, 1193, 1193, 1194, 1193, + /* 660 */ 569, 459, 33, 375, 235, 126, 127, 81, 1217, 1217, + /* 670 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, + /* 680 */ 1437, 962, 572, 230, 961, 123, 123, 123, 123, 122, + /* 690 */ 122, 121, 121, 121, 120, 117, 448, 1159, 230, 1193, + /* 700 */ 158, 1193, 1194, 1193, 1556, 13, 13, 303, 960, 1233, + /* 710 */ 1159, 154, 411, 1159, 375, 1584, 1177, 5, 371, 1581, + /* 720 */ 431, 1239, 3, 960, 123, 123, 123, 123, 122, 122, + /* 730 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, + /* 740 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, + /* 750 */ 125, 411, 210, 571, 1193, 1032, 1193, 1194, 1193, 1193, + /* 760 */ 390, 855, 156, 1555, 376, 404, 1101, 1101, 492, 572, + /* 770 */ 469, 344, 1322, 1322, 1555, 126, 127, 81, 1217, 1217, + /* 780 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, + /* 790 */ 130, 572, 13, 13, 532, 123, 123, 123, 123, 122, + /* 800 */ 122, 121, 121, 121, 120, 117, 448, 304, 572, 457, + /* 810 */ 229, 1193, 1194, 1193, 13, 13, 1193, 1194, 1193, 1300, + /* 820 */ 467, 1270, 411, 1320, 1320, 1555, 1015, 457, 456, 436, + /* 830 */ 301, 72, 72, 1268, 123, 123, 123, 123, 122, 122, + /* 840 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, + /* 850 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, + /* 860 */ 125, 411, 384, 1076, 1159, 286, 286, 421, 314, 280, + /* 870 */ 280, 287, 287, 461, 408, 407, 1539, 1159, 569, 572, + /* 880 */ 1159, 1196, 569, 409, 569, 126, 127, 81, 1217, 1217, + /* 890 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, + /* 900 */ 457, 1485, 13, 13, 1541, 123, 123, 123, 123, 122, + /* 910 */ 122, 121, 121, 121, 120, 117, 448, 202, 572, 462, + /* 920 */ 1587, 578, 2, 1248, 843, 844, 845, 1563, 319, 409, + /* 930 */ 147, 6, 411, 257, 256, 255, 208, 1330, 9, 1196, + /* 940 */ 264, 72, 72, 1436, 123, 123, 123, 123, 122, 122, + /* 950 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, + /* 960 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, + /* 970 */ 125, 572, 286, 286, 572, 1213, 411, 577, 315, 1248, + /* 980 */ 421, 371, 1581, 356, 319, 569, 147, 495, 529, 1644, + /* 990 */ 397, 935, 495, 1330, 71, 71, 934, 72, 72, 242, + /* 1000 */ 1328, 105, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, + /* 1010 */ 124, 125, 125, 125, 125, 123, 123, 123, 123, 122, + /* 1020 */ 122, 121, 121, 121, 120, 117, 448, 1117, 286, 286, + /* 1030 */ 1422, 452, 1528, 1213, 443, 286, 286, 1492, 1355, 313, + /* 1040 */ 478, 569, 1118, 454, 351, 495, 354, 1266, 569, 209, + /* 1050 */ 572, 418, 179, 572, 1031, 242, 385, 1119, 523, 123, + /* 1060 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, + /* 1070 */ 448, 1020, 108, 72, 72, 1019, 13, 13, 915, 572, + /* 1080 */ 1498, 572, 286, 286, 98, 530, 1537, 452, 916, 1334, + /* 1090 */ 1329, 203, 411, 286, 286, 569, 152, 211, 1498, 1500, + /* 1100 */ 426, 569, 56, 56, 57, 57, 569, 1019, 1019, 1021, + /* 1110 */ 447, 572, 411, 531, 12, 297, 126, 127, 81, 1217, + /* 1120 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, + /* 1130 */ 125, 572, 411, 867, 15, 15, 126, 127, 81, 1217, + /* 1140 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, + /* 1150 */ 125, 373, 529, 264, 44, 44, 126, 115, 81, 1217, + /* 1160 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, + /* 1170 */ 125, 1498, 478, 1271, 417, 123, 123, 123, 123, 122, + /* 1180 */ 122, 121, 121, 121, 120, 117, 448, 205, 1213, 495, + /* 1190 */ 430, 867, 468, 322, 495, 123, 123, 123, 123, 122, + /* 1200 */ 122, 121, 121, 121, 120, 117, 448, 572, 557, 1140, + /* 1210 */ 1642, 1422, 1642, 543, 572, 123, 123, 123, 123, 122, + /* 1220 */ 122, 121, 121, 121, 120, 117, 448, 572, 1422, 572, + /* 1230 */ 13, 13, 542, 323, 1325, 411, 334, 58, 58, 349, + /* 1240 */ 1422, 1170, 326, 286, 286, 549, 1213, 300, 895, 530, + /* 1250 */ 45, 45, 59, 59, 1140, 1643, 569, 1643, 565, 417, + /* 1260 */ 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, + /* 1270 */ 125, 125, 125, 125, 1367, 373, 500, 290, 1193, 512, + /* 1280 */ 1366, 427, 394, 394, 393, 275, 391, 896, 1138, 852, + /* 1290 */ 478, 258, 1422, 1170, 463, 1159, 12, 331, 428, 333, + /* 1300 */ 1117, 460, 236, 258, 325, 460, 544, 1544, 1159, 1098, + /* 1310 */ 491, 1159, 324, 1098, 440, 1118, 335, 516, 123, 123, + /* 1320 */ 123, 123, 122, 122, 121, 121, 121, 120, 117, 448, + /* 1330 */ 1119, 318, 563, 1138, 572, 1193, 1194, 1193, 112, 564, + /* 1340 */ 201, 4, 238, 433, 935, 490, 285, 228, 1517, 934, + /* 1350 */ 170, 560, 572, 142, 1516, 567, 572, 60, 60, 572, + /* 1360 */ 416, 572, 441, 572, 535, 302, 875, 8, 487, 572, + /* 1370 */ 237, 572, 416, 572, 485, 61, 61, 572, 449, 62, + /* 1380 */ 62, 332, 63, 63, 46, 46, 47, 47, 361, 572, + /* 1390 */ 561, 572, 48, 48, 50, 50, 51, 51, 572, 295, + /* 1400 */ 64, 64, 482, 295, 539, 412, 471, 1031, 572, 538, + /* 1410 */ 318, 563, 65, 65, 66, 66, 409, 475, 572, 1031, + /* 1420 */ 572, 14, 14, 875, 1020, 110, 110, 409, 1019, 572, + /* 1430 */ 474, 67, 67, 111, 455, 449, 573, 449, 98, 317, + /* 1440 */ 1019, 132, 132, 133, 133, 572, 1561, 572, 974, 409, + /* 1450 */ 6, 1562, 68, 68, 1560, 6, 975, 572, 6, 1559, + /* 1460 */ 1019, 1019, 1021, 6, 346, 218, 101, 531, 53, 53, + /* 1470 */ 69, 69, 1019, 1019, 1021, 1022, 28, 1586, 1181, 451, + /* 1480 */ 70, 70, 290, 87, 215, 31, 1363, 394, 394, 393, + /* 1490 */ 275, 391, 350, 109, 852, 107, 572, 112, 564, 483, + /* 1500 */ 4, 1212, 572, 239, 153, 572, 39, 236, 1299, 325, + /* 1510 */ 112, 564, 1298, 4, 567, 572, 32, 324, 572, 54, + /* 1520 */ 54, 572, 1135, 353, 398, 165, 165, 567, 166, 166, + /* 1530 */ 572, 291, 355, 572, 17, 357, 572, 449, 77, 77, + /* 1540 */ 1313, 55, 55, 1297, 73, 73, 572, 238, 470, 561, + /* 1550 */ 449, 472, 364, 135, 135, 170, 74, 74, 142, 163, + /* 1560 */ 163, 374, 561, 539, 572, 321, 572, 886, 540, 137, + /* 1570 */ 137, 339, 1353, 422, 298, 237, 539, 572, 1031, 572, + /* 1580 */ 340, 538, 101, 369, 110, 110, 162, 131, 131, 164, + /* 1590 */ 164, 1031, 111, 368, 449, 573, 449, 110, 110, 1019, + /* 1600 */ 157, 157, 141, 141, 572, 111, 572, 449, 573, 449, + /* 1610 */ 412, 288, 1019, 572, 882, 318, 563, 572, 219, 572, + /* 1620 */ 241, 1012, 477, 263, 263, 894, 893, 140, 140, 138, + /* 1630 */ 138, 1019, 1019, 1021, 1022, 28, 139, 139, 525, 455, + /* 1640 */ 76, 76, 78, 78, 1019, 1019, 1021, 1022, 28, 1181, + /* 1650 */ 451, 572, 1083, 290, 112, 564, 1575, 4, 394, 394, + /* 1660 */ 393, 275, 391, 572, 1023, 852, 572, 479, 345, 263, + /* 1670 */ 101, 567, 882, 1376, 75, 75, 1421, 501, 236, 260, + /* 1680 */ 325, 112, 564, 359, 4, 101, 43, 43, 324, 49, + /* 1690 */ 49, 901, 902, 161, 449, 101, 977, 978, 567, 1079, + /* 1700 */ 1349, 260, 965, 932, 263, 114, 561, 1095, 517, 1095, + /* 1710 */ 1083, 1094, 865, 1094, 151, 933, 1144, 114, 238, 1361, + /* 1720 */ 558, 449, 1023, 559, 1426, 1278, 170, 1269, 1257, 142, + /* 1730 */ 1601, 1256, 1258, 561, 1594, 1031, 496, 278, 213, 1346, + /* 1740 */ 310, 110, 110, 939, 311, 312, 237, 11, 234, 111, + /* 1750 */ 221, 449, 573, 449, 293, 395, 1019, 1408, 337, 1403, + /* 1760 */ 1396, 338, 1031, 299, 343, 1413, 1412, 481, 110, 110, + /* 1770 */ 506, 402, 225, 1296, 206, 367, 111, 1358, 449, 573, + /* 1780 */ 449, 412, 1359, 1019, 1489, 1488, 318, 563, 1019, 1019, + /* 1790 */ 1021, 1022, 28, 562, 207, 220, 80, 564, 389, 4, + /* 1800 */ 1597, 1357, 552, 1356, 1233, 181, 267, 232, 1536, 1534, + /* 1810 */ 455, 1230, 420, 567, 82, 1019, 1019, 1021, 1022, 28, + /* 1820 */ 86, 217, 85, 1494, 190, 175, 183, 465, 185, 466, + /* 1830 */ 36, 1409, 186, 187, 188, 499, 449, 244, 37, 99, + /* 1840 */ 400, 1415, 1414, 488, 1417, 194, 473, 403, 561, 1483, + /* 1850 */ 248, 92, 1505, 494, 198, 279, 112, 564, 250, 4, + /* 1860 */ 348, 497, 405, 352, 1259, 251, 252, 515, 1316, 434, + /* 1870 */ 1315, 1314, 94, 567, 1307, 886, 1306, 1031, 226, 406, + /* 1880 */ 1611, 1610, 438, 110, 110, 1580, 1286, 524, 439, 308, + /* 1890 */ 266, 111, 1285, 449, 573, 449, 449, 309, 1019, 366, + /* 1900 */ 1284, 1609, 265, 1566, 1565, 442, 372, 1381, 561, 129, + /* 1910 */ 550, 1380, 10, 1470, 383, 106, 316, 551, 100, 35, + /* 1920 */ 534, 575, 212, 1339, 381, 387, 1187, 1338, 274, 276, + /* 1930 */ 1019, 1019, 1021, 1022, 28, 277, 413, 1031, 576, 1254, + /* 1940 */ 388, 1521, 1249, 110, 110, 167, 1522, 168, 148, 1520, + /* 1950 */ 1519, 111, 306, 449, 573, 449, 222, 223, 1019, 839, + /* 1960 */ 169, 79, 450, 214, 414, 233, 320, 145, 1093, 1091, + /* 1970 */ 328, 182, 171, 1212, 918, 184, 240, 336, 243, 1107, + /* 1980 */ 189, 172, 173, 423, 425, 88, 180, 191, 89, 90, + /* 1990 */ 1019, 1019, 1021, 1022, 28, 91, 174, 1110, 245, 1106, + /* 2000 */ 246, 159, 18, 247, 347, 1099, 263, 195, 1227, 493, + /* 2010 */ 249, 196, 38, 854, 498, 368, 253, 360, 897, 197, + /* 2020 */ 502, 93, 19, 20, 507, 884, 363, 510, 95, 307, + /* 2030 */ 160, 96, 518, 97, 1175, 1060, 1146, 40, 21, 227, + /* 2040 */ 176, 1145, 282, 284, 969, 200, 963, 114, 262, 1165, + /* 2050 */ 22, 23, 24, 1161, 1169, 25, 1163, 1150, 34, 26, + /* 2060 */ 1168, 546, 27, 204, 101, 103, 104, 1074, 7, 1061, + /* 2070 */ 1059, 1063, 1116, 1064, 1115, 268, 269, 29, 41, 270, + /* 2080 */ 1024, 866, 113, 30, 568, 392, 1183, 144, 178, 1182, + /* 2090 */ 271, 928, 1245, 1245, 1245, 1245, 1245, 1245, 1245, 1602, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 193, 193, 193, 274, 275, 276, 193, 274, 275, 276, @@ -168596,7 +171186,7 @@ static const YYCODETYPE yy_lookahead[] = { /* 730 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 740 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, /* 750 */ 57, 19, 193, 193, 59, 23, 116, 117, 118, 59, - /* 760 */ 201, 21, 241, 304, 22, 206, 127, 128, 129, 193, + /* 760 */ 201, 21, 241, 304, 193, 206, 127, 128, 129, 193, /* 770 */ 128, 129, 235, 236, 304, 43, 44, 45, 46, 47, /* 780 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, /* 790 */ 22, 193, 216, 217, 193, 102, 103, 104, 105, 106, @@ -168607,129 +171197,129 @@ static const YYCODETYPE yy_lookahead[] = { /* 840 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 850 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, /* 860 */ 57, 19, 193, 123, 76, 239, 240, 193, 253, 239, - /* 870 */ 240, 239, 240, 193, 106, 107, 193, 89, 252, 193, - /* 880 */ 92, 59, 252, 141, 252, 43, 44, 45, 46, 47, + /* 870 */ 240, 239, 240, 244, 106, 107, 193, 89, 252, 193, + /* 880 */ 92, 59, 252, 254, 252, 43, 44, 45, 46, 47, /* 890 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, /* 900 */ 284, 161, 216, 217, 193, 102, 103, 104, 105, 106, - /* 910 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 16, - /* 920 */ 187, 188, 189, 190, 7, 8, 9, 309, 195, 25, + /* 910 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 244, + /* 920 */ 187, 188, 189, 190, 7, 8, 9, 309, 195, 254, /* 930 */ 197, 313, 19, 127, 128, 129, 262, 204, 22, 117, - /* 940 */ 24, 216, 217, 263, 102, 103, 104, 105, 106, 107, + /* 940 */ 24, 216, 217, 273, 102, 103, 104, 105, 106, 107, /* 950 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, /* 960 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, /* 970 */ 57, 193, 239, 240, 193, 59, 19, 188, 253, 190, - /* 980 */ 77, 226, 79, 193, 195, 252, 197, 193, 19, 301, - /* 990 */ 302, 193, 193, 204, 216, 217, 226, 216, 217, 266, + /* 980 */ 193, 311, 312, 16, 195, 252, 197, 193, 19, 301, + /* 990 */ 302, 135, 193, 204, 216, 217, 140, 216, 217, 266, /* 1000 */ 204, 159, 45, 46, 47, 48, 49, 50, 51, 52, /* 1010 */ 53, 54, 55, 56, 57, 102, 103, 104, 105, 106, /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 12, 239, 240, - /* 1030 */ 232, 298, 238, 117, 253, 239, 240, 238, 259, 260, - /* 1040 */ 193, 252, 27, 31, 193, 193, 142, 204, 252, 193, - /* 1050 */ 193, 39, 262, 193, 100, 266, 278, 42, 204, 102, + /* 1030 */ 193, 298, 238, 117, 253, 239, 240, 238, 259, 260, + /* 1040 */ 193, 252, 27, 193, 77, 193, 79, 204, 252, 262, + /* 1050 */ 193, 299, 300, 193, 100, 266, 278, 42, 204, 102, /* 1060 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, /* 1070 */ 113, 117, 159, 216, 217, 121, 216, 217, 63, 193, - /* 1080 */ 193, 193, 239, 240, 115, 116, 193, 298, 73, 238, + /* 1080 */ 193, 193, 239, 240, 115, 116, 193, 298, 73, 240, /* 1090 */ 238, 231, 19, 239, 240, 252, 22, 24, 211, 212, - /* 1100 */ 24, 193, 216, 217, 216, 217, 252, 153, 154, 155, - /* 1110 */ 253, 16, 19, 144, 213, 268, 43, 44, 45, 46, + /* 1100 */ 263, 252, 216, 217, 216, 217, 252, 153, 154, 155, + /* 1110 */ 253, 193, 19, 144, 213, 268, 43, 44, 45, 46, /* 1120 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1130 */ 57, 238, 19, 59, 193, 59, 43, 44, 45, 46, + /* 1130 */ 57, 193, 19, 59, 216, 217, 43, 44, 45, 46, /* 1140 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1150 */ 57, 22, 23, 193, 25, 193, 43, 44, 45, 46, + /* 1150 */ 57, 193, 19, 24, 216, 217, 43, 44, 45, 46, /* 1160 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1170 */ 57, 284, 77, 193, 79, 102, 103, 104, 105, 106, - /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 286, 193, 193, - /* 1190 */ 193, 117, 291, 117, 232, 102, 103, 104, 105, 106, - /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 204, 22, 23, - /* 1210 */ 66, 25, 216, 217, 35, 102, 103, 104, 105, 106, - /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 193, 268, 85, - /* 1230 */ 101, 193, 309, 309, 240, 19, 313, 313, 94, 208, - /* 1240 */ 209, 193, 239, 240, 193, 66, 252, 19, 268, 244, - /* 1250 */ 216, 217, 193, 74, 213, 252, 161, 19, 263, 254, + /* 1170 */ 57, 284, 193, 208, 209, 102, 103, 104, 105, 106, + /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 286, 59, 193, + /* 1190 */ 232, 117, 291, 193, 193, 102, 103, 104, 105, 106, + /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 193, 204, 22, + /* 1210 */ 23, 193, 25, 66, 193, 102, 103, 104, 105, 106, + /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 193, 193, 193, + /* 1230 */ 216, 217, 85, 193, 238, 19, 16, 216, 217, 238, + /* 1240 */ 193, 94, 193, 239, 240, 231, 117, 268, 35, 116, + /* 1250 */ 216, 217, 216, 217, 22, 23, 252, 25, 208, 209, /* 1260 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 1270 */ 54, 55, 56, 57, 193, 216, 217, 5, 59, 193, - /* 1280 */ 19, 244, 10, 11, 12, 13, 14, 101, 309, 17, - /* 1290 */ 146, 254, 313, 193, 193, 76, 115, 216, 217, 309, - /* 1300 */ 12, 263, 30, 313, 32, 46, 87, 46, 89, 130, - /* 1310 */ 193, 92, 40, 22, 263, 27, 216, 217, 102, 103, + /* 1270 */ 54, 55, 56, 57, 193, 193, 19, 5, 59, 66, + /* 1280 */ 193, 263, 10, 11, 12, 13, 14, 74, 101, 17, + /* 1290 */ 193, 46, 193, 146, 193, 76, 213, 77, 263, 79, + /* 1300 */ 12, 260, 30, 46, 32, 264, 87, 193, 89, 29, + /* 1310 */ 263, 92, 40, 33, 232, 27, 193, 108, 102, 103, /* 1320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - /* 1330 */ 42, 150, 291, 216, 217, 116, 117, 118, 19, 20, - /* 1340 */ 193, 22, 70, 260, 116, 193, 24, 264, 193, 263, - /* 1350 */ 78, 63, 61, 81, 116, 36, 193, 260, 193, 29, - /* 1360 */ 193, 264, 193, 33, 145, 193, 59, 48, 216, 217, - /* 1370 */ 98, 216, 217, 193, 115, 193, 115, 193, 59, 216, - /* 1380 */ 217, 216, 217, 216, 217, 216, 217, 255, 216, 217, - /* 1390 */ 71, 193, 131, 193, 25, 65, 216, 217, 216, 217, - /* 1400 */ 216, 217, 208, 209, 85, 133, 193, 100, 193, 90, - /* 1410 */ 138, 139, 138, 139, 216, 217, 216, 217, 193, 100, - /* 1420 */ 193, 108, 135, 116, 117, 106, 107, 140, 121, 216, - /* 1430 */ 217, 216, 217, 114, 162, 116, 117, 118, 299, 300, - /* 1440 */ 121, 216, 217, 216, 217, 193, 244, 193, 135, 244, - /* 1450 */ 193, 256, 257, 140, 244, 193, 254, 193, 193, 254, - /* 1460 */ 153, 154, 155, 141, 254, 149, 150, 258, 216, 217, + /* 1330 */ 42, 138, 139, 101, 193, 116, 117, 118, 19, 20, + /* 1340 */ 255, 22, 70, 130, 135, 65, 256, 257, 193, 140, + /* 1350 */ 78, 63, 193, 81, 193, 36, 193, 216, 217, 193, + /* 1360 */ 115, 193, 263, 193, 145, 268, 59, 48, 193, 193, + /* 1370 */ 98, 193, 115, 193, 291, 216, 217, 193, 59, 216, + /* 1380 */ 217, 161, 216, 217, 216, 217, 216, 217, 131, 193, + /* 1390 */ 71, 193, 216, 217, 216, 217, 216, 217, 193, 260, + /* 1400 */ 216, 217, 19, 264, 85, 133, 244, 100, 193, 90, + /* 1410 */ 138, 139, 216, 217, 216, 217, 254, 244, 193, 100, + /* 1420 */ 193, 216, 217, 116, 117, 106, 107, 254, 121, 193, + /* 1430 */ 115, 216, 217, 114, 162, 116, 117, 118, 115, 244, + /* 1440 */ 121, 216, 217, 216, 217, 193, 309, 193, 31, 254, + /* 1450 */ 313, 309, 216, 217, 309, 313, 39, 193, 313, 309, + /* 1460 */ 153, 154, 155, 313, 193, 150, 25, 144, 216, 217, /* 1470 */ 216, 217, 153, 154, 155, 156, 157, 0, 1, 2, - /* 1480 */ 216, 217, 5, 115, 158, 193, 160, 10, 11, 12, - /* 1490 */ 13, 14, 193, 59, 17, 126, 193, 19, 20, 129, - /* 1500 */ 22, 193, 22, 22, 24, 193, 23, 30, 25, 32, - /* 1510 */ 19, 20, 144, 22, 36, 216, 217, 40, 193, 216, - /* 1520 */ 217, 193, 152, 129, 216, 217, 193, 36, 216, 217, - /* 1530 */ 193, 99, 193, 193, 53, 193, 193, 59, 23, 193, - /* 1540 */ 25, 216, 217, 193, 216, 217, 152, 70, 59, 71, - /* 1550 */ 59, 117, 193, 216, 217, 78, 216, 217, 81, 216, - /* 1560 */ 217, 318, 71, 85, 193, 133, 193, 193, 90, 23, - /* 1570 */ 23, 25, 25, 120, 121, 98, 85, 193, 100, 193, - /* 1580 */ 23, 90, 25, 121, 106, 107, 19, 216, 217, 216, + /* 1480 */ 216, 217, 5, 149, 150, 22, 193, 10, 11, 12, + /* 1490 */ 13, 14, 193, 158, 17, 160, 193, 19, 20, 116, + /* 1500 */ 22, 25, 193, 24, 22, 193, 24, 30, 226, 32, + /* 1510 */ 19, 20, 226, 22, 36, 193, 53, 40, 193, 216, + /* 1520 */ 217, 193, 23, 193, 25, 216, 217, 36, 216, 217, + /* 1530 */ 193, 99, 193, 193, 22, 193, 193, 59, 216, 217, + /* 1540 */ 193, 216, 217, 193, 216, 217, 193, 70, 129, 71, + /* 1550 */ 59, 129, 193, 216, 217, 78, 216, 217, 81, 216, + /* 1560 */ 217, 193, 71, 85, 193, 133, 193, 126, 90, 216, + /* 1570 */ 217, 152, 258, 61, 152, 98, 85, 193, 100, 193, + /* 1580 */ 23, 90, 25, 121, 106, 107, 23, 216, 217, 216, /* 1590 */ 217, 100, 114, 131, 116, 117, 118, 106, 107, 121, - /* 1600 */ 216, 217, 216, 217, 193, 114, 117, 116, 117, 118, - /* 1610 */ 133, 193, 121, 193, 193, 138, 139, 193, 23, 193, - /* 1620 */ 25, 23, 23, 25, 25, 7, 8, 216, 217, 193, - /* 1630 */ 193, 153, 154, 155, 156, 157, 216, 217, 193, 162, + /* 1600 */ 216, 217, 216, 217, 193, 114, 193, 116, 117, 118, + /* 1610 */ 133, 22, 121, 193, 59, 138, 139, 193, 142, 193, + /* 1620 */ 141, 23, 23, 25, 25, 120, 121, 216, 217, 216, + /* 1630 */ 217, 153, 154, 155, 156, 157, 216, 217, 19, 162, /* 1640 */ 216, 217, 216, 217, 153, 154, 155, 156, 157, 1, - /* 1650 */ 2, 193, 193, 5, 19, 20, 59, 22, 10, 11, - /* 1660 */ 12, 13, 14, 193, 97, 17, 193, 23, 193, 25, - /* 1670 */ 288, 36, 193, 242, 216, 217, 236, 23, 30, 25, + /* 1650 */ 2, 193, 59, 5, 19, 20, 318, 22, 10, 11, + /* 1660 */ 12, 13, 14, 193, 59, 17, 193, 23, 23, 25, + /* 1670 */ 25, 36, 117, 193, 216, 217, 193, 23, 30, 25, /* 1680 */ 32, 19, 20, 23, 22, 25, 216, 217, 40, 216, - /* 1690 */ 217, 216, 217, 193, 59, 216, 217, 193, 36, 83, - /* 1700 */ 84, 153, 153, 155, 155, 23, 71, 25, 23, 193, - /* 1710 */ 25, 193, 193, 193, 117, 193, 193, 193, 70, 193, - /* 1720 */ 193, 59, 193, 255, 255, 287, 78, 255, 243, 81, - /* 1730 */ 191, 255, 297, 71, 271, 100, 293, 245, 267, 214, - /* 1740 */ 246, 106, 107, 108, 246, 271, 98, 245, 293, 114, - /* 1750 */ 220, 116, 117, 118, 267, 271, 121, 271, 225, 219, - /* 1760 */ 229, 219, 100, 219, 259, 259, 259, 259, 106, 107, - /* 1770 */ 249, 196, 60, 280, 141, 243, 114, 249, 116, 117, - /* 1780 */ 118, 133, 245, 121, 200, 297, 138, 139, 153, 154, - /* 1790 */ 155, 156, 157, 297, 200, 38, 19, 20, 151, 22, - /* 1800 */ 200, 150, 140, 294, 294, 22, 272, 43, 234, 18, - /* 1810 */ 162, 270, 200, 36, 237, 153, 154, 155, 156, 157, - /* 1820 */ 237, 283, 237, 237, 18, 199, 149, 246, 272, 270, - /* 1830 */ 272, 200, 158, 246, 246, 234, 59, 234, 246, 199, - /* 1840 */ 290, 62, 289, 200, 199, 22, 221, 115, 71, 200, - /* 1850 */ 200, 199, 199, 221, 218, 218, 19, 20, 64, 22, - /* 1860 */ 218, 227, 22, 224, 126, 224, 165, 221, 24, 305, - /* 1870 */ 200, 113, 312, 36, 218, 220, 218, 100, 282, 218, - /* 1880 */ 91, 218, 317, 106, 107, 221, 227, 282, 317, 82, - /* 1890 */ 148, 114, 265, 116, 117, 118, 59, 145, 121, 22, - /* 1900 */ 277, 158, 200, 265, 25, 202, 147, 250, 71, 279, - /* 1910 */ 13, 146, 194, 194, 249, 248, 250, 140, 247, 246, - /* 1920 */ 6, 192, 192, 192, 303, 303, 213, 207, 300, 213, - /* 1930 */ 153, 154, 155, 156, 157, 213, 213, 100, 213, 222, - /* 1940 */ 207, 214, 214, 106, 107, 4, 222, 207, 3, 22, - /* 1950 */ 163, 114, 15, 116, 117, 118, 16, 23, 121, 23, - /* 1960 */ 139, 151, 130, 25, 142, 16, 24, 20, 144, 1, - /* 1970 */ 142, 130, 130, 61, 53, 53, 37, 151, 53, 53, - /* 1980 */ 130, 116, 34, 1, 141, 5, 22, 115, 161, 141, - /* 1990 */ 153, 154, 155, 156, 157, 25, 68, 68, 75, 41, - /* 2000 */ 115, 24, 131, 20, 19, 125, 22, 96, 22, 22, - /* 2010 */ 67, 23, 22, 67, 59, 24, 22, 28, 67, 23, - /* 2020 */ 22, 22, 149, 23, 23, 23, 116, 23, 25, 37, - /* 2030 */ 97, 141, 23, 23, 22, 143, 25, 75, 88, 34, - /* 2040 */ 34, 34, 34, 86, 75, 93, 23, 34, 22, 34, - /* 2050 */ 25, 24, 34, 25, 23, 142, 23, 142, 44, 23, - /* 2060 */ 23, 23, 11, 23, 25, 22, 22, 22, 15, 23, - /* 2070 */ 23, 22, 22, 25, 1, 1, 141, 25, 23, 135, - /* 2080 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2090 */ 319, 319, 319, 319, 141, 141, 319, 319, 319, 319, + /* 1690 */ 217, 7, 8, 23, 59, 25, 83, 84, 36, 23, + /* 1700 */ 193, 25, 23, 23, 25, 25, 71, 153, 145, 155, + /* 1710 */ 117, 153, 23, 155, 25, 23, 97, 25, 70, 193, + /* 1720 */ 193, 59, 117, 236, 193, 193, 78, 193, 193, 81, + /* 1730 */ 141, 193, 193, 71, 193, 100, 288, 287, 242, 255, + /* 1740 */ 255, 106, 107, 108, 255, 255, 98, 243, 297, 114, + /* 1750 */ 214, 116, 117, 118, 245, 191, 121, 271, 293, 267, + /* 1760 */ 267, 246, 100, 246, 245, 271, 271, 293, 106, 107, + /* 1770 */ 220, 271, 229, 225, 249, 219, 114, 259, 116, 117, + /* 1780 */ 118, 133, 259, 121, 219, 219, 138, 139, 153, 154, + /* 1790 */ 155, 156, 157, 280, 249, 243, 19, 20, 245, 22, + /* 1800 */ 196, 259, 140, 259, 60, 297, 141, 297, 200, 200, + /* 1810 */ 162, 38, 200, 36, 294, 153, 154, 155, 156, 157, + /* 1820 */ 151, 150, 294, 283, 22, 43, 234, 18, 237, 200, + /* 1830 */ 270, 272, 237, 237, 237, 18, 59, 199, 270, 149, + /* 1840 */ 246, 272, 272, 200, 234, 234, 246, 246, 71, 246, + /* 1850 */ 199, 158, 290, 62, 22, 200, 19, 20, 199, 22, + /* 1860 */ 289, 221, 221, 200, 200, 199, 199, 115, 218, 64, + /* 1870 */ 218, 218, 22, 36, 227, 126, 227, 100, 165, 221, + /* 1880 */ 224, 224, 24, 106, 107, 312, 218, 305, 113, 282, + /* 1890 */ 91, 114, 220, 116, 117, 118, 59, 282, 121, 218, + /* 1900 */ 218, 218, 200, 317, 317, 82, 221, 265, 71, 148, + /* 1910 */ 145, 265, 22, 277, 200, 158, 279, 140, 147, 25, + /* 1920 */ 146, 202, 248, 250, 249, 247, 13, 250, 194, 194, + /* 1930 */ 153, 154, 155, 156, 157, 6, 303, 100, 192, 192, + /* 1940 */ 246, 213, 192, 106, 107, 207, 213, 207, 222, 213, + /* 1950 */ 213, 114, 222, 116, 117, 118, 214, 214, 121, 4, + /* 1960 */ 207, 213, 3, 22, 303, 15, 163, 16, 23, 23, + /* 1970 */ 139, 151, 130, 25, 20, 142, 24, 16, 144, 1, + /* 1980 */ 142, 130, 130, 61, 37, 53, 300, 151, 53, 53, + /* 1990 */ 153, 154, 155, 156, 157, 53, 130, 116, 34, 1, + /* 2000 */ 141, 5, 22, 115, 161, 68, 25, 68, 75, 41, + /* 2010 */ 141, 115, 24, 20, 19, 131, 125, 23, 28, 22, + /* 2020 */ 67, 22, 22, 22, 67, 59, 24, 96, 22, 67, + /* 2030 */ 23, 149, 22, 25, 23, 23, 23, 22, 34, 141, + /* 2040 */ 37, 97, 23, 23, 116, 22, 143, 25, 34, 75, + /* 2050 */ 34, 34, 34, 88, 75, 34, 86, 23, 22, 34, + /* 2060 */ 93, 24, 34, 25, 25, 142, 142, 23, 44, 23, + /* 2070 */ 23, 23, 23, 11, 23, 25, 22, 22, 22, 141, + /* 2080 */ 23, 23, 22, 22, 25, 15, 1, 23, 25, 1, + /* 2090 */ 141, 135, 319, 319, 319, 319, 319, 319, 319, 141, /* 2100 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2110 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2120 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, @@ -168748,176 +171338,177 @@ static const YYCODETYPE yy_lookahead[] = { /* 2250 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2260 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2270 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2280 */ 319, + /* 2280 */ 319, 319, 319, 319, 319, }; -#define YY_SHIFT_COUNT (574) +#define YY_SHIFT_COUNT (578) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2074) +#define YY_SHIFT_MAX (2088) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 1648, 1477, 1272, 322, 322, 1, 1319, 1478, 1491, 1837, /* 10 */ 1837, 1837, 471, 0, 0, 214, 1093, 1837, 1837, 1837, /* 20 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 30 */ 271, 271, 1219, 1219, 216, 88, 1, 1, 1, 1, - /* 40 */ 1, 40, 111, 258, 361, 469, 512, 583, 622, 693, - /* 50 */ 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, 1093, + /* 30 */ 1837, 271, 271, 1219, 1219, 216, 88, 1, 1, 1, + /* 40 */ 1, 1, 40, 111, 258, 361, 469, 512, 583, 622, + /* 50 */ 693, 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, /* 60 */ 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, - /* 70 */ 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1635, 1662, - /* 80 */ 1777, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, + /* 70 */ 1093, 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1635, + /* 80 */ 1662, 1777, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, /* 90 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, /* 100 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, /* 110 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, /* 120 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 130 */ 137, 181, 181, 181, 181, 181, 181, 181, 94, 430, - /* 140 */ 66, 65, 112, 366, 533, 533, 740, 1261, 533, 533, - /* 150 */ 79, 79, 533, 412, 412, 412, 77, 412, 123, 113, - /* 160 */ 113, 22, 22, 2096, 2096, 328, 328, 328, 239, 468, - /* 170 */ 468, 468, 468, 1015, 1015, 409, 366, 1129, 1186, 533, - /* 180 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 190 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 969, - /* 200 */ 621, 621, 533, 642, 788, 788, 1228, 1228, 822, 822, - /* 210 */ 67, 1274, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 1307, - /* 220 */ 954, 954, 585, 472, 640, 387, 695, 538, 541, 700, - /* 230 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 240 */ 222, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 250 */ 533, 533, 533, 1179, 1179, 1179, 533, 533, 533, 565, - /* 260 */ 533, 533, 533, 916, 1144, 533, 533, 1288, 533, 533, - /* 270 */ 533, 533, 533, 533, 533, 533, 639, 1330, 209, 1076, - /* 280 */ 1076, 1076, 1076, 580, 209, 209, 1313, 768, 917, 649, - /* 290 */ 1181, 1316, 405, 1316, 1238, 249, 1181, 1181, 249, 1181, - /* 300 */ 405, 1238, 1369, 464, 1259, 1012, 1012, 1012, 1368, 1368, - /* 310 */ 1368, 1368, 184, 184, 1326, 904, 1287, 1480, 1712, 1712, - /* 320 */ 1633, 1633, 1757, 1757, 1633, 1647, 1651, 1783, 1764, 1791, - /* 330 */ 1791, 1791, 1791, 1633, 1806, 1677, 1651, 1651, 1677, 1783, - /* 340 */ 1764, 1677, 1764, 1677, 1633, 1806, 1674, 1779, 1633, 1806, - /* 350 */ 1823, 1633, 1806, 1633, 1806, 1823, 1732, 1732, 1732, 1794, - /* 360 */ 1840, 1840, 1823, 1732, 1738, 1732, 1794, 1732, 1732, 1701, - /* 370 */ 1844, 1758, 1758, 1823, 1633, 1789, 1789, 1807, 1807, 1742, - /* 380 */ 1752, 1877, 1633, 1743, 1742, 1759, 1765, 1677, 1879, 1897, - /* 390 */ 1897, 1914, 1914, 1914, 2096, 2096, 2096, 2096, 2096, 2096, - /* 400 */ 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 207, - /* 410 */ 1095, 331, 620, 903, 806, 1074, 1483, 1432, 1481, 1322, - /* 420 */ 1370, 1394, 1515, 1291, 1546, 1547, 1557, 1595, 1598, 1599, - /* 430 */ 1434, 1453, 1618, 1462, 1567, 1489, 1644, 1654, 1616, 1660, - /* 440 */ 1548, 1549, 1682, 1685, 1597, 742, 1941, 1945, 1927, 1787, - /* 450 */ 1937, 1940, 1934, 1936, 1821, 1810, 1832, 1938, 1938, 1942, - /* 460 */ 1822, 1947, 1824, 1949, 1968, 1828, 1841, 1938, 1842, 1912, - /* 470 */ 1939, 1938, 1826, 1921, 1922, 1925, 1926, 1850, 1865, 1948, - /* 480 */ 1843, 1982, 1980, 1964, 1872, 1827, 1928, 1970, 1929, 1923, - /* 490 */ 1958, 1848, 1885, 1977, 1983, 1985, 1871, 1880, 1984, 1943, - /* 500 */ 1986, 1987, 1988, 1990, 1946, 1955, 1991, 1911, 1989, 1994, - /* 510 */ 1951, 1992, 1996, 1873, 1998, 2000, 2001, 2002, 2003, 2004, - /* 520 */ 1999, 1933, 1890, 2009, 2010, 1910, 2005, 2012, 1892, 2011, - /* 530 */ 2006, 2007, 2008, 2013, 1950, 1962, 1957, 2014, 1969, 1952, - /* 540 */ 2015, 2023, 2026, 2027, 2025, 2028, 2018, 1913, 1915, 2031, - /* 550 */ 2011, 2033, 2036, 2037, 2038, 2039, 2040, 2043, 2051, 2044, - /* 560 */ 2045, 2046, 2047, 2049, 2050, 2048, 1944, 1935, 1953, 1954, - /* 570 */ 2052, 2055, 2053, 2073, 2074, + /* 130 */ 1837, 137, 181, 181, 181, 181, 181, 181, 181, 94, + /* 140 */ 430, 66, 65, 112, 366, 533, 533, 740, 1257, 533, + /* 150 */ 533, 79, 79, 533, 412, 412, 412, 77, 412, 123, + /* 160 */ 113, 113, 113, 22, 22, 2100, 2100, 328, 328, 328, + /* 170 */ 239, 468, 468, 468, 468, 1015, 1015, 409, 366, 1187, + /* 180 */ 1232, 533, 533, 533, 533, 533, 533, 533, 533, 533, + /* 190 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, + /* 200 */ 533, 969, 621, 621, 533, 642, 788, 788, 1133, 1133, + /* 210 */ 822, 822, 67, 1193, 2100, 2100, 2100, 2100, 2100, 2100, + /* 220 */ 2100, 1307, 954, 954, 585, 472, 640, 387, 695, 538, + /* 230 */ 541, 700, 533, 533, 533, 533, 533, 533, 533, 533, + /* 240 */ 533, 533, 222, 533, 533, 533, 533, 533, 533, 533, + /* 250 */ 533, 533, 533, 533, 533, 1213, 1213, 1213, 533, 533, + /* 260 */ 533, 565, 533, 533, 533, 916, 1147, 533, 533, 1288, + /* 270 */ 533, 533, 533, 533, 533, 533, 533, 533, 639, 1280, + /* 280 */ 209, 1129, 1129, 1129, 1129, 580, 209, 209, 1209, 768, + /* 290 */ 917, 649, 1315, 1334, 405, 1334, 1383, 249, 1315, 1315, + /* 300 */ 249, 1315, 405, 1383, 1441, 464, 1245, 1417, 1417, 1417, + /* 310 */ 1323, 1323, 1323, 1323, 184, 184, 1335, 1476, 856, 1482, + /* 320 */ 1744, 1744, 1665, 1665, 1773, 1773, 1665, 1669, 1671, 1802, + /* 330 */ 1782, 1809, 1809, 1809, 1809, 1665, 1817, 1690, 1671, 1671, + /* 340 */ 1690, 1802, 1782, 1690, 1782, 1690, 1665, 1817, 1693, 1791, + /* 350 */ 1665, 1817, 1832, 1665, 1817, 1665, 1817, 1832, 1752, 1752, + /* 360 */ 1752, 1805, 1850, 1850, 1832, 1752, 1749, 1752, 1805, 1752, + /* 370 */ 1752, 1713, 1858, 1775, 1775, 1832, 1665, 1799, 1799, 1823, + /* 380 */ 1823, 1761, 1765, 1890, 1665, 1757, 1761, 1771, 1774, 1690, + /* 390 */ 1894, 1913, 1913, 1929, 1929, 1929, 2100, 2100, 2100, 2100, + /* 400 */ 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, + /* 410 */ 2100, 207, 1220, 331, 620, 967, 806, 1074, 1499, 1432, + /* 420 */ 1463, 1479, 1419, 1422, 1557, 1512, 1598, 1599, 1644, 1645, + /* 430 */ 1654, 1660, 1555, 1505, 1684, 1462, 1670, 1563, 1619, 1593, + /* 440 */ 1676, 1679, 1613, 1680, 1554, 1558, 1689, 1692, 1605, 1589, + /* 450 */ 1955, 1959, 1941, 1803, 1950, 1951, 1945, 1946, 1831, 1820, + /* 460 */ 1842, 1948, 1948, 1952, 1833, 1954, 1834, 1961, 1978, 1838, + /* 470 */ 1851, 1948, 1852, 1922, 1947, 1948, 1836, 1932, 1935, 1936, + /* 480 */ 1942, 1866, 1881, 1964, 1859, 1998, 1996, 1980, 1888, 1843, + /* 490 */ 1937, 1981, 1939, 1933, 1968, 1869, 1896, 1988, 1993, 1995, + /* 500 */ 1884, 1891, 1997, 1953, 1999, 2000, 1994, 2001, 1957, 1966, + /* 510 */ 2002, 1931, 1990, 2006, 1962, 2003, 2007, 2004, 1882, 2010, + /* 520 */ 2011, 2012, 2008, 2013, 2015, 1944, 1898, 2019, 2020, 1928, + /* 530 */ 2014, 2023, 1903, 2022, 2016, 2017, 2018, 2021, 1965, 1974, + /* 540 */ 1970, 2024, 1979, 1967, 2025, 2034, 2036, 2037, 2038, 2039, + /* 550 */ 2028, 1923, 1924, 2044, 2022, 2046, 2047, 2048, 2049, 2050, + /* 560 */ 2051, 2054, 2062, 2055, 2056, 2057, 2058, 2060, 2061, 2059, + /* 570 */ 1956, 1938, 1949, 1958, 2063, 2064, 2070, 2085, 2088, }; -#define YY_REDUCE_COUNT (408) +#define YY_REDUCE_COUNT (410) #define YY_REDUCE_MIN (-271) -#define YY_REDUCE_MAX (1740) +#define YY_REDUCE_MAX (1753) static const short yy_reduce_ofst[] = { /* 0 */ -125, 733, 789, 241, 293, -123, -193, -191, -183, -187, /* 10 */ 166, 238, 133, -207, -199, -267, -176, -6, 204, 489, - /* 20 */ 576, -175, 598, 686, 615, 725, 860, 778, 781, 857, - /* 30 */ 616, 887, 87, 240, -192, 408, 626, 796, 843, 854, - /* 40 */ 1003, -271, -271, -271, -271, -271, -271, -271, -271, -271, + /* 20 */ 576, 598, -175, 686, 860, 615, 725, 1014, 778, 781, + /* 30 */ 857, 616, 887, 87, 240, -192, 408, 626, 796, 843, + /* 40 */ 854, 1004, -271, -271, -271, -271, -271, -271, -271, -271, /* 50 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, /* 60 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 70 */ -271, -271, -271, -271, -271, -271, -271, -271, 80, 83, - /* 80 */ 313, 886, 888, 996, 1034, 1059, 1081, 1100, 1117, 1152, - /* 90 */ 1155, 1163, 1165, 1167, 1169, 1172, 1180, 1182, 1184, 1198, - /* 100 */ 1200, 1213, 1215, 1225, 1227, 1252, 1254, 1264, 1299, 1303, - /* 110 */ 1308, 1312, 1325, 1328, 1337, 1340, 1343, 1371, 1373, 1384, - /* 120 */ 1386, 1411, 1420, 1424, 1426, 1458, 1470, 1473, 1475, 1479, - /* 130 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 140 */ -271, 138, 459, 396, -158, 470, 302, -212, 521, 201, - /* 150 */ -195, -92, 559, 630, 632, 630, -271, 632, 901, 63, - /* 160 */ 407, -271, -271, -271, -271, 161, 161, 161, 251, 335, - /* 170 */ 847, 960, 980, 537, 588, 618, 628, 688, 688, -166, - /* 180 */ -161, 674, 790, 794, 799, 851, 852, -122, 680, -120, - /* 190 */ 995, 1038, 415, 1051, 893, 798, 962, 400, 1086, 779, - /* 200 */ 923, 924, 263, 1041, 979, 990, 1083, 1097, 1031, 1194, - /* 210 */ 362, 994, 1139, 1005, 1037, 1202, 1205, 1195, 1210, -194, - /* 220 */ 56, 185, -135, 232, 522, 560, 601, 617, 669, 683, - /* 230 */ 711, 856, 908, 941, 1048, 1101, 1147, 1257, 1262, 1265, - /* 240 */ 392, 1292, 1333, 1339, 1342, 1346, 1350, 1359, 1374, 1418, - /* 250 */ 1421, 1436, 1437, 593, 755, 770, 997, 1445, 1459, 1209, - /* 260 */ 1500, 1504, 1516, 1132, 1243, 1518, 1519, 1440, 1520, 560, - /* 270 */ 1522, 1523, 1524, 1526, 1527, 1529, 1382, 1438, 1431, 1468, - /* 280 */ 1469, 1472, 1476, 1209, 1431, 1431, 1485, 1525, 1539, 1435, - /* 290 */ 1463, 1471, 1492, 1487, 1443, 1494, 1474, 1484, 1498, 1486, - /* 300 */ 1502, 1455, 1530, 1531, 1533, 1540, 1542, 1544, 1505, 1506, - /* 310 */ 1507, 1508, 1521, 1528, 1493, 1537, 1532, 1575, 1488, 1496, - /* 320 */ 1584, 1594, 1509, 1510, 1600, 1538, 1534, 1541, 1574, 1577, - /* 330 */ 1583, 1585, 1586, 1612, 1626, 1581, 1556, 1558, 1587, 1559, - /* 340 */ 1601, 1588, 1603, 1592, 1631, 1640, 1550, 1553, 1643, 1645, - /* 350 */ 1625, 1649, 1652, 1650, 1653, 1632, 1636, 1637, 1642, 1634, - /* 360 */ 1639, 1641, 1646, 1656, 1655, 1658, 1659, 1661, 1663, 1560, - /* 370 */ 1564, 1596, 1605, 1664, 1670, 1565, 1571, 1627, 1638, 1657, - /* 380 */ 1665, 1623, 1702, 1630, 1666, 1667, 1671, 1673, 1703, 1718, - /* 390 */ 1719, 1729, 1730, 1731, 1621, 1622, 1628, 1720, 1713, 1716, - /* 400 */ 1722, 1723, 1733, 1717, 1724, 1727, 1728, 1725, 1740, + /* 70 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, 80, + /* 80 */ 83, 313, 886, 888, 918, 938, 1021, 1034, 1036, 1141, + /* 90 */ 1159, 1163, 1166, 1168, 1170, 1176, 1178, 1180, 1184, 1196, + /* 100 */ 1198, 1205, 1215, 1225, 1227, 1236, 1252, 1254, 1264, 1303, + /* 110 */ 1309, 1312, 1322, 1325, 1328, 1337, 1340, 1343, 1353, 1371, + /* 120 */ 1373, 1384, 1386, 1411, 1413, 1420, 1424, 1426, 1458, 1470, + /* 130 */ 1473, -271, -271, -271, -271, -271, -271, -271, -271, -271, + /* 140 */ -271, -271, 138, 459, 396, -158, 470, 302, -212, 521, + /* 150 */ 201, -195, -92, 559, 630, 632, 630, -271, 632, 901, + /* 160 */ 63, 407, 670, -271, -271, -271, -271, 161, 161, 161, + /* 170 */ 251, 335, 847, 979, 1097, 537, 588, 618, 628, 688, + /* 180 */ 688, -166, -161, 674, 787, 794, 799, 852, 996, -122, + /* 190 */ 837, -120, 1018, 1035, 415, 1047, 1001, 958, 1082, 400, + /* 200 */ 1099, 779, 1137, 1142, 263, 1083, 1145, 1150, 1041, 1139, + /* 210 */ 965, 1050, 362, 849, 752, 629, 675, 1162, 1173, 1090, + /* 220 */ 1195, -194, 56, 185, -135, 232, 522, 560, 571, 601, + /* 230 */ 617, 669, 683, 711, 850, 893, 1000, 1040, 1049, 1081, + /* 240 */ 1087, 1101, 392, 1114, 1123, 1155, 1161, 1175, 1271, 1293, + /* 250 */ 1299, 1330, 1339, 1342, 1347, 593, 1282, 1286, 1350, 1359, + /* 260 */ 1368, 1314, 1480, 1483, 1507, 1085, 1338, 1526, 1527, 1487, + /* 270 */ 1531, 560, 1532, 1534, 1535, 1538, 1539, 1541, 1448, 1450, + /* 280 */ 1496, 1484, 1485, 1489, 1490, 1314, 1496, 1496, 1504, 1536, + /* 290 */ 1564, 1451, 1486, 1492, 1509, 1493, 1465, 1515, 1494, 1495, + /* 300 */ 1517, 1500, 1519, 1474, 1550, 1543, 1548, 1556, 1565, 1566, + /* 310 */ 1518, 1523, 1542, 1544, 1525, 1545, 1513, 1553, 1552, 1604, + /* 320 */ 1508, 1510, 1608, 1609, 1520, 1528, 1612, 1540, 1559, 1560, + /* 330 */ 1592, 1591, 1595, 1596, 1597, 1629, 1638, 1594, 1569, 1570, + /* 340 */ 1600, 1568, 1610, 1601, 1611, 1603, 1643, 1651, 1562, 1571, + /* 350 */ 1655, 1659, 1640, 1663, 1666, 1664, 1667, 1641, 1650, 1652, + /* 360 */ 1653, 1647, 1656, 1657, 1658, 1668, 1672, 1681, 1649, 1682, + /* 370 */ 1683, 1573, 1582, 1607, 1615, 1685, 1702, 1586, 1587, 1642, + /* 380 */ 1646, 1673, 1675, 1636, 1714, 1637, 1677, 1674, 1678, 1694, + /* 390 */ 1719, 1734, 1735, 1746, 1747, 1750, 1633, 1661, 1686, 1738, + /* 400 */ 1728, 1733, 1736, 1737, 1740, 1726, 1730, 1742, 1743, 1748, + /* 410 */ 1753, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1641, 1641, 1641, 1469, 1236, 1347, 1236, 1236, 1236, 1469, - /* 10 */ 1469, 1469, 1236, 1377, 1377, 1522, 1269, 1236, 1236, 1236, - /* 20 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1468, 1236, 1236, - /* 30 */ 1236, 1236, 1557, 1557, 1236, 1236, 1236, 1236, 1236, 1236, - /* 40 */ 1236, 1236, 1386, 1236, 1393, 1236, 1236, 1236, 1236, 1236, - /* 50 */ 1470, 1471, 1236, 1236, 1236, 1521, 1523, 1486, 1400, 1399, - /* 60 */ 1398, 1397, 1504, 1365, 1391, 1384, 1388, 1465, 1466, 1464, - /* 70 */ 1619, 1471, 1470, 1236, 1387, 1433, 1449, 1432, 1236, 1236, - /* 80 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 90 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 100 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 110 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 120 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 130 */ 1441, 1448, 1447, 1446, 1455, 1445, 1442, 1435, 1434, 1436, - /* 140 */ 1437, 1236, 1236, 1260, 1236, 1236, 1257, 1311, 1236, 1236, - /* 150 */ 1236, 1236, 1236, 1541, 1540, 1236, 1438, 1236, 1269, 1427, - /* 160 */ 1426, 1452, 1439, 1451, 1450, 1529, 1593, 1592, 1487, 1236, - /* 170 */ 1236, 1236, 1236, 1236, 1236, 1557, 1236, 1236, 1236, 1236, - /* 180 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 190 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1367, - /* 200 */ 1557, 1557, 1236, 1269, 1557, 1557, 1368, 1368, 1265, 1265, - /* 210 */ 1371, 1236, 1536, 1338, 1338, 1338, 1338, 1347, 1338, 1236, - /* 220 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 230 */ 1236, 1236, 1236, 1236, 1526, 1524, 1236, 1236, 1236, 1236, - /* 240 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 250 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 260 */ 1236, 1236, 1236, 1343, 1236, 1236, 1236, 1236, 1236, 1236, - /* 270 */ 1236, 1236, 1236, 1236, 1236, 1586, 1236, 1499, 1325, 1343, - /* 280 */ 1343, 1343, 1343, 1345, 1326, 1324, 1337, 1270, 1243, 1633, - /* 290 */ 1403, 1392, 1344, 1392, 1630, 1390, 1403, 1403, 1390, 1403, - /* 300 */ 1344, 1630, 1286, 1608, 1281, 1377, 1377, 1377, 1367, 1367, - /* 310 */ 1367, 1367, 1371, 1371, 1467, 1344, 1337, 1236, 1633, 1633, - /* 320 */ 1353, 1353, 1632, 1632, 1353, 1487, 1616, 1412, 1314, 1320, - /* 330 */ 1320, 1320, 1320, 1353, 1254, 1390, 1616, 1616, 1390, 1412, - /* 340 */ 1314, 1390, 1314, 1390, 1353, 1254, 1503, 1627, 1353, 1254, - /* 350 */ 1477, 1353, 1254, 1353, 1254, 1477, 1312, 1312, 1312, 1301, - /* 360 */ 1236, 1236, 1477, 1312, 1286, 1312, 1301, 1312, 1312, 1575, - /* 370 */ 1236, 1481, 1481, 1477, 1353, 1567, 1567, 1380, 1380, 1385, - /* 380 */ 1371, 1472, 1353, 1236, 1385, 1383, 1381, 1390, 1304, 1589, - /* 390 */ 1589, 1585, 1585, 1585, 1638, 1638, 1536, 1601, 1269, 1269, - /* 400 */ 1269, 1269, 1601, 1288, 1288, 1270, 1270, 1269, 1601, 1236, - /* 410 */ 1236, 1236, 1236, 1236, 1236, 1596, 1236, 1531, 1488, 1357, - /* 420 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 430 */ 1236, 1236, 1236, 1236, 1542, 1236, 1236, 1236, 1236, 1236, - /* 440 */ 1236, 1236, 1236, 1236, 1236, 1417, 1236, 1239, 1533, 1236, - /* 450 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1394, 1395, 1358, - /* 460 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1409, 1236, 1236, - /* 470 */ 1236, 1404, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 480 */ 1629, 1236, 1236, 1236, 1236, 1236, 1236, 1502, 1501, 1236, - /* 490 */ 1236, 1355, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 500 */ 1236, 1236, 1236, 1236, 1236, 1284, 1236, 1236, 1236, 1236, - /* 510 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 520 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1382, - /* 530 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 540 */ 1236, 1236, 1236, 1236, 1572, 1372, 1236, 1236, 1236, 1236, - /* 550 */ 1620, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, - /* 560 */ 1236, 1236, 1236, 1236, 1236, 1612, 1328, 1418, 1236, 1421, - /* 570 */ 1258, 1236, 1248, 1236, 1236, + /* 0 */ 1648, 1648, 1648, 1478, 1243, 1354, 1243, 1243, 1243, 1478, + /* 10 */ 1478, 1478, 1243, 1384, 1384, 1531, 1276, 1243, 1243, 1243, + /* 20 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1477, 1243, + /* 30 */ 1243, 1243, 1243, 1564, 1564, 1243, 1243, 1243, 1243, 1243, + /* 40 */ 1243, 1243, 1243, 1393, 1243, 1400, 1243, 1243, 1243, 1243, + /* 50 */ 1243, 1479, 1480, 1243, 1243, 1243, 1530, 1532, 1495, 1407, + /* 60 */ 1406, 1405, 1404, 1513, 1372, 1398, 1391, 1395, 1474, 1475, + /* 70 */ 1473, 1626, 1480, 1479, 1243, 1394, 1442, 1458, 1441, 1243, + /* 80 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 90 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 100 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 110 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 120 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 130 */ 1243, 1450, 1457, 1456, 1455, 1464, 1454, 1451, 1444, 1443, + /* 140 */ 1445, 1446, 1243, 1243, 1267, 1243, 1243, 1264, 1318, 1243, + /* 150 */ 1243, 1243, 1243, 1243, 1550, 1549, 1243, 1447, 1243, 1276, + /* 160 */ 1435, 1434, 1433, 1461, 1448, 1460, 1459, 1538, 1600, 1599, + /* 170 */ 1496, 1243, 1243, 1243, 1243, 1243, 1243, 1564, 1243, 1243, + /* 180 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 190 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 200 */ 1243, 1374, 1564, 1564, 1243, 1276, 1564, 1564, 1375, 1375, + /* 210 */ 1272, 1272, 1378, 1243, 1545, 1345, 1345, 1345, 1345, 1354, + /* 220 */ 1345, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 230 */ 1243, 1243, 1243, 1243, 1243, 1243, 1535, 1533, 1243, 1243, + /* 240 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 250 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 260 */ 1243, 1243, 1243, 1243, 1243, 1350, 1243, 1243, 1243, 1243, + /* 270 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1593, 1243, 1508, + /* 280 */ 1332, 1350, 1350, 1350, 1350, 1352, 1333, 1331, 1344, 1277, + /* 290 */ 1250, 1640, 1410, 1399, 1351, 1399, 1637, 1397, 1410, 1410, + /* 300 */ 1397, 1410, 1351, 1637, 1293, 1615, 1288, 1384, 1384, 1384, + /* 310 */ 1374, 1374, 1374, 1374, 1378, 1378, 1476, 1351, 1344, 1243, + /* 320 */ 1640, 1640, 1360, 1360, 1639, 1639, 1360, 1496, 1623, 1419, + /* 330 */ 1321, 1327, 1327, 1327, 1327, 1360, 1261, 1397, 1623, 1623, + /* 340 */ 1397, 1419, 1321, 1397, 1321, 1397, 1360, 1261, 1512, 1634, + /* 350 */ 1360, 1261, 1486, 1360, 1261, 1360, 1261, 1486, 1319, 1319, + /* 360 */ 1319, 1308, 1243, 1243, 1486, 1319, 1293, 1319, 1308, 1319, + /* 370 */ 1319, 1582, 1243, 1490, 1490, 1486, 1360, 1574, 1574, 1387, + /* 380 */ 1387, 1392, 1378, 1481, 1360, 1243, 1392, 1390, 1388, 1397, + /* 390 */ 1311, 1596, 1596, 1592, 1592, 1592, 1645, 1645, 1545, 1608, + /* 400 */ 1276, 1276, 1276, 1276, 1608, 1295, 1295, 1277, 1277, 1276, + /* 410 */ 1608, 1243, 1243, 1243, 1243, 1243, 1243, 1603, 1243, 1540, + /* 420 */ 1497, 1364, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 430 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1551, 1243, + /* 440 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1424, + /* 450 */ 1243, 1246, 1542, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 460 */ 1243, 1401, 1402, 1365, 1243, 1243, 1243, 1243, 1243, 1243, + /* 470 */ 1243, 1416, 1243, 1243, 1243, 1411, 1243, 1243, 1243, 1243, + /* 480 */ 1243, 1243, 1243, 1243, 1636, 1243, 1243, 1243, 1243, 1243, + /* 490 */ 1243, 1511, 1510, 1243, 1243, 1362, 1243, 1243, 1243, 1243, + /* 500 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1291, + /* 510 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 520 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, + /* 530 */ 1243, 1243, 1243, 1389, 1243, 1243, 1243, 1243, 1243, 1243, + /* 540 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1579, 1379, + /* 550 */ 1243, 1243, 1243, 1243, 1627, 1243, 1243, 1243, 1243, 1243, + /* 560 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1619, + /* 570 */ 1335, 1425, 1243, 1428, 1265, 1243, 1255, 1243, 1243, }; /********** End of lemon-generated parsing tables *****************************/ @@ -169724,135 +172315,135 @@ static const char *const yyRuleName[] = { /* 185 */ "expr ::= expr COLLATE ID|STRING", /* 186 */ "expr ::= CAST LP expr AS typetoken RP", /* 187 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", - /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", - /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", - /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", - /* 191 */ "term ::= CTIME_KW", - /* 192 */ "expr ::= LP nexprlist COMMA expr RP", - /* 193 */ "expr ::= expr AND expr", - /* 194 */ "expr ::= expr OR expr", - /* 195 */ "expr ::= expr LT|GT|GE|LE expr", - /* 196 */ "expr ::= expr EQ|NE expr", - /* 197 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 198 */ "expr ::= expr PLUS|MINUS expr", - /* 199 */ "expr ::= expr STAR|SLASH|REM expr", - /* 200 */ "expr ::= expr CONCAT expr", - /* 201 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 202 */ "expr ::= expr likeop expr", - /* 203 */ "expr ::= expr likeop expr ESCAPE expr", - /* 204 */ "expr ::= expr ISNULL|NOTNULL", - /* 205 */ "expr ::= expr NOT NULL", - /* 206 */ "expr ::= expr IS expr", - /* 207 */ "expr ::= expr IS NOT expr", - /* 208 */ "expr ::= expr IS NOT DISTINCT FROM expr", - /* 209 */ "expr ::= expr IS DISTINCT FROM expr", - /* 210 */ "expr ::= NOT expr", - /* 211 */ "expr ::= BITNOT expr", - /* 212 */ "expr ::= PLUS|MINUS expr", - /* 213 */ "expr ::= expr PTR expr", - /* 214 */ "between_op ::= BETWEEN", - /* 215 */ "between_op ::= NOT BETWEEN", - /* 216 */ "expr ::= expr between_op expr AND expr", - /* 217 */ "in_op ::= IN", - /* 218 */ "in_op ::= NOT IN", - /* 219 */ "expr ::= expr in_op LP exprlist RP", - /* 220 */ "expr ::= LP select RP", - /* 221 */ "expr ::= expr in_op LP select RP", - /* 222 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 223 */ "expr ::= EXISTS LP select RP", - /* 224 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 225 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 226 */ "case_exprlist ::= WHEN expr THEN expr", - /* 227 */ "case_else ::= ELSE expr", - /* 228 */ "case_else ::=", - /* 229 */ "case_operand ::=", - /* 230 */ "exprlist ::=", - /* 231 */ "nexprlist ::= nexprlist COMMA expr", - /* 232 */ "nexprlist ::= expr", - /* 233 */ "paren_exprlist ::=", - /* 234 */ "paren_exprlist ::= LP exprlist RP", - /* 235 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 236 */ "uniqueflag ::= UNIQUE", - /* 237 */ "uniqueflag ::=", - /* 238 */ "eidlist_opt ::=", - /* 239 */ "eidlist_opt ::= LP eidlist RP", - /* 240 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 241 */ "eidlist ::= nm collate sortorder", - /* 242 */ "collate ::=", - /* 243 */ "collate ::= COLLATE ID|STRING", - /* 244 */ "cmd ::= DROP INDEX ifexists fullname", - /* 245 */ "cmd ::= VACUUM vinto", - /* 246 */ "cmd ::= VACUUM nm vinto", - /* 247 */ "vinto ::= INTO expr", - /* 248 */ "vinto ::=", - /* 249 */ "cmd ::= PRAGMA nm dbnm", - /* 250 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 251 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 254 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 255 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 256 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 257 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 258 */ "trigger_time ::= BEFORE|AFTER", - /* 259 */ "trigger_time ::= INSTEAD OF", - /* 260 */ "trigger_time ::=", - /* 261 */ "trigger_event ::= DELETE|INSERT", - /* 262 */ "trigger_event ::= UPDATE", - /* 263 */ "trigger_event ::= UPDATE OF idlist", - /* 264 */ "when_clause ::=", - /* 265 */ "when_clause ::= WHEN expr", - /* 266 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 267 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 268 */ "trnm ::= nm DOT nm", - /* 269 */ "tridxby ::= INDEXED BY nm", - /* 270 */ "tridxby ::= NOT INDEXED", - /* 271 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 272 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 273 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 274 */ "trigger_cmd ::= scanpt select scanpt", - /* 275 */ "expr ::= RAISE LP IGNORE RP", - /* 276 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 277 */ "raisetype ::= ROLLBACK", - /* 278 */ "raisetype ::= ABORT", - /* 279 */ "raisetype ::= FAIL", - /* 280 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 281 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 282 */ "cmd ::= DETACH database_kw_opt expr", - /* 283 */ "key_opt ::=", - /* 284 */ "key_opt ::= KEY expr", - /* 285 */ "cmd ::= REINDEX", - /* 286 */ "cmd ::= REINDEX nm dbnm", - /* 287 */ "cmd ::= ANALYZE", - /* 288 */ "cmd ::= ANALYZE nm dbnm", - /* 289 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 290 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 291 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 292 */ "add_column_fullname ::= fullname", - /* 293 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 294 */ "cmd ::= create_vtab", - /* 295 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 296 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 297 */ "vtabarg ::=", - /* 298 */ "vtabargtoken ::= ANY", - /* 299 */ "vtabargtoken ::= lp anylist RP", - /* 300 */ "lp ::= LP", - /* 301 */ "with ::= WITH wqlist", - /* 302 */ "with ::= WITH RECURSIVE wqlist", - /* 303 */ "wqas ::= AS", - /* 304 */ "wqas ::= AS MATERIALIZED", - /* 305 */ "wqas ::= AS NOT MATERIALIZED", - /* 306 */ "wqitem ::= nm eidlist_opt wqas LP select RP", - /* 307 */ "wqlist ::= wqitem", - /* 308 */ "wqlist ::= wqlist COMMA wqitem", - /* 309 */ "windowdefn_list ::= windowdefn", - /* 310 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 311 */ "windowdefn ::= nm AS LP window RP", - /* 312 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 313 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 314 */ "window ::= ORDER BY sortlist frame_opt", - /* 315 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 316 */ "window ::= frame_opt", + /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", + /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 193 */ "term ::= CTIME_KW", + /* 194 */ "expr ::= LP nexprlist COMMA expr RP", + /* 195 */ "expr ::= expr AND expr", + /* 196 */ "expr ::= expr OR expr", + /* 197 */ "expr ::= expr LT|GT|GE|LE expr", + /* 198 */ "expr ::= expr EQ|NE expr", + /* 199 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 200 */ "expr ::= expr PLUS|MINUS expr", + /* 201 */ "expr ::= expr STAR|SLASH|REM expr", + /* 202 */ "expr ::= expr CONCAT expr", + /* 203 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 204 */ "expr ::= expr likeop expr", + /* 205 */ "expr ::= expr likeop expr ESCAPE expr", + /* 206 */ "expr ::= expr ISNULL|NOTNULL", + /* 207 */ "expr ::= expr NOT NULL", + /* 208 */ "expr ::= expr IS expr", + /* 209 */ "expr ::= expr IS NOT expr", + /* 210 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 211 */ "expr ::= expr IS DISTINCT FROM expr", + /* 212 */ "expr ::= NOT expr", + /* 213 */ "expr ::= BITNOT expr", + /* 214 */ "expr ::= PLUS|MINUS expr", + /* 215 */ "expr ::= expr PTR expr", + /* 216 */ "between_op ::= BETWEEN", + /* 217 */ "between_op ::= NOT BETWEEN", + /* 218 */ "expr ::= expr between_op expr AND expr", + /* 219 */ "in_op ::= IN", + /* 220 */ "in_op ::= NOT IN", + /* 221 */ "expr ::= expr in_op LP exprlist RP", + /* 222 */ "expr ::= LP select RP", + /* 223 */ "expr ::= expr in_op LP select RP", + /* 224 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 225 */ "expr ::= EXISTS LP select RP", + /* 226 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 227 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 228 */ "case_exprlist ::= WHEN expr THEN expr", + /* 229 */ "case_else ::= ELSE expr", + /* 230 */ "case_else ::=", + /* 231 */ "case_operand ::=", + /* 232 */ "exprlist ::=", + /* 233 */ "nexprlist ::= nexprlist COMMA expr", + /* 234 */ "nexprlist ::= expr", + /* 235 */ "paren_exprlist ::=", + /* 236 */ "paren_exprlist ::= LP exprlist RP", + /* 237 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 238 */ "uniqueflag ::= UNIQUE", + /* 239 */ "uniqueflag ::=", + /* 240 */ "eidlist_opt ::=", + /* 241 */ "eidlist_opt ::= LP eidlist RP", + /* 242 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 243 */ "eidlist ::= nm collate sortorder", + /* 244 */ "collate ::=", + /* 245 */ "collate ::= COLLATE ID|STRING", + /* 246 */ "cmd ::= DROP INDEX ifexists fullname", + /* 247 */ "cmd ::= VACUUM vinto", + /* 248 */ "cmd ::= VACUUM nm vinto", + /* 249 */ "vinto ::= INTO expr", + /* 250 */ "vinto ::=", + /* 251 */ "cmd ::= PRAGMA nm dbnm", + /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 256 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 257 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 260 */ "trigger_time ::= BEFORE|AFTER", + /* 261 */ "trigger_time ::= INSTEAD OF", + /* 262 */ "trigger_time ::=", + /* 263 */ "trigger_event ::= DELETE|INSERT", + /* 264 */ "trigger_event ::= UPDATE", + /* 265 */ "trigger_event ::= UPDATE OF idlist", + /* 266 */ "when_clause ::=", + /* 267 */ "when_clause ::= WHEN expr", + /* 268 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 269 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 270 */ "trnm ::= nm DOT nm", + /* 271 */ "tridxby ::= INDEXED BY nm", + /* 272 */ "tridxby ::= NOT INDEXED", + /* 273 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 274 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 275 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 276 */ "trigger_cmd ::= scanpt select scanpt", + /* 277 */ "expr ::= RAISE LP IGNORE RP", + /* 278 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 279 */ "raisetype ::= ROLLBACK", + /* 280 */ "raisetype ::= ABORT", + /* 281 */ "raisetype ::= FAIL", + /* 282 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 283 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 284 */ "cmd ::= DETACH database_kw_opt expr", + /* 285 */ "key_opt ::=", + /* 286 */ "key_opt ::= KEY expr", + /* 287 */ "cmd ::= REINDEX", + /* 288 */ "cmd ::= REINDEX nm dbnm", + /* 289 */ "cmd ::= ANALYZE", + /* 290 */ "cmd ::= ANALYZE nm dbnm", + /* 291 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 292 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 293 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 294 */ "add_column_fullname ::= fullname", + /* 295 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 296 */ "cmd ::= create_vtab", + /* 297 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 298 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 299 */ "vtabarg ::=", + /* 300 */ "vtabargtoken ::= ANY", + /* 301 */ "vtabargtoken ::= lp anylist RP", + /* 302 */ "lp ::= LP", + /* 303 */ "with ::= WITH wqlist", + /* 304 */ "with ::= WITH RECURSIVE wqlist", + /* 305 */ "wqas ::= AS", + /* 306 */ "wqas ::= AS MATERIALIZED", + /* 307 */ "wqas ::= AS NOT MATERIALIZED", + /* 308 */ "wqitem ::= nm eidlist_opt wqas LP select RP", + /* 309 */ "wqlist ::= wqitem", + /* 310 */ "wqlist ::= wqlist COMMA wqitem", + /* 311 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 312 */ "windowdefn ::= nm AS LP window RP", + /* 313 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 314 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 315 */ "window ::= ORDER BY sortlist frame_opt", + /* 316 */ "window ::= nm ORDER BY sortlist frame_opt", /* 317 */ "window ::= nm frame_opt", /* 318 */ "frame_opt ::=", /* 319 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", @@ -169939,6 +172530,8 @@ static const char *const yyRuleName[] = { /* 400 */ "anylist ::= anylist LP anylist RP", /* 401 */ "anylist ::= anylist ANY", /* 402 */ "with ::=", + /* 403 */ "windowdefn_list ::= windowdefn", + /* 404 */ "window ::= frame_opt", }; #endif /* NDEBUG */ @@ -170633,135 +173226,135 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 217, /* (185) expr ::= expr COLLATE ID|STRING */ 217, /* (186) expr ::= CAST LP expr AS typetoken RP */ 217, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - 217, /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - 217, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - 217, /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - 216, /* (191) term ::= CTIME_KW */ - 217, /* (192) expr ::= LP nexprlist COMMA expr RP */ - 217, /* (193) expr ::= expr AND expr */ - 217, /* (194) expr ::= expr OR expr */ - 217, /* (195) expr ::= expr LT|GT|GE|LE expr */ - 217, /* (196) expr ::= expr EQ|NE expr */ - 217, /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - 217, /* (198) expr ::= expr PLUS|MINUS expr */ - 217, /* (199) expr ::= expr STAR|SLASH|REM expr */ - 217, /* (200) expr ::= expr CONCAT expr */ - 274, /* (201) likeop ::= NOT LIKE_KW|MATCH */ - 217, /* (202) expr ::= expr likeop expr */ - 217, /* (203) expr ::= expr likeop expr ESCAPE expr */ - 217, /* (204) expr ::= expr ISNULL|NOTNULL */ - 217, /* (205) expr ::= expr NOT NULL */ - 217, /* (206) expr ::= expr IS expr */ - 217, /* (207) expr ::= expr IS NOT expr */ - 217, /* (208) expr ::= expr IS NOT DISTINCT FROM expr */ - 217, /* (209) expr ::= expr IS DISTINCT FROM expr */ - 217, /* (210) expr ::= NOT expr */ - 217, /* (211) expr ::= BITNOT expr */ - 217, /* (212) expr ::= PLUS|MINUS expr */ - 217, /* (213) expr ::= expr PTR expr */ - 275, /* (214) between_op ::= BETWEEN */ - 275, /* (215) between_op ::= NOT BETWEEN */ - 217, /* (216) expr ::= expr between_op expr AND expr */ - 276, /* (217) in_op ::= IN */ - 276, /* (218) in_op ::= NOT IN */ - 217, /* (219) expr ::= expr in_op LP exprlist RP */ - 217, /* (220) expr ::= LP select RP */ - 217, /* (221) expr ::= expr in_op LP select RP */ - 217, /* (222) expr ::= expr in_op nm dbnm paren_exprlist */ - 217, /* (223) expr ::= EXISTS LP select RP */ - 217, /* (224) expr ::= CASE case_operand case_exprlist case_else END */ - 279, /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - 279, /* (226) case_exprlist ::= WHEN expr THEN expr */ - 280, /* (227) case_else ::= ELSE expr */ - 280, /* (228) case_else ::= */ - 278, /* (229) case_operand ::= */ - 261, /* (230) exprlist ::= */ - 253, /* (231) nexprlist ::= nexprlist COMMA expr */ - 253, /* (232) nexprlist ::= expr */ - 277, /* (233) paren_exprlist ::= */ - 277, /* (234) paren_exprlist ::= LP exprlist RP */ - 190, /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - 281, /* (236) uniqueflag ::= UNIQUE */ - 281, /* (237) uniqueflag ::= */ - 221, /* (238) eidlist_opt ::= */ - 221, /* (239) eidlist_opt ::= LP eidlist RP */ - 232, /* (240) eidlist ::= eidlist COMMA nm collate sortorder */ - 232, /* (241) eidlist ::= nm collate sortorder */ - 282, /* (242) collate ::= */ - 282, /* (243) collate ::= COLLATE ID|STRING */ - 190, /* (244) cmd ::= DROP INDEX ifexists fullname */ - 190, /* (245) cmd ::= VACUUM vinto */ - 190, /* (246) cmd ::= VACUUM nm vinto */ - 283, /* (247) vinto ::= INTO expr */ - 283, /* (248) vinto ::= */ - 190, /* (249) cmd ::= PRAGMA nm dbnm */ - 190, /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */ - 190, /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - 190, /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */ - 190, /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - 211, /* (254) plus_num ::= PLUS INTEGER|FLOAT */ - 212, /* (255) minus_num ::= MINUS INTEGER|FLOAT */ - 190, /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - 285, /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - 287, /* (258) trigger_time ::= BEFORE|AFTER */ - 287, /* (259) trigger_time ::= INSTEAD OF */ - 287, /* (260) trigger_time ::= */ - 288, /* (261) trigger_event ::= DELETE|INSERT */ - 288, /* (262) trigger_event ::= UPDATE */ - 288, /* (263) trigger_event ::= UPDATE OF idlist */ - 290, /* (264) when_clause ::= */ - 290, /* (265) when_clause ::= WHEN expr */ - 286, /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - 286, /* (267) trigger_cmd_list ::= trigger_cmd SEMI */ - 292, /* (268) trnm ::= nm DOT nm */ - 293, /* (269) tridxby ::= INDEXED BY nm */ - 293, /* (270) tridxby ::= NOT INDEXED */ - 291, /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 291, /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 291, /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 291, /* (274) trigger_cmd ::= scanpt select scanpt */ - 217, /* (275) expr ::= RAISE LP IGNORE RP */ - 217, /* (276) expr ::= RAISE LP raisetype COMMA nm RP */ - 236, /* (277) raisetype ::= ROLLBACK */ - 236, /* (278) raisetype ::= ABORT */ - 236, /* (279) raisetype ::= FAIL */ - 190, /* (280) cmd ::= DROP TRIGGER ifexists fullname */ - 190, /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 190, /* (282) cmd ::= DETACH database_kw_opt expr */ - 295, /* (283) key_opt ::= */ - 295, /* (284) key_opt ::= KEY expr */ - 190, /* (285) cmd ::= REINDEX */ - 190, /* (286) cmd ::= REINDEX nm dbnm */ - 190, /* (287) cmd ::= ANALYZE */ - 190, /* (288) cmd ::= ANALYZE nm dbnm */ - 190, /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 190, /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - 190, /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 296, /* (292) add_column_fullname ::= fullname */ - 190, /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 190, /* (294) cmd ::= create_vtab */ - 190, /* (295) cmd ::= create_vtab LP vtabarglist RP */ - 298, /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 300, /* (297) vtabarg ::= */ - 301, /* (298) vtabargtoken ::= ANY */ - 301, /* (299) vtabargtoken ::= lp anylist RP */ - 302, /* (300) lp ::= LP */ - 266, /* (301) with ::= WITH wqlist */ - 266, /* (302) with ::= WITH RECURSIVE wqlist */ - 305, /* (303) wqas ::= AS */ - 305, /* (304) wqas ::= AS MATERIALIZED */ - 305, /* (305) wqas ::= AS NOT MATERIALIZED */ - 304, /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */ - 241, /* (307) wqlist ::= wqitem */ - 241, /* (308) wqlist ::= wqlist COMMA wqitem */ - 306, /* (309) windowdefn_list ::= windowdefn */ - 306, /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 307, /* (311) windowdefn ::= nm AS LP window RP */ - 308, /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (314) window ::= ORDER BY sortlist frame_opt */ - 308, /* (315) window ::= nm ORDER BY sortlist frame_opt */ - 308, /* (316) window ::= frame_opt */ + 217, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + 217, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 217, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 217, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + 217, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 216, /* (193) term ::= CTIME_KW */ + 217, /* (194) expr ::= LP nexprlist COMMA expr RP */ + 217, /* (195) expr ::= expr AND expr */ + 217, /* (196) expr ::= expr OR expr */ + 217, /* (197) expr ::= expr LT|GT|GE|LE expr */ + 217, /* (198) expr ::= expr EQ|NE expr */ + 217, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 217, /* (200) expr ::= expr PLUS|MINUS expr */ + 217, /* (201) expr ::= expr STAR|SLASH|REM expr */ + 217, /* (202) expr ::= expr CONCAT expr */ + 274, /* (203) likeop ::= NOT LIKE_KW|MATCH */ + 217, /* (204) expr ::= expr likeop expr */ + 217, /* (205) expr ::= expr likeop expr ESCAPE expr */ + 217, /* (206) expr ::= expr ISNULL|NOTNULL */ + 217, /* (207) expr ::= expr NOT NULL */ + 217, /* (208) expr ::= expr IS expr */ + 217, /* (209) expr ::= expr IS NOT expr */ + 217, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ + 217, /* (211) expr ::= expr IS DISTINCT FROM expr */ + 217, /* (212) expr ::= NOT expr */ + 217, /* (213) expr ::= BITNOT expr */ + 217, /* (214) expr ::= PLUS|MINUS expr */ + 217, /* (215) expr ::= expr PTR expr */ + 275, /* (216) between_op ::= BETWEEN */ + 275, /* (217) between_op ::= NOT BETWEEN */ + 217, /* (218) expr ::= expr between_op expr AND expr */ + 276, /* (219) in_op ::= IN */ + 276, /* (220) in_op ::= NOT IN */ + 217, /* (221) expr ::= expr in_op LP exprlist RP */ + 217, /* (222) expr ::= LP select RP */ + 217, /* (223) expr ::= expr in_op LP select RP */ + 217, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ + 217, /* (225) expr ::= EXISTS LP select RP */ + 217, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ + 279, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 279, /* (228) case_exprlist ::= WHEN expr THEN expr */ + 280, /* (229) case_else ::= ELSE expr */ + 280, /* (230) case_else ::= */ + 278, /* (231) case_operand ::= */ + 261, /* (232) exprlist ::= */ + 253, /* (233) nexprlist ::= nexprlist COMMA expr */ + 253, /* (234) nexprlist ::= expr */ + 277, /* (235) paren_exprlist ::= */ + 277, /* (236) paren_exprlist ::= LP exprlist RP */ + 190, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 281, /* (238) uniqueflag ::= UNIQUE */ + 281, /* (239) uniqueflag ::= */ + 221, /* (240) eidlist_opt ::= */ + 221, /* (241) eidlist_opt ::= LP eidlist RP */ + 232, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ + 232, /* (243) eidlist ::= nm collate sortorder */ + 282, /* (244) collate ::= */ + 282, /* (245) collate ::= COLLATE ID|STRING */ + 190, /* (246) cmd ::= DROP INDEX ifexists fullname */ + 190, /* (247) cmd ::= VACUUM vinto */ + 190, /* (248) cmd ::= VACUUM nm vinto */ + 283, /* (249) vinto ::= INTO expr */ + 283, /* (250) vinto ::= */ + 190, /* (251) cmd ::= PRAGMA nm dbnm */ + 190, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 190, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 190, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 190, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 211, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ + 212, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ + 190, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 285, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 287, /* (260) trigger_time ::= BEFORE|AFTER */ + 287, /* (261) trigger_time ::= INSTEAD OF */ + 287, /* (262) trigger_time ::= */ + 288, /* (263) trigger_event ::= DELETE|INSERT */ + 288, /* (264) trigger_event ::= UPDATE */ + 288, /* (265) trigger_event ::= UPDATE OF idlist */ + 290, /* (266) when_clause ::= */ + 290, /* (267) when_clause ::= WHEN expr */ + 286, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 286, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ + 292, /* (270) trnm ::= nm DOT nm */ + 293, /* (271) tridxby ::= INDEXED BY nm */ + 293, /* (272) tridxby ::= NOT INDEXED */ + 291, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 291, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 291, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 291, /* (276) trigger_cmd ::= scanpt select scanpt */ + 217, /* (277) expr ::= RAISE LP IGNORE RP */ + 217, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ + 236, /* (279) raisetype ::= ROLLBACK */ + 236, /* (280) raisetype ::= ABORT */ + 236, /* (281) raisetype ::= FAIL */ + 190, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ + 190, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 190, /* (284) cmd ::= DETACH database_kw_opt expr */ + 295, /* (285) key_opt ::= */ + 295, /* (286) key_opt ::= KEY expr */ + 190, /* (287) cmd ::= REINDEX */ + 190, /* (288) cmd ::= REINDEX nm dbnm */ + 190, /* (289) cmd ::= ANALYZE */ + 190, /* (290) cmd ::= ANALYZE nm dbnm */ + 190, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 190, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 190, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 296, /* (294) add_column_fullname ::= fullname */ + 190, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 190, /* (296) cmd ::= create_vtab */ + 190, /* (297) cmd ::= create_vtab LP vtabarglist RP */ + 298, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 300, /* (299) vtabarg ::= */ + 301, /* (300) vtabargtoken ::= ANY */ + 301, /* (301) vtabargtoken ::= lp anylist RP */ + 302, /* (302) lp ::= LP */ + 266, /* (303) with ::= WITH wqlist */ + 266, /* (304) with ::= WITH RECURSIVE wqlist */ + 305, /* (305) wqas ::= AS */ + 305, /* (306) wqas ::= AS MATERIALIZED */ + 305, /* (307) wqas ::= AS NOT MATERIALIZED */ + 304, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ + 241, /* (309) wqlist ::= wqitem */ + 241, /* (310) wqlist ::= wqlist COMMA wqitem */ + 306, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 307, /* (312) windowdefn ::= nm AS LP window RP */ + 308, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 308, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 308, /* (315) window ::= ORDER BY sortlist frame_opt */ + 308, /* (316) window ::= nm ORDER BY sortlist frame_opt */ 308, /* (317) window ::= nm frame_opt */ 309, /* (318) frame_opt ::= */ 309, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ @@ -170848,6 +173441,8 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 303, /* (400) anylist ::= anylist LP anylist RP */ 303, /* (401) anylist ::= anylist ANY */ 266, /* (402) with ::= */ + 306, /* (403) windowdefn_list ::= windowdefn */ + 308, /* (404) window ::= frame_opt */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -171041,135 +173636,135 @@ static const signed char yyRuleInfoNRhs[] = { -3, /* (185) expr ::= expr COLLATE ID|STRING */ -6, /* (186) expr ::= CAST LP expr AS typetoken RP */ -5, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - -4, /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - -6, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - -5, /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - -1, /* (191) term ::= CTIME_KW */ - -5, /* (192) expr ::= LP nexprlist COMMA expr RP */ - -3, /* (193) expr ::= expr AND expr */ - -3, /* (194) expr ::= expr OR expr */ - -3, /* (195) expr ::= expr LT|GT|GE|LE expr */ - -3, /* (196) expr ::= expr EQ|NE expr */ - -3, /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - -3, /* (198) expr ::= expr PLUS|MINUS expr */ - -3, /* (199) expr ::= expr STAR|SLASH|REM expr */ - -3, /* (200) expr ::= expr CONCAT expr */ - -2, /* (201) likeop ::= NOT LIKE_KW|MATCH */ - -3, /* (202) expr ::= expr likeop expr */ - -5, /* (203) expr ::= expr likeop expr ESCAPE expr */ - -2, /* (204) expr ::= expr ISNULL|NOTNULL */ - -3, /* (205) expr ::= expr NOT NULL */ - -3, /* (206) expr ::= expr IS expr */ - -4, /* (207) expr ::= expr IS NOT expr */ - -6, /* (208) expr ::= expr IS NOT DISTINCT FROM expr */ - -5, /* (209) expr ::= expr IS DISTINCT FROM expr */ - -2, /* (210) expr ::= NOT expr */ - -2, /* (211) expr ::= BITNOT expr */ - -2, /* (212) expr ::= PLUS|MINUS expr */ - -3, /* (213) expr ::= expr PTR expr */ - -1, /* (214) between_op ::= BETWEEN */ - -2, /* (215) between_op ::= NOT BETWEEN */ - -5, /* (216) expr ::= expr between_op expr AND expr */ - -1, /* (217) in_op ::= IN */ - -2, /* (218) in_op ::= NOT IN */ - -5, /* (219) expr ::= expr in_op LP exprlist RP */ - -3, /* (220) expr ::= LP select RP */ - -5, /* (221) expr ::= expr in_op LP select RP */ - -5, /* (222) expr ::= expr in_op nm dbnm paren_exprlist */ - -4, /* (223) expr ::= EXISTS LP select RP */ - -5, /* (224) expr ::= CASE case_operand case_exprlist case_else END */ - -5, /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - -4, /* (226) case_exprlist ::= WHEN expr THEN expr */ - -2, /* (227) case_else ::= ELSE expr */ - 0, /* (228) case_else ::= */ - 0, /* (229) case_operand ::= */ - 0, /* (230) exprlist ::= */ - -3, /* (231) nexprlist ::= nexprlist COMMA expr */ - -1, /* (232) nexprlist ::= expr */ - 0, /* (233) paren_exprlist ::= */ - -3, /* (234) paren_exprlist ::= LP exprlist RP */ - -12, /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - -1, /* (236) uniqueflag ::= UNIQUE */ - 0, /* (237) uniqueflag ::= */ - 0, /* (238) eidlist_opt ::= */ - -3, /* (239) eidlist_opt ::= LP eidlist RP */ - -5, /* (240) eidlist ::= eidlist COMMA nm collate sortorder */ - -3, /* (241) eidlist ::= nm collate sortorder */ - 0, /* (242) collate ::= */ - -2, /* (243) collate ::= COLLATE ID|STRING */ - -4, /* (244) cmd ::= DROP INDEX ifexists fullname */ - -2, /* (245) cmd ::= VACUUM vinto */ - -3, /* (246) cmd ::= VACUUM nm vinto */ - -2, /* (247) vinto ::= INTO expr */ - 0, /* (248) vinto ::= */ - -3, /* (249) cmd ::= PRAGMA nm dbnm */ - -5, /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */ - -6, /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - -5, /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */ - -6, /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - -2, /* (254) plus_num ::= PLUS INTEGER|FLOAT */ - -2, /* (255) minus_num ::= MINUS INTEGER|FLOAT */ - -5, /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - -11, /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - -1, /* (258) trigger_time ::= BEFORE|AFTER */ - -2, /* (259) trigger_time ::= INSTEAD OF */ - 0, /* (260) trigger_time ::= */ - -1, /* (261) trigger_event ::= DELETE|INSERT */ - -1, /* (262) trigger_event ::= UPDATE */ - -3, /* (263) trigger_event ::= UPDATE OF idlist */ - 0, /* (264) when_clause ::= */ - -2, /* (265) when_clause ::= WHEN expr */ - -3, /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - -2, /* (267) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (268) trnm ::= nm DOT nm */ - -3, /* (269) tridxby ::= INDEXED BY nm */ - -2, /* (270) tridxby ::= NOT INDEXED */ - -9, /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (274) trigger_cmd ::= scanpt select scanpt */ - -4, /* (275) expr ::= RAISE LP IGNORE RP */ - -6, /* (276) expr ::= RAISE LP raisetype COMMA nm RP */ - -1, /* (277) raisetype ::= ROLLBACK */ - -1, /* (278) raisetype ::= ABORT */ - -1, /* (279) raisetype ::= FAIL */ - -4, /* (280) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (282) cmd ::= DETACH database_kw_opt expr */ - 0, /* (283) key_opt ::= */ - -2, /* (284) key_opt ::= KEY expr */ - -1, /* (285) cmd ::= REINDEX */ - -3, /* (286) cmd ::= REINDEX nm dbnm */ - -1, /* (287) cmd ::= ANALYZE */ - -3, /* (288) cmd ::= ANALYZE nm dbnm */ - -6, /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - -6, /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (292) add_column_fullname ::= fullname */ - -8, /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (294) cmd ::= create_vtab */ - -4, /* (295) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (297) vtabarg ::= */ - -1, /* (298) vtabargtoken ::= ANY */ - -3, /* (299) vtabargtoken ::= lp anylist RP */ - -1, /* (300) lp ::= LP */ - -2, /* (301) with ::= WITH wqlist */ - -3, /* (302) with ::= WITH RECURSIVE wqlist */ - -1, /* (303) wqas ::= AS */ - -2, /* (304) wqas ::= AS MATERIALIZED */ - -3, /* (305) wqas ::= AS NOT MATERIALIZED */ - -6, /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */ - -1, /* (307) wqlist ::= wqitem */ - -3, /* (308) wqlist ::= wqlist COMMA wqitem */ - -1, /* (309) windowdefn_list ::= windowdefn */ - -3, /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (311) windowdefn ::= nm AS LP window RP */ - -5, /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (314) window ::= ORDER BY sortlist frame_opt */ - -5, /* (315) window ::= nm ORDER BY sortlist frame_opt */ - -1, /* (316) window ::= frame_opt */ + -8, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + -4, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -9, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + -5, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (193) term ::= CTIME_KW */ + -5, /* (194) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (195) expr ::= expr AND expr */ + -3, /* (196) expr ::= expr OR expr */ + -3, /* (197) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (198) expr ::= expr EQ|NE expr */ + -3, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (200) expr ::= expr PLUS|MINUS expr */ + -3, /* (201) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (202) expr ::= expr CONCAT expr */ + -2, /* (203) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (204) expr ::= expr likeop expr */ + -5, /* (205) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (206) expr ::= expr ISNULL|NOTNULL */ + -3, /* (207) expr ::= expr NOT NULL */ + -3, /* (208) expr ::= expr IS expr */ + -4, /* (209) expr ::= expr IS NOT expr */ + -6, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (211) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (212) expr ::= NOT expr */ + -2, /* (213) expr ::= BITNOT expr */ + -2, /* (214) expr ::= PLUS|MINUS expr */ + -3, /* (215) expr ::= expr PTR expr */ + -1, /* (216) between_op ::= BETWEEN */ + -2, /* (217) between_op ::= NOT BETWEEN */ + -5, /* (218) expr ::= expr between_op expr AND expr */ + -1, /* (219) in_op ::= IN */ + -2, /* (220) in_op ::= NOT IN */ + -5, /* (221) expr ::= expr in_op LP exprlist RP */ + -3, /* (222) expr ::= LP select RP */ + -5, /* (223) expr ::= expr in_op LP select RP */ + -5, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (225) expr ::= EXISTS LP select RP */ + -5, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (228) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (229) case_else ::= ELSE expr */ + 0, /* (230) case_else ::= */ + 0, /* (231) case_operand ::= */ + 0, /* (232) exprlist ::= */ + -3, /* (233) nexprlist ::= nexprlist COMMA expr */ + -1, /* (234) nexprlist ::= expr */ + 0, /* (235) paren_exprlist ::= */ + -3, /* (236) paren_exprlist ::= LP exprlist RP */ + -12, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (238) uniqueflag ::= UNIQUE */ + 0, /* (239) uniqueflag ::= */ + 0, /* (240) eidlist_opt ::= */ + -3, /* (241) eidlist_opt ::= LP eidlist RP */ + -5, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (243) eidlist ::= nm collate sortorder */ + 0, /* (244) collate ::= */ + -2, /* (245) collate ::= COLLATE ID|STRING */ + -4, /* (246) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (247) cmd ::= VACUUM vinto */ + -3, /* (248) cmd ::= VACUUM nm vinto */ + -2, /* (249) vinto ::= INTO expr */ + 0, /* (250) vinto ::= */ + -3, /* (251) cmd ::= PRAGMA nm dbnm */ + -5, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (260) trigger_time ::= BEFORE|AFTER */ + -2, /* (261) trigger_time ::= INSTEAD OF */ + 0, /* (262) trigger_time ::= */ + -1, /* (263) trigger_event ::= DELETE|INSERT */ + -1, /* (264) trigger_event ::= UPDATE */ + -3, /* (265) trigger_event ::= UPDATE OF idlist */ + 0, /* (266) when_clause ::= */ + -2, /* (267) when_clause ::= WHEN expr */ + -3, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (270) trnm ::= nm DOT nm */ + -3, /* (271) tridxby ::= INDEXED BY nm */ + -2, /* (272) tridxby ::= NOT INDEXED */ + -9, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (276) trigger_cmd ::= scanpt select scanpt */ + -4, /* (277) expr ::= RAISE LP IGNORE RP */ + -6, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (279) raisetype ::= ROLLBACK */ + -1, /* (280) raisetype ::= ABORT */ + -1, /* (281) raisetype ::= FAIL */ + -4, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (284) cmd ::= DETACH database_kw_opt expr */ + 0, /* (285) key_opt ::= */ + -2, /* (286) key_opt ::= KEY expr */ + -1, /* (287) cmd ::= REINDEX */ + -3, /* (288) cmd ::= REINDEX nm dbnm */ + -1, /* (289) cmd ::= ANALYZE */ + -3, /* (290) cmd ::= ANALYZE nm dbnm */ + -6, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (294) add_column_fullname ::= fullname */ + -8, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (296) cmd ::= create_vtab */ + -4, /* (297) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (299) vtabarg ::= */ + -1, /* (300) vtabargtoken ::= ANY */ + -3, /* (301) vtabargtoken ::= lp anylist RP */ + -1, /* (302) lp ::= LP */ + -2, /* (303) with ::= WITH wqlist */ + -3, /* (304) with ::= WITH RECURSIVE wqlist */ + -1, /* (305) wqas ::= AS */ + -2, /* (306) wqas ::= AS MATERIALIZED */ + -3, /* (307) wqas ::= AS NOT MATERIALIZED */ + -6, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ + -1, /* (309) wqlist ::= wqitem */ + -3, /* (310) wqlist ::= wqlist COMMA wqitem */ + -3, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (312) windowdefn ::= nm AS LP window RP */ + -5, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (315) window ::= ORDER BY sortlist frame_opt */ + -5, /* (316) window ::= nm ORDER BY sortlist frame_opt */ -2, /* (317) window ::= nm frame_opt */ 0, /* (318) frame_opt ::= */ -3, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ @@ -171256,6 +173851,8 @@ static const signed char yyRuleInfoNRhs[] = { -4, /* (400) anylist ::= anylist LP anylist RP */ -2, /* (401) anylist ::= anylist ANY */ 0, /* (402) with ::= */ + -1, /* (403) windowdefn_list ::= windowdefn */ + -1, /* (404) window ::= frame_opt */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -171298,10 +173895,10 @@ static YYACTIONTYPE yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* explain ::= EXPLAIN */ -{ pParse->explain = 1; } +{ if( pParse->pReprepare==0 ) pParse->explain = 1; } break; case 1: /* explain ::= EXPLAIN QUERY PLAN */ -{ pParse->explain = 2; } +{ if( pParse->pReprepare==0 ) pParse->explain = 2; } break; case 2: /* cmdx ::= cmd */ { sqlite3FinishCoding(pParse); } @@ -171352,7 +173949,7 @@ static YYACTIONTYPE yy_reduce( case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); case 81: /* ifexists ::= */ yytestcase(yyruleno==81); case 98: /* distinct ::= */ yytestcase(yyruleno==98); - case 242: /* collate ::= */ yytestcase(yyruleno==242); + case 244: /* collate ::= */ yytestcase(yyruleno==244); {yymsp[1].minor.yy394 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ @@ -171536,9 +174133,9 @@ static YYACTIONTYPE yy_reduce( break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); - case 215: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==215); - case 218: /* in_op ::= NOT IN */ yytestcase(yyruleno==218); - case 243: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==243); + case 217: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==217); + case 220: /* in_op ::= NOT IN */ yytestcase(yyruleno==220); + case 245: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==245); {yymsp[-1].minor.yy394 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ @@ -171611,7 +174208,6 @@ static YYACTIONTYPE yy_reduce( if( p ){ parserDoubleLinkSelect(pParse, p); } - yymsp[0].minor.yy47 = p; /*A-overwrites-X*/ } break; case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ @@ -171688,9 +174284,9 @@ static YYACTIONTYPE yy_reduce( case 99: /* sclp ::= */ case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); - case 230: /* exprlist ::= */ yytestcase(yyruleno==230); - case 233: /* paren_exprlist ::= */ yytestcase(yyruleno==233); - case 238: /* eidlist_opt ::= */ yytestcase(yyruleno==238); + case 232: /* exprlist ::= */ yytestcase(yyruleno==232); + case 235: /* paren_exprlist ::= */ yytestcase(yyruleno==235); + case 240: /* eidlist_opt ::= */ yytestcase(yyruleno==240); {yymsp[1].minor.yy322 = 0;} break; case 100: /* selcollist ::= sclp scanpt expr scanpt as */ @@ -171703,21 +174299,24 @@ static YYACTIONTYPE yy_reduce( case 101: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); + sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy322, p); } break; case 102: /* selcollist ::= sclp scanpt nm DOT STAR */ { - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); - Expr *pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + Expr *pRight, *pLeft, *pDot; + pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); + pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); + pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, pDot); } break; case 103: /* as ::= AS nm */ case 115: /* dbnm ::= DOT nm */ yytestcase(yyruleno==115); - case 254: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==254); - case 255: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==255); + case 256: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==256); + case 257: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==257); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; case 105: /* from ::= */ @@ -171889,16 +174488,16 @@ static YYACTIONTYPE yy_reduce( case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); case 151: /* where_opt ::= */ yytestcase(yyruleno==151); case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); - case 228: /* case_else ::= */ yytestcase(yyruleno==228); - case 229: /* case_operand ::= */ yytestcase(yyruleno==229); - case 248: /* vinto ::= */ yytestcase(yyruleno==248); + case 230: /* case_else ::= */ yytestcase(yyruleno==230); + case 231: /* case_operand ::= */ yytestcase(yyruleno==231); + case 250: /* vinto ::= */ yytestcase(yyruleno==250); {yymsp[1].minor.yy528 = 0;} break; case 145: /* having_opt ::= HAVING expr */ case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); - case 227: /* case_else ::= ELSE expr */ yytestcase(yyruleno==227); - case 247: /* vinto ::= INTO expr */ yytestcase(yyruleno==247); + case 229: /* case_else ::= ELSE expr */ yytestcase(yyruleno==229); + case 249: /* vinto ::= INTO expr */ yytestcase(yyruleno==249); {yymsp[-1].minor.yy528 = yymsp[0].minor.yy528;} break; case 147: /* limit_opt ::= LIMIT expr */ @@ -172084,33 +174683,48 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 188: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + case 188: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ +{ + yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy322, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy394); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-1].minor.yy322); +} + yymsp[-7].minor.yy528 = yylhsminor.yy528; + break; + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } yymsp[-3].minor.yy528 = yylhsminor.yy528; break; - case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy322, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy394); sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); } yymsp[-5].minor.yy528 = yylhsminor.yy528; break; - case 190: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + case 191: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ +{ + yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy322, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy394); + sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-2].minor.yy322); +} + yymsp[-8].minor.yy528 = yylhsminor.yy528; + break; + case 192: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 191: /* term ::= CTIME_KW */ + case 193: /* term ::= CTIME_KW */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } yymsp[0].minor.yy528 = yylhsminor.yy528; break; - case 192: /* expr ::= LP nexprlist COMMA expr RP */ + case 194: /* expr ::= LP nexprlist COMMA expr RP */ { ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528); yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); @@ -172124,22 +174738,22 @@ static YYACTIONTYPE yy_reduce( } } break; - case 193: /* expr ::= expr AND expr */ + case 195: /* expr ::= expr AND expr */ {yymsp[-2].minor.yy528=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} break; - case 194: /* expr ::= expr OR expr */ - case 195: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==195); - case 196: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==196); - case 197: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==197); - case 198: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==198); - case 199: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==200); + case 196: /* expr ::= expr OR expr */ + case 197: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==197); + case 198: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==198); + case 199: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==202); {yymsp[-2].minor.yy528=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} break; - case 201: /* likeop ::= NOT LIKE_KW|MATCH */ + case 203: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 202: /* expr ::= expr likeop expr */ + case 204: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; @@ -172151,7 +174765,7 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-2].minor.yy528 ) yymsp[-2].minor.yy528->flags |= EP_InfixFunc; } break; - case 203: /* expr ::= expr likeop expr ESCAPE expr */ + case 205: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; @@ -172164,47 +174778,47 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-4].minor.yy528 ) yymsp[-4].minor.yy528->flags |= EP_InfixFunc; } break; - case 204: /* expr ::= expr ISNULL|NOTNULL */ + case 206: /* expr ::= expr ISNULL|NOTNULL */ {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy528,0);} break; - case 205: /* expr ::= expr NOT NULL */ + case 207: /* expr ::= expr NOT NULL */ {yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy528,0);} break; - case 206: /* expr ::= expr IS expr */ + case 208: /* expr ::= expr IS expr */ { yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-2].minor.yy528, TK_ISNULL); } break; - case 207: /* expr ::= expr IS NOT expr */ + case 209: /* expr ::= expr IS NOT expr */ { yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-3].minor.yy528, TK_NOTNULL); } break; - case 208: /* expr ::= expr IS NOT DISTINCT FROM expr */ + case 210: /* expr ::= expr IS NOT DISTINCT FROM expr */ { yymsp[-5].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-5].minor.yy528, TK_ISNULL); } break; - case 209: /* expr ::= expr IS DISTINCT FROM expr */ + case 211: /* expr ::= expr IS DISTINCT FROM expr */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-4].minor.yy528, TK_NOTNULL); } break; - case 210: /* expr ::= NOT expr */ - case 211: /* expr ::= BITNOT expr */ yytestcase(yyruleno==211); + case 212: /* expr ::= NOT expr */ + case 213: /* expr ::= BITNOT expr */ yytestcase(yyruleno==213); {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy528, 0);/*A-overwrites-B*/} break; - case 212: /* expr ::= PLUS|MINUS expr */ + case 214: /* expr ::= PLUS|MINUS expr */ { yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy528, 0); /*A-overwrites-B*/ } break; - case 213: /* expr ::= expr PTR expr */ + case 215: /* expr ::= expr PTR expr */ { ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy528); pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy528); @@ -172212,11 +174826,11 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy528 = yylhsminor.yy528; break; - case 214: /* between_op ::= BETWEEN */ - case 217: /* in_op ::= IN */ yytestcase(yyruleno==217); + case 216: /* between_op ::= BETWEEN */ + case 219: /* in_op ::= IN */ yytestcase(yyruleno==219); {yymsp[0].minor.yy394 = 0;} break; - case 216: /* expr ::= expr between_op expr AND expr */ + case 218: /* expr ::= expr between_op expr AND expr */ { ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); @@ -172229,7 +174843,7 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 219: /* expr ::= expr in_op LP exprlist RP */ + case 221: /* expr ::= expr in_op LP exprlist RP */ { if( yymsp[-1].minor.yy322==0 ){ /* Expressions of the form @@ -172275,20 +174889,20 @@ static YYACTIONTYPE yy_reduce( } } break; - case 220: /* expr ::= LP select RP */ + case 222: /* expr ::= LP select RP */ { yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy528, yymsp[-1].minor.yy47); } break; - case 221: /* expr ::= expr in_op LP select RP */ + case 223: /* expr ::= expr in_op LP select RP */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, yymsp[-1].minor.yy47); if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 222: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 224: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); @@ -172298,14 +174912,14 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 223: /* expr ::= EXISTS LP select RP */ + case 225: /* expr ::= EXISTS LP select RP */ { Expr *p; p = yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy47); } break; - case 224: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 226: /* expr ::= CASE case_operand case_exprlist case_else END */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy528, 0); if( yymsp[-4].minor.yy528 ){ @@ -172317,29 +174931,29 @@ static YYACTIONTYPE yy_reduce( } } break; - case 225: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 227: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[0].minor.yy528); } break; - case 226: /* case_exprlist ::= WHEN expr THEN expr */ + case 228: /* case_exprlist ::= WHEN expr THEN expr */ { yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, yymsp[0].minor.yy528); } break; - case 231: /* nexprlist ::= nexprlist COMMA expr */ + case 233: /* nexprlist ::= nexprlist COMMA expr */ {yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy528);} break; - case 232: /* nexprlist ::= expr */ + case 234: /* nexprlist ::= expr */ {yymsp[0].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy528); /*A-overwrites-Y*/} break; - case 234: /* paren_exprlist ::= LP exprlist RP */ - case 239: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==239); + case 236: /* paren_exprlist ::= LP exprlist RP */ + case 241: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==241); {yymsp[-2].minor.yy322 = yymsp[-1].minor.yy322;} break; - case 235: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 237: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy322, yymsp[-10].minor.yy394, @@ -172349,48 +174963,48 @@ static YYACTIONTYPE yy_reduce( } } break; - case 236: /* uniqueflag ::= UNIQUE */ - case 278: /* raisetype ::= ABORT */ yytestcase(yyruleno==278); + case 238: /* uniqueflag ::= UNIQUE */ + case 280: /* raisetype ::= ABORT */ yytestcase(yyruleno==280); {yymsp[0].minor.yy394 = OE_Abort;} break; - case 237: /* uniqueflag ::= */ + case 239: /* uniqueflag ::= */ {yymsp[1].minor.yy394 = OE_None;} break; - case 240: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 242: /* eidlist ::= eidlist COMMA nm collate sortorder */ { yymsp[-4].minor.yy322 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); } break; - case 241: /* eidlist ::= nm collate sortorder */ + case 243: /* eidlist ::= nm collate sortorder */ { yymsp[-2].minor.yy322 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); /*A-overwrites-Y*/ } break; - case 244: /* cmd ::= DROP INDEX ifexists fullname */ + case 246: /* cmd ::= DROP INDEX ifexists fullname */ {sqlite3DropIndex(pParse, yymsp[0].minor.yy131, yymsp[-1].minor.yy394);} break; - case 245: /* cmd ::= VACUUM vinto */ + case 247: /* cmd ::= VACUUM vinto */ {sqlite3Vacuum(pParse,0,yymsp[0].minor.yy528);} break; - case 246: /* cmd ::= VACUUM nm vinto */ + case 248: /* cmd ::= VACUUM nm vinto */ {sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy528);} break; - case 249: /* cmd ::= PRAGMA nm dbnm */ + case 251: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 250: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 251: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 256: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; @@ -172398,50 +175012,50 @@ static YYACTIONTYPE yy_reduce( sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy33, &all); } break; - case 257: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy394, yymsp[-4].minor.yy180.a, yymsp[-4].minor.yy180.b, yymsp[-2].minor.yy131, yymsp[0].minor.yy528, yymsp[-10].minor.yy394, yymsp[-8].minor.yy394); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 258: /* trigger_time ::= BEFORE|AFTER */ + case 260: /* trigger_time ::= BEFORE|AFTER */ { yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 259: /* trigger_time ::= INSTEAD OF */ + case 261: /* trigger_time ::= INSTEAD OF */ { yymsp[-1].minor.yy394 = TK_INSTEAD;} break; - case 260: /* trigger_time ::= */ + case 262: /* trigger_time ::= */ { yymsp[1].minor.yy394 = TK_BEFORE; } break; - case 261: /* trigger_event ::= DELETE|INSERT */ - case 262: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==262); + case 263: /* trigger_event ::= DELETE|INSERT */ + case 264: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==264); {yymsp[0].minor.yy180.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy180.b = 0;} break; - case 263: /* trigger_event ::= UPDATE OF idlist */ + case 265: /* trigger_event ::= UPDATE OF idlist */ {yymsp[-2].minor.yy180.a = TK_UPDATE; yymsp[-2].minor.yy180.b = yymsp[0].minor.yy254;} break; - case 264: /* when_clause ::= */ - case 283: /* key_opt ::= */ yytestcase(yyruleno==283); + case 266: /* when_clause ::= */ + case 285: /* key_opt ::= */ yytestcase(yyruleno==285); { yymsp[1].minor.yy528 = 0; } break; - case 265: /* when_clause ::= WHEN expr */ - case 284: /* key_opt ::= KEY expr */ yytestcase(yyruleno==284); + case 267: /* when_clause ::= WHEN expr */ + case 286: /* key_opt ::= KEY expr */ yytestcase(yyruleno==286); { yymsp[-1].minor.yy528 = yymsp[0].minor.yy528; } break; - case 266: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 268: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { assert( yymsp[-2].minor.yy33!=0 ); yymsp[-2].minor.yy33->pLast->pNext = yymsp[-1].minor.yy33; yymsp[-2].minor.yy33->pLast = yymsp[-1].minor.yy33; } break; - case 267: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 269: /* trigger_cmd_list ::= trigger_cmd SEMI */ { assert( yymsp[-1].minor.yy33!=0 ); yymsp[-1].minor.yy33->pLast = yymsp[-1].minor.yy33; } break; - case 268: /* trnm ::= nm DOT nm */ + case 270: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -172449,39 +175063,39 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 269: /* tridxby ::= INDEXED BY nm */ + case 271: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 270: /* tridxby ::= NOT INDEXED */ + case 272: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 271: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + case 273: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ {yylhsminor.yy33 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy131, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528, yymsp[-7].minor.yy394, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy522);} yymsp[-8].minor.yy33 = yylhsminor.yy33; break; - case 272: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 274: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { yylhsminor.yy33 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy254,yymsp[-2].minor.yy47,yymsp[-6].minor.yy394,yymsp[-1].minor.yy444,yymsp[-7].minor.yy522,yymsp[0].minor.yy522);/*yylhsminor.yy33-overwrites-yymsp[-6].minor.yy394*/ } yymsp[-7].minor.yy33 = yylhsminor.yy33; break; - case 273: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + case 275: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ {yylhsminor.yy33 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy528, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy522);} yymsp[-5].minor.yy33 = yylhsminor.yy33; break; - case 274: /* trigger_cmd ::= scanpt select scanpt */ + case 276: /* trigger_cmd ::= scanpt select scanpt */ {yylhsminor.yy33 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy47, yymsp[-2].minor.yy522, yymsp[0].minor.yy522); /*yylhsminor.yy33-overwrites-yymsp[-1].minor.yy47*/} yymsp[-2].minor.yy33 = yylhsminor.yy33; break; - case 275: /* expr ::= RAISE LP IGNORE RP */ + case 277: /* expr ::= RAISE LP IGNORE RP */ { yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); if( yymsp[-3].minor.yy528 ){ @@ -172489,7 +175103,7 @@ static YYACTIONTYPE yy_reduce( } } break; - case 276: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 278: /* expr ::= RAISE LP raisetype COMMA nm RP */ { yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); if( yymsp[-5].minor.yy528 ) { @@ -172497,118 +175111,114 @@ static YYACTIONTYPE yy_reduce( } } break; - case 277: /* raisetype ::= ROLLBACK */ + case 279: /* raisetype ::= ROLLBACK */ {yymsp[0].minor.yy394 = OE_Rollback;} break; - case 279: /* raisetype ::= FAIL */ + case 281: /* raisetype ::= FAIL */ {yymsp[0].minor.yy394 = OE_Fail;} break; - case 280: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 282: /* cmd ::= DROP TRIGGER ifexists fullname */ { sqlite3DropTrigger(pParse,yymsp[0].minor.yy131,yymsp[-1].minor.yy394); } break; - case 281: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 283: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { sqlite3Attach(pParse, yymsp[-3].minor.yy528, yymsp[-1].minor.yy528, yymsp[0].minor.yy528); } break; - case 282: /* cmd ::= DETACH database_kw_opt expr */ + case 284: /* cmd ::= DETACH database_kw_opt expr */ { sqlite3Detach(pParse, yymsp[0].minor.yy528); } break; - case 285: /* cmd ::= REINDEX */ + case 287: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 286: /* cmd ::= REINDEX nm dbnm */ + case 288: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 287: /* cmd ::= ANALYZE */ + case 289: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 288: /* cmd ::= ANALYZE nm dbnm */ + case 290: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 289: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 291: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy131,&yymsp[0].minor.yy0); } break; - case 290: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 292: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 291: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + case 293: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy131, &yymsp[0].minor.yy0); } break; - case 292: /* add_column_fullname ::= fullname */ + case 294: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy131); } break; - case 293: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 295: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy131, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 294: /* cmd ::= create_vtab */ + case 296: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 295: /* cmd ::= create_vtab LP vtabarglist RP */ + case 297: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 296: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 298: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy394); } break; - case 297: /* vtabarg ::= */ + case 299: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 298: /* vtabargtoken ::= ANY */ - case 299: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==299); - case 300: /* lp ::= LP */ yytestcase(yyruleno==300); + case 300: /* vtabargtoken ::= ANY */ + case 301: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==301); + case 302: /* lp ::= LP */ yytestcase(yyruleno==302); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 301: /* with ::= WITH wqlist */ - case 302: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==302); + case 303: /* with ::= WITH wqlist */ + case 304: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==304); { sqlite3WithPush(pParse, yymsp[0].minor.yy521, 1); } break; - case 303: /* wqas ::= AS */ + case 305: /* wqas ::= AS */ {yymsp[0].minor.yy516 = M10d_Any;} break; - case 304: /* wqas ::= AS MATERIALIZED */ + case 306: /* wqas ::= AS MATERIALIZED */ {yymsp[-1].minor.yy516 = M10d_Yes;} break; - case 305: /* wqas ::= AS NOT MATERIALIZED */ + case 307: /* wqas ::= AS NOT MATERIALIZED */ {yymsp[-2].minor.yy516 = M10d_No;} break; - case 306: /* wqitem ::= nm eidlist_opt wqas LP select RP */ + case 308: /* wqitem ::= nm eidlist_opt wqas LP select RP */ { yymsp[-5].minor.yy385 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy322, yymsp[-1].minor.yy47, yymsp[-3].minor.yy516); /*A-overwrites-X*/ } break; - case 307: /* wqlist ::= wqitem */ + case 309: /* wqlist ::= wqitem */ { yymsp[0].minor.yy521 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy385); /*A-overwrites-X*/ } break; - case 308: /* wqlist ::= wqlist COMMA wqitem */ + case 310: /* wqlist ::= wqlist COMMA wqitem */ { yymsp[-2].minor.yy521 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy521, yymsp[0].minor.yy385); } break; - case 309: /* windowdefn_list ::= windowdefn */ -{ yylhsminor.yy41 = yymsp[0].minor.yy41; } - yymsp[0].minor.yy41 = yylhsminor.yy41; - break; - case 310: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 311: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { assert( yymsp[0].minor.yy41!=0 ); sqlite3WindowChain(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy41); @@ -172617,7 +175227,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy41 = yylhsminor.yy41; break; - case 311: /* windowdefn ::= nm AS LP window RP */ + case 312: /* windowdefn ::= nm AS LP window RP */ { if( ALWAYS(yymsp[-1].minor.yy41) ){ yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); @@ -172626,35 +175236,28 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy41 = yylhsminor.yy41; break; - case 312: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 313: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { yymsp[-4].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, 0); } break; - case 313: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 314: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, &yymsp[-5].minor.yy0); } yymsp[-5].minor.yy41 = yylhsminor.yy41; break; - case 314: /* window ::= ORDER BY sortlist frame_opt */ + case 315: /* window ::= ORDER BY sortlist frame_opt */ { yymsp[-3].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, 0); } break; - case 315: /* window ::= nm ORDER BY sortlist frame_opt */ + case 316: /* window ::= nm ORDER BY sortlist frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); } yymsp[-4].minor.yy41 = yylhsminor.yy41; break; - case 316: /* window ::= frame_opt */ - case 335: /* filter_over ::= over_clause */ yytestcase(yyruleno==335); -{ - yylhsminor.yy41 = yymsp[0].minor.yy41; -} - yymsp[0].minor.yy41 = yylhsminor.yy41; - break; case 317: /* window ::= nm frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, 0, &yymsp[-1].minor.yy0); @@ -172720,6 +175323,12 @@ static YYACTIONTYPE yy_reduce( } yymsp[-1].minor.yy41 = yylhsminor.yy41; break; + case 335: /* filter_over ::= over_clause */ +{ + yylhsminor.yy41 = yymsp[0].minor.yy41; +} + yymsp[0].minor.yy41 = yylhsminor.yy41; + break; case 336: /* filter_over ::= filter_clause */ { yylhsminor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); @@ -172813,6 +175422,8 @@ static YYACTIONTYPE yy_reduce( /* (400) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==400); /* (401) anylist ::= anylist ANY */ yytestcase(yyruleno==401); /* (402) with ::= */ yytestcase(yyruleno==402); + /* (403) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=403); + /* (404) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=404); break; /********** End reduce actions ************************************************/ }; @@ -173601,180 +176212,179 @@ static const unsigned char aKWCode[148] = {0, static int keywordCode(const char *z, int n, int *pType){ int i, j; const char *zKW; - if( n>=2 ){ - i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; - for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){ - if( aKWLen[i]!=n ) continue; - zKW = &zKWText[aKWOffset[i]]; + assert( n>=2 ); + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; + for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){ + if( aKWLen[i]!=n ) continue; + zKW = &zKWText[aKWOffset[i]]; #ifdef SQLITE_ASCII - if( (z[0]&~0x20)!=zKW[0] ) continue; - if( (z[1]&~0x20)!=zKW[1] ) continue; - j = 2; - while( j=2 ) keywordCode((char*)z, n, &id); return id; } #define SQLITE_N_KEYWORD 147 @@ -174079,7 +176689,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); - testcase( z[0]=='9' ); + testcase( z[0]=='9' ); testcase( z[0]=='.' ); *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ @@ -174151,7 +176761,8 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ return i; } case CC_KYWD0: { - for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( aiClass[z[1]]>CC_KYWD ){ i = 1; break; } + for(i=2; aiClass[z[i]]<=CC_KYWD; i++){} if( IdChar(z[i]) ){ /* This token started out using characters that can appear in keywords, ** but z[i] is a character not allowed within keywords, so this must @@ -174930,30 +177541,20 @@ static int sqlite3TestExtInit(sqlite3 *db){ ** Forward declarations of external module initializer functions ** for modules that need them. */ -#ifdef SQLITE_ENABLE_FTS1 -SQLITE_PRIVATE int sqlite3Fts1Init(sqlite3*); -#endif -#ifdef SQLITE_ENABLE_FTS2 -SQLITE_PRIVATE int sqlite3Fts2Init(sqlite3*); -#endif #ifdef SQLITE_ENABLE_FTS5 SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); #endif #ifdef SQLITE_ENABLE_STMTVTAB SQLITE_PRIVATE int sqlite3StmtVtabInit(sqlite3*); #endif - +#ifdef SQLITE_EXTRA_AUTOEXT +int SQLITE_EXTRA_AUTOEXT(sqlite3*); +#endif /* ** An array of pointers to extension initializer functions for ** built-in extensions. */ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { -#ifdef SQLITE_ENABLE_FTS1 - sqlite3Fts1Init, -#endif -#ifdef SQLITE_ENABLE_FTS2 - sqlite3Fts2Init, -#endif #ifdef SQLITE_ENABLE_FTS3 sqlite3Fts3Init, #endif @@ -174982,6 +177583,9 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { #ifdef SQLITE_ENABLE_BYTECODE_VTAB sqlite3VdbeBytecodeVtabInit, #endif +#ifdef SQLITE_EXTRA_AUTOEXT + SQLITE_EXTRA_AUTOEXT, +#endif }; #ifndef SQLITE_AMALGAMATION @@ -175055,6 +177659,32 @@ SQLITE_API char *sqlite3_temp_directory = 0; */ SQLITE_API char *sqlite3_data_directory = 0; +/* +** Determine whether or not high-precision (long double) floating point +** math works correctly on CPU currently running. +*/ +static SQLITE_NOINLINE int hasHighPrecisionDouble(int rc){ + if( sizeof(LONGDOUBLE_TYPE)<=8 ){ + /* If the size of "long double" is not more than 8, then + ** high-precision math is not possible. */ + return 0; + }else{ + /* Just because sizeof(long double)>8 does not mean that the underlying + ** hardware actually supports high-precision floating point. For example, + ** clearing the 0x100 bit in the floating-point control word on Intel + ** processors will make long double work like double, even though long + ** double takes up more space. The only way to determine if long double + ** actually works is to run an experiment. */ + LONGDOUBLE_TYPE a, b, c; + rc++; + a = 1.0+rc*0.1; + b = 1.0e+18+rc*25.0; + c = a+b; + return b!=c; + } +} + + /* ** Initialize SQLite. ** @@ -175250,6 +177880,12 @@ SQLITE_API int sqlite3_initialize(void){ } #endif + /* Experimentally determine if high-precision floating point is + ** available. */ +#ifndef SQLITE_OMIT_WSD + sqlite3Config.bUseLongDouble = hasHighPrecisionDouble(rc); +#endif + return rc; } @@ -175820,6 +178456,10 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ va_list ap; int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); va_start(ap, op); switch( op ){ @@ -176149,6 +178789,14 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ } #endif + while( db->pDbData ){ + DbClientData *p = db->pDbData; + db->pDbData = p->pNext; + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + sqlite3_free(p); + } + /* Convert the connection into a zombie and then close it. */ db->eOpenState = SQLITE_STATE_ZOMBIE; @@ -176566,9 +179214,9 @@ static int sqliteDefaultBusyCallback( void *ptr, /* Database connection */ int count /* Number of times table has been busy */ ){ -#if SQLITE_OS_WIN || HAVE_USLEEP +#if SQLITE_OS_WIN || !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP /* This case is for systems that have support for sleeping for fractions of - ** a second. Examples: All windows systems, unix systems with usleep() */ + ** a second. Examples: All windows systems, unix systems with nanosleep() */ static const u8 delays[] = { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; static const u8 totals[] = @@ -177223,6 +179871,12 @@ SQLITE_API void *sqlite3_preupdate_hook( void *pArg /* First callback argument */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 ){ + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pPreUpdateArg; db->xPreUpdateCallback = xCallback; @@ -177369,7 +180023,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -178206,7 +180860,7 @@ static int openDatabase( ** 0 off off ** ** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) -** and so that is the default. But developers are encouranged to use +** and so that is the default. But developers are encouraged to use ** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. */ #if !defined(SQLITE_DQS) @@ -178606,6 +181260,69 @@ SQLITE_API int sqlite3_collation_needed16( } #endif /* SQLITE_OMIT_UTF16 */ +/* +** Find existing client data. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3 *db, const char *zName){ + DbClientData *p; + sqlite3_mutex_enter(db->mutex); + for(p=db->pDbData; p; p=p->pNext){ + if( strcmp(p->zName, zName)==0 ){ + void *pResult = p->pData; + sqlite3_mutex_leave(db->mutex); + return pResult; + } + } + sqlite3_mutex_leave(db->mutex); + return 0; +} + +/* +** Add new client data to a database connection. +*/ +SQLITE_API int sqlite3_set_clientdata( + sqlite3 *db, /* Attach client data to this connection */ + const char *zName, /* Name of the client data */ + void *pData, /* The client data itself */ + void (*xDestructor)(void*) /* Destructor */ +){ + DbClientData *p, **pp; + sqlite3_mutex_enter(db->mutex); + pp = &db->pDbData; + for(p=db->pDbData; p && strcmp(p->zName,zName); p=p->pNext){ + pp = &p->pNext; + } + if( p ){ + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + if( pData==0 ){ + *pp = p->pNext; + sqlite3_free(p); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + } + }else if( pData==0 ){ + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + }else{ + size_t n = strlen(zName); + p = sqlite3_malloc64( sizeof(DbClientData)+n+1 ); + if( p==0 ){ + if( xDestructor ) xDestructor(pData); + sqlite3_mutex_leave(db->mutex); + return SQLITE_NOMEM; + } + memcpy(p->zName, zName, n+1); + p->pNext = db->pDbData; + db->pDbData = p; + } + p->pData = pData; + p->xDestructor = xDestructor; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + + #ifndef SQLITE_OMIT_DEPRECATED /* ** This function is now an anachronism. It used to be used to recover from a @@ -178741,7 +181458,7 @@ SQLITE_API int sqlite3_table_column_metadata( /* Find the column for which info is requested */ if( zColumnName==0 ){ - /* Query for existance of table only */ + /* Query for existence of table only */ }else{ for(iCol=0; iColnCol; iCol++){ pCol = &pTab->aCol[iCol]; @@ -178955,6 +181672,28 @@ SQLITE_API int sqlite3_test_control(int op, ...){ } #endif + /* sqlite3_test_control(SQLITE_TESTCTRL_FK_NO_ACTION, sqlite3 *db, int b); + ** + ** If b is true, then activate the SQLITE_FkNoAction setting. If b is + ** false then clearn that setting. If the SQLITE_FkNoAction setting is + ** abled, all foreign key ON DELETE and ON UPDATE actions behave as if + ** they were NO ACTION, regardless of how they are defined. + ** + ** NB: One must usually run "PRAGMA writable_schema=RESET" after + ** using this test-control, before it will take full effect. failing + ** to reset the schema can result in some unexpected behavior. + */ + case SQLITE_TESTCTRL_FK_NO_ACTION: { + sqlite3 *db = va_arg(ap, sqlite3*); + int b = va_arg(ap, int); + if( b ){ + db->flags |= SQLITE_FkNoAction; + }else{ + db->flags &= ~SQLITE_FkNoAction; + } + break; + } + /* ** sqlite3_test_control(BITVEC_TEST, size, program) ** @@ -179061,10 +181800,12 @@ SQLITE_API int sqlite3_test_control(int op, ...){ sqlite3ShowSrcList(0); sqlite3ShowWith(0); sqlite3ShowUpsert(0); +#ifndef SQLITE_OMIT_TRIGGER sqlite3ShowTriggerStep(0); sqlite3ShowTriggerStepList(0); sqlite3ShowTrigger(0); sqlite3ShowTriggerList(0); +#endif #ifndef SQLITE_OMIT_WINDOWFUNC sqlite3ShowWindow(0); sqlite3ShowWinFunc(0); @@ -179181,7 +181922,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** formed and never corrupt. This flag is clear by default, indicating that ** database files might have arbitrary corruption. Setting the flag during ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. + ** that demonstrate invariants on well-formed database files. */ case SQLITE_TESTCTRL_NEVER_CORRUPT: { sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); @@ -179335,7 +182076,7 @@ SQLITE_API int sqlite3_test_control(int op, ...){ ** ** op==0 Store the current sqlite3TreeTrace in *ptr ** op==1 Set sqlite3TreeTrace to the value *ptr - ** op==3 Store the current sqlite3WhereTrace in *ptr + ** op==2 Store the current sqlite3WhereTrace in *ptr ** op==3 Set sqlite3WhereTrace to the value *ptr */ case SQLITE_TESTCTRL_TRACEFLAGS: { @@ -179371,6 +182112,23 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } +#if !defined(SQLITE_OMIT_WSD) + /* sqlite3_test_control(SQLITE_TESTCTRL_USELONGDOUBLE, int X); + ** + ** X<0 Make no changes to the bUseLongDouble. Just report value. + ** X==0 Disable bUseLongDouble + ** X==1 Enable bUseLongDouble + ** X>=2 Set bUseLongDouble to its default value for this platform + */ + case SQLITE_TESTCTRL_USELONGDOUBLE: { + int b = va_arg(ap, int); + if( b>=2 ) b = hasHighPrecisionDouble(b); + if( b>=0 ) sqlite3Config.bUseLongDouble = b>0; + rc = sqlite3Config.bUseLongDouble!=0; + break; + } +#endif + #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) @@ -179671,7 +182429,7 @@ SQLITE_API int sqlite3_snapshot_get( } /* -** Open a read-transaction on the snapshot idendified by pSnapshot. +** Open a read-transaction on the snapshot identified by pSnapshot. */ SQLITE_API int sqlite3_snapshot_open( sqlite3 *db, @@ -179778,7 +182536,7 @@ SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ int nOpt; const char **azCompileOpt; -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( zOptName==0 ){ (void)SQLITE_MISUSE_BKPT; return 0; @@ -179973,6 +182731,9 @@ SQLITE_API int sqlite3_unlock_notify( ){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); enterMutex(); @@ -180994,6 +183755,7 @@ struct Fts3Table { int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + int iSavepoint; /* ** The following array of hash tables is used to buffer pending index @@ -181737,6 +184499,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS); /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); @@ -184986,6 +187749,8 @@ static int fts3RenameMethod( rc = sqlite3Fts3PendingTermsFlush(p); } + p->bIgnoreSavepoint = 1; + if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", @@ -185013,6 +187778,8 @@ static int fts3RenameMethod( "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", p->zDb, p->zName, zName ); + + p->bIgnoreSavepoint = 0; return rc; } @@ -185023,12 +187790,28 @@ static int fts3RenameMethod( */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint<=iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + + if( pTab->bIgnoreSavepoint==0 ){ + if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){ + char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", + pTab->zDb, pTab->zName, pTab->zName + ); + if( zSql ){ + pTab->bIgnoreSavepoint = 1; + rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0); + pTab->bIgnoreSavepoint = 0; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } } return rc; } @@ -185039,12 +187822,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint >= iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint-1 ); + pTab->iSavepoint = iSavepoint; return SQLITE_OK; } @@ -185054,11 +187836,13 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** Discard the contents of the pending terms table. */ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; + Fts3Table *pTab = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); + assert( pTab->inTransaction ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + sqlite3Fts3PendingTermsClear(pTab); + } return SQLITE_OK; } @@ -185077,8 +187861,49 @@ static int fts3ShadowName(const char *zName){ return 0; } +/* +** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual +** table. +*/ +static int fts3Integrity( + sqlite3_vtab *pVtab, /* The virtual table to be checked */ + const char *zSchema, /* Name of schema in which pVtab lives */ + const char *zTabname, /* Name of the pVTab table */ + int isQuick, /* True if this is a quick_check */ + char **pzErr /* Write error message here */ +){ + Fts3Table *p = (Fts3Table*)pVtab; + char *zSql; + int rc; + char *zErr = 0; + + assert( pzErr!=0 ); + assert( *pzErr==0 ); + UNUSED_PARAMETER(isQuick); + zSql = sqlite3_mprintf( + "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');", + zSchema, zTabname, zTabname); + if( zSql==0 ){ + return SQLITE_NOMEM; + } + rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr); + sqlite3_free(zSql); + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", + p->bFts4 ? 4 : 3, zSchema, zTabname); + }else if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS%d table %s.%s: %s", + p->bFts4 ? 4 : 3, zSchema, zTabname, zErr); + } + sqlite3_free(zErr); + return SQLITE_OK; +} + + + static const sqlite3_module fts3Module = { - /* iVersion */ 3, + /* iVersion */ 4, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, @@ -185102,6 +187927,7 @@ static const sqlite3_module fts3Module = { /* xRelease */ fts3ReleaseMethod, /* xRollbackTo */ fts3RollbackToMethod, /* xShadowName */ fts3ShadowName, + /* xIntegrity */ fts3Integrity, }; /* @@ -187777,7 +190603,8 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ @@ -191343,7 +194170,8 @@ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestr 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ @@ -194684,7 +197512,6 @@ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - sqlite3Fts3PendingTermsClear(p); /* Determine the auto-incr-merge setting if unknown. If enabled, ** estimate the number of leaf blocks of content to be written @@ -194706,6 +197533,10 @@ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ rc = sqlite3_reset(pStmt); } } + + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } return rc; } @@ -195337,6 +198168,8 @@ static int fts3AppendToNode( blobGrowBuffer(pPrev, nTerm, &rc); if( rc!=SQLITE_OK ) return rc; + assert( pPrev!=0 ); + assert( pPrev->a!=0 ); nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); nSuffix = nTerm - nPrefix; @@ -195393,9 +198226,13 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; /* If the current block is not empty, and if adding this term/doclist - ** to the current block would make it larger than Fts3Table.nNodeSize - ** bytes, write this block out to the database. */ - if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + ** to the current block would make it larger than Fts3Table.nNodeSize bytes, + ** and if there is still room for another leaf page, write this block out to + ** the database. */ + if( pLeaf->block.n>0 + && (pLeaf->block.n + nSpace)>p->nNodeSize + && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst) + ){ rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); pWriter->nWork++; @@ -195706,6 +198543,7 @@ static int fts3IncrmergeLoad( for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ NodeReader reader; + memset(&reader, 0, sizeof(reader)); pNode = &pWriter->aNodeWriter[i]; if( pNode->block.a){ @@ -195726,7 +198564,7 @@ static int fts3IncrmergeLoad( rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0); blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc - ); + ); if( rc==SQLITE_OK ){ memcpy(pNode->block.a, aBlock, nBlock); pNode->block.n = nBlock; @@ -196576,7 +199414,7 @@ static u64 fts3ChecksumIndex( int rc; u64 cksum = 0; - assert( *pRc==SQLITE_OK ); + if( *pRc ) return 0; memset(&filter, 0, sizeof(filter)); memset(&csr, 0, sizeof(csr)); @@ -196791,8 +199629,11 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoIncrmerge(p, &zVal[6]); }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ rc = fts3DoAutoincrmerge(p, &zVal[10]); + }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - }else{ + else{ int v; if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ v = atoi(&zVal[9]); @@ -196810,8 +199651,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; rc = SQLITE_OK; } -#endif } +#endif return rc; } @@ -199752,25 +202593,51 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ ** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; #define fast_isspace(x) (jsonIsSpace[(unsigned char)x]) +/* +** Characters that are special to JSON. Control charaters, +** '"' and '\\'. +*/ +static const char jsonIsOk[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +}; + + #if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) # define VVA(X) #else @@ -199781,6 +202648,7 @@ static const char jsonIsSpace[] = { typedef struct JsonString JsonString; typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; +typedef struct JsonCleanup JsonCleanup; /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator @@ -199796,16 +202664,26 @@ struct JsonString { char zSpace[100]; /* Initial static space */ }; +/* A deferred cleanup task. A list of JsonCleanup objects might be +** run when the JsonParse object is destroyed. +*/ +struct JsonCleanup { + JsonCleanup *pJCNext; /* Next in a list */ + void (*xOp)(void*); /* Routine to run */ + void *pArg; /* Argument to xOp() */ +}; + /* JSON type values */ -#define JSON_NULL 0 -#define JSON_TRUE 1 -#define JSON_FALSE 2 -#define JSON_INT 3 -#define JSON_REAL 4 -#define JSON_STRING 5 -#define JSON_ARRAY 6 -#define JSON_OBJECT 7 +#define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ +#define JSON_NULL 1 +#define JSON_TRUE 2 +#define JSON_FALSE 3 +#define JSON_INT 4 +#define JSON_REAL 5 +#define JSON_STRING 6 +#define JSON_ARRAY 7 +#define JSON_OBJECT 8 /* The "subtype" set for JSON values */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ @@ -199814,52 +202692,87 @@ struct JsonString { ** Names of the various JSON types: */ static const char * const jsonType[] = { + "subst", "null", "true", "false", "integer", "real", "text", "array", "object" }; /* Bit values for the JsonNode.jnFlag field */ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ -#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ -#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ -#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Target of a JSON_SUBST node */ +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x20 /* Is a label of an object */ +#define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ -/* A single node of parsed JSON +/* A single node of parsed JSON. An array of these nodes describes +** a parse of JSON + edits. +** +** Use the json_parse() SQL function (available when compiled with +** -DSQLITE_DEBUG) to see a dump of complete JsonParse objects, including +** a complete listing and decoding of the array of JsonNodes. */ struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ u8 eU; /* Which union element to use */ - u32 n; /* Bytes of content, or number of sub-nodes */ + u32 n; /* Bytes of content for INT, REAL or STRING + ** Number of sub-nodes for ARRAY and OBJECT + ** Node that SUBST applies to */ union { const char *zJContent; /* 1: Content for INT, REAL, and STRING */ u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ - u32 iReplace; /* 4: Replacement content for JNODE_REPLACE */ - JsonNode *pPatch; /* 5: Node chain of patch for JNODE_PATCH */ + u32 iPrev; /* 4: Previous SUBST node, or 0 */ } u; }; -/* A completely parsed JSON string + +/* A parsed and possibly edited JSON string. Lifecycle: +** +** 1. JSON comes in and is parsed into an array aNode[]. The original +** JSON text is stored in zJson. +** +** 2. Zero or more changes are made (via json_remove() or json_replace() +** or similar) to the aNode[] array. +** +** 3. A new, edited and mimified JSON string is generated from aNode +** and stored in zAlt. The JsonParse object always owns zAlt. +** +** Step 1 always happens. Step 2 and 3 may or may not happen, depending +** on the operation. +** +** aNode[].u.zJContent entries typically point into zJson. Hence zJson +** must remain valid for the lifespan of the parse. For edits, +** aNode[].u.zJContent might point to malloced space other than zJson. +** Entries in pClup are responsible for freeing that extra malloced space. +** +** When walking the parse tree in aNode[], edits are ignored if useMod is +** false. */ struct JsonParse { u32 nNode; /* Number of slots of aNode[] used */ u32 nAlloc; /* Number of slots of aNode[] allocated */ JsonNode *aNode; /* Array of nodes containing the parse */ - const char *zJson; /* Original JSON string */ + char *zJson; /* Original JSON string (before edits) */ + char *zAlt; /* Revised and/or mimified JSON */ u32 *aUp; /* Index of parent of each node */ + JsonCleanup *pClup;/* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ + u8 useMod; /* Actually use the edits contain inside aNode */ + u8 hasMod; /* aNode contains edits from the original zJson */ + u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ + int nAlt; /* Length of alternative JSON string zAlt, in bytes */ u32 iErr; /* Error location in zJson[] */ - u32 iHold; /* Replace cache line with the lowest iHold value */ + u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ + u32 iHold; /* Age of this entry in the cache for LRU replacement */ }; /* @@ -199892,16 +202805,14 @@ static void jsonInit(JsonString *p, sqlite3_context *pCtx){ jsonZero(p); } - /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ static void jsonReset(JsonString *p){ - if( !p->bStatic ) sqlite3_free(p->zBuf); + if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); jsonZero(p); } - /* Report an out-of-memory (OOM) condition */ static void jsonOom(JsonString *p){ @@ -199918,7 +202829,7 @@ static int jsonGrow(JsonString *p, u32 N){ char *zNew; if( p->bStatic ){ if( p->bErr ) return 1; - zNew = sqlite3_malloc64(nTotal); + zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ jsonOom(p); return SQLITE_NOMEM; @@ -199927,12 +202838,12 @@ static int jsonGrow(JsonString *p, u32 N){ p->zBuf = zNew; p->bStatic = 0; }else{ - zNew = sqlite3_realloc64(p->zBuf, nTotal); - if( zNew==0 ){ - jsonOom(p); + p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); + if( p->zBuf==0 ){ + p->bErr = 1; + jsonZero(p); return SQLITE_NOMEM; } - p->zBuf = zNew; } p->nAlloc = nTotal; return SQLITE_OK; @@ -199940,12 +202851,35 @@ static int jsonGrow(JsonString *p, u32 N){ /* Append N bytes from zIn onto the end of the JsonString string. */ -static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ - if( N==0 ) return; - if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; +static SQLITE_NOINLINE void jsonAppendExpand( + JsonString *p, + const char *zIn, + u32 N +){ + assert( N>0 ); + if( jsonGrow(p,N) ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( N+p->nUsed >= p->nAlloc ){ + jsonAppendExpand(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} +static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ + assert( N>0 ); + if( N+p->nUsed >= p->nAlloc ){ + jsonAppendExpand(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} + /* Append formatted text (not to exceed N bytes) to the JsonString. */ @@ -199960,10 +202894,35 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ /* Append a single character */ -static void jsonAppendChar(JsonString *p, char c){ - if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; +static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ + if( jsonGrow(p,1) ) return; p->zBuf[p->nUsed++] = c; } +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc ){ + jsonAppendCharExpand(p,c); + }else{ + p->zBuf[p->nUsed++] = c; + } +} + +/* Try to force the string to be a zero-terminated RCStr string. +** +** Return true on success. Return false if an OOM prevents this +** from happening. +*/ +static int jsonForceRCStr(JsonString *p){ + jsonAppendChar(p, 0); + if( p->bErr ) return 0; + p->nUsed--; + if( p->bStatic==0 ) return 1; + p->nAlloc = 0; + p->nUsed++; + jsonGrow(p, p->nUsed); + p->nUsed--; + return p->bStatic==0; +} + /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. @@ -199972,7 +202931,8 @@ static void jsonAppendSeparator(JsonString *p){ char c; if( p->nUsed==0 ) return; c = p->zBuf[p->nUsed-1]; - if( c!='[' && c!='{' ) jsonAppendChar(p, ','); + if( c=='[' || c=='{' ) return; + jsonAppendChar(p, ','); } /* Append the N-byte string in zIn to the end of the JsonString string @@ -199986,11 +202946,16 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; for(i=0; izBuf[p->nUsed++] = c; + }else if( c=='"' || c=='\\' ){ json_simple_escape: if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; - }else if( c<=0x1f ){ + p->zBuf[p->nUsed++] = c; + }else if( c=='\'' ){ + p->zBuf[p->nUsed++] = c; + }else{ static const char aSpecial[] = { 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 @@ -200001,6 +202966,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ assert( aSpecial['\n']=='n' ); assert( aSpecial['\r']=='r' ); assert( aSpecial['\t']=='t' ); + assert( c>=0 && czBuf[p->nUsed++] = 'u'; p->zBuf[p->nUsed++] = '0'; p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0' + (c>>4); - c = "0123456789abcdef"[c&0xf]; + p->zBuf[p->nUsed++] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed++] = "0123456789abcdef"[c&0xf]; } - p->zBuf[p->nUsed++] = c; } p->zBuf[p->nUsed++] = '"'; assert( p->nUsednAlloc ); @@ -200032,7 +202997,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ while( N>0 ){ for(i=0; i0 ){ - jsonAppendRaw(p, zIn, i); + jsonAppendRawNZ(p, zIn, i); zIn += i; N -= i; if( N==0 ) break; @@ -200043,16 +203008,16 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendChar(p, '\''); break; case 'v': - jsonAppendRaw(p, "\\u0009", 6); + jsonAppendRawNZ(p, "\\u0009", 6); break; case 'x': - jsonAppendRaw(p, "\\u00", 4); - jsonAppendRaw(p, &zIn[2], 2); + jsonAppendRawNZ(p, "\\u00", 4); + jsonAppendRawNZ(p, &zIn[2], 2); zIn += 2; N -= 2; break; case '0': - jsonAppendRaw(p, "\\u0000", 6); + jsonAppendRawNZ(p, "\\u0000", 6); break; case '\r': if( zIn[2]=='\n' ){ @@ -200070,7 +203035,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ N -= 2; break; default: - jsonAppendRaw(p, zIn, 2); + jsonAppendRawNZ(p, zIn, 2); break; } zIn += 2; @@ -200100,11 +203065,12 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ jsonPrintf(100,p,"%lld",i); }else{ assert( rc==2 ); - jsonAppendRaw(p, "9.0e999", 7); + jsonAppendRawNZ(p, "9.0e999", 7); } return; } - jsonAppendRaw(p, zIn, N); + assert( N>0 ); + jsonAppendRawNZ(p, zIn, N); } /* @@ -200136,7 +203102,7 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ } } if( N>0 ){ - jsonAppendRaw(p, zIn, N); + jsonAppendRawNZ(p, zIn, N); } } @@ -200152,7 +203118,7 @@ static void jsonAppendValue( ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { - jsonAppendRaw(p, "null", 4); + jsonAppendRawNZ(p, "null", 4); break; } case SQLITE_FLOAT: { @@ -200188,15 +203154,25 @@ static void jsonAppendValue( /* Make the JSON in p the result of the SQL function. +** +** The JSON string is reset. */ static void jsonResult(JsonString *p){ if( p->bErr==0 ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, - SQLITE_UTF8); - jsonZero(p); + if( p->bStatic ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + }else if( jsonForceRCStr(p) ){ + sqlite3RCStrRef(p->zBuf); + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + sqlite3RCStrUnref, + SQLITE_UTF8); + } + } + if( p->bErr==1 ){ + sqlite3_result_error_nomem(p->pCtx); } - assert( p->bStatic ); + jsonReset(p); } /************************************************************************** @@ -200221,20 +203197,73 @@ static u32 jsonNodeSize(JsonNode *pNode){ ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; + while( pParse->pClup ){ + JsonCleanup *pTask = pParse->pClup; + pParse->pClup = pTask->pJCNext; + pTask->xOp(pTask->pArg); + sqlite3_free(pTask); + } + assert( pParse->nJPRef<=1 ); + if( pParse->aNode ){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + } pParse->nNode = 0; pParse->nAlloc = 0; - sqlite3_free(pParse->aUp); - pParse->aUp = 0; + if( pParse->aUp ){ + sqlite3_free(pParse->aUp); + pParse->aUp = 0; + } + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->bJsonIsRCStr = 0; + } + if( pParse->zAlt ){ + sqlite3RCStrUnref(pParse->zAlt); + pParse->zAlt = 0; + } } /* ** Free a JsonParse object that was obtained from sqlite3_malloc(). +** +** Note that destroying JsonParse might call sqlite3RCStrUnref() to +** destroy the zJson value. The RCStr object might recursively invoke +** JsonParse to destroy this pParse object again. Take care to ensure +** that this recursive destructor sequence terminates harmlessly. */ static void jsonParseFree(JsonParse *pParse){ - jsonParseReset(pParse); - sqlite3_free(pParse); + if( pParse->nJPRef>1 ){ + pParse->nJPRef--; + }else{ + jsonParseReset(pParse); + sqlite3_free(pParse); + } +} + +/* +** Add a cleanup task to the JsonParse object. +** +** If an OOM occurs, the cleanup operation happens immediately +** and this function returns SQLITE_NOMEM. +*/ +static int jsonParseAddCleanup( + JsonParse *pParse, /* Add the cleanup task to this parser */ + void(*xOp)(void*), /* The cleanup task */ + void *pArg /* Argument to the cleanup */ +){ + JsonCleanup *pTask = sqlite3_malloc64( sizeof(*pTask) ); + if( pTask==0 ){ + pParse->oom = 1; + xOp(pArg); + return SQLITE_ERROR; + } + pTask->pJCNext = pParse->pClup; + pParse->pClup = pTask; + pTask->xOp = xOp; + pTask->pArg = pArg; + return SQLITE_OK; } /* @@ -200243,32 +203272,38 @@ static void jsonParseFree(JsonParse *pParse){ ** the number of JsonNode objects that are encoded. */ static void jsonRenderNode( + JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ - JsonString *pOut, /* Write JSON here */ - sqlite3_value **aReplace /* Replacement values */ + JsonString *pOut /* Write JSON here */ ){ assert( pNode!=0 ); - if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ - if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){ - assert( pNode->eU==4 ); - jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); - return; + while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ + u32 idx = (u32)(pNode - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pNode = &pParse->aNode[i+1]; + break; + } + i = pParse->aNode[i].u.iPrev; } - assert( pNode->eU==5 ); - pNode = pNode->u.pPatch; } switch( pNode->eType ){ default: { assert( pNode->eType==JSON_NULL ); - jsonAppendRaw(pOut, "null", 4); + jsonAppendRawNZ(pOut, "null", 4); break; } case JSON_TRUE: { - jsonAppendRaw(pOut, "true", 4); + jsonAppendRawNZ(pOut, "true", 4); break; } case JSON_FALSE: { - jsonAppendRaw(pOut, "false", 5); + jsonAppendRawNZ(pOut, "false", 5); break; } case JSON_STRING: { @@ -200284,7 +203319,8 @@ static void jsonRenderNode( }else if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); } break; } @@ -200293,7 +203329,8 @@ static void jsonRenderNode( if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); } break; } @@ -200302,7 +203339,8 @@ static void jsonRenderNode( if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); } break; } @@ -200311,15 +203349,16 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; + pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; } jsonAppendChar(pOut, ']'); @@ -200330,17 +203369,18 @@ static void jsonRenderNode( jsonAppendChar(pOut, '{'); for(;;){ while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); - jsonRenderNode(&pNode[j+1], pOut, aReplace); + jsonRenderNode(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; + pNode = &pParse->aNode[pNode->u.iAppend]; j = 1; } jsonAppendChar(pOut, '}'); @@ -200350,18 +203390,29 @@ static void jsonRenderNode( } /* -** Return a JsonNode and all its descendents as a JSON string. +** Return a JsonNode and all its descendants as a JSON string. */ static void jsonReturnJson( + JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ + int bGenerateAlt /* Also store the rendered text in zAlt */ ){ JsonString s; - jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s, aReplace); - jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( pParse->nErr==0 ){ + jsonInit(&s, pCtx); + jsonRenderNode(pParse, pNode, &s); + if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ + pParse->zAlt = sqlite3RCStrRef(s.zBuf); + pParse->nAlt = s.nUsed; + } + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); + } } /* @@ -200399,9 +203450,9 @@ static u32 jsonHexToInt4(const char *z){ ** Make the JsonNode the return value of the function. */ static void jsonReturn( + JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ + sqlite3_context *pCtx /* Return value for this function */ ){ switch( pNode->eType ){ default: { @@ -200423,7 +203474,6 @@ static void jsonReturn( int bNeg = 0; const char *z; - assert( pNode->eU==1 ); z = pNode->u.zJContent; if( z[0]=='-' ){ z++; bNeg = 1; } @@ -200548,7 +203598,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace); + jsonReturnJson(pParse, pNode, pCtx, 0); break; } } @@ -200570,6 +203620,12 @@ static int jsonParseAddNode(JsonParse*,u32,u32,const char*); #endif +/* +** Add a single node to pParse->aNode after first expanding the +** size of the aNode array. Return the index of the new node. +** +** If an OOM error occurs, set pParse->oom and return -1. +*/ static JSON_NOINLINE int jsonParseAddNodeExpand( JsonParse *pParse, /* Append the node to this object */ u32 eType, /* Node type */ @@ -200586,7 +203642,7 @@ static JSON_NOINLINE int jsonParseAddNodeExpand( pParse->oom = 1; return -1; } - pParse->nAlloc = nNew; + pParse->nAlloc = sqlite3_msize(pNew)/sizeof(JsonNode); pParse->aNode = pNew; assert( pParse->nNodenAlloc ); return jsonParseAddNode(pParse, eType, n, zContent); @@ -200604,10 +203660,13 @@ static int jsonParseAddNode( const char *zContent /* Content */ ){ JsonNode *p; - if( pParse->aNode==0 || pParse->nNode>=pParse->nAlloc ){ + assert( pParse->aNode!=0 || pParse->nNode>=pParse->nAlloc ); + if( pParse->nNode>=pParse->nAlloc ){ return jsonParseAddNodeExpand(pParse, eType, n, zContent); } + assert( pParse->aNode!=0 ); p = &pParse->aNode[pParse->nNode]; + assert( p!=0 ); p->eType = (u8)(eType & 0xff); p->jnFlags = (u8)(eType >> 8); VVA( p->eU = zContent ? 1 : 0 ); @@ -200616,6 +203675,52 @@ static int jsonParseAddNode( return pParse->nNode++; } +/* +** Add an array of new nodes to the current pParse->aNode array. +** Return the index of the first node added. +** +** If an OOM error occurs, set pParse->oom. +*/ +static void jsonParseAddNodeArray( + JsonParse *pParse, /* Append the node to this object */ + JsonNode *aNode, /* Array of nodes to add */ + u32 nNode /* Number of elements in aNew */ +){ + assert( aNode!=0 ); + assert( nNode>=1 ); + if( pParse->nNode + nNode > pParse->nAlloc ){ + u32 nNew = pParse->nNode + nNode; + JsonNode *aNew = sqlite3_realloc64(pParse->aNode, nNew*sizeof(JsonNode)); + if( aNew==0 ){ + pParse->oom = 1; + return; + } + pParse->nAlloc = sqlite3_msize(aNew)/sizeof(JsonNode); + pParse->aNode = aNew; + } + memcpy(&pParse->aNode[pParse->nNode], aNode, nNode*sizeof(JsonNode)); + pParse->nNode += nNode; +} + +/* +** Add a new JSON_SUBST node. The node immediately following +** this new node will be the substitute content for iNode. +*/ +static int jsonParseAddSubstNode( + JsonParse *pParse, /* Add the JSON_SUBST here */ + u32 iNode /* References this node */ +){ + int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0); + if( pParse->oom ) return -1; + pParse->aNode[iNode].jnFlags |= JNODE_REPLACE; + pParse->aNode[idx].eU = 4; + pParse->aNode[idx].u.iPrev = pParse->iSubst; + pParse->iSubst = idx; + pParse->hasMod = 1; + pParse->useMod = 1; + return idx; +} + /* ** Return true if z[] begins with 2 (or more) hexadecimal digits */ @@ -200782,7 +203887,7 @@ static const struct NanInfName { ** ** Special return values: ** -** 0 End if input +** 0 End of input ** -1 Syntax error ** -2 '}' seen ** -3 ']' seen @@ -200957,15 +204062,12 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ jnFlags = 0; parse_string: cDelim = z[i]; - j = i+1; - for(;;){ + for(j=i+1; 1; j++){ + if( jsonIsOk[(unsigned char)z[j]] ) continue; c = z[j]; - if( (c & ~0x1f)==0 ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; - } - if( c=='\\' ){ + if( c==cDelim ){ + break; + }else if( c=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' @@ -200985,10 +204087,11 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ pParse->iErr = j; return -1; } - }else if( c==cDelim ){ - break; + }else if( c<=0x1f ){ + /* Control characters are not allowed in strings */ + pParse->iErr = j; + return -1; } - j++; } jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); return j+1; @@ -201224,20 +204327,18 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ /* ** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory associated with -** pParse. +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. ** -** pParse is uninitialized when this routine is called. +** pParse must be initialized to an empty parse object prior to calling +** this routine. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ + sqlite3_context *pCtx /* Report errors here */ ){ int i; - memset(pParse, 0, sizeof(*pParse)); - if( zJson==0 ) return 1; - pParse->zJson = zJson; + const char *zJson = pParse->zJson; i = jsonParseValue(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ @@ -201266,6 +204367,7 @@ static int jsonParse( return 0; } + /* Mark node i of pParse as being a child of iParent. Call recursively ** to fill in all the descendants of node i. */ @@ -201315,35 +204417,49 @@ static int jsonParseFindParents(JsonParse *pParse){ #define JSON_CACHE_SZ 4 /* Max number of cache entries */ /* -** Obtain a complete parse of the JSON found in the first argument -** of the argv array. Use the sqlite3_get_auxdata() cache for this -** parse if it is available. If the cache is not available or if it -** is no longer valid, parse the JSON again and return the new parse, -** and also register the new parse so that it will be available for +** Obtain a complete parse of the JSON found in the pJson argument +** +** Use the sqlite3_get_auxdata() cache to find a preexisting parse +** if it is available. If the cache is not available or if it +** is no longer valid, parse the JSON again and return the new parse. +** Also register the new parse so that it will be available for ** future sqlite3_get_auxdata() calls. ** ** If an error occurs and pErrCtx!=0 then report the error on pErrCtx ** and return NULL. ** -** If an error occurs and pErrCtx==0 then return the Parse object with -** JsonParse.nErr non-zero. If the caller invokes this routine with -** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller -** is responsible for invoking jsonParseFree() on the returned value. -** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. +** The returned pointer (if it is not NULL) is owned by the cache in +** most cases, not the caller. The caller does NOT need to invoke +** jsonParseFree(), in most cases. +** +** Except, if an error occurs and pErrCtx==0 then return the JsonParse +** object with JsonParse.nErr non-zero and the caller will own the JsonParse +** object. In that case, it will be the responsibility of the caller to +** invoke jsonParseFree(). To summarize: +** +** pErrCtx!=0 || p->nErr==0 ==> Return value p is owned by the +** cache. Call does not need to +** free it. +** +** pErrCtx==0 && p->nErr!=0 ==> Return value is owned by the caller +** and so the caller must free it. */ static JsonParse *jsonParseCached( - sqlite3_context *pCtx, - sqlite3_value **argv, - sqlite3_context *pErrCtx + sqlite3_context *pCtx, /* Context to use for cache search */ + sqlite3_value *pJson, /* Function param containing JSON text */ + sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ + int bUnedited /* No prior edits allowed */ ){ - const char *zJson = (const char*)sqlite3_value_text(argv[0]); - int nJson = sqlite3_value_bytes(argv[0]); + char *zJson = (char*)sqlite3_value_text(pJson); + int nJson = sqlite3_value_bytes(pJson); JsonParse *p; JsonParse *pMatch = 0; int iKey; int iMinKey = 0; u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; + int bJsonRCStr; + if( zJson==0 ) return 0; for(iKey=0; iKeynJson==nJson - && memcmp(p->zJson,zJson,nJson)==0 + && (p->hasMod==0 || bUnedited==0) + && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0) + ){ + p->nErr = 0; + p->useMod = 0; + pMatch = p; + }else + if( pMatch==0 + && p->zAlt!=0 + && bUnedited==0 + && p->nAlt==nJson + && memcmp(p->zAlt, zJson, nJson)==0 ){ p->nErr = 0; + p->useMod = 1; pMatch = p; }else if( p->iHoldiHold; @@ -201366,28 +204494,44 @@ static JsonParse *jsonParseCached( } } if( pMatch ){ + /* The input JSON text was found in the cache. Use the preexisting + ** parse of this JSON */ pMatch->nErr = 0; pMatch->iHold = iMaxHold+1; + assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ return pMatch; } - p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); + + /* The input JSON was not found anywhere in the cache. We will need + ** to parse it ourselves and generate a new JsonParse object. + */ + bJsonRCStr = sqlite3ValueIsOfClass(pJson,sqlite3RCStrUnref); + p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) ); if( p==0 ){ sqlite3_result_error_nomem(pCtx); return 0; } memset(p, 0, sizeof(*p)); - p->zJson = (char*)&p[1]; - memcpy((char*)p->zJson, zJson, nJson+1); - if( jsonParse(p, pErrCtx, p->zJson) ){ + if( bJsonRCStr ){ + p->zJson = sqlite3RCStrRef(zJson); + p->bJsonIsRCStr = 1; + }else{ + p->zJson = (char*)&p[1]; + memcpy(p->zJson, zJson, nJson+1); + } + p->nJPRef = 1; + if( jsonParse(p, pErrCtx) ){ if( pErrCtx==0 ){ p->nErr = 1; + assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ return p; } - sqlite3_free(p); + jsonParseFree(p); return 0; } p->nJson = nJson; p->iHold = iMaxHold+1; + /* Transfer ownership of the new JsonParse to the cache */ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, (void(*)(void*))jsonParseFree); return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); @@ -201438,9 +204582,31 @@ static JsonNode *jsonLookupStep( ){ u32 i, j, nKey; const char *zKey; - JsonNode *pRoot = &pParse->aNode[iRoot]; + JsonNode *pRoot; + if( pParse->oom ) return 0; + pRoot = &pParse->aNode[iRoot]; + if( pRoot->jnFlags & (JNODE_REPLACE|JNODE_REMOVE) && pParse->useMod ){ + while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){ + u32 idx = (u32)(pRoot - pParse->aNode); + i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pRoot = &pParse->aNode[i+1]; + iRoot = i+1; + break; + } + i = pParse->aNode[i].u.iPrev; + } + } + if( pRoot->jnFlags & JNODE_REMOVE ){ + return 0; + } + } if( zPath[0]==0 ) return pRoot; - if( pRoot->jnFlags & JNODE_REPLACE ) return 0; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; zPath++; @@ -201474,14 +204640,16 @@ static JsonNode *jsonLookupStep( j += jsonNodeSize(&pRoot[j]); } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; + iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; j = 1; } if( pApnd ){ u32 iStart, iLabel; JsonNode *pNode; + assert( pParse->useMod ); iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); zPath += i; @@ -201490,7 +204658,7 @@ static JsonNode *jsonLookupStep( if( pNode ){ pRoot = &pParse->aNode[iRoot]; assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; + pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; VVA( pRoot->eU = 2 ); pParse->aNode[iLabel].jnFlags |= JNODE_RAW; @@ -201511,12 +204679,13 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; j += jsonNodeSize(&pBase[j]); } if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pBase->eU==2 ); - iBase += pBase->u.iAppend; + iBase = pBase->u.iAppend; pBase = &pParse->aNode[iBase]; j = 1; } @@ -201544,13 +204713,17 @@ static JsonNode *jsonLookupStep( zPath += j + 1; j = 1; for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + while( j<=pRoot->n + && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) + ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; j += jsonNodeSize(&pRoot[j]); } + if( i==0 && j<=pRoot->n ) break; if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; + iRoot = pRoot->u.iAppend; pRoot = &pParse->aNode[iRoot]; j = 1; } @@ -201560,13 +204733,14 @@ static JsonNode *jsonLookupStep( if( i==0 && pApnd ){ u32 iStart; JsonNode *pNode; + assert( pParse->useMod ); iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); if( pParse->oom ) return 0; if( pNode ){ pRoot = &pParse->aNode[iRoot]; assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; + pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; VVA( pRoot->eU = 2 ); } @@ -201693,47 +204867,90 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ ** SQL functions used for testing and debugging ****************************************************************************/ +#if SQLITE_DEBUG +/* +** Print N node entries. +*/ +static void jsonDebugPrintNodeEntries( + JsonNode *aNode, /* First node entry to print */ + int N /* Number of node entries to print */ +){ + int i; + for(i=0; iaNode, p->nNode); +} +static void jsonDebugPrintNode(JsonNode *pNode){ + jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode)); +} +#else + /* The usual case */ +# define jsonDebugPrintNode(X) +# define jsonDebugPrintParse(X) +#endif + #ifdef SQLITE_DEBUG /* -** The json_parse(JSON) function returns a string which describes -** a parse of the JSON provided. Or it returns NULL if JSON is not -** well-formed. +** SQL function: json_parse(JSON) +** +** Parse JSON using jsonParseCached(). Then print a dump of that +** parse on standard output. Return the mimified JSON result, just +** like the json() function. */ static void jsonParseFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - JsonString s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ - u32 i; + JsonParse *p; /* The parse */ assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonParseFindParents(&x); - jsonInit(&s, ctx); - for(i=0; inNode); + printf("nAlloc = %u\n", p->nAlloc); + printf("nJson = %d\n", p->nJson); + printf("nAlt = %d\n", p->nAlt); + printf("nErr = %u\n", p->nErr); + printf("oom = %u\n", p->oom); + printf("hasNonstd = %u\n", p->hasNonstd); + printf("useMod = %u\n", p->useMod); + printf("hasMod = %u\n", p->hasMod); + printf("nJPRef = %u\n", p->nJPRef); + printf("iSubst = %u\n", p->iSubst); + printf("iHold = %u\n", p->iHold); + jsonDebugPrintNodeEntries(p->aNode, p->nNode); + jsonReturnJson(p, p->aNode, ctx, 1); } /* @@ -201817,7 +205034,7 @@ static void jsonArrayLengthFunc( u32 i; JsonNode *pNode; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; assert( p->nNode ); if( argc==2 ){ @@ -201830,9 +205047,16 @@ static void jsonArrayLengthFunc( return; } if( pNode->eType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); + while( 1 /*exit-by-break*/ ){ + i = 1; + while( i<=pNode->n ){ + if( (pNode[i].jnFlags & JNODE_REMOVE)==0 ) n++; + i += jsonNodeSize(&pNode[i]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( p->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &p->aNode[pNode->u.iAppend]; } } sqlite3_result_int64(ctx, n); @@ -201879,7 +205103,7 @@ static void jsonExtractFunc( JsonString jx; if( argc<2 ) return; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ /* With a single PATH argument */ @@ -201897,11 +205121,11 @@ static void jsonExtractFunc( */ jsonInit(&jx, ctx); if( sqlite3Isdigit(zPath[0]) ){ - jsonAppendRaw(&jx, "$[", 2); + jsonAppendRawNZ(&jx, "$[", 2); jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendRaw(&jx, "]", 2); + jsonAppendRawNZ(&jx, "]", 2); }else{ - jsonAppendRaw(&jx, "$.", 1 + (zPath[0]!='[')); + jsonAppendRawNZ(&jx, "$.", 1 + (zPath[0]!='[')); jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); jsonAppendChar(&jx, 0); } @@ -201912,15 +205136,15 @@ static void jsonExtractFunc( } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(pNode, ctx, 0); + jsonReturnJson(p, pNode, ctx, 0); }else{ - jsonReturn(pNode, ctx, 0); + jsonReturn(p, pNode, ctx); sqlite3_result_subtype(ctx, 0); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0); + if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx); } }else{ /* Two or more PATH arguments results in a JSON array with each @@ -201934,9 +205158,9 @@ static void jsonExtractFunc( if( p->nErr ) break; jsonAppendSeparator(&jx); if( pNode ){ - jsonRenderNode(pNode, &jx, 0); + jsonRenderNode(p, pNode, &jx); }else{ - jsonAppendRaw(&jx, "null", 4); + jsonAppendRawNZ(&jx, "null", 4); } } if( i==argc ){ @@ -201981,45 +205205,38 @@ static JsonNode *jsonMergePatch( assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ) break; if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; }else{ JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); if( pNew==0 ) return 0; - pTarget = &pParse->aNode[iTarget]; - if( pNew!=&pTarget[j+1] ){ - assert( pTarget[j+1].eU==0 - || pTarget[j+1].eU==1 - || pTarget[j+1].eU==2 ); - testcase( pTarget[j+1].eU==1 ); - testcase( pTarget[j+1].eU==2 ); - VVA( pTarget[j+1].eU = 5 ); - pTarget[j+1].u.pPatch = pNew; - pTarget[j+1].jnFlags |= JNODE_PATCH; + if( pNew!=&pParse->aNode[iTarget+j+1] ){ + jsonParseAddSubstNode(pParse, iTarget+j+1); + jsonParseAddNodeArray(pParse, pNew, jsonNodeSize(pNew)); } + pTarget = &pParse->aNode[iTarget]; } break; } } if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ - int iStart, iPatch; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + int iStart; + JsonNode *pApnd; + u32 nApnd; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + pApnd = &pPatch[i+1]; + if( pApnd->eType==JSON_OBJECT ) jsonRemoveAllNulls(pApnd); + nApnd = jsonNodeSize(pApnd); + jsonParseAddNodeArray(pParse, pApnd, jsonNodeSize(pApnd)); if( pParse->oom ) return 0; - jsonRemoveAllNulls(pPatch); - pTarget = &pParse->aNode[iTarget]; - assert( pParse->aNode[iRoot].eU==0 || pParse->aNode[iRoot].eU==2 ); - testcase( pParse->aNode[iRoot].eU==2 ); + pParse->aNode[iStart].n = 1+nApnd; pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart; VVA( pParse->aNode[iRoot].eU = 2 ); - pParse->aNode[iRoot].u.iAppend = iStart - iRoot; iRoot = iStart; - assert( pParse->aNode[iPatch].eU==0 ); - VVA( pParse->aNode[iPatch].eU = 5 ); - pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; - pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + pTarget = &pParse->aNode[iTarget]; } } return pTarget; @@ -202035,25 +205252,28 @@ static void jsonPatchFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The JSON that is being patched */ - JsonParse y; /* The patch */ + JsonParse *pX; /* The JSON that is being patched */ + JsonParse *pY; /* The patch */ JsonNode *pResult; /* The result of the merge */ UNUSED_PARAMETER(argc); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ - jsonParseReset(&x); - return; - } - pResult = jsonMergePatch(&x, 0, y.aNode); - assert( pResult!=0 || x.oom ); - if( pResult ){ - jsonReturnJson(pResult, ctx, 0); + pX = jsonParseCached(ctx, argv[0], ctx, 1); + if( pX==0 ) return; + assert( pX->hasMod==0 ); + pX->hasMod = 1; + pY = jsonParseCached(ctx, argv[1], ctx, 1); + if( pY==0 ) return; + pX->useMod = 1; + pY->useMod = 1; + pResult = jsonMergePatch(pX, 0, pY->aNode); + assert( pResult!=0 || pX->oom ); + if( pResult && pX->oom==0 ){ + jsonDebugPrintParse(pX); + jsonDebugPrintNode(pResult); + jsonReturnJson(pX, pResult, ctx, 0); }else{ sqlite3_result_error_nomem(ctx); } - jsonParseReset(&x); - jsonParseReset(&y); } @@ -202109,26 +205329,120 @@ static void jsonRemoveFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; if( argc<1 ) return; - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) goto remove_done; - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto remove_done; - if( pNode ) pNode->jnFlags |= JNODE_REMOVE; + pNode = jsonLookup(pParse, zPath, 0, ctx); + if( pParse->nErr ) goto remove_done; + if( pNode ){ + pNode->jnFlags |= JNODE_REMOVE; + pParse->hasMod = 1; + pParse->useMod = 1; + } } - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0); + if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(pParse, pParse->aNode, ctx, 1); } remove_done: - jsonParseReset(&x); + jsonDebugPrintParse(p); +} + +/* +** Substitute the value at iNode with the pValue parameter. +*/ +static void jsonReplaceNode( + sqlite3_context *pCtx, + JsonParse *p, + int iNode, + sqlite3_value *pValue +){ + int idx = jsonParseAddSubstNode(p, iNode); + if( idx<=0 ){ + assert( p->oom ); + return; + } + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonParseAddNode(p, JSON_NULL, 0, 0); + break; + } + case SQLITE_FLOAT: { + char *z = sqlite3_mprintf("%!0.15g", sqlite3_value_double(pValue)); + int n; + if( z==0 ){ + p->oom = 1; + break; + } + n = sqlite3Strlen30(z); + jsonParseAddNode(p, JSON_REAL, n, z); + jsonParseAddCleanup(p, sqlite3_free, z); + break; + } + case SQLITE_INTEGER: { + char *z = sqlite3_mprintf("%lld", sqlite3_value_int64(pValue)); + int n; + if( z==0 ){ + p->oom = 1; + break; + } + n = sqlite3Strlen30(z); + jsonParseAddNode(p, JSON_INT, n, z); + jsonParseAddCleanup(p, sqlite3_free, z); + + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( z==0 ){ + p->oom = 1; + break; + } + if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ + char *zCopy = sqlite3_malloc64( n+1 ); + int k; + if( zCopy ){ + memcpy(zCopy, z, n); + zCopy[n] = 0; + jsonParseAddCleanup(p, sqlite3_free, zCopy); + }else{ + p->oom = 1; + sqlite3_result_error_nomem(pCtx); + } + k = jsonParseAddNode(p, JSON_STRING, n, zCopy); + assert( k>0 || p->oom ); + if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; + }else{ + JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); + if( pPatch==0 ){ + p->oom = 1; + break; + } + jsonParseAddNodeArray(p, pPatch->aNode, pPatch->nNode); + /* The nodes copied out of pPatch and into p likely contain + ** u.zJContent pointers into pPatch->zJson. So preserve the + ** content of pPatch until p is destroyed. */ + assert( pPatch->nJPRef>=1 ); + pPatch->nJPRef++; + jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch); + } + break; + } + default: { + jsonParseAddNode(p, JSON_NULL, 0, 0); + sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); + p->nErr++; + break; + } + } } /* @@ -202142,7 +205456,7 @@ static void jsonReplaceFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; @@ -202152,28 +205466,22 @@ static void jsonReplaceFunc( jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; + pParse->nJPRef++; for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto replace_err; + pParse->useMod = 1; + pNode = jsonLookup(pParse, zPath, 0, ctx); + if( pParse->nErr ) goto replace_err; if( pNode ){ - assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 ); - testcase( pNode->eU!=0 && pNode->eU!=1 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - VVA( pNode->eU = 4 ); - pNode->u.iReplace = i + 1; + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } + jsonReturnJson(pParse, pParse->aNode, ctx, 1); replace_err: - jsonParseReset(&x); + jsonDebugPrintParse(pParse); + jsonParseFree(pParse); } @@ -202194,7 +205502,7 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ + JsonParse *pParse; /* The parse */ JsonNode *pNode; const char *zPath; u32 i; @@ -202206,33 +205514,27 @@ static void jsonSetFunc( jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); + if( pParse==0 ) return; + pParse->nJPRef++; for(i=1; i<(u32)argc; i+=2){ zPath = (const char*)sqlite3_value_text(argv[i]); bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx); - if( x.oom ){ + pParse->useMod = 1; + pNode = jsonLookup(pParse, zPath, &bApnd, ctx); + if( pParse->oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; - }else if( x.nErr ){ + }else if( pParse->nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - testcase( pNode->eU!=0 && pNode->eU!=1 ); - assert( pNode->eU!=3 && pNode->eU!=5 ); - VVA( pNode->eU = 4 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->u.iReplace = i + 1; + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } + jsonDebugPrintParse(pParse); + jsonReturnJson(pParse, pParse->aNode, ctx, 1); jsonSetDone: - jsonParseReset(&x); + jsonParseFree(pParse); } /* @@ -202251,7 +205553,7 @@ static void jsonTypeFunc( const char *zPath; JsonNode *pNode; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); @@ -202277,13 +205579,19 @@ static void jsonValidFunc( ){ JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ +#ifdef SQLITE_LEGACY_JSON_VALID + /* Incorrect legacy behavior was to return FALSE for a NULL input */ + sqlite3_result_int(ctx, 0); +#endif + return; + } + p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); }else{ - sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); + sqlite3_result_int(ctx, p->nErr==0 && (p->hasNonstd==0 || p->useMod)); if( p->nErr ) jsonParseFree(p); } } @@ -202324,7 +205632,7 @@ static void jsonErrorFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); + p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); sqlite3_free(p); @@ -202333,7 +205641,7 @@ static void jsonErrorFunc( }else{ int n = 1; u32 i; - const char *z = p->zJson; + const char *z = (const char*)sqlite3_value_text(argv[0]); for(i=0; iiErr && ALWAYS(z[i]); i++){ if( (z[i]&0xc0)!=0x80 ) n++; } @@ -202381,7 +205689,8 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); @@ -202422,7 +205731,7 @@ static void jsonGroupInverse( pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); #ifdef NEVER /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will - ** always have been called to initalize it */ + ** always have been called to initialize it */ if( NEVER(!pStr) ) return; #endif z = pStr->zBuf; @@ -202489,7 +205798,8 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); @@ -202600,7 +205910,6 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zJson); sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); p->iRowid = 0; @@ -202738,7 +206047,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0); + jsonReturn(&p->sParse, pThis, ctx); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -202754,7 +206063,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0); + jsonReturn(&p->sParse, pThis, ctx); break; } case JEACH_TYPE: { @@ -202765,7 +206074,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0); + jsonReturn(&p->sParse, pThis, ctx); break; } case JEACH_ID: { @@ -202920,11 +206229,19 @@ static int jsonEachFilter( if( idxNum==0 ) return SQLITE_OK; z = (const char*)sqlite3_value_text(argv[0]); if( z==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc64( n+1 ); - if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ + memset(&p->sParse, 0, sizeof(p->sParse)); + p->sParse.nJPRef = 1; + if( sqlite3ValueIsOfClass(argv[0], sqlite3RCStrUnref) ){ + p->sParse.zJson = sqlite3RCStrRef((char*)z); + }else{ + n = sqlite3_value_bytes(argv[0]); + p->sParse.zJson = sqlite3RCStrNew( n+1 ); + if( p->sParse.zJson==0 ) return SQLITE_NOMEM; + memcpy(p->sParse.zJson, z, (size_t)n+1); + } + p->sParse.bJsonIsRCStr = 1; + p->zJson = p->sParse.zJson; + if( jsonParse(&p->sParse, 0) ){ int rc = SQLITE_NOMEM; if( p->sParse.oom==0 ){ sqlite3_free(cur->pVtab->zErrMsg); @@ -203009,7 +206326,8 @@ static sqlite3_module jsonEachModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; /* The methods of the json_tree virtual table. */ @@ -203037,7 +206355,8 @@ static sqlite3_module jsonTreeModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ #endif /* !defined(SQLITE_OMIT_JSON) */ @@ -203202,6 +206521,11 @@ typedef unsigned int u32; #endif #endif /* !defined(SQLITE_AMALGAMATION) */ +/* Macro to check for 4-byte alignment. Only used inside of assert() */ +#ifdef SQLITE_DEBUG +# define FOUR_BYTE_ALIGNED(X) ((((char*)(X) - (char*)0) & 3)==0) +#endif + /* #include */ /* #include */ /* #include */ @@ -203267,6 +206591,7 @@ struct Rtree { int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ + char *zNodeName; /* Name of the %_node table */ u32 nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ u32 nCursor; /* Number of open cursors */ @@ -203279,7 +206604,6 @@ struct Rtree { ** headed by the node (leaf nodes have RtreeNode.iNode==0). */ RtreeNode *pDeleted; - int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ /* Blob I/O on xxx_node */ sqlite3_blob *pNodeBlob; @@ -203576,15 +206900,20 @@ struct RtreeMatchArg { ** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined ** at run-time. */ -#ifndef SQLITE_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ +#ifndef SQLITE_BYTEORDER /* Replicate changes at tag-20230904a */ +# if defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define SQLITE_BYTEORDER 4321 +# elif defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define SQLITE_BYTEORDER 1234 +# elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1 +# define SQLITE_BYTEORDER 4321 +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) -# define SQLITE_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) || \ - defined(__ARMEB__) || defined(__AARCH64EB__) -# define SQLITE_BYTEORDER 4321 +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 # else # define SQLITE_BYTEORDER 0 # endif @@ -203608,7 +206937,7 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ - assert( (((sqlite3_uint64)p)&3)==0 ); /* p is always 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(p) ); #if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 pCoord->u = _byteswap_ulong(*(u32*)p); #elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 @@ -203662,7 +206991,7 @@ static void writeInt16(u8 *p, int i){ } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; - assert( (((sqlite3_uint64)p)&3)==0 ); /* p is always 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(p) ); assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); #if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 @@ -203833,11 +207162,9 @@ static int nodeAcquire( } } if( pRtree->pNodeBlob==0 ){ - char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); - if( zTab==0 ) return SQLITE_NOMEM; - rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, pRtree->zNodeName, + "data", iNode, 0, &pRtree->pNodeBlob); - sqlite3_free(zTab); } if( rc ){ nodeBlobReset(pRtree); @@ -204390,7 +207717,7 @@ static void rtreeNonleafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); - assert( (((sqlite3_uint64)pCellData)&3)==0 ); /* 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(pCellData) ); switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ case RTREE_FALSE: break; /* Never satisfied */ @@ -204443,7 +207770,7 @@ static void rtreeLeafConstraint( || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE || p->op==RTREE_FALSE ); pCellData += 8 + p->iCoord*4; - assert( (((sqlite3_uint64)pCellData)&3)==0 ); /* 4-byte aligned */ + assert( FOUR_BYTE_ALIGNED(pCellData) ); RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ case RTREE_TRUE: return; /* Always satisfied */ @@ -205013,7 +208340,20 @@ static int rtreeFilter( p->pInfo->nCoord = pRtree->nDim2; p->pInfo->anQueue = pCsr->anQueue; p->pInfo->mxLevel = pRtree->iDepth + 1; - }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + }else if( eType==SQLITE_INTEGER ){ + sqlite3_int64 iVal = sqlite3_value_int64(argv[ii]); +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = iVal; +#else + p->u.rValue = (double)iVal; + if( iVal>=((sqlite3_int64)1)<<48 + || iVal<=-(((sqlite3_int64)1)<<48) + ){ + if( p->op==RTREE_LT ) p->op = RTREE_LE; + if( p->op==RTREE_GT ) p->op = RTREE_GE; + } +#endif + }else if( eType==SQLITE_FLOAT ){ #ifdef SQLITE_RTREE_INT_ONLY p->u.rValue = sqlite3_value_int64(argv[ii]); #else @@ -205144,11 +208484,12 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ u8 op; + u8 doOmit = 1; switch( p->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; - case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; doOmit = 0; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; doOmit = 0; break; case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; - case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; doOmit = 0; break; case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; case SQLITE_INDEX_CONSTRAINT_MATCH: op = RTREE_MATCH; break; default: op = 0; break; @@ -205157,15 +208498,19 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ zIdxStr[iIdx++] = op; zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); - pIdxInfo->aConstraintUsage[ii].omit = 1; + pIdxInfo->aConstraintUsage[ii].omit = doOmit; } } } pIdxInfo->idxNum = 2; pIdxInfo->needToFreeIdxStr = 1; - if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ - return SQLITE_NOMEM; + if( iIdx>0 ){ + pIdxInfo->idxStr = sqlite3_malloc( iIdx+1 ); + if( pIdxInfo->idxStr==0 ){ + return SQLITE_NOMEM; + } + memcpy(pIdxInfo->idxStr, zIdxStr, iIdx+1); } nRow = pRtree->nRowEst >> (iIdx/2); @@ -205244,31 +208589,22 @@ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ */ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ int ii; - int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); - for(ii=0; iinDim2; ii+=2){ - RtreeCoord *a1 = &p1->aCoord[ii]; - RtreeCoord *a2 = &p2->aCoord[ii]; - if( (!isInt && (a2[0].fa1[1].f)) - || ( isInt && (a2[0].ia1[1].i)) - ){ - return 0; + if( pRtree->eCoordType==RTREE_COORD_INT32 ){ + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( a2[0].ia1[1].i ) return 0; + } + }else{ + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( a2[0].fa1[1].f ) return 0; } } return 1; } -/* -** Return the amount cell p would grow by if it were unioned with pCell. -*/ -static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ - RtreeDValue area; - RtreeCell cell; - memcpy(&cell, p, sizeof(RtreeCell)); - area = cellArea(pRtree, &cell); - cellUnion(pRtree, &cell, pCell); - return (cellArea(pRtree, &cell)-area); -} - static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, @@ -205315,38 +208651,52 @@ static int ChooseLeaf( for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ int iCell; sqlite3_int64 iBest = 0; - + int bFound = 0; RtreeDValue fMinGrowth = RTREE_ZERO; RtreeDValue fMinArea = RTREE_ZERO; - int nCell = NCELL(pNode); - RtreeCell cell; RtreeNode *pChild = 0; - RtreeCell *aCell = 0; - - /* Select the child node which will be enlarged the least if pCell - ** is inserted into it. Resolve ties by choosing the entry with - ** the smallest area. + /* First check to see if there is are any cells in pNode that completely + ** contains pCell. If two or more cells in pNode completely contain pCell + ** then pick the smallest. */ for(iCell=0; iCell1 ){ - int iLeft = 0; - int iRight = 0; - - int nLeft = nIdx/2; - int nRight = nIdx-nLeft; - int *aLeft = aIdx; - int *aRight = &aIdx[nLeft]; - - SortByDistance(aLeft, nLeft, aDistance, aSpare); - SortByDistance(aRight, nRight, aDistance, aSpare); - - memcpy(aSpare, aLeft, sizeof(int)*nLeft); - aLeft = aSpare; - - while( iLeftnDim; iDim++){ - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); - } - } - for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); - } - - for(ii=0; iinDim; iDim++){ - RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2])); - aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); - } - } - - SortByDistance(aOrder, nCell, aDistance, aSpare); - nodeZero(pRtree, pNode); - - for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ - RtreeCell *p = &aCell[aOrder[ii]]; - nodeInsertCell(pRtree, pNode, p); - if( p->iRowid==pCell->iRowid ){ - if( iHeight==0 ){ - rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, p->iRowid, pNode->iNode); - } - } - } - if( rc==SQLITE_OK ){ - rc = fixBoundingBox(pRtree, pNode); - } - for(; rc==SQLITE_OK && iiiNode currently contains - ** the height of the sub-tree headed by the cell. - */ - RtreeNode *pInsert; - RtreeCell *p = &aCell[aOrder[ii]]; - rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); - if( rc==SQLITE_OK ){ - int rc2; - rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); - rc2 = nodeRelease(pRtree, pInsert); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - } - - sqlite3_free(aCell); - return rc; -} - /* ** Insert cell pCell into node pNode. Node pNode is the head of a ** subtree iHeight high (leaf nodes have iHeight==0). @@ -206095,12 +209273,7 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ - if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ - rc = SplitNode(pRtree, pNode, pCell, iHeight); - }else{ - pRtree->iReinsertHeight = iHeight; - rc = Reinsert(pRtree, pNode, pCell, iHeight); - } + rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ rc = AdjustTree(pRtree, pNode, pCell); if( ALWAYS(rc==SQLITE_OK) ){ @@ -206443,7 +209616,6 @@ static int rtreeUpdate( } if( rc==SQLITE_OK ){ int rc2; - pRtree->iReinsertHeight = -1; rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ @@ -206584,8 +209756,11 @@ static int rtreeShadowName(const char *zName){ return 0; } +/* Forward declaration */ +static int rtreeIntegrity(sqlite3_vtab*, const char*, const char*, int, char**); + static sqlite3_module rtreeModule = { - 3, /* iVersion */ + 4, /* iVersion */ rtreeCreate, /* xCreate - create a table */ rtreeConnect, /* xConnect - connect to an existing table */ rtreeBestIndex, /* xBestIndex - Determine search strategy */ @@ -206608,7 +209783,8 @@ static sqlite3_module rtreeModule = { rtreeSavepoint, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - rtreeShadowName /* xShadowName */ + rtreeShadowName, /* xShadowName */ + rtreeIntegrity /* xIntegrity */ }; static int rtreeSqlInit( @@ -206864,22 +210040,27 @@ static int rtreeInit( } sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + /* Allocate the sqlite3_vtab structure */ nDb = (int)strlen(argv[1]); nName = (int)strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8); if( !pRtree ){ return SQLITE_NOMEM; } - memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8); pRtree->nBusy = 1; pRtree->base.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->zNodeName = &pRtree->zName[nName+1]; pRtree->eCoordType = (u8)eCoordType; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); + memcpy(pRtree->zNodeName, argv[2], nName); + memcpy(&pRtree->zNodeName[nName], "_node", 6); /* Create/Connect to the underlying relational database schema. If @@ -207376,7 +210557,6 @@ static int rtreeCheckTable( ){ RtreeCheck check; /* Common context for various routines */ sqlite3_stmt *pStmt = 0; /* Used to find column count of rtree table */ - int bEnd = 0; /* True if transaction should be closed */ int nAux = 0; /* Number of extra columns. */ /* Initialize the context object */ @@ -207385,14 +210565,6 @@ static int rtreeCheckTable( check.zDb = zDb; check.zTab = zTab; - /* If there is not already an open transaction, open one now. This is - ** to ensure that the queries run as part of this integrity-check operate - ** on a consistent snapshot. */ - if( sqlite3_get_autocommit(db) ){ - check.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); - bEnd = 1; - } - /* Find the number of auxiliary columns */ if( check.rc==SQLITE_OK ){ pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.'%q_rowid'", zDb, zTab); @@ -207433,15 +210605,34 @@ static int rtreeCheckTable( sqlite3_finalize(check.aCheckMapping[0]); sqlite3_finalize(check.aCheckMapping[1]); - /* If one was opened, close the transaction */ - if( bEnd ){ - int rc = sqlite3_exec(db, "END", 0, 0, 0); - if( check.rc==SQLITE_OK ) check.rc = rc; - } *pzReport = check.zReport; return check.rc; } +/* +** Implementation of the xIntegrity method for Rtree. +*/ +static int rtreeIntegrity( + sqlite3_vtab *pVtab, /* The virtual table to check */ + const char *zSchema, /* Schema in which the virtual table lives */ + const char *zName, /* Name of the virtual table */ + int isQuick, /* True for a quick_check */ + char **pzErr /* Write results here */ +){ + Rtree *pRtree = (Rtree*)pVtab; + int rc; + assert( pzErr!=0 && *pzErr==0 ); + UNUSED_PARAMETER(zSchema); + UNUSED_PARAMETER(zName); + UNUSED_PARAMETER(isQuick); + rc = rtreeCheckTable(pRtree->db, pRtree->zDb, pRtree->zName, pzErr); + if( rc==SQLITE_OK && *pzErr ){ + *pzErr = sqlite3_mprintf("In RTree %s.%s:\n%z", + pRtree->zDb, pRtree->zName, *pzErr); + } + return rc; +} + /* ** Usage: ** @@ -208763,24 +211954,28 @@ static int geopolyInit( (void)pAux; sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); nName = strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8); if( !pRtree ){ return SQLITE_NOMEM; } - memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8); pRtree->nBusy = 1; pRtree->base.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->zNodeName = &pRtree->zName[nName+1]; pRtree->eCoordType = RTREE_COORD_REAL32; pRtree->nDim = 2; pRtree->nDim2 = 4; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); + memcpy(pRtree->zNodeName, argv[2], nName); + memcpy(&pRtree->zNodeName[nName], "_node", 6); /* Create/Connect to the underlying relational database schema. If @@ -209194,7 +212389,6 @@ static int geopolyUpdate( } if( rc==SQLITE_OK ){ int rc2; - pRtree->iReinsertHeight = -1; rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ @@ -209291,7 +212485,8 @@ static sqlite3_module geopolyModule = { rtreeSavepoint, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - rtreeShadowName /* xShadowName */ + rtreeShadowName, /* xShadowName */ + rtreeIntegrity /* xIntegrity */ }; static int sqlite3_geopoly_init(sqlite3 *db){ @@ -217305,7 +220500,8 @@ SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "dbstat", &dbstat_module, 0); } @@ -217742,7 +220938,8 @@ SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0); } @@ -217873,6 +221070,18 @@ struct sqlite3_changeset_iter { ** The data associated with each hash-table entry is a structure containing ** a subset of the initial values that the modified row contained at the ** start of the session. Or no initial values if the row was inserted. +** +** pDfltStmt: +** This is only used by the sqlite3changegroup_xxx() APIs, not by +** regular sqlite3_session objects. It is a SELECT statement that +** selects the default value for each table column. For example, +** if the table is +** +** CREATE TABLE xx(a DEFAULT 1, b, c DEFAULT 'abc') +** +** then this variable is the compiled version of: +** +** SELECT 1, NULL, 'abc' */ struct SessionTable { SessionTable *pNext; @@ -217881,10 +221090,12 @@ struct SessionTable { int bStat1; /* True if this is sqlite_stat1 */ int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ + const char **azDflt; /* Default value expressions */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ int nChange; /* Size of apChange[] array */ SessionChange **apChange; /* Hash table buckets */ + sqlite3_stmt *pDfltStmt; }; /* @@ -218053,6 +221264,7 @@ struct SessionTable { struct SessionChange { u8 op; /* One of UPDATE, DELETE, INSERT */ u8 bIndirect; /* True if this change is "indirect" */ + u16 nRecordField; /* Number of fields in aRecord[] */ int nMaxSize; /* Max size of eventual changeset record */ int nRecord; /* Number of bytes in buffer aRecord[] */ u8 *aRecord; /* Buffer containing old.* record */ @@ -218078,7 +221290,7 @@ static int sessionVarintLen(int iVal){ ** Read a varint value from aBuf[] into *piVal. Return the number of ** bytes read. */ -static int sessionVarintGet(u8 *aBuf, int *piVal){ +static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } @@ -218341,9 +221553,11 @@ static int sessionPreupdateHash( ** Return the number of bytes of space occupied by the value (including ** the type byte). */ -static int sessionSerialLen(u8 *a){ - int e = *a; +static int sessionSerialLen(const u8 *a){ + int e; int n; + assert( a!=0 ); + e = *a; if( e==0 || e==0xFF ) return 1; if( e==SQLITE_NULL ) return 1; if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; @@ -218645,6 +221859,7 @@ static int sessionPreupdateEqual( rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); } assert( rc==SQLITE_OK ); + (void)rc; /* Suppress warning about unused variable */ if( sqlite3_value_type(pVal)!=eType ) return 0; /* A SessionChange object never has a NULL value in a PK column */ @@ -218747,13 +221962,14 @@ static int sessionGrowHash( ** ** For example, if the table is declared as: ** -** CREATE TABLE tbl1(w, x, y, z, PRIMARY KEY(w, z)); +** CREATE TABLE tbl1(w, x DEFAULT 'abc', y, z, PRIMARY KEY(w, z)); ** -** Then the four output variables are populated as follows: +** Then the five output variables are populated as follows: ** ** *pnCol = 4 ** *pzTab = "tbl1" ** *pazCol = {"w", "x", "y", "z"} +** *pazDflt = {NULL, 'abc', NULL, NULL} ** *pabPK = {1, 0, 0, 1} ** ** All returned buffers are part of the same single allocation, which must @@ -218767,6 +221983,7 @@ static int sessionTableInfo( int *pnCol, /* OUT: number of columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ + const char ***pazDflt, /* OUT: Array of default value expressions */ u8 **pabPK, /* OUT: Array of booleans - true for PK col */ int *pbRowid /* OUT: True if only PK is a rowid */ ){ @@ -218779,11 +221996,18 @@ static int sessionTableInfo( int i; u8 *pAlloc = 0; char **azCol = 0; + char **azDflt = 0; u8 *abPK = 0; int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); + *pazCol = 0; + *pabPK = 0; + *pnCol = 0; + if( pzTab ) *pzTab = 0; + if( pazDflt ) *pazDflt = 0; + nThis = sqlite3Strlen30(zThis); if( nThis==12 && 0==sqlite3_stricmp("sqlite_stat1", zThis) ){ rc = sqlite3_table_column_metadata(db, zDb, zThis, 0, 0, 0, 0, 0, 0); @@ -218797,39 +222021,28 @@ static int sessionTableInfo( }else if( rc==SQLITE_ERROR ){ zPragma = sqlite3_mprintf(""); }else{ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; return rc; } }else{ zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis); } if( !zPragma ){ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; return SQLITE_NOMEM; } rc = sqlite3_prepare_v2(db, zPragma, -1, &pStmt, 0); sqlite3_free(zPragma); if( rc!=SQLITE_OK ){ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; return rc; } nByte = nThis + 1; bRowid = (pbRowid!=0); while( SQLITE_ROW==sqlite3_step(pStmt) ){ - nByte += sqlite3_column_bytes(pStmt, 1); + nByte += sqlite3_column_bytes(pStmt, 1); /* name */ + nByte += sqlite3_column_bytes(pStmt, 4); /* dflt_value */ nDbCol++; - if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; + if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; /* pk */ } if( nDbCol==0 ) bRowid = 0; nDbCol += bRowid; @@ -218837,15 +222050,18 @@ static int sessionTableInfo( rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ - nByte += nDbCol * (sizeof(const char *) + sizeof(u8) + 1); + nByte += nDbCol * (sizeof(const char *)*2 + sizeof(u8) + 1 + 1); pAlloc = sessionMalloc64(pSession, nByte); if( pAlloc==0 ){ rc = SQLITE_NOMEM; + }else{ + memset(pAlloc, 0, nByte); } } if( rc==SQLITE_OK ){ azCol = (char **)pAlloc; - pAlloc = (u8 *)&azCol[nDbCol]; + azDflt = (char**)&azCol[nDbCol]; + pAlloc = (u8 *)&azDflt[nDbCol]; abPK = (u8 *)pAlloc; pAlloc = &abPK[nDbCol]; if( pzTab ){ @@ -218865,11 +222081,21 @@ static int sessionTableInfo( } while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nName = sqlite3_column_bytes(pStmt, 1); + int nDflt = sqlite3_column_bytes(pStmt, 4); const unsigned char *zName = sqlite3_column_text(pStmt, 1); + const unsigned char *zDflt = sqlite3_column_text(pStmt, 4); + if( zName==0 ) break; memcpy(pAlloc, zName, nName+1); azCol[i] = (char *)pAlloc; pAlloc += nName+1; + if( zDflt ){ + memcpy(pAlloc, zDflt, nDflt+1); + azDflt[i] = (char *)pAlloc; + pAlloc += nDflt+1; + }else{ + azDflt[i] = 0; + } abPK[i] = sqlite3_column_int(pStmt, 5); i++; } @@ -218880,14 +222106,11 @@ static int sessionTableInfo( ** free any allocation made. An error code will be returned in this case. */ if( rc==SQLITE_OK ){ - *pazCol = (const char **)azCol; + *pazCol = (const char**)azCol; + if( pazDflt ) *pazDflt = (const char**)azDflt; *pabPK = abPK; *pnCol = nDbCol; }else{ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; sessionFree(pSession, azCol); } if( pbRowid ) *pbRowid = bRowid; @@ -218896,10 +222119,9 @@ static int sessionTableInfo( } /* -** This function is only called from within a pre-update handler for a -** write to table pTab, part of session pSession. If this is the first -** write to this table, initalize the SessionTable.nCol, azCol[] and -** abPK[] arrays accordingly. +** This function is called to initialize the SessionTable.nCol, azCol[] +** abPK[] and azDflt[] members of SessionTable object pTab. If these +** fields are already initilialized, this function is a no-op. ** ** If an error occurs, an error code is stored in sqlite3_session.rc and ** non-zero returned. Or, if no error occurs but the table has no primary @@ -218907,15 +222129,22 @@ static int sessionTableInfo( ** indicate that updates on this table should be ignored. SessionTable.abPK ** is set to NULL in this case. */ -static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ +static int sessionInitTable( + sqlite3_session *pSession, /* Optional session handle */ + SessionTable *pTab, /* Table object to initialize */ + sqlite3 *db, /* Database handle to read schema from */ + const char *zDb /* Name of db - "main", "temp" etc. */ +){ + int rc = SQLITE_OK; + if( pTab->nCol==0 ){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); - pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK, - (pSession->bImplicitPK ? &pTab->bRowid : 0) + rc = sessionTableInfo(pSession, db, zDb, + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &pTab->azDflt, &abPK, + ((pSession==0 || pSession->bImplicitPK) ? &pTab->bRowid : 0) ); - if( pSession->rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ int i; for(i=0; inCol; i++){ if( abPK[i] ){ @@ -218927,14 +222156,321 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ pTab->bStat1 = 1; } - if( pSession->bEnableSize ){ + if( pSession && pSession->bEnableSize ){ pSession->nMaxChangesetSize += ( 1 + sessionVarintLen(pTab->nCol) + pTab->nCol + strlen(pTab->zName)+1 ); } } } - return (pSession->rc || pTab->abPK==0); + + if( pSession ){ + pSession->rc = rc; + return (rc || pTab->abPK==0); + } + return rc; +} + +/* +** Re-initialize table object pTab. +*/ +static int sessionReinitTable(sqlite3_session *pSession, SessionTable *pTab){ + int nCol = 0; + const char **azCol = 0; + const char **azDflt = 0; + u8 *abPK = 0; + int bRowid = 0; + + assert( pSession->rc==SQLITE_OK ); + + pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, + pTab->zName, &nCol, 0, &azCol, &azDflt, &abPK, + (pSession->bImplicitPK ? &bRowid : 0) + ); + if( pSession->rc==SQLITE_OK ){ + if( pTab->nCol>nCol || pTab->bRowid!=bRowid ){ + pSession->rc = SQLITE_SCHEMA; + }else{ + int ii; + int nOldCol = pTab->nCol; + for(ii=0; iinCol ){ + if( pTab->abPK[ii]!=abPK[ii] ){ + pSession->rc = SQLITE_SCHEMA; + } + }else if( abPK[ii] ){ + pSession->rc = SQLITE_SCHEMA; + } + } + + if( pSession->rc==SQLITE_OK ){ + const char **a = pTab->azCol; + pTab->azCol = azCol; + pTab->nCol = nCol; + pTab->azDflt = azDflt; + pTab->abPK = abPK; + azCol = a; + } + if( pSession->bEnableSize ){ + pSession->nMaxChangesetSize += (nCol - nOldCol); + pSession->nMaxChangesetSize += sessionVarintLen(nCol); + pSession->nMaxChangesetSize -= sessionVarintLen(nOldCol); + } + } + } + + sqlite3_free((char*)azCol); + return pSession->rc; +} + +/* +** Session-change object (*pp) contains an old.* record with fewer than +** nCol fields. This function updates it with the default values for +** the missing fields. +*/ +static void sessionUpdateOneChange( + sqlite3_session *pSession, /* For memory accounting */ + int *pRc, /* IN/OUT: Error code */ + SessionChange **pp, /* IN/OUT: Change object to update */ + int nCol, /* Number of columns now in table */ + sqlite3_stmt *pDflt /* SELECT */ +){ + SessionChange *pOld = *pp; + + while( pOld->nRecordFieldnRecordField; + int eType = sqlite3_column_type(pDflt, iField); + switch( eType ){ + case SQLITE_NULL: + nIncr = 1; + break; + case SQLITE_INTEGER: + case SQLITE_FLOAT: + nIncr = 9; + break; + default: { + int n = sqlite3_column_bytes(pDflt, iField); + nIncr = 1 + sessionVarintLen(n) + n; + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + break; + } + } + + nByte = nIncr + (sizeof(SessionChange) + pOld->nRecord); + pNew = sessionMalloc64(pSession, nByte); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return; + }else{ + memcpy(pNew, pOld, sizeof(SessionChange)); + pNew->aRecord = (u8*)&pNew[1]; + memcpy(pNew->aRecord, pOld->aRecord, pOld->nRecord); + pNew->aRecord[pNew->nRecord++] = (u8)eType; + switch( eType ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_column_int64(pDflt, iField); + sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + pNew->nRecord += 8; + break; + } + + case SQLITE_FLOAT: { + double rVal = sqlite3_column_double(pDflt, iField); + i64 iVal = 0; + memcpy(&iVal, &rVal, sizeof(rVal)); + sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + pNew->nRecord += 8; + break; + } + + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pDflt, iField); + const char *z = (const char*)sqlite3_column_text(pDflt, iField); + pNew->nRecord += sessionVarintPut(&pNew->aRecord[pNew->nRecord], n); + memcpy(&pNew->aRecord[pNew->nRecord], z, n); + pNew->nRecord += n; + break; + } + + case SQLITE_BLOB: { + int n = sqlite3_column_bytes(pDflt, iField); + const u8 *z = (const u8*)sqlite3_column_blob(pDflt, iField); + pNew->nRecord += sessionVarintPut(&pNew->aRecord[pNew->nRecord], n); + memcpy(&pNew->aRecord[pNew->nRecord], z, n); + pNew->nRecord += n; + break; + } + + default: + assert( eType==SQLITE_NULL ); + break; + } + + sessionFree(pSession, pOld); + *pp = pOld = pNew; + pNew->nRecordField++; + pNew->nMaxSize += nIncr; + if( pSession ){ + pSession->nMaxChangesetSize += nIncr; + } + } + } +} + +/* +** Ensure that there is room in the buffer to append nByte bytes of data. +** If not, use sqlite3_realloc() to grow the buffer so that there is. +** +** If successful, return zero. Otherwise, if an OOM condition is encountered, +** set *pRc to SQLITE_NOMEM and return non-zero. +*/ +static int sessionBufferGrow(SessionBuffer *p, i64 nByte, int *pRc){ +#define SESSION_MAX_BUFFER_SZ (0x7FFFFF00 - 1) + i64 nReq = p->nBuf + nByte; + if( *pRc==SQLITE_OK && nReq>p->nAlloc ){ + u8 *aNew; + i64 nNew = p->nAlloc ? p->nAlloc : 128; + + do { + nNew = nNew*2; + }while( nNewSESSION_MAX_BUFFER_SZ ){ + nNew = SESSION_MAX_BUFFER_SZ; + if( nNewaBuf, nNew); + if( 0==aNew ){ + *pRc = SQLITE_NOMEM; + }else{ + p->aBuf = aNew; + p->nAlloc = nNew; + } + } + return (*pRc!=SQLITE_OK); +} + + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a string to the buffer. All bytes in the string +** up to (but not including) the nul-terminator are written to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendStr( + SessionBuffer *p, + const char *zStr, + int *pRc +){ + int nStr = sqlite3Strlen30(zStr); + if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ + memcpy(&p->aBuf[p->nBuf], zStr, nStr); + p->nBuf += nStr; + p->aBuf[p->nBuf] = 0x00; + } +} + +/* +** Format a string using printf() style formatting and then append it to the +** buffer using sessionAppendString(). +*/ +static void sessionAppendPrintf( + SessionBuffer *p, /* Buffer to append to */ + int *pRc, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zApp = 0; + va_list ap; + va_start(ap, zFmt); + zApp = sqlite3_vmprintf(zFmt, ap); + if( zApp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sessionAppendStr(p, zApp, pRc); + } + va_end(ap); + sqlite3_free(zApp); + } +} + +/* +** Prepare a statement against database handle db that SELECTs a single +** row containing the default values for each column in table pTab. For +** example, if pTab is declared as: +** +** CREATE TABLE pTab(a PRIMARY KEY, b DEFAULT 123, c DEFAULT 'abcd'); +** +** Then this function prepares and returns the SQL statement: +** +** SELECT NULL, 123, 'abcd'; +*/ +static int sessionPrepareDfltStmt( + sqlite3 *db, /* Database handle */ + SessionTable *pTab, /* Table to prepare statement for */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + SessionBuffer sql = {0,0,0}; + int rc = SQLITE_OK; + const char *zSep = " "; + int ii = 0; + + *ppStmt = 0; + sessionAppendPrintf(&sql, &rc, "SELECT"); + for(ii=0; iinCol; ii++){ + const char *zDflt = pTab->azDflt[ii] ? pTab->azDflt[ii] : "NULL"; + sessionAppendPrintf(&sql, &rc, "%s%s", zSep, zDflt); + zSep = ", "; + } + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (const char*)sql.aBuf, -1, ppStmt, 0); + } + sqlite3_free(sql.aBuf); + + return rc; +} + +/* +** Table pTab has one or more existing change-records with old.* records +** with fewer than pTab->nCol columns. This function updates all such +** change-records with the default values for the missing columns. +*/ +static int sessionUpdateChanges(sqlite3_session *pSession, SessionTable *pTab){ + sqlite3_stmt *pStmt = 0; + int rc = pSession->rc; + + rc = sessionPrepareDfltStmt(pSession->db, pTab, &pStmt); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int ii = 0; + SessionChange **pp = 0; + for(ii=0; iinChange; ii++){ + for(pp=&pTab->apChange[ii]; *pp; pp=&((*pp)->pNext)){ + if( (*pp)->nRecordField!=pTab->nCol ){ + sessionUpdateOneChange(pSession, &rc, pp, pTab->nCol, pStmt); + } + } + } + } + + pSession->rc = rc; + rc = sqlite3_finalize(pStmt); + if( pSession->rc==SQLITE_OK ) pSession->rc = rc; + return pSession->rc; } /* @@ -219097,16 +222633,22 @@ static void sessionPreupdateOneChange( int iHash; int bNull = 0; int rc = SQLITE_OK; + int nExpect = 0; SessionStat1Ctx stat1 = {{0,0,0,0,0},0}; if( pSession->rc ) return; /* Load table details if required */ - if( sessionInitTable(pSession, pTab) ) return; + if( sessionInitTable(pSession, pTab, pSession->db, pSession->zDb) ) return; /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ - if( (pTab->nCol-pTab->bRowid)!=pSession->hook.xCount(pSession->hook.pCtx) ){ + nExpect = pSession->hook.xCount(pSession->hook.pCtx); + if( (pTab->nCol-pTab->bRowid)nCol-pTab->bRowid)!=nExpect ){ pSession->rc = SQLITE_SCHEMA; return; } @@ -219183,7 +222725,7 @@ static void sessionPreupdateOneChange( } /* Allocate the change object */ - pC = (SessionChange *)sessionMalloc64(pSession, nByte); + pC = (SessionChange*)sessionMalloc64(pSession, nByte); if( !pC ){ rc = SQLITE_NOMEM; goto error_out; @@ -219216,6 +222758,7 @@ static void sessionPreupdateOneChange( if( pSession->bIndirect || pSession->hook.xDepth(pSession->hook.pCtx) ){ pC->bIndirect = 1; } + pC->nRecordField = pTab->nCol; pC->nRecord = nByte; pC->op = op; pC->pNext = pTab->apChange[iHash]; @@ -219595,7 +223138,7 @@ SQLITE_API int sqlite3session_diff( /* Locate and if necessary initialize the target table object */ rc = sessionFindTable(pSession, zTbl, &pTo); if( pTo==0 ) goto diff_out; - if( sessionInitTable(pSession, pTo) ){ + if( sessionInitTable(pSession, pTo, pSession->db, pSession->zDb) ){ rc = pSession->rc; goto diff_out; } @@ -219608,7 +223151,7 @@ SQLITE_API int sqlite3session_diff( int bRowid = 0; u8 *abPK; const char **azCol = 0; - rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK, + rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, 0, &abPK, pSession->bImplicitPK ? &bRowid : 0 ); if( rc==SQLITE_OK ){ @@ -219723,6 +223266,7 @@ static void sessionDeleteTable(sqlite3_session *pSession, SessionTable *pList){ sessionFree(pSession, p); } } + sqlite3_finalize(pTab->pDfltStmt); sessionFree(pSession, (char*)pTab->azCol); /* cast works around VC++ bug */ sessionFree(pSession, pTab->apChange); sessionFree(pSession, pTab); @@ -219757,7 +223301,7 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession){ /* Assert that all allocations have been freed and then free the ** session object itself. */ - assert( pSession->nMalloc==0 ); + // assert( pSession->nMalloc==0 ); sqlite3_free(pSession); } @@ -219828,48 +223372,6 @@ SQLITE_API int sqlite3session_attach( return rc; } -/* -** Ensure that there is room in the buffer to append nByte bytes of data. -** If not, use sqlite3_realloc() to grow the buffer so that there is. -** -** If successful, return zero. Otherwise, if an OOM condition is encountered, -** set *pRc to SQLITE_NOMEM and return non-zero. -*/ -static int sessionBufferGrow(SessionBuffer *p, i64 nByte, int *pRc){ -#define SESSION_MAX_BUFFER_SZ (0x7FFFFF00 - 1) - i64 nReq = p->nBuf + nByte; - if( *pRc==SQLITE_OK && nReq>p->nAlloc ){ - u8 *aNew; - i64 nNew = p->nAlloc ? p->nAlloc : 128; - - do { - nNew = nNew*2; - }while( nNewSESSION_MAX_BUFFER_SZ ){ - nNew = SESSION_MAX_BUFFER_SZ; - if( nNewaBuf, nNew); - if( 0==aNew ){ - *pRc = SQLITE_NOMEM; - }else{ - p->aBuf = aNew; - p->nAlloc = nNew; - } - } - return (*pRc!=SQLITE_OK); -} - /* ** Append the value passed as the second argument to the buffer passed ** as the first. @@ -219938,27 +223440,6 @@ static void sessionAppendBlob( } } -/* -** This function is a no-op if *pRc is other than SQLITE_OK when it is -** called. Otherwise, append a string to the buffer. All bytes in the string -** up to (but not including) the nul-terminator are written to the buffer. -** -** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before -** returning. -*/ -static void sessionAppendStr( - SessionBuffer *p, - const char *zStr, - int *pRc -){ - int nStr = sqlite3Strlen30(zStr); - if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ - memcpy(&p->aBuf[p->nBuf], zStr, nStr); - p->nBuf += nStr; - p->aBuf[p->nBuf] = 0x00; - } -} - /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string representation of integer iVal @@ -219977,27 +223458,6 @@ static void sessionAppendInteger( sessionAppendStr(p, aBuf, pRc); } -static void sessionAppendPrintf( - SessionBuffer *p, /* Buffer to append to */ - int *pRc, - const char *zFmt, - ... -){ - if( *pRc==SQLITE_OK ){ - char *zApp = 0; - va_list ap; - va_start(ap, zFmt); - zApp = sqlite3_vmprintf(zFmt, ap); - if( zApp==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - sessionAppendStr(p, zApp, pRc); - } - va_end(ap); - sqlite3_free(zApp); - } -} - /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string zStr enclosed in quotes (") and @@ -220488,26 +223948,16 @@ static int sessionGenerateChangeset( for(pTab=pSession->pTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ if( pTab->nEntry ){ const char *zName = pTab->zName; - int nCol = 0; /* Number of columns in table */ - u8 *abPK = 0; /* Primary key array */ - const char **azCol = 0; /* Table columns */ int i; /* Used to iterate through hash buckets */ sqlite3_stmt *pSel = 0; /* SELECT statement to query table pTab */ int nRewind = buf.nBuf; /* Initial size of write buffer */ int nNoop; /* Size of buffer after writing tbl header */ - int bRowid = 0; + int nOldCol = pTab->nCol; /* Check the table schema is still Ok. */ - rc = sessionTableInfo( - 0, db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK, - (pSession->bImplicitPK ? &bRowid : 0) - ); - if( rc==SQLITE_OK && ( - pTab->nCol!=nCol - || pTab->bRowid!=bRowid - || memcmp(abPK, pTab->abPK, nCol) - )){ - rc = SQLITE_SCHEMA; + rc = sessionReinitTable(pSession, pTab); + if( rc==SQLITE_OK && pTab->nCol!=nOldCol ){ + rc = sessionUpdateChanges(pSession, pTab); } /* Write a table header */ @@ -220515,8 +223965,8 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ - rc = sessionSelectStmt( - db, 0, pSession->zDb, zName, bRowid, nCol, azCol, abPK, &pSel + rc = sessionSelectStmt(db, 0, pSession->zDb, + zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel ); } @@ -220525,22 +223975,22 @@ static int sessionGenerateChangeset( SessionChange *p; /* Used to iterate through changes */ for(p=pTab->apChange[i]; rc==SQLITE_OK && p; p=p->pNext){ - rc = sessionSelectBind(pSel, nCol, abPK, p); + rc = sessionSelectBind(pSel, pTab->nCol, pTab->abPK, p); if( rc!=SQLITE_OK ) continue; if( sqlite3_step(pSel)==SQLITE_ROW ){ if( p->op==SQLITE_INSERT ){ int iCol; sessionAppendByte(&buf, SQLITE_INSERT, &rc); sessionAppendByte(&buf, p->bIndirect, &rc); - for(iCol=0; iColnCol; iCol++){ sessionAppendCol(&buf, pSel, iCol, &rc); } }else{ - assert( abPK!=0 ); /* Because sessionSelectStmt() returned ok */ - rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, abPK); + assert( pTab->abPK!=0 ); + rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, pTab->abPK); } }else if( p->op!=SQLITE_INSERT ){ - rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK); + rc = sessionAppendDelete(&buf, bPatchset, p, pTab->nCol,pTab->abPK); } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pSel); @@ -220565,7 +224015,6 @@ static int sessionGenerateChangeset( if( buf.nBuf==nNoop ){ buf.nBuf = nRewind; } - sqlite3_free((char*)azCol); /* cast works around VC++ bug */ } } @@ -220989,15 +224438,19 @@ static int sessionReadRecord( } } if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - sqlite3_int64 v = sessionGetI64(aVal); - if( eType==SQLITE_INTEGER ){ - sqlite3VdbeMemSetInt64(apOut[i], v); + if( (pIn->nData-pIn->iNext)<8 ){ + rc = SQLITE_CORRUPT_BKPT; }else{ - double d; - memcpy(&d, &v, 8); - sqlite3VdbeMemSetDouble(apOut[i], d); + sqlite3_int64 v = sessionGetI64(aVal); + if( eType==SQLITE_INTEGER ){ + sqlite3VdbeMemSetInt64(apOut[i], v); + }else{ + double d; + memcpy(&d, &v, 8); + sqlite3VdbeMemSetDouble(apOut[i], d); + } + pIn->iNext += 8; } - pIn->iNext += 8; } } } @@ -222690,7 +226143,7 @@ static int sessionChangesetApply( sqlite3changeset_pk(pIter, &abPK, 0); rc = sessionTableInfo(0, db, "main", zNew, - &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK, &sApply.bRowid + &sApply.nCol, &zTab, &sApply.azCol, 0, &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; for(i=0; iflags & SQLITE_FkNoAction; + + if( flags & SQLITE_CHANGESETAPPLY_FKNOACTION ){ + db->flags |= ((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } + if( rc==SQLITE_OK ){ rc = sessionChangesetApply( db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags ); } + + if( (flags & SQLITE_CHANGESETAPPLY_FKNOACTION) && savedFlag==0 ){ + assert( db->flags & SQLITE_FkNoAction ); + db->flags &= ~((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } return rc; } @@ -222914,6 +226380,9 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + + sqlite3 *db; /* Configured by changegroup_schema() */ + char *zDb; /* Configured by changegroup_schema() */ }; /* @@ -222934,6 +226403,7 @@ static int sessionChangeMerge( ){ SessionChange *pNew = 0; int rc = SQLITE_OK; + assert( aRec!=0 ); if( !pExist ){ pNew = (SessionChange *)sqlite3_malloc64(sizeof(SessionChange) + nRec); @@ -223099,6 +226569,114 @@ static int sessionChangeMerge( return rc; } +/* +** Check if a changeset entry with nCol columns and the PK array passed +** as the final argument to this function is compatible with SessionTable +** pTab. If so, return 1. Otherwise, if they are incompatible in some way, +** return 0. +*/ +static int sessionChangesetCheckCompat( + SessionTable *pTab, + int nCol, + u8 *abPK +){ + if( pTab->azCol && nColnCol ){ + int ii; + for(ii=0; iinCol; ii++){ + u8 bPK = (ii < nCol) ? abPK[ii] : 0; + if( pTab->abPK[ii]!=bPK ) return 0; + } + return 1; + } + return (pTab->nCol==nCol && 0==memcmp(abPK, pTab->abPK, nCol)); +} + +static int sessionChangesetExtendRecord( + sqlite3_changegroup *pGrp, + SessionTable *pTab, + int nCol, + int op, + const u8 *aRec, + int nRec, + SessionBuffer *pOut +){ + int rc = SQLITE_OK; + int ii = 0; + + assert( pTab->azCol ); + assert( nColnCol ); + + pOut->nBuf = 0; + if( op==SQLITE_INSERT || (op==SQLITE_DELETE && pGrp->bPatch==0) ){ + /* Append the missing default column values to the record. */ + sessionAppendBlob(pOut, aRec, nRec, &rc); + if( rc==SQLITE_OK && pTab->pDfltStmt==0 ){ + rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt); + } + for(ii=nCol; rc==SQLITE_OK && iinCol; ii++){ + int eType = sqlite3_column_type(pTab->pDfltStmt, ii); + sessionAppendByte(pOut, eType, &rc); + switch( eType ){ + case SQLITE_FLOAT: + case SQLITE_INTEGER: { + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + }else{ + double rVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + memcpy(&iVal, &rVal, sizeof(i64)); + } + if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ + sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + } + break; + } + + case SQLITE_BLOB: + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pTab->pDfltStmt, ii); + sessionAppendVarint(pOut, n, &rc); + if( eType==SQLITE_TEXT ){ + const u8 *z = (const u8*)sqlite3_column_text(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + }else{ + const u8 *z = (const u8*)sqlite3_column_blob(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + } + break; + } + + default: + assert( eType==SQLITE_NULL ); + break; + } + } + }else if( op==SQLITE_UPDATE ){ + /* Append missing "undefined" entries to the old.* record. And, if this + ** is an UPDATE, to the new.* record as well. */ + int iOff = 0; + if( pGrp->bPatch==0 ){ + for(ii=0; iinCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); + } + } + + sessionAppendBlob(pOut, &aRec[iOff], nRec-iOff, &rc); + for(ii=0; ii<(pTab->nCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); + } + }else{ + assert( op==SQLITE_DELETE && pGrp->bPatch ); + sessionAppendBlob(pOut, aRec, nRec, &rc); + } + + return rc; +} + /* ** Add all changes in the changeset traversed by the iterator passed as ** the first argument to the changegroup hash tables. @@ -223112,6 +226690,7 @@ static int sessionChangesetToHash( int nRec; int rc = SQLITE_OK; SessionTable *pTab = 0; + SessionBuffer rec = {0, 0, 0}; while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){ const char *zNew; @@ -223123,6 +226702,9 @@ static int sessionChangesetToHash( SessionChange *pExist = 0; SessionChange **pp; + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ if( pGrp->pList==0 ){ pGrp->bPatch = pIter->bPatchset; }else if( pIter->bPatchset!=pGrp->bPatch ){ @@ -223155,18 +226737,38 @@ static int sessionChangesetToHash( pTab->zName = (char*)&pTab->abPK[nCol]; memcpy(pTab->zName, zNew, nNew+1); + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + if( rc ){ + assert( pTab->azCol==0 ); + sqlite3_free(pTab); + break; + } + } + /* The new object must be linked on to the end of the list, not ** simply added to the start of it. This is to ensure that the ** tables within the output of sqlite3changegroup_output() are in ** the right order. */ for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); *ppTab = pTab; - }else if( pTab->nCol!=nCol || memcmp(pTab->abPK, abPK, nCol) ){ + } + + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ rc = SQLITE_SCHEMA; break; } } + if( nColnCol ){ + assert( pGrp->db ); + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, &rec); + if( rc ) break; + aRec = rec.aBuf; + nRec = rec.nBuf; + } + if( sessionGrowHash(0, pIter->bPatchset, pTab) ){ rc = SQLITE_NOMEM; break; @@ -223204,6 +226806,7 @@ static int sessionChangesetToHash( } } + sqlite3_free(rec.aBuf); if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -223290,6 +226893,31 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp){ return rc; } +/* +** Provide a database schema to the changegroup object. +*/ +SQLITE_API int sqlite3changegroup_schema( + sqlite3_changegroup *pGrp, + sqlite3 *db, + const char *zDb +){ + int rc = SQLITE_OK; + + if( pGrp->pList || pGrp->db ){ + /* Cannot add a schema after one or more calls to sqlite3changegroup_add(), + ** or after sqlite3changegroup_schema() has already been called. */ + rc = SQLITE_MISUSE; + }else{ + pGrp->zDb = sqlite3_mprintf("%s", zDb); + if( pGrp->zDb==0 ){ + rc = SQLITE_NOMEM; + }else{ + pGrp->db = db; + } + } + return rc; +} + /* ** Add the changeset currently stored in buffer pData, size nData bytes, ** to changeset-group p. @@ -223353,6 +226981,7 @@ SQLITE_API int sqlite3changegroup_output_strm( */ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ + sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); sqlite3_free(pGrp); } @@ -224060,7 +227689,7 @@ struct Fts5PhraseIter { ** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 2 */ void *(*xUserData)(Fts5Context*); @@ -224289,8 +227918,8 @@ struct Fts5ExtensionApi { ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; @@ -224338,7 +227967,7 @@ struct fts5_api { int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -224347,7 +227976,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -224355,7 +227984,7 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); @@ -224527,6 +228156,10 @@ typedef struct Fts5Config Fts5Config; ** attempt to merge together. A value of 1 sets the object to use the ** compile time default. Zero disables auto-merge altogether. ** +** bContentlessDelete: +** True if the contentless_delete option was present in the CREATE +** VIRTUAL TABLE statement. +** ** zContent: ** ** zContentRowid: @@ -224561,6 +228194,7 @@ struct Fts5Config { int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ + int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ @@ -224582,6 +228216,7 @@ struct Fts5Config { char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ int bSecureDelete; /* 'secure-delete' */ + int nDeleteMerge; /* 'deletemerge' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -224904,6 +228539,9 @@ static int sqlite3Fts5IndexReset(Fts5Index *p); static int sqlite3Fts5IndexLoadConfig(Fts5Index *p); +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -224988,6 +228626,11 @@ static int sqlite3Fts5HashWrite( */ static void sqlite3Fts5HashClear(Fts5Hash*); +/* +** Return true if the hash is empty, false otherwise. +*/ +static int sqlite3Fts5HashIsEmpty(Fts5Hash*); + static int sqlite3Fts5HashQuery( Fts5Hash*, /* Hash table to query */ int nPre, @@ -225009,6 +228652,7 @@ static void sqlite3Fts5HashScanEntry(Fts5Hash *, ); + /* ** End of interface to code in fts5_hash.c. **************************************************************************/ @@ -225252,7 +228896,8 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); #define FTS5_STAR 15 /* This file is automatically generated by Lemon from input grammar -** source file "fts5parse.y". */ +** source file "fts5parse.y". +*/ /* ** 2000-05-29 ** @@ -226842,15 +230487,19 @@ static int fts5CInstIterInit( */ typedef struct HighlightContext HighlightContext; struct HighlightContext { - CInstIter iter; /* Coalesced Instance Iterator */ - int iPos; /* Current token offset in zIn[] */ + /* Constant parameters to fts5HighlightCb() */ int iRangeStart; /* First token to include */ int iRangeEnd; /* If non-zero, last token to include */ const char *zOpen; /* Opening highlight */ const char *zClose; /* Closing highlight */ const char *zIn; /* Input text */ int nIn; /* Size of input text in bytes */ - int iOff; /* Current offset within zIn[] */ + + /* Variables modified by fts5HighlightCb() */ + CInstIter iter; /* Coalesced Instance Iterator */ + int iPos; /* Current token offset in zIn[] */ + int iOff; /* Have copied up to this offset in zIn[] */ + int bOpen; /* True if highlight is open */ char *zOut; /* Output value */ }; @@ -226883,8 +230532,8 @@ static int fts5HighlightCb( int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStartOff, /* Start offset of token */ - int iEndOff /* End offset of token */ + int iStartOff, /* Start byte offset of token */ + int iEndOff /* End byte offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; @@ -226900,30 +230549,47 @@ static int fts5HighlightCb( if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } - if( iPos==p->iter.iStart ){ + /* If the parenthesis is open, and this token is not part of the current + ** phrase, and the starting byte offset of this token is past the point + ** that has currently been copied into the output buffer, close the + ** parenthesis. */ + if( p->bOpen + && (iPos<=p->iter.iStart || p->iter.iStart<0) + && iStartOff>p->iOff + ){ + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; + } + + /* If this is the start of a new phrase, and the highlight is not open: + ** + ** * copy text from the input up to the start of the phrase, and + ** * open the highlight. + */ + if( iPos==p->iter.iStart && p->bOpen==0 ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); fts5HighlightAppend(&rc, p, p->zOpen, -1); p->iOff = iStartOff; + p->bOpen = 1; } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd>=0 && p->iter.iStartiRangeStart ){ + if( p->bOpen==0 ){ + assert( p->iRangeEnd>=0 ); fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->bOpen = 1; } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); - fts5HighlightAppend(&rc, p, p->zClose, -1); p->iOff = iEndOff; + if( rc==SQLITE_OK ){ rc = fts5CInstIterNext(&p->iter); } } - if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){ + if( iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; - if( iPos>=p->iter.iStart && iPositer.iEnd ){ - fts5HighlightAppend(&rc, p, p->zClose, -1); - } } return rc; @@ -226964,6 +230630,9 @@ static void fts5HighlightFunction( if( rc==SQLITE_OK ){ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); + } fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); if( rc==SQLITE_OK ){ @@ -227242,6 +230911,9 @@ static void fts5SnippetFunction( if( rc==SQLITE_OK ){ rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); + } if( ctx.iRangeEnd>=(nColSize-1) ){ fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); }else{ @@ -227880,6 +231552,8 @@ static void sqlite3Fts5TermsetFree(Fts5Termset *p){ #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) +#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */ + /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (64*1024) @@ -228210,6 +231884,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessDelete = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ if( pConfig->zContentRowid ){ *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); @@ -228454,6 +232138,28 @@ static int sqlite3Fts5ConfigParse( sqlite3_free(zTwo); } + /* We only allow contentless_delete=1 if the table is indeed contentless. */ + if( rc==SQLITE_OK + && pRet->bContentlessDelete + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 requires a contentless table" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_delete=1 if columnsize=0 is not present. + ** + ** This restriction may be removed at some point. + */ + if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 is incompatible with columnsize=0" + ); + rc = SQLITE_ERROR; + } + /* If a tokenizer= option was successfully parsed, the tokenizer has ** already been allocated. Otherwise, allocate an instance of the default ** tokenizer (unicode61) now. */ @@ -228748,6 +232454,18 @@ static int sqlite3Fts5ConfigSetValue( } } + else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){ + int nVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nVal = sqlite3_value_int(pVal); + }else{ + *pbBadkey = 1; + } + if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE; + if( nVal>100 ) nVal = 0; + pConfig->nDeleteMerge = nVal; + } + else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; @@ -228796,6 +232514,7 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ @@ -231319,7 +235038,7 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( return pRet; } -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ sqlite3_int64 nByte = 0; Fts5ExprTerm *p; @@ -231425,6 +235144,8 @@ static char *fts5ExprPrintTcl( if( zRet==0 ) return 0; } + }else if( pExpr->eType==0 ){ + zRet = sqlite3_mprintf("{}"); }else{ char const *zOp = 0; int i; @@ -231686,14 +235407,14 @@ static void fts5ExprFold( sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); } } -#endif /* ifdef SQLITE_TEST */ +#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called during initialization to register the fts5_expr() scalar ** UDF with the SQLite handle passed as the only argument. */ static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) struct Fts5ExprFunc { const char *z; void (*x)(sqlite3_context*,int,sqlite3_value**); @@ -232453,7 +236174,6 @@ static int fts5HashEntrySort( pList = fts5HashEntryMerge(pList, ap[i]); } - pHash->nEntry = 0; sqlite3_free(ap); *ppSorted = pList; return SQLITE_OK; @@ -232507,6 +236227,28 @@ static int sqlite3Fts5HashScanInit( return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); } +#ifdef SQLITE_DEBUG +static int fts5HashCount(Fts5Hash *pHash){ + int nEntry = 0; + int ii; + for(ii=0; iinSlot; ii++){ + Fts5HashEntry *p = 0; + for(p=pHash->aSlot[ii]; p; p=p->pHashNext){ + nEntry++; + } + } + return nEntry; +} +#endif + +/* +** Return true if the hash table is empty, false otherwise. +*/ +static int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){ + assert( pHash->nEntry==fts5HashCount(pHash) ); + return pHash->nEntry==0; +} + static void sqlite3Fts5HashScanNext(Fts5Hash *p){ assert( !sqlite3Fts5HashScanEof(p) ); p->pScan = p->pScan->pScanNext; @@ -232595,6 +236337,24 @@ static void sqlite3Fts5HashScanEntry( #define FTS5_MAX_LEVEL 64 +/* +** There are two versions of the format used for the structure record: +** +** 1. the legacy format, that may be read by all fts5 versions, and +** +** 2. the V2 format, which is used by contentless_delete=1 databases. +** +** Both begin with a 4-byte "configuration cookie" value. Then, a legacy +** format structure record contains a varint - the number of levels in +** the structure. Whereas a V2 structure record contains the constant +** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a +** varint has to be at least 16256 to begin with "0xFF". And the default +** maximum number of levels is 64. +** +** See below for more on structure record formats. +*/ +#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01" + /* ** Details: ** @@ -232602,7 +236362,7 @@ static void sqlite3Fts5HashScanEntry( ** ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ** -** , contains the following 5 types of records. See the comments surrounding +** , contains the following 6 types of records. See the comments surrounding ** the FTS5_*_ROWID macros below for a description of how %_data rowids are ** assigned to each fo them. ** @@ -232611,12 +236371,12 @@ static void sqlite3Fts5HashScanEntry( ** The set of segments that make up an index - the index structure - are ** recorded in a single record within the %_data table. The record consists ** of a single 32-bit configuration cookie value followed by a list of -** SQLite varints. If the FTS table features more than one index (because -** there are one or more prefix indexes), it is guaranteed that all share -** the same cookie value. +** SQLite varints. +** +** If the structure record is a V2 record, the configuration cookie is +** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. ** -** Immediately following the configuration cookie, the record begins with -** three varints: +** Next, the record continues with three varints: ** ** + number of levels, ** + total number of segments on all levels, @@ -232631,6 +236391,12 @@ static void sqlite3Fts5HashScanEntry( ** + first leaf page number (often 1, always greater than 0) ** + final leaf page number ** +** Then, for V2 structures only: +** +** + lower origin counter value, +** + upper origin counter value, +** + the number of tombstone hash pages. +** ** 2. The Averages Record: ** ** A single record within the %_data table. The data is a list of varints. @@ -232746,6 +236512,38 @@ static void sqlite3Fts5HashScanEntry( ** * A list of delta-encoded varints - the first rowid on each subsequent ** child page. ** +** 6. Tombstone Hash Page +** +** These records are only ever present in contentless_delete=1 tables. +** There are zero or more of these associated with each segment. They +** are used to store the tombstone rowids for rows contained in the +** associated segments. +** +** The set of nHashPg tombstone hash pages associated with a single +** segment together form a single hash table containing tombstone rowids. +** To find the page of the hash on which a key might be stored: +** +** iPg = (rowid % nHashPg) +** +** Then, within page iPg, which has nSlot slots: +** +** iSlot = (rowid / nHashPg) % nSlot +** +** Each tombstone hash page begins with an 8 byte header: +** +** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8. +** 1-byte: rowid-0-tombstone flag. This flag is only valid on the +** first tombstone hash page for each segment (iPg=0). If set, +** the hash table contains rowid 0. If clear, it does not. +** Rowid 0 is handled specially. +** 2-bytes: unused. +** 4-bytes: Big-endian integer containing number of entries on page. +** +** Following this are nSlot 4 or 8 byte slots (depending on the key-size +** in the first byte of the page header). The number of slots may be +** determined based on the size of the page record and the key-size: +** +** nSlot = (nByte - 8) / key-size */ /* @@ -232779,6 +236577,7 @@ static void sqlite3Fts5HashScanEntry( #define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) +#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid+(1<<16), 0, 0, ipg) #ifdef SQLITE_DEBUG static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } @@ -232814,6 +236613,12 @@ struct Fts5Data { /* ** One object per %_data table. +** +** nContentlessDelete: +** The number of contentless delete operations since the most recent +** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked +** so that extra auto-merge work can be done by fts5IndexFlush() to +** account for the delete operations. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ @@ -232828,6 +236633,8 @@ struct Fts5Index { int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ int bDelete; /* Current write is a delete */ + int nContentlessDelete; /* Number of contentless delete ops */ + int nPendingRow; /* Number of INSERT in hash table */ /* Error state. */ int rc; /* Current error code */ @@ -232862,11 +236669,23 @@ struct Fts5DoclistIter { ** The contents of the "structure" record for each index are represented ** using an Fts5Structure record in memory. Which uses instances of the ** other Fts5StructureXXX types as components. +** +** nOriginCntr: +** This value is set to non-zero for structure records created for +** contentlessdelete=1 tables only. In that case it represents the +** origin value to apply to the next top-level segment created. */ struct Fts5StructureSegment { int iSegid; /* Segment id */ int pgnoFirst; /* First leaf page number in segment */ int pgnoLast; /* Last leaf page number in segment */ + + /* contentlessdelete=1 tables only: */ + u64 iOrigin1; + u64 iOrigin2; + int nPgTombstone; /* Number of tombstone hash table pages */ + u64 nEntryTombstone; /* Number of tombstone entries that "count" */ + u64 nEntry; /* Number of rows in this segment */ }; struct Fts5StructureLevel { int nMerge; /* Number of segments in incr-merge */ @@ -232876,6 +236695,7 @@ struct Fts5StructureLevel { struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ + u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ @@ -232964,6 +236784,13 @@ struct Fts5CResult { ** ** iTermIdx: ** Index of current term on iTermLeafPgno. +** +** apTombstone/nTombstone: +** These are used for contentless_delete=1 tables only. When the cursor +** is first allocated, the apTombstone[] array is allocated so that it +** is large enough for all tombstones hash pages associated with the +** segment. The pages themselves are loaded lazily from the database as +** they are required. */ struct Fts5SegIter { Fts5StructureSegment *pSeg; /* Segment to iterate through */ @@ -232972,6 +236799,8 @@ struct Fts5SegIter { Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ i64 iLeafOffset; /* Byte offset within current leaf */ + Fts5Data **apTombstone; /* Array of tombstone pages */ + int nTombstone; /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); @@ -233101,6 +236930,60 @@ static u16 fts5GetU16(const u8 *aIn){ return ((u16)aIn[0] << 8) + aIn[1]; } +/* +** The only argument points to a buffer at least 8 bytes in size. This +** function interprets the first 8 bytes of the buffer as a 64-bit big-endian +** unsigned integer and returns the result. +*/ +static u64 fts5GetU64(u8 *a){ + return ((u64)a[0] << 56) + + ((u64)a[1] << 48) + + ((u64)a[2] << 40) + + ((u64)a[3] << 32) + + ((u64)a[4] << 24) + + ((u64)a[5] << 16) + + ((u64)a[6] << 8) + + ((u64)a[7] << 0); +} + +/* +** The only argument points to a buffer at least 4 bytes in size. This +** function interprets the first 4 bytes of the buffer as a 32-bit big-endian +** unsigned integer and returns the result. +*/ +static u32 fts5GetU32(const u8 *a){ + return ((u32)a[0] << 24) + + ((u32)a[1] << 16) + + ((u32)a[2] << 8) + + ((u32)a[3] << 0); +} + +/* +** Write iVal, formated as a 64-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU64(u8 *a, u64 iVal){ + a[0] = ((iVal >> 56) & 0xFF); + a[1] = ((iVal >> 48) & 0xFF); + a[2] = ((iVal >> 40) & 0xFF); + a[3] = ((iVal >> 32) & 0xFF); + a[4] = ((iVal >> 24) & 0xFF); + a[5] = ((iVal >> 16) & 0xFF); + a[6] = ((iVal >> 8) & 0xFF); + a[7] = ((iVal >> 0) & 0xFF); +} + +/* +** Write iVal, formated as a 32-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU32(u8 *a, u32 iVal){ + a[0] = ((iVal >> 24) & 0xFF); + a[1] = ((iVal >> 16) & 0xFF); + a[2] = ((iVal >> 8) & 0xFF); + a[3] = ((iVal >> 0) & 0xFF); +} + /* ** Allocate and return a buffer at least nByte bytes in size. ** @@ -233328,10 +237211,17 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ /* ** Remove all records associated with segment iSegid. */ -static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ +static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){ + int iSegid = pSeg->iSegid; i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; fts5DataDelete(p, iFirst, iLast); + + if( pSeg->nPgTombstone ){ + i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0); + i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1); + fts5DataDelete(p, iTomb1, iTomb2); + } if( p->pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( @@ -233442,11 +237332,19 @@ static int fts5StructureDecode( int nSegment = 0; sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ Fts5Structure *pRet = 0; /* Structure object to return */ + int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */ + u64 nOriginCntr = 0; /* Largest origin value seen so far */ /* Grab the cookie value */ if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); i = 4; + /* Check if this is a V2 structure record. Set bStructureV2 if it is. */ + if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){ + i += 4; + bStructureV2 = 1; + } + /* Read the total number of levels and segments from the start of the ** structure record. */ i += fts5GetVarint32(&pData[i], nLevel); @@ -233497,6 +237395,14 @@ static int fts5StructureDecode( i += fts5GetVarint32(&pData[i], pSeg->iSegid); i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); + if( bStructureV2 ){ + i += fts5GetVarint(&pData[i], &pSeg->iOrigin1); + i += fts5GetVarint(&pData[i], &pSeg->iOrigin2); + i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntry); + nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); + } if( pSeg->pgnoLastpgnoFirst ){ rc = FTS5_CORRUPT; break; @@ -233507,6 +237413,9 @@ static int fts5StructureDecode( } } if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; + if( bStructureV2 ){ + pRet->nOriginCntr = nOriginCntr+1; + } if( rc!=SQLITE_OK ){ fts5StructureRelease(pRet); @@ -233719,6 +237628,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ + int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9)); assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); memset(&buf, 0, sizeof(Fts5Buffer)); @@ -233727,9 +237637,12 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ iCookie = p->pConfig->iCookie; if( iCookie<0 ) iCookie = 0; - if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){ sqlite3Fts5Put32(buf.p, iCookie); buf.n = 4; + if( pStruct->nOriginCntr>0 ){ + fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4); + } fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); @@ -233743,9 +237656,17 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ assert( pLvl->nMerge<=pLvl->nSeg ); for(iSeg=0; iSegnSeg; iSeg++){ - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast); + if( pStruct->nOriginCntr>0 ){ + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry); + } } } @@ -234268,6 +238189,23 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ } } +/* +** Allocate a tombstone hash page array (pIter->apTombstone) for the +** iterator passed as the second argument. If an OOM error occurs, leave +** an error in the Fts5Index object. +*/ +static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ + const int nTomb = pIter->pSeg->nPgTombstone; + if( nTomb>0 ){ + Fts5Data **apTomb = 0; + apTomb = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data)*nTomb); + if( apTomb ){ + pIter->apTombstone = apTomb; + pIter->nTombstone = nTomb; + } + } +} + /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when @@ -234309,6 +238247,7 @@ static void fts5SegIterInit( pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); + fts5SegIterAllocTombstone(p, pIter); } } @@ -235010,6 +238949,7 @@ static void fts5SegIterSeekInit( } fts5SegIterSetNext(p, pIter); + fts5SegIterAllocTombstone(p, pIter); /* Either: ** @@ -235090,6 +239030,20 @@ static void fts5SegIterHashInit( fts5SegIterSetNext(p, pIter); } +/* +** Array ap[] contains n elements. Release each of these elements using +** fts5DataRelease(). Then free the array itself using sqlite3_free(). +*/ +static void fts5IndexFreeArray(Fts5Data **ap, int n){ + if( ap ){ + int ii; + for(ii=0; iiterm); fts5DataRelease(pIter->pLeaf); fts5DataRelease(pIter->pNextLeaf); + fts5IndexFreeArray(pIter->apTombstone, pIter->nTombstone); fts5DlidxIterFree(pIter->pDlidx); sqlite3_free(pIter->aRowidOffset); memset(pIter, 0, sizeof(Fts5SegIter)); @@ -235230,7 +239185,6 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ assert_nc( i2!=0 ); pRes->bTermEq = 1; if( p1->iRowid==p2->iRowid ){ - p1->bDel = p2->bDel; return i2; } res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; @@ -235434,6 +239388,84 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){ pIter->iSwitchRowid = pSeg->iRowid; } +/* +** The argument to this macro must be an Fts5Data structure containing a +** tombstone hash page. This macro returns the key-size of the hash-page. +*/ +#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8) + +#define TOMBSTONE_NSLOT(pPg) \ + ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1) + +/* +** Query a single tombstone hash table for rowid iRowid. Return true if +** it is found or false otherwise. The tombstone hash table is one of +** nHashTable tables. +*/ +static int fts5IndexTombstoneQuery( + Fts5Data *pHash, /* Hash table page to query */ + int nHashTable, /* Number of pages attached to segment */ + u64 iRowid /* Rowid to query hash for */ +){ + const int szKey = TOMBSTONE_KEYSIZE(pHash); + const int nSlot = TOMBSTONE_NSLOT(pHash); + int iSlot = (iRowid / nHashTable) % nSlot; + int nCollide = nSlot; + + if( iRowid==0 ){ + return pHash->p[1]; + }else if( szKey==4 ){ + u32 *aSlot = (u32*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + }else{ + u64 *aSlot = (u64*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + } + + return 0; +} + +/* +** Return true if the iterator passed as the only argument points +** to an segment entry for which there is a tombstone. Return false +** if there is no tombstone or if the iterator is already at EOF. +*/ +static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + + if( pSeg->pLeaf && pSeg->nTombstone ){ + /* Figure out which page the rowid might be present on. */ + int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone; + assert( iPg>=0 ); + + /* If tombstone hash page iPg has not yet been loaded from the + ** database, load it now. */ + if( pSeg->apTombstone[iPg]==0 ){ + pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex, + FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) + ); + if( pSeg->apTombstone[iPg]==0 ) return 0; + } + + return fts5IndexTombstoneQuery( + pSeg->apTombstone[iPg], + pSeg->nTombstone, + pSeg->iRowid + ); + } + + return 0; +} + /* ** Move the iterator to the next entry. ** @@ -235471,7 +239503,9 @@ static void fts5MultiIterNext( fts5AssertMultiIterSetup(p, pIter); assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); - if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + if( (pIter->bSkipEmpty==0 || pSeg->nPos) + && 0==fts5MultiIterIsDeleted(pIter) + ){ pIter->xSetOutputs(pIter, pSeg); return; } @@ -235503,7 +239537,9 @@ static void fts5MultiIterNext2( } fts5AssertMultiIterSetup(p, pIter); - }while( fts5MultiIterIsEmpty(p, pIter) ); + }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) + && (p->rc==SQLITE_OK) + ); } } @@ -235516,7 +239552,7 @@ static Fts5Iter *fts5MultiIterAlloc( int nSeg ){ Fts5Iter *pNew; - int nSlot; /* Power of two >= nSeg */ + i64 nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlotbSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ + if( (pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew)) + || fts5MultiIterIsDeleted(pNew) + ){ fts5MultiIterNext(p, pNew, 0, 0); }else if( pNew->base.bEof==0 ){ Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; @@ -236236,7 +240274,9 @@ static void fts5IndexDiscardData(Fts5Index *p){ if( p->pHash ){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; + p->nPendingRow = 0; } + p->nContentlessDelete = 0; } /* @@ -236873,6 +240913,12 @@ static void fts5IndexMergeLevel( /* Read input from all segments in the input level */ nInput = pLvl->nSeg; + + /* Set the range of origins that will go into the output segment. */ + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1; + pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2; + } } bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); @@ -236932,8 +240978,11 @@ static void fts5IndexMergeLevel( int i; /* Remove the redundant segments from the %_data table */ + assert( pSeg->nEntry==0 ); for(i=0; iaSeg[i].iSegid); + Fts5StructureSegment *pOld = &pLvl->aSeg[i]; + pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone); + fts5DataRemoveSegment(p, pOld); } /* Remove the redundant segments from the input level */ @@ -236959,6 +241008,43 @@ static void fts5IndexMergeLevel( if( pnRem ) *pnRem -= writer.nLeafWritten; } +/* +** If this is not a contentless_delete=1 table, or if the 'deletemerge' +** configuration option is set to 0, then this function always returns -1. +** Otherwise, it searches the structure object passed as the second argument +** for a level suitable for merging due to having a large number of +** tombstones in the tombstone hash. If one is found, its index is returned. +** Otherwise, if there is no suitable level, -1. +*/ +static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ + Fts5Config *pConfig = p->pConfig; + int iRet = -1; + if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){ + int ii; + int nBest = 0; + + for(ii=0; iinLevel; ii++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[ii]; + i64 nEntry = 0; + i64 nTomb = 0; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + nEntry += pLvl->aSeg[iSeg].nEntry; + nTomb += pLvl->aSeg[iSeg].nEntryTombstone; + } + assert_nc( nEntry>0 || pLvl->nSeg==0 ); + if( nEntry>0 ){ + int nPercent = (nTomb * 100) / nEntry; + if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){ + iRet = ii; + nBest = nPercent; + } + } + } + } + return iRet; +} + /* ** Do up to nPg pages of automerge work on the index. ** @@ -236978,14 +241064,15 @@ static int fts5IndexMerge( int iBestLvl = 0; /* Level offering the most input segments */ int nBest = 0; /* Number of input segments on best level */ - /* Set iBestLvl to the level to read input segments from. */ + /* Set iBestLvl to the level to read input segments from. Or to -1 if + ** there is no level suitable to merge segments from. */ assert( pStruct->nLevel>0 ); for(iLvl=0; iLvlnLevel; iLvl++){ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; if( pLvl->nMerge ){ if( pLvl->nMerge>nBest ){ iBestLvl = iLvl; - nBest = pLvl->nMerge; + nBest = nMin; } break; } @@ -236994,22 +241081,18 @@ static int fts5IndexMerge( iBestLvl = iLvl; } } - - /* If nBest is still 0, then the index must be empty. */ -#ifdef SQLITE_DEBUG - for(iLvl=0; nBest==0 && iLvlnLevel; iLvl++){ - assert( pStruct->aLevel[iLvl].nSeg==0 ); + if( nBestaLevel[iBestLvl].nMerge==0 ){ - break; - } + if( iBestLvl<0 ) break; bRet = 1; fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } + + if( nMin==1 ) nMin = 2; } *ppStruct = pStruct; return bRet; @@ -237175,7 +241258,7 @@ static void fts5SecureDeleteOverflow( pLeaf = 0; }else if( bDetailNone ){ break; - }else if( iNext>=pLeaf->szLeaf || iNext<4 ){ + }else if( iNext>=pLeaf->szLeaf || pLeaf->nnszLeaf || iNext<4 ){ p->rc = FTS5_CORRUPT; break; }else{ @@ -237194,9 +241277,13 @@ static void fts5SecureDeleteOverflow( int i1 = pLeaf->szLeaf; int i2 = 0; + i1 += fts5GetVarint32(&aPg[i1], iFirst); + if( iFirstrc = FTS5_CORRUPT; + break; + } aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); if( aIdx==0 ) break; - i1 += fts5GetVarint32(&aPg[i1], iFirst); i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); if( i1nn ){ memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); @@ -237241,7 +241328,6 @@ static void fts5DoSecureDelete( int iPgIdx = pSeg->pLeaf->szLeaf; u64 iDelta = 0; - u64 iNextDelta = 0; int iNextOff = 0; int iOff = 0; int nIdx = 0; @@ -237249,8 +241335,6 @@ static void fts5DoSecureDelete( int bLastInDoclist = 0; int iIdx = 0; int iStart = 0; - int iKeyOff = 0; - int iPrevKeyOff = 0; int iDelKeyOff = 0; /* Offset of deleted key, if any */ nIdx = nPg-iPgIdx; @@ -237275,10 +241359,21 @@ static void fts5DoSecureDelete( ** This block sets the following variables: ** ** iStart: + ** The offset of the first byte of the rowid or delta-rowid + ** value for the doclist entry being removed. + ** ** iDelta: + ** The value of the rowid or delta-rowid value for the doclist + ** entry being removed. + ** + ** iNextOff: + ** The offset of the next entry following the position list + ** for the one being removed. If the position list for this + ** entry overflows onto the next leaf page, this value will be + ** greater than pLeaf->szLeaf. */ { - int iSOP; + int iSOP; /* Start-Of-Position-list */ if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ iStart = pSeg->iTermLeafOffset; }else{ @@ -237314,47 +241409,75 @@ static void fts5DoSecureDelete( } iOff = iStart; - if( iNextOff>=iPgIdx ){ - int pgno = pSeg->iLeafPgno+1; - fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); - iNextOff = iPgIdx; - }else{ - /* Set bLastInDoclist to true if the entry being removed is the last - ** in its doclist. */ - for(iIdx=0, iKeyOff=0; iIdxbDel==0 ){ + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + }else{ + /* Loop through the page-footer. If iNextOff (offset of the + ** entry following the one we are removing) is equal to the + ** offset of a key on this page, then the entry is the last + ** in its doclist. */ + int iKeyOff = 0; + for(iIdx=0; iIdxbDel ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta); + aPg[iOff++] = 0x01; + }else if( bLastInDoclist==0 ){ if( iNextOff!=iPgIdx ){ + u64 iNextDelta = 0; iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta); iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta); } }else if( - iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + pSeg->iLeafPgno==pSeg->iTermLeafPgno + && iStart==pSeg->iTermLeafOffset ){ /* The entry being removed was the only position list in its ** doclist. Therefore the term needs to be removed as well. */ int iKey = 0; - for(iIdx=0, iKeyOff=0; iIdx(u32)iStart ) break; iKeyOff += iVal; } + assert_nc( iKey>=1 ); + /* Set iDelKeyOff to the value of the footer entry to remove from + ** the page. */ iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + /* This is the only position-list associated with the term, and there + ** is another term following it on this page. So the subsequent term + ** needs to be moved to replace the term associated with the entry + ** being removed. */ int nPrefix = 0; int nSuffix = 0; int nPrefix2 = 0; @@ -237379,7 +241502,9 @@ static void fts5DoSecureDelete( iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>nPrefix ){ + if( nPrefix2>pSeg->term.n ){ + p->rc = FTS5_CORRUPT; + }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); iOff += (nPrefix2-nPrefix); } @@ -237389,80 +241514,88 @@ static void fts5DoSecureDelete( } } }else if( iStart==4 ){ - int iPgno; - - assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); - /* The entry being removed may be the only position list in - ** its doclist. */ - for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ - Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); - int bEmpty = (pPg && pPg->nn==4); - fts5DataRelease(pPg); - if( bEmpty==0 ) break; - } - - if( iPgno==pSeg->iTermLeafPgno ){ - i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); - Fts5Data *pTerm = fts5DataRead(p, iId); - if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ - u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; - int nTermIdx = pTerm->nn - pTerm->szLeaf; - int iTermIdx = 0; - int iTermOff = 0; - - while( 1 ){ - u32 iVal = 0; - int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); - iTermOff += iVal; - if( (iTermIdx+nByte)>=nTermIdx ) break; - iTermIdx += nByte; - } - nTermIdx = iTermIdx; + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); - } + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); } - fts5DataRelease(pTerm); } + fts5DataRelease(pTerm); } + } - if( p->rc==SQLITE_OK ){ - const int nMove = nPg - iNextOff; - int nShift = 0; + /* Assuming no error has occurred, this block does final edits to the + ** leaf page before writing it back to disk. Input variables are: + ** + ** nPg: Total initial size of leaf page. + ** iPgIdx: Initial offset of page footer. + ** + ** iOff: Offset to move data to + ** iNextOff: Offset to move data from + */ + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; /* Number of bytes to move */ + int nShift = iNextOff - iOff; /* Distance to move them */ - memmove(&aPg[iOff], &aPg[iNextOff], nMove); - iPgIdx -= (iNextOff - iOff); - nPg = iPgIdx; - fts5PutU16(&aPg[2], iPgIdx); + int iPrevKeyOut = 0; + int iKeyIn = 0; - nShift = iNextOff - iOff; - for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ - iKeyOff -= nShift; - nShift = 0; - } - nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); - iPrevKeyOff = iKeyOff; - } - } + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= nShift; + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); - if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + for(iIdx=0; iIdxiOff ? nShift : 0)); + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut); + iPrevKeyOut = iKeyOut; } + } - assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); - fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); } - sqlite3_free(aIdx); + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg); + } + sqlite3_free(aIdx); } /* @@ -237516,184 +241649,198 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); - iSegid = fts5AllocateSegid(p, pStruct); fts5StructureInvalidate(p); - if( iSegid ){ - const int pgsz = p->pConfig->pgsz; - int eDetail = p->pConfig->eDetail; - int bSecureDelete = p->pConfig->bSecureDelete; - Fts5StructureSegment *pSeg; /* New segment within pStruct */ - Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ - Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ - - Fts5SegWriter writer; - fts5WriteInit(p, &writer, iSegid); - - pBuf = &writer.writer.buf; - pPgidx = &writer.writer.pgidx; - - /* fts5WriteInit() should have initialized the buffers to (most likely) - ** the maximum space required. */ - assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + if( sqlite3Fts5HashIsEmpty(pHash)==0 ){ + iSegid = fts5AllocateSegid(p, pStruct); + if( iSegid ){ + const int pgsz = p->pConfig->pgsz; + int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); + + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); + } + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ + + /* Get the term and doclist for this entry. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); + nTerm = (int)strlen(zTerm); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } - /* Begin scanning through hash table entries. This loop runs once for each - ** term/doclist currently stored within the hash table. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); - } - while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ - const char *zTerm; /* Buffer containing term */ - int nTerm; /* Size of zTerm in bytes */ - const u8 *pDoclist; /* Pointer to doclist for this term */ - int nDoclist; /* Size of doclist in bytes */ - - /* Get the term and doclist for this entry. */ - sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - nTerm = (int)strlen(zTerm); - if( bSecureDelete==0 ){ - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; - assert( writer.bFirstRowidInPage==0 ); - } - - if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ - /* The entire doclist will fit on the current leaf. */ - fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); - }else{ - int bTermWritten = !bSecureDelete; - i64 iRowid = 0; - i64 iPrev = 0; - int iOff = 0; - - /* The entire doclist will not fit on this leaf. The following - ** loop iterates through the poslists that make up the current - ** doclist. */ - while( p->rc==SQLITE_OK && iOff=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + int bTermWritten = !bSecureDelete; + i64 iRowid = 0; + i64 iPrev = 0; + int iOff = 0; + + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ iOff++; - nDoclist = 0; - }else{ continue; } } - }else if( (pDoclist[iOff] & 0x01) ){ - fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); - if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ - iOff++; - continue; - } } - } - if( p->rc==SQLITE_OK && bTermWritten==0 ){ - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); - bTermWritten = 1; - assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); - } + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } - if( writer.bFirstRowidInPage ){ - fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); - writer.bFirstRowidInPage = 0; - fts5WriteDlidxAppend(p, &writer, iRowid); - }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); - } - if( p->rc!=SQLITE_OK ) break; - assert( pBuf->n<=pBuf->nSpace ); - iPrev = iRowid; + if( writer.bFirstRowidInPage ){ + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + }else{ + u64 iRowidDelta = (u64)iRowid - (u64)iPrev; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta); + } + if( p->rc!=SQLITE_OK ) break; + assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; - if( eDetail==FTS5_DETAIL_NONE ){ - if( iOffp[pBuf->n++] = 0; - iOff++; + if( eDetail==FTS5_DETAIL_NONE ){ if( iOffp[pBuf->n++] = 0; iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); } - } - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - }else{ - int bDummy; - int nPos; - int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); - nCopy += nPos; - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); - } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); + int bDel = 0; + int nPos = 0; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel); + if( bDel && bSecureDelete ){ + fts5BufferAppendVarint(&p->rc, pBuf, nPos*2); + iOff += nCopy; + nCopy = nPos; + }else{ + nCopy += nPos; + } + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; } - if( iPos>=nCopy ) break; } + iOff += nCopy; } - iOff += nCopy; } } - } - /* TODO2: Doclist terminator written here. */ - /* pBuf->p[pBuf->n++] = '\0'; */ - assert( pBuf->n<=pBuf->nSpace ); - if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); - } - sqlite3Fts5HashClear(pHash); - fts5WriteFinish(p, &writer, &pgnoLast); - - assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); - if( pgnoLast>0 ){ - /* Update the Fts5Structure. It is written back to the database by the - ** fts5StructureRelease() call below. */ - if( pStruct->nLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; + fts5WriteFinish(p, &writer, &pgnoLast); + + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pStruct->nOriginCntr; + pSeg->iOrigin2 = pStruct->nOriginCntr; + pSeg->nEntry = p->nPendingRow; + pStruct->nOriginCntr++; + } + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); } - fts5StructurePromote(p, 0, pStruct); } } - fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); @@ -237704,10 +241851,15 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ - if( p->nPendingData ){ + if( p->nPendingData || p->nContentlessDelete ){ assert( p->pHash ); - p->nPendingData = 0; fts5FlushOneHash(p); + if( p->rc==SQLITE_OK ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + p->nPendingRow = 0; + p->nContentlessDelete = 0; + } } } @@ -237723,17 +241875,22 @@ static Fts5Structure *fts5IndexOptimizeStruct( /* Figure out if this structure requires optimization. A structure does ** not require optimization if either: ** - ** + it consists of fewer than two segments, or - ** + all segments are on the same level, or - ** + all segments except one are currently inputs to a merge operation. + ** 1. it consists of fewer than two segments, or + ** 2. all segments are on the same level, or + ** 3. all segments except one are currently inputs to a merge operation. ** - ** In the first case, return NULL. In the second, increment the ref-count - ** on *pStruct and return a copy of the pointer to it. + ** In the first case, if there are no tombstone hash pages, return NULL. In + ** the second, increment the ref-count on *pStruct and return a copy of the + ** pointer to it. */ - if( nSeg<2 ) return 0; + if( nSeg==0 ) return 0; for(i=0; inLevel; i++){ int nThis = pStruct->aLevel[i].nSeg; - if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + int nMerge = pStruct->aLevel[i].nMerge; + if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){ + if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){ + return 0; + } fts5StructureRef(pStruct); return pStruct; } @@ -237749,6 +241906,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; + pNew->nOriginCntr = pStruct->nOriginCntr; pLvl = &pNew->aLevel[pNew->nLevel-1]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ @@ -237779,6 +241937,7 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); + assert( p->nContentlessDelete==0 ); pStruct = fts5StructureRead(p); fts5StructureInvalidate(p); @@ -237808,7 +241967,10 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ ** INSERT command. */ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ - Fts5Structure *pStruct = fts5StructureRead(p); + Fts5Structure *pStruct = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); if( pStruct ){ int nMin = p->pConfig->nUsermerge; fts5StructureInvalidate(p); @@ -237816,7 +241978,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); fts5StructureRelease(pStruct); pStruct = pNew; - nMin = 2; + nMin = 1; nMerge = nMerge*-1; } if( pStruct && pStruct->nLevel ){ @@ -238330,6 +242492,9 @@ static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ p->iWriteRowid = iRowid; p->bDelete = bDelete; + if( bDelete==0 ){ + p->nPendingRow++; + } return fts5IndexReturn(p); } @@ -238367,6 +242532,9 @@ static int sqlite3Fts5IndexReinit(Fts5Index *p){ fts5StructureInvalidate(p); fts5IndexDiscardData(p); memset(&s, 0, sizeof(Fts5Structure)); + if( p->pConfig->bContentlessDelete ){ + s.nOriginCntr = 1; + } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); return fts5IndexReturn(p); @@ -238758,6 +242926,347 @@ static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ return fts5IndexReturn(p); } +/* +** Retrieve the origin value that will be used for the segment currently +** being accumulated in the in-memory hash table when it is flushed to +** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to +** the queried value. Or, if an error occurs, an error code is returned +** and the final value of (*piOrigin) is undefined. +*/ +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + *piOrigin = pStruct->nOriginCntr; + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +/* +** Buffer pPg contains a page of a tombstone hash table - one of nPg pages +** associated with the same segment. This function adds rowid iRowid to +** the hash table. The caller is required to guarantee that there is at +** least one free slot on the page. +** +** If parameter bForce is false and the hash table is deemed to be full +** (more than half of the slots are occupied), then non-zero is returned +** and iRowid not inserted. Or, if bForce is true or if the hash table page +** is not full, iRowid is inserted and zero returned. +*/ +static int fts5IndexTombstoneAddToPage( + Fts5Data *pPg, + int bForce, + int nPg, + u64 iRowid +){ + const int szKey = TOMBSTONE_KEYSIZE(pPg); + const int nSlot = TOMBSTONE_NSLOT(pPg); + const int nElem = fts5GetU32(&pPg->p[4]); + int iSlot = (iRowid / nPg) % nSlot; + int nCollide = nSlot; + + if( szKey==4 && iRowid>0xFFFFFFFF ) return 2; + if( iRowid==0 ){ + pPg->p[1] = 0x01; + return 0; + } + + if( bForce==0 && nElem>=(nSlot/2) ){ + return 1; + } + + fts5PutU32(&pPg->p[4], nElem+1); + if( szKey==4 ){ + u32 *aSlot = (u32*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid); + }else{ + u64 *aSlot = (u64*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU64((u8*)&aSlot[iSlot], iRowid); + } + + return 0; +} + +/* +** This function attempts to build a new hash containing all the keys +** currently in the tombstone hash table for segment pSeg. The new +** hash will be stored in the nOut buffers passed in array apOut[]. +** All pages of the new hash use key-size szKey (4 or 8). +** +** Return 0 if the hash is successfully rebuilt into the nOut pages. +** Or non-zero if it is not (because one page became overfull). In this +** case the caller should retry with a larger nOut parameter. +** +** Parameter pData1 is page iPg1 of the hash table being rebuilt. +*/ +static int fts5IndexTombstoneRehash( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int nOut, /* Number of output pages */ + Fts5Data **apOut /* Array of output hash pages */ +){ + int ii; + int res = 0; + + /* Initialize the headers of all the output pages */ + for(ii=0; iip[0] = szKey; + fts5PutU32(&apOut[ii]->p[4], 0); + } + + /* Loop through the current pages of the hash table. */ + for(ii=0; res==0 && iinPgTombstone; ii++){ + Fts5Data *pData = 0; /* Page ii of the current hash table */ + Fts5Data *pFree = 0; /* Free this at the end of the loop */ + + if( iPg1==ii ){ + pData = pData1; + }else{ + pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)); + } + + if( pData ){ + int szKeyIn = TOMBSTONE_KEYSIZE(pData); + int nSlotIn = (pData->nn - 8) / szKeyIn; + int iIn; + for(iIn=0; iInp[8]; + if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]); + }else{ + u64 *aSlot = (u64*)&pData->p[8]; + if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]); + } + + /* If iVal is not 0 at this point, insert it into the new hash table */ + if( iVal ){ + Fts5Data *pPg = apOut[(iVal % nOut)]; + res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal); + if( res ) break; + } + } + + /* If this is page 0 of the old hash, copy the rowid-0-flag from the + ** old hash to the new. */ + if( ii==0 ){ + apOut[0]->p[1] = pData->p[1]; + } + } + fts5DataRelease(pFree); + } + + return res; +} + +/* +** This is called to rebuild the hash table belonging to segment pSeg. +** If parameter pData1 is not NULL, then one page of the existing hash table +** has already been loaded - pData1, which is page iPg1. The key-size for +** the new hash table is szKey (4 or 8). +** +** If successful, the new hash table is not written to disk. Instead, +** output parameter (*pnOut) is set to the number of pages in the new +** hash table, and (*papOut) to point to an array of buffers containing +** the new page data. +** +** If an error occurs, an error code is left in the Fts5Index object and +** both output parameters set to 0 before returning. +*/ +static void fts5IndexTombstoneRebuild( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int *pnOut, /* OUT: Number of output pages */ + Fts5Data ***papOut /* OUT: Output hash pages */ +){ + const int MINSLOT = 32; + int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey); + int nSlot = 0; /* Number of slots in each output page */ + int nOut = 0; + + /* Figure out how many output pages (nOut) and how many slots per + ** page (nSlot). There are three possibilities: + ** + ** 1. The hash table does not yet exist. In this case the new hash + ** table will consist of a single page with MINSLOT slots. + ** + ** 2. The hash table exists but is currently a single page. In this + ** case an attempt is made to grow the page to accommodate the new + ** entry. The page is allowed to grow up to nSlotPerPage (see above) + ** slots. + ** + ** 3. The hash table already consists of more than one page, or of + ** a single page already so large that it cannot be grown. In this + ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage + ** slots each, where nPg is the current number of pages in the + ** hash table. + */ + if( pSeg->nPgTombstone==0 ){ + /* Case 1. */ + nOut = 1; + nSlot = MINSLOT; + }else if( pSeg->nPgTombstone==1 ){ + /* Case 2. */ + int nElem = (int)fts5GetU32(&pData1->p[4]); + assert( pData1 && iPg1==0 ); + nOut = 1; + nSlot = MAX(nElem*4, MINSLOT); + if( nSlot>nSlotPerPage ) nOut = 0; + } + if( nOut==0 ){ + /* Case 3. */ + nOut = (pSeg->nPgTombstone * 2 + 1); + nSlot = nSlotPerPage; + } + + /* Allocate the required array and output pages */ + while( 1 ){ + int res = 0; + int ii = 0; + int szPage = 0; + Fts5Data **apOut = 0; + + /* Allocate space for the new hash table */ + assert( nSlot>=MINSLOT ); + apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut); + szPage = 8 + nSlot*szKey; + for(ii=0; iirc, + sizeof(Fts5Data)+szPage + ); + if( pNew ){ + pNew->nn = szPage; + pNew->p = (u8*)&pNew[1]; + apOut[ii] = pNew; + } + } + + /* Rebuild the hash table. */ + if( p->rc==SQLITE_OK ){ + res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut); + } + if( res==0 ){ + if( p->rc ){ + fts5IndexFreeArray(apOut, nOut); + apOut = 0; + nOut = 0; + } + *pnOut = nOut; + *papOut = apOut; + break; + } + + /* If control flows to here, it was not possible to rebuild the hash + ** table. Free all buffers and then try again with more pages. */ + assert( p->rc==SQLITE_OK ); + fts5IndexFreeArray(apOut, nOut); + nSlot = nSlotPerPage; + nOut = nOut*2 + 1; + } +} + + +/* +** Add a tombstone for rowid iRowid to segment pSeg. +*/ +static void fts5IndexTombstoneAdd( + Fts5Index *p, + Fts5StructureSegment *pSeg, + u64 iRowid +){ + Fts5Data *pPg = 0; + int iPg = -1; + int szKey = 0; + int nHash = 0; + Fts5Data **apHash = 0; + + p->nContentlessDelete++; + + if( pSeg->nPgTombstone>0 ){ + iPg = iRowid % pSeg->nPgTombstone; + pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)); + if( pPg==0 ){ + assert( p->rc!=SQLITE_OK ); + return; + } + + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ + fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); + fts5DataRelease(pPg); + return; + } + } + + /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */ + szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4; + if( iRowid>0xFFFFFFFF ) szKey = 8; + + /* Rebuild the hash table */ + fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash); + assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) ); + + /* If all has succeeded, write the new rowid into one of the new hash + ** table pages, then write them all out to disk. */ + if( nHash ){ + int ii = 0; + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); + for(ii=0; iiiSegid, ii); + fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); + } + pSeg->nPgTombstone = nHash; + fts5StructureWrite(p, p->pStruct); + } + + fts5DataRelease(pPg); + fts5IndexFreeArray(apHash, nHash); +} + +/* +** Add iRowid to the tombstone list of the segment or segments that contain +** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite +** error code otherwise. +*/ +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + int bFound = 0; /* True after pSeg->nEntryTombstone incr. */ + int iLvl; + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + int iSeg; + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){ + if( bFound==0 ){ + pSeg->nEntryTombstone++; + bFound = 1; + } + fts5IndexTombstoneAdd(p, pSeg, iRowid); + } + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} /************************************************************************* ************************************************************************** @@ -239309,13 +243818,14 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum ** function only. */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Decode a segment-data rowid from the %_data table. This function is ** the opposite of macro FTS5_SEGMENT_ROWID(). */ static void fts5DecodeRowid( i64 iRowid, /* Rowid from %_data table */ + int *pbTombstone, /* OUT: Tombstone hash flag */ int *piSegid, /* OUT: Segment id */ int *pbDlidx, /* OUT: Dlidx flag */ int *piHeight, /* OUT: Height */ @@ -239331,13 +243841,16 @@ static void fts5DecodeRowid( iRowid >>= FTS5_DATA_DLI_B; *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); + iRowid >>= FTS5_DATA_ID_B; + + *pbTombstone = (int)(iRowid & 0x0001); } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ - fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ + fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ @@ -239347,14 +243860,16 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ } } else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", - bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", + bTomb ? "tombstone " : "", + iSegid, iHeight, iPgno ); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugStructure( int *pRc, /* IN/OUT: error code */ Fts5Buffer *pBuf, @@ -239369,16 +243884,22 @@ static void fts5DebugStructure( ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d", pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast ); + if( pSeg->iOrigin1>0 ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld", + pSeg->iOrigin1, pSeg->iOrigin2 + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -239403,9 +243924,9 @@ static void fts5DecodeStructure( fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -239428,9 +243949,9 @@ static void fts5DecodeAverages( zSpace = " "; } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Buffer (a/n) is assumed to contain a list of serialized varints. Read ** each varint and append its string representation to buffer pBuf. Return @@ -239447,9 +243968,9 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ } return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The start of buffer (a/n) contains the start of a doclist. The doclist ** may or may not finish within the buffer. This function appends a text @@ -239482,9 +244003,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This function is part of the fts5_decode() debugging function. It is ** only ever used with detail=none tables. @@ -239525,9 +244046,9 @@ static void fts5DecodeRowidList( sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The implementation of user-defined scalar function fts5_decode(). */ @@ -239538,6 +244059,7 @@ static void fts5DecodeFunction( ){ i64 iRowid; /* Rowid for record being decoded */ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ + int bTomb; const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ @@ -239560,7 +244082,7 @@ static void fts5DecodeFunction( if( a==0 ) goto decode_out; if( n>0 ) memcpy(a, aBlob, n); - fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); + fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); fts5DebugRowid(&rc, &s, iRowid); if( bDlidx ){ @@ -239579,6 +244101,28 @@ static void fts5DecodeFunction( " %d(%lld)", lvl.iLeafPgno, lvl.iRowid ); } + }else if( bTomb ){ + u32 nElem = fts5GetU32(&a[4]); + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; + int nSlot = (n - 8) / szKey; + int ii; + sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); + if( aBlob[1] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); + } + for(ii=0; iiszLeaf ){ + rc = FTS5_CORRUPT; + }else{ + fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff); + } iOff = iTermOff; if( iOffestimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + break; + } + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5StructVtab *p = (Fts5StructVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + int rc = SQLITE_OK; + Fts5StructVcsr *pNew = 0; + + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + *ppCsr = (sqlite3_vtab_cursor*)pNew; + + return SQLITE_OK; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + fts5StructureRelease(pCsr->pStruct); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int fts5structNextMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + + assert( pCsr->pStruct ); + pCsr->iSeg++; + pCsr->iRowid++; + while( pCsr->iLevelnLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){ + pCsr->iLevel++; + pCsr->iSeg = 0; + } + if( pCsr->iLevel>=p->nLevel ){ + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fts5structEofMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + return pCsr->pStruct==0; +} + +static int fts5structRowidMethod( + sqlite3_vtab_cursor *cur, + sqlite_int64 *piRowid +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + *piRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int fts5structColumnMethod( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg]; + + switch( i ){ + case 0: /* level */ + sqlite3_result_int(ctx, pCsr->iLevel); + break; + case 1: /* segment */ + sqlite3_result_int(ctx, pCsr->iSeg); + break; + case 2: /* merge */ + sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge); + break; + case 3: /* segid */ + sqlite3_result_int(ctx, pSeg->iSegid); + break; + case 4: /* leaf1 */ + sqlite3_result_int(ctx, pSeg->pgnoFirst); + break; + case 5: /* leaf2 */ + sqlite3_result_int(ctx, pSeg->pgnoLast); + break; + case 6: /* origin1 */ + sqlite3_result_int64(ctx, pSeg->iOrigin1); + break; + case 7: /* origin2 */ + sqlite3_result_int64(ctx, pSeg->iOrigin2); + break; + case 8: /* npgtombstone */ + sqlite3_result_int(ctx, pSeg->nPgTombstone); + break; + case 9: /* nentrytombstone */ + sqlite3_result_int64(ctx, pSeg->nEntryTombstone); + break; + case 10: /* nentry */ + sqlite3_result_int64(ctx, pSeg->nEntry); + break; + } + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int fts5structFilterMethod( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor; + int rc = SQLITE_OK; + + const u8 *aBlob = 0; + int nBlob = 0; + + assert( argc==1 ); + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + + nBlob = sqlite3_value_bytes(argv[0]); + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct); + if( rc==SQLITE_OK ){ + pCsr->iLevel = 0; + pCsr->iRowid = 0; + pCsr->iSeg = -1; + rc = fts5structNextMethod(pVtabCursor); + } + + return rc; +} + +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called as part of registering the FTS5 module with database @@ -239783,7 +244558,7 @@ static void fts5RowidFunction( ** SQLite error code is returned instead. */ static int sqlite3Fts5IndexInit(sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); @@ -239800,6 +244575,37 @@ static int sqlite3Fts5IndexInit(sqlite3 *db){ db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } + + if( rc==SQLITE_OK ){ + static const sqlite3_module fts5structure_module = { + 0, /* iVersion */ + 0, /* xCreate */ + fts5structConnectMethod, /* xConnect */ + fts5structBestIndexMethod, /* xBestIndex */ + fts5structDisconnectMethod, /* xDisconnect */ + 0, /* xDestroy */ + fts5structOpenMethod, /* xOpen */ + fts5structCloseMethod, /* xClose */ + fts5structFilterMethod, /* xFilter */ + fts5structNextMethod, /* xNext */ + fts5structEofMethod, /* xEof */ + fts5structColumnMethod, /* xColumn */ + fts5structRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0); + } return rc; #else return SQLITE_OK; @@ -239935,6 +244741,8 @@ struct Fts5FullTable { Fts5Storage *pStorage; /* Document store */ Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ + int iSavepoint; /* Successful xSavepoint()+1 */ + int bInSavepoint; #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif @@ -240223,6 +245031,13 @@ static int fts5InitVtab( pConfig->pzErrmsg = 0; } + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + if( rc!=SQLITE_OK ){ fts5FreeVtab(pTab); pTab = 0; @@ -241147,6 +245962,9 @@ static int fts5FilterMethod( pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); } + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) goto filter_out; + if( pTab->pSortCsr ){ /* If pSortCsr is non-NULL, then this call is being made as part of ** processing for a "... MATCH ORDER BY rank" query (ePlan is @@ -241169,6 +245987,7 @@ static int fts5FilterMethod( pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); }else if( pCsr->pExpr ){ + assert( rc==SQLITE_OK ); rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ if( bOrderByRank ){ @@ -241340,6 +246159,7 @@ static int fts5SpecialInsert( Fts5Config *pConfig = pTab->p.pConfig; int rc = SQLITE_OK; int bError = 0; + int bLoadConfig = 0; if( 0==sqlite3_stricmp("delete-all", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ @@ -241351,6 +246171,7 @@ static int fts5SpecialInsert( }else{ rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NONE ){ fts5SetVtabError(pTab, @@ -241360,6 +246181,7 @@ static int fts5SpecialInsert( }else{ rc = sqlite3Fts5StorageRebuild(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ rc = sqlite3Fts5StorageOptimize(pTab->pStorage); }else if( 0==sqlite3_stricmp("merge", zCmd) ){ @@ -241372,6 +246194,8 @@ static int fts5SpecialInsert( }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ pConfig->bPrefixIndex = sqlite3_value_int(pVal); #endif + }else if( 0==sqlite3_stricmp("flush", zCmd) ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); }else{ rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); if( rc==SQLITE_OK ){ @@ -241385,6 +246209,12 @@ static int fts5SpecialInsert( } } } + + if( rc==SQLITE_OK && bLoadConfig ){ + pTab->p.pConfig->iCookie--; + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } + return rc; } @@ -241443,7 +246273,6 @@ static int fts5UpdateMethod( int rc = SQLITE_OK; /* Return code */ int bUpdateOrDelete = 0; - /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); @@ -241472,7 +246301,14 @@ static int fts5UpdateMethod( if( pConfig->eContent!=FTS5_CONTENT_NORMAL && 0==sqlite3_stricmp("delete", z) ){ - rc = fts5SpecialDelete(pTab, apVal); + if( pConfig->bContentlessDelete ){ + fts5SetVtabError(pTab, + "'delete' may not be used with a contentless_delete=1 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = fts5SpecialDelete(pTab, apVal); + } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } @@ -241489,7 +246325,7 @@ static int fts5UpdateMethod( ** Cases 3 and 4 may violate the rowid constraint. */ int eConflict = SQLITE_ABORT; - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){ eConflict = sqlite3_vtab_on_conflict(pConfig->db); } @@ -241497,8 +246333,12 @@ static int fts5UpdateMethod( assert( nArg!=1 || eType0==SQLITE_INTEGER ); /* Filter out attempts to run UPDATE or DELETE on contentless tables. - ** This is not suported. */ - if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ + ** This is not suported. Except - they are both supported if the CREATE + ** VIRTUAL TABLE statement contained "contentless_delete=1". */ + if( eType0==SQLITE_INTEGER + && pConfig->eContent==FTS5_CONTENT_NONE + && pConfig->bContentlessDelete==0 + ){ pTab->p.base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName @@ -241522,7 +246362,8 @@ static int fts5UpdateMethod( } else if( eType0!=SQLITE_INTEGER ){ - /* If this is a REPLACE, first remove the current entry (if any) */ + /* An INSERT statement. If the conflict-mode is REPLACE, first remove + ** the current entry (if any). */ if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); @@ -241585,8 +246426,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_SYNC, 0); pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - fts5TripCursors(pTab); - rc = sqlite3Fts5StorageSync(pTab->pStorage); + rc = sqlite3Fts5FlushToDisk(&pTab->p); pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -242353,6 +247193,12 @@ static int fts5ColumnMethod( sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); } pConfig->pzErrmsg = 0; + }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){ + char *zErr = sqlite3_mprintf("cannot UPDATE a subset of " + "columns on fts5 contentless-delete table: %s", pConfig->zName + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); } return rc; } @@ -242391,8 +247237,12 @@ static int fts5RenameMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ const char *zName /* New name of table */ ){ + int rc; Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - return sqlite3Fts5StorageRename(pTab->pStorage, zName); + pTab->bInSavepoint = 1; + rc = sqlite3Fts5StorageRename(pTab->pStorage, zName); + pTab->bInSavepoint = 0; + return rc; } static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ @@ -242406,9 +247256,29 @@ static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ ** Flush the contents of the pending-terms table to disk. */ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint); - return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + char *zSql = 0; + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); + + if( pTab->bInSavepoint==0 ){ + zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", + pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName + ); + if( zSql ){ + pTab->bInSavepoint = 1; + rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0); + pTab->bInSavepoint = 0; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } + } + + return rc; } /* @@ -242417,9 +247287,16 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint); - return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); + if( (iSavepoint+1)iSavepoint ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint; + } + } + return rc; } /* @@ -242429,11 +247306,14 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ */ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + int rc = SQLITE_OK; fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); pTab->p.pConfig->pgsz = 0; - return sqlite3Fts5StorageRollback(pTab->pStorage); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + } + return rc; } /* @@ -242635,7 +247515,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301", -1, SQLITE_TRANSIENT); } /* @@ -242653,9 +247533,46 @@ static int fts5ShadowName(const char *zName){ return 0; } +/* +** Run an integrity check on the FTS5 data structures. Return a string +** if anything is found amiss. Return a NULL pointer if everything is +** OK. +*/ +static int fts5Integrity( + sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */ + const char *zSchema, /* Name of schema in which this table lives */ + const char *zTabname, /* Name of the table itself */ + int isQuick, /* True if this is a quick-check */ + char **pzErr /* Write error message here */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + Fts5Config *pConfig = pTab->p.pConfig; + char *zSql; + char *zErr = 0; + int rc; + assert( pzErr!=0 && *pzErr==0 ); + UNUSED_PARAM(isQuick); + zSql = sqlite3_mprintf( + "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');", + zSchema, zTabname, pConfig->zName); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr); + sqlite3_free(zSql); + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + }else if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, zErr); + } + sqlite3_free(zErr); + return SQLITE_OK; +} + static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { - /* iVersion */ 3, + /* iVersion */ 4, /* xCreate */ fts5CreateMethod, /* xConnect */ fts5ConnectMethod, /* xBestIndex */ fts5BestIndexMethod, @@ -242678,7 +247595,8 @@ static int fts5Init(sqlite3 *db){ /* xSavepoint */ fts5SavepointMethod, /* xRelease */ fts5ReleaseMethod, /* xRollbackTo */ fts5RollbackToMethod, - /* xShadowName */ fts5ShadowName + /* xShadowName */ fts5ShadowName, + /* xIntegrity */ fts5Integrity }; int rc; @@ -242848,10 +247766,10 @@ static int fts5StorageGetStmt( "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ - "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */ "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ - "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ "SELECT %s FROM %s AS T", /* SCAN */ @@ -242899,6 +247817,19 @@ static int fts5StorageGetStmt( break; } + case FTS5_STMT_REPLACE_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, + (pC->bContentlessDelete ? ",?" : "") + ); + break; + + case FTS5_STMT_LOOKUP_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], + (pC->bContentlessDelete ? ",origin" : ""), + pC->zDb, pC->zName + ); + break; + default: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); break; @@ -243088,9 +248019,11 @@ static int sqlite3Fts5StorageOpen( } if( rc==SQLITE_OK && pConfig->bColumnsize ){ - rc = sqlite3Fts5CreateTable( - pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr - ); + const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB"; + if( pConfig->bContentlessDelete ){ + zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER"; + } + rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( @@ -243167,7 +248100,7 @@ static int fts5StorageDeleteFromIndex( ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int rc2; /* sqlite3_reset() return code */ int iCol; Fts5InsertCtx ctx; @@ -243183,7 +248116,6 @@ static int fts5StorageDeleteFromIndex( ctx.pStorage = p; ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ const char *zText; @@ -243220,6 +248152,37 @@ static int fts5StorageDeleteFromIndex( return rc; } +/* +** This function is called to process a DELETE on a contentless_delete=1 +** table. It adds the tombstone required to delete the entry with rowid +** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, +** an SQLite error code. +*/ +static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ + i64 iOrigin = 0; + sqlite3_stmt *pLookup = 0; + int rc = SQLITE_OK; + + assert( p->pConfig->bContentlessDelete ); + assert( p->pConfig->eContent==FTS5_CONTENT_NONE ); + + /* Look up the origin of the document in the %_docsize table. Store + ** this in stack variable iOrigin. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLookup, 1, iDel); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + iOrigin = sqlite3_column_int64(pLookup, 1); + } + rc = sqlite3_reset(pLookup); + } + + if( rc==SQLITE_OK && iOrigin!=0 ){ + rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); + } + + return rc; +} /* ** Insert a record into the %_docsize table. Specifically, do: @@ -243240,10 +248203,17 @@ static int fts5StorageInsertDocsize( rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pReplace, 1, iRowid); - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); - sqlite3_bind_null(pReplace, 2); + if( p->pConfig->bContentlessDelete ){ + i64 iOrigin = 0; + rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); + sqlite3_bind_int64(pReplace, 3, iOrigin); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } } } return rc; @@ -243307,7 +248277,15 @@ static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **ap /* Delete the index records */ if( rc==SQLITE_OK ){ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + } + + if( rc==SQLITE_OK ){ + if( p->pConfig->bContentlessDelete ){ + rc = fts5StorageContentlessDelete(p, iDel); + }else{ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + } } /* Delete the %_docsize record */ @@ -243895,7 +248873,9 @@ static int sqlite3Fts5StorageSync(Fts5Storage *p){ i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); if( p->bTotalsValid ){ rc = fts5StorageSaveTotals(p); - p->bTotalsValid = 0; + if( rc==SQLITE_OK ){ + p->bTotalsValid = 0; + } } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexSync(p->pIndex); @@ -247263,7 +252243,8 @@ static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, - /* xShadowName */ 0 + /* xShadowName */ 0, + /* xIntegrity */ 0 }; void *p = (void*)pGlobal; @@ -247592,6 +252573,7 @@ static sqlite3_module stmtModule = { 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/core/src/sqlite/sqlite3.h b/core/src/sqlite/sqlite3.h index 48effe202..d4f1c810c 100644 --- a/core/src/sqlite/sqlite3.h +++ b/core/src/sqlite/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.42.0" -#define SQLITE_VERSION_NUMBER 3042000 -#define SQLITE_SOURCE_ID "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0" +#define SQLITE_VERSION "3.44.0" +#define SQLITE_VERSION_NUMBER 3044000 +#define SQLITE_SOURCE_ID "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -528,6 +528,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -1190,7 +1191,7 @@ struct sqlite3_io_methods { ** by clients within the current process, only within other processes. ** **
      4. [[SQLITE_FCNTL_CKSM_FILE]] -** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use interally by the +** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the ** [checksum VFS shim] only. ** **
      5. [[SQLITE_FCNTL_RESET_CACHE]] @@ -2126,7 +2127,7 @@ struct sqlite3_mem_methods { ** is stored in each sorted record and the required column values loaded ** from the database as records are returned in sorted order. The default ** value for this option is to never use this optimization. Specifying a -** negative value for this option restores the default behaviour. +** negative value for this option restores the default behavior. ** This option is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. ** @@ -2301,7 +2302,7 @@ struct sqlite3_mem_methods { ** database handle, SQLite checks if this will mean that there are now no ** connections at all to the database. If so, it performs a checkpoint ** operation before closing the connection. This option may be used to -** override this behaviour. The first parameter passed to this operation +** override this behavior. The first parameter passed to this operation ** is an integer - positive to disable checkpoints-on-close, or zero (the ** default) to enable them, and negative to leave the setting unchanged. ** The second parameter is a pointer to an integer @@ -2454,7 +2455,7 @@ struct sqlite3_mem_methods { ** the [VACUUM] command will fail with an obscure error when attempting to ** process a table with generated columns and a descending index. This is ** not considered a bug since SQLite versions 3.3.0 and earlier do not support -** either generated columns or decending indexes. +** either generated columns or descending indexes. **
      ** ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] @@ -2735,6 +2736,7 @@ SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); ** ** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether ** or not an interrupt is currently in effect for [database connection] D. +** It returns 1 if an interrupt is currently in effect, or 0 otherwise. */ SQLITE_API void sqlite3_interrupt(sqlite3*); SQLITE_API int sqlite3_is_interrupted(sqlite3*); @@ -3388,8 +3390,10 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, ** M argument should be the bitwise OR-ed combination of ** zero or more [SQLITE_TRACE] constants. ** -** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides -** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P) +** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or +** sqlite3_trace_v2(D,M,X,P) for the [database connection] D. Each +** database connection may have at most one trace callback. ** ** ^The X callback is invoked whenever any of the events identified by ** mask M occur. ^The integer return value from the callback is currently @@ -3758,7 +3762,7 @@ SQLITE_API int sqlite3_open_v2( ** as F) must be one of: **
        **
      • A database filename pointer created by the SQLite core and -** passed into the xOpen() method of a VFS implemention, or +** passed into the xOpen() method of a VFS implementation, or **
      • A filename obtained from [sqlite3_db_filename()], or **
      • A new filename constructed using [sqlite3_create_filename()]. **
      @@ -3871,7 +3875,7 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); /* ** CAPI3REF: Create and Destroy VFS Filenames ** -** These interfces are provided for use by [VFS shim] implementations and +** These interfaces are provided for use by [VFS shim] implementations and ** are not useful outside of that context. ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of @@ -3951,6 +3955,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language ** text that describes the error, as either UTF-8 or UTF-16 respectively. +** (See how SQLite handles [invalid UTF] for exceptions to this rule.) ** ^(Memory to hold the error message string is managed internally. ** The application does not need to worry about freeing the result. ** However, the error string might be overwritten or deallocated by @@ -4418,6 +4423,41 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN +** setting for [prepared statement] S. If E is zero, then S becomes +** a normal prepared statement. If E is 1, then S behaves as if +** its SQL text began with "[EXPLAIN]". If E is 2, then S behaves as if +** its SQL text began with "[EXPLAIN QUERY PLAN]". +** +** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared. +** SQLite tries to avoid a reprepare, but a reprepare might be necessary +** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode. +** +** Because of the potential need to reprepare, a call to +** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be +** reprepared because it was created using [sqlite3_prepare()] instead of +** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and +** hence has no saved SQL text with which to reprepare. +** +** Changing the explain setting for a prepared statement does not change +** the original SQL text for the statement. Hence, if the SQL text originally +** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0) +** is called to convert the statement into an ordinary statement, the EXPLAIN +** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S) +** output, even though the statement now acts like a normal SQL statement. +** +** This routine returns SQLITE_OK if the explain mode is successfully +** changed, or an error code if the explain mode could not be changed. +** The explain mode cannot be changed while a statement is active. +** Hence, it is good practice to call [sqlite3_reset(S)] +** immediately prior to calling sqlite3_stmt_explain(S,E). +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt @@ -4581,7 +4621,7 @@ typedef struct sqlite3_context sqlite3_context; ** with it may be passed. ^It is called to dispose of the BLOB or string even ** if the call to the bind API fails, except the destructor is not called if ** the third parameter is a NULL pointer or the fourth parameter is negative. -** ^ (2) The special constant, [SQLITE_STATIC], may be passsed to indicate that +** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that ** the application remains responsible for disposing of the object. ^In this ** case, the object and the provided pointer to it must remain valid until ** either the prepared statement is finalized or the same SQL parameter is @@ -5260,20 +5300,33 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S ** back to the beginning of its program. ** -** ^If the most recent call to [sqlite3_step(S)] for the -** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], -** or if [sqlite3_step(S)] has never before been called on S, -** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** ^The return code from [sqlite3_reset(S)] indicates whether or not +** the previous evaluation of prepared statement S completed successfully. +** ^If [sqlite3_step(S)] has never before been called on S or if +** [sqlite3_step(S)] has not been called since the previous call +** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return +** [SQLITE_OK]. ** ** ^If the most recent call to [sqlite3_step(S)] for the ** [prepared statement] S indicated an error, then ** [sqlite3_reset(S)] returns an appropriate [error code]. +** ^The [sqlite3_reset(S)] interface might also return an [error code] +** if there were no prior errors but the process of resetting +** the prepared statement caused a new error. ^For example, if an +** [INSERT] statement with a [RETURNING] clause is only stepped one time, +** that one call to [sqlite3_step(S)] might return SQLITE_ROW but +** the overall statement might still fail and the [sqlite3_reset(S)] call +** might return SQLITE_BUSY if locking constraints prevent the +** database change from committing. Therefore, it is important that +** applications check the return code from [sqlite3_reset(S)] even if +** no prior call to [sqlite3_step(S)] indicated a problem. ** ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} @@ -5484,7 +5537,7 @@ SQLITE_API int sqlite3_create_window_function( ** [application-defined SQL function] ** that has side-effects or that could potentially leak sensitive information. ** This will prevent attacks in which an application is tricked -** into using a database file that has had its schema surreptiously +** into using a database file that has had its schema surreptitiously ** modified to invoke the application-defined function in ways that are ** harmful. **

      @@ -5828,32 +5881,32 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to -** associate metadata with argument values. If the same value is passed to -** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated metadata may be preserved. An example -** of where this might be useful is in a regular-expression matching -** function. The compiled version of the regular expression can be stored as -** metadata associated with the pattern string. +** associate auxiliary data with argument values. If the same argument +** value is passed to multiple invocations of the same SQL function during +** query execution, under some circumstances the associated auxiliary data +** might be preserved. An example of where this might be useful is in a +** regular-expression matching function. The compiled version of the regular +** expression can be stored as auxiliary data associated with the pattern string. ** Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data ** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument ** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no metadata +** function argument. ^If there is no auxiliary data ** associated with the function argument, the sqlite3_get_auxdata(C,N) interface ** returns a NULL pointer. ** -** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th -** argument of the application-defined function. ^Subsequent +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the +** N-th argument of the application-defined function. ^Subsequent ** calls to sqlite3_get_auxdata(C,N) return P from the most recent -** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or -** NULL if the metadata has been discarded. +** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or +** NULL if the auxiliary data has been discarded. ** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, ** SQLite will invoke the destructor function X with parameter P exactly -** once, when the metadata is discarded. -** SQLite is free to discard the metadata at any time, including:

        +** once, when the auxiliary data is discarded. +** SQLite is free to discard the auxiliary data at any time, including:
          **
        • ^(when the corresponding function parameter changes)^, or **
        • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the ** SQL statement)^, or @@ -5869,7 +5922,7 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** function implementation should not make any use of P after ** sqlite3_set_auxdata() has been called. ** -** ^(In practice, metadata is preserved between function calls for +** ^(In practice, auxiliary data is preserved between function calls for ** function parameters that are compile-time constants, including literal ** values and [parameters] and expressions composed from the same.)^ ** @@ -5879,10 +5932,67 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** ** These routines must be called from the same thread in which ** the SQL function is running. +** +** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()]. */ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +/* +** CAPI3REF: Database Connection Client Data +** METHOD: sqlite3 +** +** These functions are used to associate one or more named pointers +** with a [database connection]. +** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P +** to be attached to [database connection] D using name N. Subsequent +** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P +** or a NULL pointer if there were no prior calls to +** sqlite3_set_clientdata() with the same values of D and N. +** Names are compared using strcmp() and are thus case sensitive. +** +** If P and X are both non-NULL, then the destructor X is invoked with +** argument P on the first of the following occurrences: +**
            +**
          • An out-of-memory error occurs during the call to +** sqlite3_set_clientdata() which attempts to register pointer P. +**
          • A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made +** with the same D and N parameters. +**
          • The database connection closes. SQLite does not make any guarantees +** about the order in which destructors are called, only that all +** destructors will be called exactly once at some point during the +** database connection closing process. +**
          +** +** SQLite does not do anything with client data other than invoke +** destructors on the client data at the appropriate time. The intended +** use for client data is to provide a mechanism for wrapper libraries +** to store additional information about an SQLite database connection. +** +** There is no limit (other than available memory) on the number of different +** client data pointers (with different names) that can be attached to a +** single database connection. However, the implementation is optimized +** for the case of having only one or two different client data names. +** Applications and wrapper libraries are discouraged from using more than +** one client data name each. +** +** There is no way to enumerate the client data pointers +** associated with a database connection. The N parameter can be thought +** of as a secret key such that only code that knows the secret key is able +** to access the associated data. +** +** Security Warning: These interfaces should not be exposed in scripting +** languages or in other circumstances where it might be possible for an +** an attacker to invoke them. Any agent that can invoke these interfaces +** can probably also take control of the process. +** +** Database connection client data is only available for SQLite +** version 3.44.0 ([dateof:3.44.0]) and later. +** +** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()]. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3*,const char*); +SQLITE_API int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*)); /* ** CAPI3REF: Constants Defining Special Destructor Behavior @@ -6515,7 +6625,7 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); /* -** CAPI3REF: Allowed return values from [sqlite3_txn_state()] +** CAPI3REF: Allowed return values from sqlite3_txn_state() ** KEYWORDS: {transaction state} ** ** These constants define the current transaction state of a database file. @@ -6647,7 +6757,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^Each call to the sqlite3_autovacuum_pages() interface overrides all ** previous invocations for that database connection. ^If the callback ** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer, -** then the autovacuum steps callback is cancelled. The return value +** then the autovacuum steps callback is canceled. The return value ** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might ** be some other error code if something goes wrong. The current ** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other @@ -7166,6 +7276,10 @@ struct sqlite3_module { /* The methods above are in versions 1 and 2 of the sqlite_module object. ** Those below are for version 3 and greater. */ int (*xShadowName)(const char*); + /* The methods above are in versions 1 through 3 of the sqlite_module object. + ** Those below are for version 4 and greater. */ + int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema, + const char *zTabName, int mFlags, char **pzErr); }; /* @@ -7653,7 +7767,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); ** code is returned and the transaction rolled back. ** ** Calling this function with an argument that is not a NULL pointer or an -** open blob handle results in undefined behaviour. ^Calling this routine +** open blob handle results in undefined behavior. ^Calling this routine ** with a null pointer (such as would be returned by a failed call to ** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function ** is passed a valid open blob handle, the values returned by the @@ -8133,6 +8247,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 #define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_FK_NO_ACTION 7 #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 @@ -8161,7 +8276,8 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TRACEFLAGS 31 #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 -#define SQLITE_TESTCTRL_LAST 33 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 +#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -9617,7 +9733,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_DIRECTONLY]]
          SQLITE_VTAB_DIRECTONLY
          **
          Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** prohibits that virtual table from being used from within triggers and ** views. **
          @@ -9807,7 +9923,7 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*); ** communicated to the xBestIndex method as a ** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^ If xBestIndex wants to use ** this constraint, it must set the corresponding -** aConstraintUsage[].argvIndex to a postive integer. ^(Then, under +** aConstraintUsage[].argvIndex to a positive integer. ^(Then, under ** the usual mode of handling IN operators, SQLite generates [bytecode] ** that invokes the [xFilter|xFilter() method] once for each value ** on the right-hand side of the IN operator.)^ Thus the virtual table @@ -10236,7 +10352,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** When the [sqlite3_blob_write()] API is used to update a blob column, ** the pre-update hook is invoked with SQLITE_DELETE. This is because the ** in this case the new values are not available. In this case, when a -** callback made with op==SQLITE_DELETE is actuall a write using the +** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the ** pre-update hook is being invoked for some other reason, including a @@ -10497,6 +10613,13 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const c ** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy ** of the database exists. ** +** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set, +** the returned buffer content will remain accessible and unchanged +** until either the next write operation on the connection or when +** the connection is closed, and applications must not modify the +** buffer. If the bit had been clear, the returned buffer will not +** be accessed by SQLite after the call. +** ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory ** allocation error occurs. @@ -10545,6 +10668,9 @@ SQLITE_API unsigned char *sqlite3_serialize( ** SQLite will try to increase the buffer size using sqlite3_realloc64() ** if writes on the database cause it to grow larger than M bytes. ** +** Applications must not modify the buffer P or invalidate it before +** the database connection D is closed. +** ** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the ** database is currently in a read transaction or is involved in a backup ** operation. @@ -10553,6 +10679,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** +** The deserialized database should not be in [WAL mode]. If the database +** is in WAL mode, then any attempt to use the database file will result +** in an [SQLITE_CANTOPEN] error. The application can set the +** [file format version numbers] (bytes 18 and 19) of the input database P +** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the +** database file into rollback mode and work around this limitation. +** ** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the ** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then ** [sqlite3_free()] is invoked on argument P prior to returning. @@ -11625,6 +11758,18 @@ SQLITE_API int sqlite3changeset_concat( ); +/* +** CAPI3REF: Upgrade the Schema of a Changeset/Patchset +*/ +SQLITE_API int sqlite3changeset_upgrade( + sqlite3 *db, + const char *zDb, + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + + + /* ** CAPI3REF: Changegroup Handle ** @@ -11671,6 +11816,38 @@ typedef struct sqlite3_changegroup sqlite3_changegroup; */ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); +/* +** CAPI3REF: Add a Schema to a Changegroup +** METHOD: sqlite3_changegroup_schema +** +** This method may be used to optionally enforce the rule that the changesets +** added to the changegroup handle must match the schema of database zDb +** ("main", "temp", or the name of an attached database). If +** sqlite3changegroup_add() is called to add a changeset that is not compatible +** with the configured schema, SQLITE_SCHEMA is returned and the changegroup +** object is left in an undefined state. +** +** A changeset schema is considered compatible with the database schema in +** the same way as for sqlite3changeset_apply(). Specifically, for each +** table in the changeset, there exists a database table with: +** +**
            +**
          • The name identified by the changeset, and +**
          • at least as many columns as recorded in the changeset, and +**
          • the primary key columns in the same position as recorded in +** the changeset. +**
          +** +** The output of the changegroup object always has the same schema as the +** database nominated using this function. In cases where changesets passed +** to sqlite3changegroup_add() have fewer columns than the corresponding table +** in the database schema, these are filled in using the default column +** values from the database schema. This makes it possible to combined +** changesets that have different numbers of columns for a single table +** within a changegroup, provided that they are otherwise compatible. +*/ +SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb); + /* ** CAPI3REF: Add A Changeset To A Changegroup ** METHOD: sqlite3_changegroup @@ -11739,13 +11916,18 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); ** If the new changeset contains changes to a table that is already present ** in the changegroup, then the number of columns and the position of the ** primary key columns for the table must be consistent. If this is not the -** case, this function fails with SQLITE_SCHEMA. If the input changeset -** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is -** returned. Or, if an out-of-memory condition occurs during processing, this -** function returns SQLITE_NOMEM. In all cases, if an error occurs the state -** of the final contents of the changegroup is undefined. +** case, this function fails with SQLITE_SCHEMA. Except, if the changegroup +** object has been configured with a database schema using the +** sqlite3changegroup_schema() API, then it is possible to combine changesets +** with different numbers of columns for a single table, provided that +** they are otherwise compatible. ** -** If no error occurs, SQLITE_OK is returned. +** If the input changeset appears to be corrupt and the corruption is +** detected, SQLITE_CORRUPT is returned. Or, if an out-of-memory condition +** occurs during processing, this function returns SQLITE_NOMEM. +** +** In all cases, if an error occurs the state of the final contents of the +** changegroup is undefined. If no error occurs, SQLITE_OK is returned. */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); @@ -12010,10 +12192,17 @@ SQLITE_API int sqlite3changeset_apply_v2( **
        • an insert change if all fields of the conflicting row match ** the row being inserted. **
        +** +**
        SQLITE_CHANGESETAPPLY_FKNOACTION
        +** If this flag it set, then all foreign key constraints in the target +** database behave as if they were declared with "ON UPDATE NO ACTION ON +** DELETE NO ACTION", even if they are actually CASCADE, RESTRICT, SET NULL +** or SET DEFAULT. */ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 #define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 +#define SQLITE_CHANGESETAPPLY_FKNOACTION 0x0008 /* ** CAPI3REF: Constants Passed To The Conflict Handler @@ -12754,7 +12943,7 @@ struct Fts5PhraseIter { ** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 2 */ void *(*xUserData)(Fts5Context*); @@ -12983,8 +13172,8 @@ struct Fts5ExtensionApi { ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; @@ -13032,7 +13221,7 @@ struct fts5_api { int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -13041,7 +13230,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -13049,7 +13238,7 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); diff --git a/core/src/sqlite/sqlite3ext.h b/core/src/sqlite/sqlite3ext.h index 19e030028..ae0949baf 100644 --- a/core/src/sqlite/sqlite3ext.h +++ b/core/src/sqlite/sqlite3ext.h @@ -361,6 +361,11 @@ struct sqlite3_api_routines { int (*value_encoding)(sqlite3_value*); /* Version 3.41.0 and later */ int (*is_interrupted)(sqlite3*); + /* Version 3.43.0 and later */ + int (*stmt_explain)(sqlite3_stmt*,int); + /* Version 3.44.0 and later */ + void *(*get_clientdata)(sqlite3*,const char*); + int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); }; /* @@ -689,6 +694,11 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_value_encoding sqlite3_api->value_encoding /* Version 3.41.0 and later */ #define sqlite3_is_interrupted sqlite3_api->is_interrupted +/* Version 3.43.0 and later */ +#define sqlite3_stmt_explain sqlite3_api->stmt_explain +/* Version 3.44.0 and later */ +#define sqlite3_get_clientdata sqlite3_api->get_clientdata +#define sqlite3_set_clientdata sqlite3_api->set_clientdata #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/py/correctness/src/crsql_correctness/__init__.py b/py/correctness/src/crsql_correctness/__init__.py index 64a101249..9b8682ac6 100644 --- a/py/correctness/src/crsql_correctness/__init__.py +++ b/py/correctness/src/crsql_correctness/__init__.py @@ -7,6 +7,7 @@ def connect(db_file, uri=False): c = sqlite3.connect(db_file, uri=uri) c.enable_load_extension(True) c.load_extension(extension) + c.execute("SELECT crsql_set_ts('1700000000')") return c @@ -22,6 +23,7 @@ def sync_left_to_right(l, r, since): print("sync_left_to_right") changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) + r.execute("SELECT crsql_set_ts('1700000000')") for change in changes: r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) diff --git a/py/correctness/tests/test_as_ordered.py b/py/correctness/tests/test_as_ordered.py index 4bd6a5527..cbe79286c 100644 --- a/py/correctness/tests/test_as_ordered.py +++ b/py/correctness/tests/test_as_ordered.py @@ -10,6 +10,7 @@ def make_simple_schema(count=-1): # O... so the type must be `ANY` for the orderable column for our triggers and the like to work... # we could cast the whatever to an int... c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, spot TEXT, list)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.execute("SELECT crsql_fract_as_ordered('foo', 'spot', 'list')") c.commit() diff --git a/py/correctness/tests/test_cl_merging.py b/py/correctness/tests/test_cl_merging.py index 0aebe8dba..6c38936cd 100644 --- a/py/correctness/tests/test_cl_merging.py +++ b/py/correctness/tests/test_cl_merging.py @@ -27,6 +27,7 @@ def make_simple_schema(count=-1): else: c = connect("test" + str(count) + ".db") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() return c @@ -35,6 +36,7 @@ def make_simple_schema(count=-1): def make_pko_schema(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() return c @@ -44,6 +46,7 @@ def sync_left_to_right(l, r, since): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -53,6 +56,7 @@ def sync_left_to_right_exact_version(l, r, db_version): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version = ?", (db_version,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -62,6 +66,7 @@ def sync_left_to_right_include_siteid(l, r, since): changes = l.execute( "SELECT [table], pk, cid, val, col_version, db_version, site_id, cl, seq, ts FROM crsql_changes WHERE db_version > ?", (since,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -78,6 +83,7 @@ def sync_left_to_right_normal_delta_state(l, r, since): largest_version = 0 for change in changes: max(largest_version, change[5]) + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -90,6 +96,7 @@ def sync_left_to_right_single_vrsn(l, r, vrsn): "SELECT [table], pk, cid, val, col_version, db_version, site_id, cl, seq, ts FROM crsql_changes WHERE db_version = ? AND site_id IS NOT ?", (vrsn, r_siteid)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -396,6 +403,7 @@ def test_resurrection_of_live_thing_via_sentinel(): # only merge over the sentinal colun to c2 sentinel_resurrect = c1.execute( "SELECT * FROM crsql_changes WHERE cid = '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_resurrect) c2.commit() @@ -411,6 +419,7 @@ def test_resurrection_of_live_thing_via_sentinel(): # now lets finish getting changes from the other node changes = c1.execute( "SELECT * FROM crsql_changes WHERE cid != '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", changes) c2.commit() @@ -456,6 +465,7 @@ def test_resurrection_of_live_thing_via_sentinel_multiple(): c3_changes = c3.execute( "SELECT * FROM crsql_changes").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", c3_changes) c2.commit() @@ -469,10 +479,12 @@ def test_resurrection_of_live_thing_via_sentinel_multiple(): # only merge over the sentinal colun to c2 sentinel_resurrect = c1.execute( "SELECT * FROM crsql_changes WHERE cid = '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_resurrect) c2.commit() + c3.execute("SELECT crsql_set_ts('1700000000')") c3.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_resurrect) c3.commit() @@ -489,10 +501,12 @@ def test_resurrection_of_live_thing_via_sentinel_multiple(): # now lets finish getting changes from the other node changes = c1.execute( "SELECT * FROM crsql_changes WHERE cid != '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", changes) c2.commit() + c3.execute("SELECT crsql_set_ts('1700000000')") c3.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", changes) c3.commit() @@ -530,10 +544,12 @@ def test_resurrection_of_live_thing_via_sentinel_out_of_order(): # only merge over the sentinal colun to c2 sentinel_resurrect = c1.execute( "SELECT * FROM crsql_changes WHERE cid = '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_resurrect) c2.commit() + c3.execute("SELECT crsql_set_ts('1700000000')") c3.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_resurrect) c3.commit() @@ -550,6 +566,7 @@ def test_resurrection_of_live_thing_via_sentinel_out_of_order(): changes_c2 = c2.execute( "SELECT * FROM crsql_changes WHERE cid != '-1'").fetchone() + c3.execute("SELECT crsql_set_ts('1700000000')") c3.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", changes_c2) c3.commit() @@ -561,10 +578,13 @@ def test_resurrection_of_live_thing_via_sentinel_out_of_order(): # now lets finish getting changes from the other node changes = c1.execute( "SELECT * FROM crsql_changes WHERE cid != '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", changes) c2.commit() + c3.execute("SELECT crsql_set_ts('1700000000')") + c3.execute("SELECT crsql_set_ts('1700000000')") c3.execute("INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", changes) c3.commit() @@ -595,6 +615,7 @@ def test_resurrection_of_live_thing_via_non_sentinel(): non_sentinel_resurrect = c1.execute( "SELECT * FROM crsql_changes WHERE cid != '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", non_sentinel_resurrect) c2.commit() @@ -630,6 +651,7 @@ def test_resurrection_of_dead_thing_via_sentinel(): sentinel_resurrect = c1.execute( "SELECT * FROM crsql_changes WHERE cid = '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_resurrect) c2.commit() @@ -660,6 +682,7 @@ def test_resurrection_of_dead_thing_via_non_sentinel(): sentinel_resurrect = c1.execute( "SELECT * FROM crsql_changes WHERE cid != '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_resurrect) c2.commit() @@ -709,6 +732,7 @@ def test_delete_via_sentinel(): sentinel_delete = c1.execute( "SELECT * FROM crsql_changes WHERE cid = '-1'").fetchone() + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", sentinel_delete) c2.commit() @@ -843,6 +867,7 @@ def get_column_names_values(column_data): def create_hypothesis_schema(c): c.execute( "CREATE TABLE item (id PRIMARY KEY NOT NULL, width INTEGER, height INTEGER, name TEXT, description TEXT, weight INTEGER)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('item')") @@ -1096,6 +1121,7 @@ def test_discord_report_corrosion(): def make_schema(): c = connect(":memory:") c.execute("CREATE TABLE foo (a primary key not null, b, c, d, e)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() return c diff --git a/py/correctness/tests/test_cl_triggers.py b/py/correctness/tests/test_cl_triggers.py index e98e20cba..c35bb3eed 100644 --- a/py/correctness/tests/test_cl_triggers.py +++ b/py/correctness/tests/test_cl_triggers.py @@ -22,6 +22,7 @@ def create_db(db_file=None): db_file = ":memory:" c = connect(db_file) c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() return c @@ -33,6 +34,7 @@ def sync_left_to_right(l, r, since): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -43,6 +45,7 @@ def test_upsert_non_existing(): c = connect(":memory:") c.execute( "CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL NOT NULL, b TEXT) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -63,6 +66,7 @@ def test_upsert_non_existing(): def test_insert_delete_insert_delete(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -85,6 +89,7 @@ def test_insert_delete_insert_delete(): def test_upsert_previously_existing(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -118,6 +123,7 @@ def test_upsert_previously_existing(): def test_upsert_currently_existing(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -149,6 +155,7 @@ def test_upsert_currently_existing(): def test_update_existing(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -173,6 +180,7 @@ def test_update_existing(): def test_insert_existing(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -193,6 +201,7 @@ def test_insert_existing(): def test_insert_or_ignore_existing(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -212,6 +221,7 @@ def test_insert_or_ignore_existing(): def test_delete_existing(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -228,6 +238,7 @@ def test_delete_existing(): def test_delete_previously_deleted(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -416,6 +427,7 @@ def test_change_primary_key_from_another_db(): def test_change_primary_key_away_from_thing_with_large_length(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -439,6 +451,7 @@ def test_change_primary_key_away_from_thing_with_large_length(): def test_insert_previously_existing(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() diff --git a/py/correctness/tests/test_commit_alter_non_destructive.py b/py/correctness/tests/test_commit_alter_non_destructive.py index 819df852c..34b2e82e6 100644 --- a/py/correctness/tests/test_commit_alter_non_destructive.py +++ b/py/correctness/tests/test_commit_alter_non_destructive.py @@ -6,6 +6,7 @@ def test_commit_alter(): c = connect(":memory:") c.execute("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, title TEXT)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -13,8 +14,10 @@ def test_commit_alter(): c.commit() start_time = time.time() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo')") c.execute("ALTER TABLE foo ADD COLUMN owner TEXT") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('main', 'foo', 1)") end_time = time.time() print(f"non-destructive alter time: {end_time - start_time}") diff --git a/py/correctness/tests/test_commit_alter_perf.py b/py/correctness/tests/test_commit_alter_perf.py index b21b3a54d..4cfddc489 100644 --- a/py/correctness/tests/test_commit_alter_perf.py +++ b/py/correctness/tests/test_commit_alter_perf.py @@ -6,6 +6,7 @@ def test_commit_alter_perf(): c = connect(":memory:") c.execute("CREATE TABLE issue (id INTEGER PRIMARY KEY NOT NULL, title TEXT, owner TEXT, status INTEGER, priority INTEGER)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('issue')") c.commit() @@ -17,14 +18,18 @@ def test_commit_alter_perf(): print(f"insert time: {end_time - start_time}") start_time = time.time() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('issue')") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('issue')") end_time = time.time() print(f"no alter alter time: {end_time - start_time}") start_time = time.time() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('issue')") c.execute("ALTER TABLE issue ADD COLUMN description TEXT") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('issue')") end_time = time.time() print(f"alter add col time: {end_time - start_time}") diff --git a/py/correctness/tests/test_crsql_changes_filters.py b/py/correctness/tests/test_crsql_changes_filters.py index f4c6692a8..7e9e3abf2 100644 --- a/py/correctness/tests/test_crsql_changes_filters.py +++ b/py/correctness/tests/test_crsql_changes_filters.py @@ -5,6 +5,7 @@ def setup_db(): c = connect(":memory:") c.execute("CREATE TABLE item (id PRIMARY KEY NOT NULL, x INTEGER, y INTEGER, desc TEXT)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('item')") c.commit() diff --git a/py/correctness/tests/test_dbversion.py b/py/correctness/tests/test_dbversion.py index 282c4a65b..be344573b 100644 --- a/py/correctness/tests/test_dbversion.py +++ b/py/correctness/tests/test_dbversion.py @@ -16,6 +16,7 @@ def test_min_on_init(): def test_increments_on_modification(): c = connect(":memory:") c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.execute("insert into foo values (1, 2)") c.execute("commit") @@ -39,6 +40,7 @@ def test_db_version_restored_from_disk(): # close and re-open to check that we work with empty clock tables c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.close() c = connect(dbfile) @@ -64,6 +66,7 @@ def test_db_version_restored_from_disk(): # create a new db and sync with it c2 = connect(":memory:") c2.execute("CREATE TABLE foo (id primary key not null, a);") + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute("SELECT crsql_as_crr('foo');") sync_left_to_right(c, c2, 0) @@ -92,6 +95,7 @@ def test_each_tx_gets_a_version(): c = connect(":memory:") c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.execute("insert into foo values (1, 2)") c.execute("insert into foo values (2, 2)") @@ -110,6 +114,7 @@ def test_rollback_does_not_move_db_version(): c = connect(":memory:") c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.execute("insert into foo values (1, 2)") @@ -139,6 +144,7 @@ def test_overwriting_keeps_track_of_true_db_version(): def create_db(): db1 = connect(":memory:") db1.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b DEFAULT 0);") + db1.execute("SELECT crsql_set_ts('1700000000')") db1.execute("SELECT crsql_as_crr('foo');") db1.commit() return db1 diff --git a/py/correctness/tests/test_insert_new_rows.py b/py/correctness/tests/test_insert_new_rows.py index 2a71d3e8d..8ca346a3a 100644 --- a/py/correctness/tests/test_insert_new_rows.py +++ b/py/correctness/tests/test_insert_new_rows.py @@ -6,6 +6,7 @@ def test_c1_c2_c3_c4_c6_c7_crr_values(): c = connect(":memory:") init_version = c.execute("SELECT crsql_db_version()").fetchone()[0] c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.execute("insert into foo values(1, 2)") diff --git a/py/correctness/tests/test_lookaside_key_creation.py b/py/correctness/tests/test_lookaside_key_creation.py index a47326505..1071b1ff2 100644 --- a/py/correctness/tests/test_lookaside_key_creation.py +++ b/py/correctness/tests/test_lookaside_key_creation.py @@ -15,6 +15,7 @@ def sync_left_to_right(l, r, since): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -23,8 +24,10 @@ def sync_left_to_right(l, r, since): def simple_schema(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b TEXT)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.execute("CREATE TABLE bar (a NOT NULL, b NOT NULL, PRIMARY KEY(a, b))") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('bar')") c.commit() return c diff --git a/py/correctness/tests/test_sandbox.py b/py/correctness/tests/test_sandbox.py index 46a107419..a172286b4 100644 --- a/py/correctness/tests/test_sandbox.py +++ b/py/correctness/tests/test_sandbox.py @@ -11,6 +11,7 @@ def sync_left_to_right(l, r, since): ret = 0 for change in changes: ret = change[5] + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) @@ -22,6 +23,7 @@ def test_sync(): def setup(): c = connect(":memory:") c.execute("CREATE TABLE item (id PRIMARY KEY NOT NULL, width INTEGER, height INTEGER, name TEXT, dscription TEXT, weight INTEGER)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('item')") c.commit() return c diff --git a/py/correctness/tests/test_schema_modification.py b/py/correctness/tests/test_schema_modification.py index cf9ece282..9c1abca06 100644 --- a/py/correctness/tests/test_schema_modification.py +++ b/py/correctness/tests/test_schema_modification.py @@ -12,16 +12,20 @@ def test_c1_4_no_primary_keys(): c = connect(":memory:") c.execute("create table foo (a)") with pytest.raises(Exception) as e_info: + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") def test_c1_3_quoted_identifiers(): c = connect(":memory:") c.execute("create table \"foo\" (a primary key not null)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.execute("create table `bar` (a primary key not null)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('bar')") c.execute("create table [baz] (a primary key not null)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('baz')") def check_clock(t): return c.execute( @@ -35,6 +39,7 @@ def check_clock(t): return c.execute( def test_c1_c5_compound_primary_key(): c = connect(":memory:") c.execute("create table foo (a not null, b not null, c, primary key (a, b))") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.execute( @@ -48,6 +53,7 @@ def test_c1_c5_compound_primary_key(): def test_c1_6_single_primary_key(): c = connect(":memory:") c.execute("create table foo (a not null, b, c, primary key (a))") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.execute( "SELECT key, col_version, col_name, db_version, site_id FROM foo__crsql_clock").fetchall() @@ -61,6 +67,7 @@ def test_c2_create_index(): # TODO: create index is silent failing in some cases? c.execute("create index foo_idx on foo (b)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") idx_info = c.execute( "select * from pragma_index_info('foo_idx')").fetchall() @@ -71,6 +78,7 @@ def test_c2_create_index(): def setup_alter_test(): c = connect(":memory:") c.execute("CREATE TABLE todo (id PRIMARY KEY NOT NULL, name, complete, list);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('todo');") c.execute("INSERT INTO todo VALUES (1, 'cook', 0, 'home');") c.commit() @@ -89,9 +97,11 @@ def test_drop_clock_on_col_remove(): assert (clock_entries == [(1, 1, 1, 'complete', 0), (1, 1, 1, 'list', 0), (1, 1, 1, 'name', 0)]) + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('todo');") # Dropping a column should remove its entries from our replication logs. c.execute("ALTER TABLE todo DROP COLUMN list;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('todo');") c.commit() @@ -115,9 +125,11 @@ def test_backfill_col_add(): # Default value colums, by the same logic then, also # do not need a backfill. c = setup_alter_test() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('todo');") c.execute("ALTER TABLE todo ADD COLUMN assignee;") c.execute("ALTER TABLE todo ADD COLUMN due_date DEFAULT '2018-01-01';") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('todo');") c.commit() @@ -164,8 +176,10 @@ def test_backfill_clocks_on_rename(): c = setup_alter_test() c.execute("INSERT INTO todo VALUES (2, 'clean', 0, 'home');") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('todo');") c.execute("ALTER TABLE todo RENAME name TO task;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('todo');") c.commit() changes = c.execute(changes_with_versions_query).fetchall() @@ -185,8 +199,10 @@ def test_delete_sentinels_not_lost(): # starting off correctly assert (changes == [('todo', b'\x01\t\x01', '-1', None, 2, 2)]) + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('todo');") c.execute("ALTER TABLE todo RENAME name TO task;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('todo');") c.commit() @@ -198,6 +214,7 @@ def test_pk_only_sentinels(): c = connect(":memory:") c.execute( "CREATE TABLE assoc (id1 NOT NULL, id2 NOT NULL, PRIMARY KEY (id1, id2));") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('assoc');") c.execute("INSERT INTO assoc VALUES (1, 2);") c.commit() @@ -206,8 +223,10 @@ def test_pk_only_sentinels(): assert ( changes == [('assoc', b'\x02\x09\x01\x09\x02', '-1', None)]) + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('assoc');") c.execute("ALTER TABLE assoc ADD COLUMN data;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('assoc');") c.commit() @@ -232,6 +251,7 @@ def test_backfill_existing_data(): c.execute("INSERT INTO foo (id) VALUES (3);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -253,6 +273,7 @@ def test_backfill_moves_dbversion(): c.execute("INSERT INTO foo VALUES (2, 'baz');") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -261,6 +282,7 @@ def test_backfill_moves_dbversion(): c.execute("INSERT INTO bar VALUES (1, 'bar');") c.execute("INSERT INTO bar (id) VALUES (3);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('bar');") c.commit() @@ -280,6 +302,7 @@ def test_backfill_for_alter_does_not_move_dbversion(): c = connect(":memory:") c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT DEFAULT NULL);") c.execute("INSERT INTO foo VALUES (1, 'bar');") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() site_id = get_site_id(c) @@ -287,6 +310,7 @@ def test_backfill_for_alter_does_not_move_dbversion(): # - change name to have a default value # - add an age column (nullable) # - add a row with a name and age + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "CREATE TABLE new_foo(id PRIMARY KEY NOT NULL, name TEXT DEFAULT 'none', age INTEGER DEFAULT NULL);") @@ -296,6 +320,7 @@ def test_backfill_for_alter_does_not_move_dbversion(): c.execute("INSERT INTO new_foo (id, name, age) VALUES (2, 'baz', 33);") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") # now: @@ -317,11 +342,14 @@ def test_add_col_with_default(): c = connect(":memory:") c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT DEFAULT NULL);") c.execute("INSERT INTO foo VALUES (1, 'bar');") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute("ALTER TABLE foo ADD COLUMN age INTEGER DEFAULT 0;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") changes = c.execute(full_changes_query).fetchall() @@ -337,11 +365,14 @@ def test_add_col_nullable(): c = connect(":memory:") c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT DEFAULT NULL);") c.execute("INSERT INTO foo VALUES (1, 'bar');") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute("ALTER TABLE foo ADD COLUMN age INTEGER DEFAULT NULL;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") changes = c.execute(full_changes_query).fetchall() @@ -353,11 +384,14 @@ def test_add_col_implicit_nullable(): c = connect(":memory:") c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT DEFAULT NULL);") c.execute("INSERT INTO foo VALUES (1, 'bar');") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute("ALTER TABLE foo ADD COLUMN age INTEGER;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") changes = c.execute(full_changes_query).fetchall() @@ -369,9 +403,11 @@ def test_add_col_through_12step(): c = connect(":memory:") c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL, name TEXT DEFAULT NULL);") c.execute("INSERT INTO foo (id) VALUES (3);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "CREATE TABLE new_foo(id PRIMARY KEY NOT NULL, name TEXT DEFAULT NULL, age INTEGER DEFAULT NULL);") @@ -382,6 +418,7 @@ def test_add_col_through_12step(): c.execute("UPDATE new_foo SET age = 44 WHERE id = 3;") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") changes = c.execute(full_changes_query).fetchall() @@ -397,6 +434,7 @@ def test_pk_only_table_backfill(): c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL);") c.execute("INSERT INTO foo VALUES (1);") c.execute("INSERT INTO foo VALUES (2);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -418,6 +456,7 @@ def test_pk_and_default_backfill(): c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL, b);") c.execute("INSERT INTO foo (id) VALUES (1);") c.execute("INSERT INTO foo (id) VALUES (2);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -431,9 +470,11 @@ def test_pk_and_default_backfill(): def test_pk_and_default_backfill_post12step_with_new_rows(): c = connect(":memory:") c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "CREATE TABLE new_foo(id PRIMARY KEY NOT NULL, b);") @@ -442,6 +483,7 @@ def test_pk_and_default_backfill_post12step_with_new_rows(): c.execute("INSERT INTO new_foo (id) VALUES (2);") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") c.commit() @@ -467,14 +509,17 @@ def test_add_column_and_set_column(): c = connect(":memory:") c.execute("CREATE TABLE foo (id PRIMARY KEY NOT NULL);") c.execute("INSERT INTO foo (id) VALUES (3);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() changes = c.execute(full_changes_query).fetchall() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "ALTER TABLE foo ADD COLUMN age INTEGER DEFAULT NULL;") c.execute("UPDATE foo SET age = 44 WHERE id = 3;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") c.commit() @@ -489,16 +534,19 @@ def test_add_column_and_set_column(): def test_remove_rows_on_alter(): c = connect(":memory:") c.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.execute("INSERT INTO foo VALUES (1, 2);") c.execute("INSERT INTO foo VALUES (3, 4);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "ALTER TABLE foo ADD COLUMN c INTEGER DEFAULT NULL;") c.execute("DELETE FROM foo WHERE a = 1;") c.execute("DELETE FROM foo WHERE a = 3;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") c.commit() @@ -535,17 +583,20 @@ def test_change_existing_values_on_alter(): def test_remove_col_from_pk(): c = connect(":memory:") c.execute("CREATE TABLE foo (a NOT NULL, b NOT NULL, c, PRIMARY KEY (a, b));") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.execute("INSERT INTO foo VALUES (1, 2, 3);") c.execute("INSERT INTO foo VALUES (4, 5, 6);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "CREATE TABLE new_foo(a PRIMARY KEY NOT NULL, b, c);") c.execute("INSERT INTO new_foo SELECT * FROM foo;") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") c.commit() @@ -564,6 +615,7 @@ def test_remove_col_from_pk(): def test_remove_pk_column(): c = connect(":memory:") c.execute("CREATE TABLE foo (a NOT NULL, b NOT NULL, c, PRIMARY KEY (a, b));") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -571,12 +623,14 @@ def test_remove_pk_column(): c.execute("INSERT INTO foo VALUES (4, 5, 6);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "CREATE TABLE new_foo(b PRIMARY KEY NOT NULL, c);") c.execute("INSERT INTO new_foo SELECT b, c FROM foo;") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") changes = c.execute(full_changes_query).fetchall() @@ -588,6 +642,7 @@ def test_remove_pk_column(): def test_add_existing_col_to_pk(): c = connect(":memory:") c.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b, c);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -595,12 +650,14 @@ def test_add_existing_col_to_pk(): c.execute("INSERT INTO foo VALUES (4, 5, 6);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "CREATE TABLE new_foo(a NOT NULL, b NOT NULL, c, PRIMARY KEY (a, b));") c.execute("INSERT INTO new_foo SELECT * FROM foo;") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") changes = c.execute(full_changes_query).fetchall() @@ -612,6 +669,7 @@ def test_add_existing_col_to_pk(): def test_add_new_col_to_pk(): c = connect(":memory:") c.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -619,12 +677,14 @@ def test_add_new_col_to_pk(): c.execute("INSERT INTO foo VALUES (4, 5);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute( "CREATE TABLE new_foo(a NOT NULL, b, c NOT NULL, PRIMARY KEY (a, c));") c.execute("INSERT INTO new_foo SELECT a, b, b + 1 FROM foo;") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") changes = c.execute(full_changes_query).fetchall() @@ -643,6 +703,7 @@ def test_add_new_col_to_pk(): def test_rename_pk_column(): c = connect(":memory:") c.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -650,11 +711,13 @@ def test_rename_pk_column(): c.execute("INSERT INTO foo VALUES (4, 5);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute("CREATE TABLE new_foo(c PRIMARY KEY NOT NULL, b)") c.execute("INSERT INTO new_foo SELECT a, b FROM foo;") c.execute("DROP TABLE foo;") c.execute("ALTER TABLE new_foo RENAME TO foo;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") c.commit() @@ -672,6 +735,7 @@ def test_pk_only_table_pk_membership(): def test_changing_values_in_primary_key_columns(): c = connect(":memory:") c.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo');") c.commit() @@ -679,8 +743,10 @@ def test_changing_values_in_primary_key_columns(): c.execute("INSERT INTO foo VALUES (4, 5);") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_begin_alter('foo');") c.execute("UPDATE foo SET a = 2 WHERE a = 1;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_commit_alter('foo');") c.commit() diff --git a/py/correctness/tests/test_sentinel_omission.py b/py/correctness/tests/test_sentinel_omission.py index 6919edf52..c628c7a32 100644 --- a/py/correctness/tests/test_sentinel_omission.py +++ b/py/correctness/tests/test_sentinel_omission.py @@ -6,6 +6,7 @@ def sync_left_to_right(l, r, since): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -14,8 +15,10 @@ def sync_left_to_right(l, r, since): def make_simple_schema(): c = connect(":memory:") c.execute("CREATE TABLE test (id INTEGER PRIMARY KEY NOT NULL, [text] TEXT);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('test')") c.execute("CREATE TABLE test2 (id INTEGER PRIMARY KEY NOT NULL, [text] TEXT);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('test2')") c.commit() return c diff --git a/py/correctness/tests/test_seq.py b/py/correctness/tests/test_seq.py index d33a0d572..76502dda9 100644 --- a/py/correctness/tests/test_seq.py +++ b/py/correctness/tests/test_seq.py @@ -5,6 +5,7 @@ def sync_left_to_right(l, r): changes = l.execute("SELECT * FROM crsql_changes") for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -13,6 +14,7 @@ def sync_left_to_right(l, r): def test_increments_by_one_in_tx(): c = connect(":memory:") c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.commit() @@ -27,6 +29,7 @@ def test_increments_by_one_in_tx(): def test_resets_on_every_tx(): c = connect(":memory:") c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.commit() @@ -49,11 +52,13 @@ def test_preserved_on_merge(): # the order we insert changes into crsql_changes should be the order they come back out when using seq c = connect(":memory:") c.execute("create table foo (id primary key not null, a)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.commit() c2 = connect(":memory:") c2.execute("create table foo (id primary key not null, a)") + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute("select crsql_as_crr('foo')") c2.commit() @@ -78,6 +83,7 @@ def test_incr_by_one(): # delete c = connect(":memory:") c.execute("create table foo (a primary key not null, b, c, d)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.commit() @@ -115,6 +121,7 @@ def test_incr_by_one(): assert (rows == [(0,), (1,), (2,)]) c.execute("create table bar (a primary key not null, b);") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('bar')") c.commit() @@ -133,6 +140,7 @@ def test_incr_by_one(): # test update of pk vals with col vals c.execute("CREATE TABLE baz (a primary key not null)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('baz')") c.commit() c.execute("INSERT INTO baz VALUES (1)") @@ -162,6 +170,7 @@ def test_incr_by_one(): def test_seq_when_re_inserting(): c = connect(":memory:") c.execute("create table foo (a primary key not null, b)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.commit() @@ -190,6 +199,7 @@ def test_seq_when_re_inserting(): def test_seq_when_updating_after_reinsert(): c = connect(":memory:") c.execute("create table foo (a primary key not null, b)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.commit() @@ -214,6 +224,7 @@ def test_seq_when_updating_after_reinsert(): def test_seq_when_resinserting_from_merge(): c = connect(":memory:") c.execute("create table foo (a primary key not null, b)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('foo')") c.commit() @@ -227,6 +238,7 @@ def test_seq_when_resinserting_from_merge(): c2 = connect(":memory:") c2.execute("create table foo (a primary key not null, b)") + c2.execute("SELECT crsql_set_ts('1700000000')") c2.execute("select crsql_as_crr('foo')") c2.commit() diff --git a/py/correctness/tests/test_site_id_lookaside.py b/py/correctness/tests/test_site_id_lookaside.py index 734427a86..900763c97 100644 --- a/py/correctness/tests/test_site_id_lookaside.py +++ b/py/correctness/tests/test_site_id_lookaside.py @@ -8,6 +8,7 @@ def make_simple_schema(): c = connect(":memory:") c.execute("CREATE TABLE foo (a INTEGER PRIMARY KEY NOT NULL, b INTEGER) STRICT;") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() return c @@ -17,6 +18,7 @@ def test_insert_site_id(): # is in lookaside # is an ordinal in actual table a = make_simple_schema() + a.execute("SELECT crsql_set_ts('1700000000')") a.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0, '0')") a.commit() @@ -38,6 +40,7 @@ def test_insert_site_id(): def test_site_id_filter(): a = make_simple_schema() + a.execute("SELECT crsql_set_ts('1700000000')") a.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0, '0')") a.commit() @@ -52,6 +55,7 @@ def test_local_changes_have_local_site(): a.execute("INSERT INTO foo VALUES (3,2)") a.execute("INSERT INTO foo VALUES (4,2)") a.commit() + a.execute("SELECT crsql_set_ts('1700000000')") a.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0, '0')") a.commit() @@ -65,7 +69,7 @@ def test_local_changes_have_local_site(): def test_site_id_ordinals_do_not_move_on_merge(): a = make_simple_schema() - + a.execute("SELECT crsql_set_ts('1700000000')") a.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0, '0')") a.commit() @@ -74,6 +78,7 @@ def test_site_id_ordinals_do_not_move_on_merge(): assert (x == [("X'1DC8D6BB7F8941088327D9439A7927A4'",)]) # insert again with the same site id + a.execute("SELECT crsql_set_ts('1700000000')") a.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010902', 'b', 1, 1, 1, x'1dc8d6bb7f8941088327d9439a7927a4', 1, 0, '0')") a.commit() @@ -84,10 +89,12 @@ def test_site_id_ordinals_do_not_move_on_merge(): ("X'1DC8D6BB7F8941088327D9439A7927A4'",)]) # insert a new site id + a.execute("SELECT crsql_set_ts('1700000000')") a.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010903', 'b', 1, 1, 1, x'2dc8d6bb7f8941088327d9439a7927a4', 1, 0, '0')") a.commit() # insert again with that new site id + a.execute("SELECT crsql_set_ts('1700000000')") a.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010904', 'b', 1, 1, 1, x'2dc8d6bb7f8941088327d9439a7927a4', 1, 0, '0')") a.commit() diff --git a/py/correctness/tests/test_siteid.py b/py/correctness/tests/test_siteid.py index 9f74af715..5e8f9a2cb 100644 --- a/py/correctness/tests/test_siteid.py +++ b/py/correctness/tests/test_siteid.py @@ -9,6 +9,7 @@ def sync_left_to_right(l, r, since): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ? AND site_id IS NOT ?", (since, r_site_id)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -47,6 +48,7 @@ def test_c3c4(): def test_site_id_for_local_writes(): c = connect(":memory:") c.execute("CREATE TABLE foo (id not null, x, y, primary key (id))") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() @@ -78,6 +80,7 @@ def simple_schema(): a = connect(":memory:") a.execute("create table foo (a primary key not null, b);") a.commit() + a.execute("SELECT crsql_set_ts('1700000000')") a.execute("SELECT crsql_as_crr('foo')") a.commit() return a diff --git a/py/correctness/tests/test_sync.py b/py/correctness/tests/test_sync.py index 7924625e0..11b004159 100644 --- a/py/correctness/tests/test_sync.py +++ b/py/correctness/tests/test_sync.py @@ -21,6 +21,7 @@ def sync_left_to_right(l, r, since): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -33,9 +34,13 @@ def create_schema(c): c.execute( "CREATE TABLE component (id primary key not null, type, slide_id, content)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('user')") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('deck')") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('slide')") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("select crsql_as_crr('component')") @@ -160,6 +165,7 @@ def create_db1(): db1 = connect(":memory:") db1.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b DEFAULT 0);") db1.execute("INSERT INTO foo (a) VALUES (1);") + db1.execute("SELECT crsql_set_ts('1700000000')") db1.execute("SELECT crsql_as_crr('foo');") db1.commit() return db1 @@ -168,6 +174,7 @@ def create_db2(): db2 = connect(":memory:") db2.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b DEFAULT 0);") db2.execute("INSERT INTO foo VALUES (1, 2);") + db2.execute("SELECT crsql_set_ts('1700000000')") db2.execute("SELECT crsql_as_crr('foo');") db2.commit() return db2 @@ -218,11 +225,13 @@ def create_dbs(): db1 = connect(":memory:") db1.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b DEFAULT 4);") db1.execute("INSERT INTO foo (a) VALUES (1);") + db1.execute("SELECT crsql_set_ts('1700000000')") db1.execute("SELECT crsql_as_crr('foo');") db1.commit() db2 = connect(":memory:") db2.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b DEFAULT 4);") + db2.execute("SELECT crsql_set_ts('1700000000')") db2.execute("SELECT crsql_as_crr('foo');") db2.commit() db2.execute("INSERT INTO foo (a,b) VALUES (1,2);") @@ -249,14 +258,17 @@ def test_merging_larger(): def test_db_version_moves_as_expected_post_alter(): db = connect(":memory:") db.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b);") + db.execute("SELECT crsql_set_ts('1700000000')") db.execute("SELECT crsql_as_crr('foo');") db.commit() db.execute("INSERT INTO foo (a, b) VALUES (1, 2);") db.commit() + db.execute("SELECT crsql_set_ts('1700000000')") db.execute("SELECT crsql_begin_alter('foo');") db.execute("ALTER TABLE foo ADD COLUMN c;") + db.execute("SELECT crsql_set_ts('1700000000')") db.execute("SELECT crsql_commit_alter('foo');") db.commit() @@ -285,16 +297,19 @@ def test_merging_on_defaults2(): def create_db1(): db1 = connect(":memory:") db1.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b DEFAULT 4);") + db1.execute("SELECT crsql_set_ts('1700000000')") db1.execute("SELECT crsql_as_crr('foo');") db1.commit() db1.execute("INSERT INTO foo (a) VALUES (1);") db1.commit() + db1.execute("SELECT crsql_set_ts('1700000000')") db1.execute("SELECT crsql_begin_alter('foo')") # Test with higher than incoming value and lower than incoming value # defaults db1.execute("ALTER TABLE foo ADD COLUMN c DEFAULT 0;") + db1.execute("SELECT crsql_set_ts('1700000000')") db1.execute("SELECT crsql_commit_alter('foo')") db1.commit() return db1 @@ -302,11 +317,14 @@ def create_db1(): def create_db2(): db2 = connect(":memory:") db2.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b DEFAULT 4);") + db2.execute("SELECT crsql_set_ts('1700000000')") db2.execute("SELECT crsql_as_crr('foo');") db2.commit() + db2.execute("SELECT crsql_set_ts('1700000000')") db2.execute("SELECT crsql_begin_alter('foo')") db2.execute("ALTER TABLE foo ADD COLUMN c DEFAULT 0;") + db2.execute("SELECT crsql_set_ts('1700000000')") db2.execute("SELECT crsql_commit_alter('foo')") db2.commit() @@ -350,6 +368,7 @@ def create_db2(): def create_basic_db(): db = connect(":memory:") db.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b);") + db.execute("SELECT crsql_set_ts('1700000000')") db.execute("SELECT crsql_as_crr('foo');") db.commit() return db diff --git a/py/correctness/tests/test_sync_bit.py b/py/correctness/tests/test_sync_bit.py index 03119735b..d4f04307a 100644 --- a/py/correctness/tests/test_sync_bit.py +++ b/py/correctness/tests/test_sync_bit.py @@ -10,6 +10,7 @@ def create_db(): c = connect(":memory:") c.execute("CREATE TABLE foo (a PRIMARY KEY NOT NULL, b)") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("SELECT crsql_as_crr('foo')") c.commit() return c @@ -18,6 +19,7 @@ def create_db(): def test_insert_row(): # db version, seq, col version, site id, cl should all be from the insertion c = create_db() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 4, 4, x'1dc8d6bb7f8941088327d9439a7927a4', 3, 6, '0')") c.commit() @@ -50,6 +52,7 @@ def test_update_row(): c = create_db() c.execute("INSERT INTO foo VALUES (1, 2)") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 3, 6, '0')") changes = c.execute("SELECT * FROM crsql_changes").fetchall() @@ -80,6 +83,8 @@ def test_delete_row(): c = create_db() c.execute("INSERT INTO foo VALUES (1, 2)") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") + c.execute("SELECT crsql_set_ts('1700000000')") c.execute("INSERT INTO crsql_changes VALUES ('foo', x'010901', '-1', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 4, 6, '0')") c.commit() changes = c.execute("SELECT * FROM crsql_changes").fetchall() @@ -102,6 +107,7 @@ def test_custom_trigger(): INSERT INTO log (b) VALUES (1); END;""") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010901', 'b', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 3, 6, '0')") c.commit() @@ -121,6 +127,7 @@ def test_custom_trigger(): INSERT INTO log (b) VALUES (1); END;""") c.commit() + c.execute("SELECT crsql_set_ts('1700000000')") c.execute( "INSERT INTO crsql_changes VALUES ('foo', x'010902', 'b', 1, 4, 4, x'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 3, 6, '0')") rows = c.execute("SELECT * FROM log").fetchall() diff --git a/py/correctness/tests/test_sync_prop.py b/py/correctness/tests/test_sync_prop.py index 46ca4e35d..2172268a2 100644 --- a/py/correctness/tests/test_sync_prop.py +++ b/py/correctness/tests/test_sync_prop.py @@ -85,6 +85,7 @@ def open_db(i): conn = connect(":memory:") conn.execute( "CREATE TABLE item (id PRIMARY KEY NOT NULL, width INTEGER, height INTEGER, name TEXT, description TEXT, weight INTEGER)") + conn.execute("SELECT crsql_set_ts('1700000000')") conn.execute("SELECT crsql_as_crr('item')") conn.commit() return (i, conn, dict()) @@ -246,6 +247,7 @@ def sync_left_to_right(l, r, since_map, since_is_rowid): change = tuple(temp) else: ret[change[0]] = change[5] + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) diff --git a/py/correctness/tests/test_update_rows.py b/py/correctness/tests/test_update_rows.py index 746406838..45769e77f 100644 --- a/py/correctness/tests/test_update_rows.py +++ b/py/correctness/tests/test_update_rows.py @@ -4,6 +4,7 @@ def sync_left_to_right(l, r, since): changes = l.execute( "SELECT * FROM crsql_changes WHERE db_version > ?", (since,)) for change in changes: + r.execute("SELECT crsql_set_ts('1700000000')") r.execute( "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", change) r.commit() @@ -12,6 +13,7 @@ def test_update_pk(): def create_db(): db = connect(":memory:") db.execute("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, a, b)") + db.execute("SELECT crsql_set_ts('1700000000')") db.execute("SELECT crsql_as_crr('foo');") db.commit() return db @@ -71,6 +73,7 @@ def test_empty_update_doesnt_change_db_version(): def create_db(): db = connect(":memory:") db.execute("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, a, b)") + db.execute("SELECT crsql_set_ts('1700000000')") db.execute("SELECT crsql_as_crr('foo');") db.commit() return db @@ -134,6 +137,7 @@ def test_ts_is_inserted(): def create_db(): db = connect(":memory:") db.execute("CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL, a, b)") + db.execute("SELECT crsql_set_ts('1700000000')") db.execute("SELECT crsql_as_crr('foo');") db.commit() return db @@ -147,8 +151,8 @@ def get_site_id(db): db1_site_id = get_site_id(db1) db2_site_id = get_site_id(db2) - # use max u64 - db1.execute("SELECT crsql_set_ts('18446744073709551615');") + # use max i64 + db1.execute("SELECT crsql_set_ts('9223372036854775807')") db1.execute("INSERT INTO foo (id, a, b) VALUES (1, 2, 3)") db1.commit() @@ -160,8 +164,8 @@ def get_site_id(db): db1_changes = db1.execute("SELECT * FROM crsql_changes").fetchall() - assert (db1_changes == [('foo', b'\x01\t\x01', 'a', 2, 1, 1, db1_site_id, 1, 0, '18446744073709551615'), - ('foo', b'\x01\t\x01', 'b', 3, 1, 1, db1_site_id, 1, 1, '18446744073709551615'), + assert (db1_changes == [('foo', b'\x01\t\x01', 'a', 2, 1, 1, db1_site_id, 1, 0, '9223372036854775807'), + ('foo', b'\x01\t\x01', 'b', 3, 1, 1, db1_site_id, 1, 1, '9223372036854775807'), ('foo', b'\x01\t\x02', 'a', 5, 1, 2, db1_site_id, 1, 0, '0'), ('foo', b'\x01\t\x02', 'b', 6, 1, 2, db1_site_id, 1, 1, '0')]) diff --git a/v2_metadata_design.md b/v2_metadata_design.md new file mode 100644 index 000000000..155d91cc9 --- /dev/null +++ b/v2_metadata_design.md @@ -0,0 +1,1068 @@ +# cr-sqlite V2 Metadata Design + +> **Minimum SQLite version**: 3.44.0 (for `ORDER BY` inside aggregate calls support). The bundled amalgamation has been upgraded from 3.42 to 3.44. + +## Design Goals + +- Alive PKs retain full original PK columns + `hashed_pk` (fast lookup by `__crsql_key` or hash). The hash is needed because delete events arrive with only `hashed_pk` (from remote nodes that have the row as dead). We must look up the alive table by hash to find and remove it. +- Dead PKs store only a `PK_HASH_SIZE`-byte truncated XXH128 hash (no original PK columns) to reduce storage in the tombstone table (which can grow large). +- Clock table: single `INT64` primary key `(oid << CRSQL_COL_ID_BITS) | col_id`, no CL stored (CL lives in `v2_pks`/`v2_tombstones`), no sentinel rows (tombstones have their own table), timestamps stored as `INTEGER` (unix seconds). +- Feed returns real PK columns for alive rows. Dead rows return `hashed_pk` in V2 wire format; in V1 compat mode, dead rows return real PK columns (from `v2_tombstone_pks`). + +## Hash Function + +XXH128 of the PK values, truncated to `PK_HASH_SIZE` bytes. + +The hash is computed over the `crsql_pack_columns()` blob of all PK columns, so it is stable across nodes (unlike `__crsql_key` which is a local auto-increment). Works for composite PKs, integer PKs, text PKs, etc. — `crsql_pack_columns` serializes all PK values into a deterministic byte sequence. + +- `PK_HASH_SIZE` is a compile-time define (default: 10 bytes / 80 bits). +- XXH128 produces 128 bits; we take the first `PK_HASH_SIZE` bytes. +- Tunable via `#define CRSQL_PK_HASH_SIZE` at build time. + +### Dependency: `xxhash-rust` + +Uses the [`xxhash-rust`](https://github.com/DoumanAsh/xxhash-rust) crate (v0.8.17), which provides `xxh3_128` returning a `u128`. + +- `no_std` compatible (the `std` feature is optional and only enables `std::io::Write` — we don't need it). +- Feature-gated: enable only `xxh3` (includes both `xxh3_64` and `xxh3_128`). +- Pure Rust, no C dependencies, no `unsafe` in the public API. +- SIMD acceleration (SSE2/AVX2/NEON) at compile time via target features. +- Compatible with `panic = "abort"` (no unwinding in hash code). +- API: `xxhash_rust::xxh3::xxh3_128(data: &[u8]) -> u128` + +Cargo.toml addition: +```toml +[dependencies.xxhash-rust] +version = "0.8.17" +default-features = false +features = ["xxh3"] +``` + +Alternatives considered: +- `twox-hash`: Also has `XxHash3_128`, `no_std` compatible, but uses a struct-based API (`XxHash3_128::oneshot(seed, data)`) vs `xxhash-rust`'s simpler one-shot function. More popular but heavier. +- `xxh3` (cberner): Only XXH3, supports runtime AVX2/NEON detection, but narrower scope and less maintained. + +`xxhash-rust` is the best fit: minimal, `no_std` by default, simple one-shot API, well-maintained. + +Exposed as SQL function: `crsql_hash_pk(pk1, pk2, ...)` + +- Takes variadic PK values directly (same signature as `crsql_pack_columns`). +- Internally: `pack_columns` → XXH128 → truncate to `PK_HASH_SIZE` bytes. +- Usage: `SELECT * FROM foo__crsql_v2_tombstones WHERE hashed_pk = crsql_hash_pk(1, 'abc')` +- Also used in Rust trigger code and merge logic (no SQL function call overhead there). +- Registered alongside `crsql_pack_columns` for ad-hoc queries, debugging, and consistency. + +## Tables in This Design + +1. `
    • __crsql_v2_col_map` — column name → 0-based col_id mapping, might differ between nodes +2. `
    __crsql_v2_clock` — packed clock (cells only, no tombstones) +3. `
    __crsql_v2_pks` — alive PKs with full columns + hash + CL +4. `
    __crsql_v2_tombstones` — dead PKs with hash only + delete metadata +5. `
    __crsql_v2_tombstone_pks` — hash → original PK cols for tombstones (V1 compat) + +> **Note:** `
    __crsql_pks` (the original PK-to-key mapping table) is created by cr-sqlite and has schema: `(__crsql_key INTEGER PRIMARY KEY, , , ...)` with a unique index on the original PK columns. In this design we do not use `__crsql_pks` directly for CL lookups; `__crsql_v2_pks` replaces it. + +## CL (Causal Length) Semantics + +CL is a per-PK monotonic counter that tracks the number of create/delete transitions for that PK. It starts at 1 (odd = alive) when a row is first created, and increments on each delete (even = dead) and each subsequent resurrection (odd = alive again). + +- `CL=1`: row created (alive) +- `CL=2`: row deleted (dead) +- `CL=3`: row resurrected (alive) +- `CL=4`: row deleted again (dead) +- ... + +Odd CL → row is alive (stored in `v2_pks`). Even CL → row is dead (stored in `v2_tombstones`). The `CHECK` constraints enforce this: `v2_pks.cl % 2 = 1`, `v2_tombstones.cl % 2 = 0`. + +Merge conflict resolution uses CL as the primary ordering key: + +- **`insert_cl < local_cl`**: incoming change is from an older causal epoch → ignore entirely. All column-level comparisons are skipped (the row has been deleted and possibly resurrected since this change was made). +- **`insert_cl > local_cl`**: incoming change is from a newer causal epoch → automatic win. If `insert_cl` is odd, the row is resurrected (moved from tombstones to `v2_pks`). If `insert_cl` is even, the row is deleted (moved from `v2_pks` to `v2_tombstones`). +- **`insert_cl == local_cl`**: same causal epoch → fall back to per-column version comparison (`col_version`, then `site_id` as tiebreaker). This is the existing `did_cid_win` logic. + +CL is stored per-PK (not per-column) in `v2_pks` (alive) or `v2_tombstones` (dead). The clock table does not store CL — it is joined from `v2_pks`/`v2_tombstones` at feed time. This matches the existing V1 design where CL lives in the PK table, not the clock table. + +### V1 vs V2 CL Representation Differences + +**V1**: CL is stored as `col_version` in a sentinel clock row (`col_name = '-1'`). The sentinel-omission optimization means the sentinel row is not written until the first delete — `get_local_cl` returns `0` when no sentinel exists, which is interpreted as "row exists, logically CL=1" (the `COALESCE(t2.col_version, 1)` in the V1 feed query). `local_cl = 0` in V1 merge means "row exists but no sentinel" (logically CL=1), not "row never seen." + +**V2**: CL is stored explicitly in `v2_pks.cl` (with `DEFAULT 1`) or `v2_tombstones.cl`. There is no sentinel-omission optimization — `v2_pks` always has the row with `cl >= 1`. `local_cl = 0` in V2 means "row not in `v2_pks` or `v2_tombstones`" (row never seen locally). The V2 merge path treats `local_cl = 0` the same as V1: `insert_cl > 0` always wins. + +**V2→V1 translation**: when writing V2 data into V1 tables, if `local_cl = 0` in V2 (row never seen), the V1 path must create the `__crsql_pks` entry and write clock entries as if it were a fresh insert. The V1 sentinel-omission optimization applies naturally — no sentinel is written until a delete arrives. + +--- + +## 1. Column Map: col_name → 0-based col_id + +Maps each tracked column name to a compact integer ID. Used to pack `(key, col_id)` into a single `cell_key` INTEGER: `(key << CRSQL_COL_ID_BITS) | col_id`. + +`CRSQL_COL_ID_BITS` is a compile-time define (default: 12 bits = up to 4096 columns per table). That's safe as [SQLite limits](https://sqlite.org/limits.html) say that the default setting for `SQLITE_MAX_COLUMN` is 2000. Tunable via `#define CRSQL_COL_ID_BITS` at build time. Reducing it frees bits for larger rowids (rowid limit = `2^(64 - CRSQL_COL_ID_BITS)`). + +```sql +CREATE TABLE IF NOT EXISTS "
    __crsql_v2_col_map" ( + col_id INTEGER PRIMARY KEY, + col_name TEXT NOT NULL +) STRICT; + +CREATE UNIQUE INDEX IF NOT EXISTS "idx_
    _v2_col_map_name" + ON "
    __crsql_v2_col_map"(col_name); +``` + +## 2. Clock Table: Packed INT64, No Sentinels/Tombstones + +One row per `(row, column)` cell that has been written. + +- `cell_key = (pk_key << CRSQL_COL_ID_BITS) | col_id` where col_id from col_map (0-based) +- `ts` stored as `INTEGER` (unix seconds) instead of `TEXT` for compactness. +- Feed index on `(site_id, db_version, seq)` for efficient change feed queries. + +```sql +CREATE TABLE IF NOT EXISTS "
    __crsql_v2_clock" ( + cell_key INTEGER PRIMARY KEY, -- (pk_key << CRSQL_COL_ID_BITS) | col_id + col_version INTEGER NOT NULL, -- column version + site_id INTEGER NOT NULL, -- which site made the change + db_version INTEGER NOT NULL, -- local DB version at time of change + seq INTEGER NOT NULL, -- sequence within db_version + ts INTEGER NOT NULL CHECK (ts > 0), -- unix timestamp (seconds); reject unset +) STRICT; + +CREATE INDEX IF NOT EXISTS "
    __crsql_v2_clock_feed_idx" + ON "
    __crsql_v2_clock"(site_id, db_version, seq); +``` + +## 3. Alive PKs: `__crsql_key` + Hash + CL + +`__crsql_key INTEGER PRIMARY KEY` (stores the underlying main table's rowid for rowid tables, auto-increment for WITHOUT ROWID base tables). Unique index on `hashed_pk` for O(1) deletion lookups (is this PK alive or dead?). + +### Two Modes Depending on Main Table Type + +- **Rowid tables**: `__crsql_key = main_table.rowid`. No PK columns stored here — PK values are fetched from the main table via `SELECT pk_cols WHERE rowid = ?`. Columns: `__crsql_key`, `hashed_pk`, `cl`. +- **WITHOUT ROWID tables**: `__crsql_key = auto-increment`. Full PK columns stored here (needed for feed emission, since there's no rowid to join back to the main table). Columns: `__crsql_key`, ``, `hashed_pk`, `cl`. + +V1 compat: PK columns (stored or fetched) needed for V1 wire emission. + +### `__crsql_key` Assignment (Rowid Reuse) + +When the main table is a rowid table with an accessible rowid — meaning at least one of the built-in aliases (`rowid`, `oid`, `_rowid_`) is not shadowed by a declared column, OR the table has an `INTEGER PRIMARY KEY` column (which is itself the rowid alias) — then `__crsql_key = main_table.rowid`. + +**Edge case** ([SQLite rowidtable docs](https://www.sqlite.org/rowidtable.html)): if a table declares columns named `rowid`, `oid`, AND `_rowid_`, all three aliases are shadowed. If there's also no `INTEGER PRIMARY KEY`, the rowid is completely inaccessible by name → fall back to auto-increment. + +Examples: +- `CREATE TABLE bar(rowid TEXT, oid TEXT, _rowid_ TEXT)` — rowid exists but is unreachable. +- `CREATE TABLE foo(rowid TEXT, oid TEXT, _rowid_ TEXT, pk INTEGER PRIMARY KEY)` — `pk` is the rowid alias, so we use `pk` directly. + +**Detection logic at `as_crr` time**: query `pragma_table_info('
    ')` to determine the mode: + +1. Check if any column has `pk > 0` AND `type = 'INTEGER'` → `INTEGER PRIMARY KEY` exists → use it as `__crsql_key` (it *is* the rowid alias). +2. Otherwise, check if none of `rowid`, `oid`, `_rowid_` appear as column names in `pragma_table_info` → at least one alias is unshadowed → use `rowid` as `__crsql_key`. +3. If all three aliases are shadowed AND no `INTEGER PRIMARY KEY` → fall back to auto-increment with stored PK columns. + +**Tests required**: write tests covering all three cases — (a) plain rowid table, (b) `INTEGER PRIMARY KEY` table, (c) all-three-aliases-shadowed table, (d) `INTEGER PRIMARY KEY` + shadowed aliases (case b wins). Verify that `__crsql_key` is correctly assigned and that feed/merge queries work in each case. + +This eliminates the `INSERT OR IGNORE` + auto-increment round-trip on local writes: +- Local insert: `NEW.rowid` is available directly in the trigger. +- Local update: `NEW.rowid` is available directly in the trigger. + +> **Note:** In trigger context, `NEW.rowid` / `OLD.rowid` are always accessible for rowid tables regardless of column shadowing. The shadowing edge case only affects ad-hoc SQL queries outside triggers (e.g., merge path doing `SELECT rowid FROM main_table WHERE pk = ?`). When rowid is inaccessible for ad-hoc queries, or the table is WITHOUT ROWID, or was explicitly configured to use the WITHOUT ROWID layout at `as_crr` time, fall back to auto-increment with stored PK columns (current V1 scheme). The clock table doesn't care — `__crsql_key` is just an `INT64` join key. + +### Merge Path (Row Not in Main Table Yet) + +The incoming change may be for a row that doesn't exist locally. Check `hashed_pk` against `v2_pks` and `v2_tombstones` first: + +- **Clean slate** (`hashed_pk` not in either table): `INSERT` directly into main table first, get `sqlite3_last_insert_rowid()` back, use it as `__crsql_key`. No stub, no fallback, no remapping. +- **Resurrection** (`hashed_pk` in `v2_tombstones` only): the row was deleted locally but an incoming change resurrects it. Same as clean-slate — `INSERT` into main table first, get `sqlite3_last_insert_rowid()`, use it as `__crsql_key`. Then delete the tombstone entry and add the alive entry with the new CL. + +For WITHOUT ROWID tables, use auto-increment fallback in both cases. + +### Large Rowid Handling + +`cell_key = (__crsql_key << CRSQL_COL_ID_BITS) | col_id` must fit in a **signed** `INT64` (SQLite's `INTEGER` type is 64-bit signed), so `cell_key` must stay positive. This means `__crsql_key` is limited to `2^(63 - CRSQL_COL_ID_BITS)`. With the default `CRSQL_COL_ID_BITS = 12`, the limit is `2^51 = 2,251,799,813,685,248`. + +If a rowid table may have rowids `>= 2^(63 - CRSQL_COL_ID_BITS)`, it should be converted to WITHOUT ROWID at CRR registration time (per-table decision). The `as_crr` call accepts an option to convert a rowid table to WITHOUT ROWID (storing the full PK as the primary key). This is a one-time schema decision made when the table is registered as a CRR. The `as_crr` call also accepts an opt-out parameter to skip the `CHECK` constraint below (for applications that know their rowids are safe and want to avoid the per-write overhead). + +`CHECK` constraint on main table (rowid tables only): + +```sql +CHECK (rowid >= 0 AND rowid < 2251799813685248) -- 2^(63 - CRSQL_COL_ID_BITS), signed INT64 safe +``` + +Ensures `cell_key = (rowid << CRSQL_COL_ID_BITS) | col_id` fits in a positive signed `INT64` without overflow. Using `2^(63 - bits)` (not `2^(64 - bits)`) because `key << 12` with `key >= 2^51` would overflow into the sign bit, producing negative `cell_key` values that sort before all positive ones — breaking `PRIMARY KEY` ordering. `rowid >= 0` allows `INTEGER PRIMARY KEY` tables that use 0 as a valid key value (`cell_key = 0 | col_id = col_id`, which is fine). Overhead is one integer comparison per write. Rowid changes (e.g., `UPDATE rowid = ...`) are handled by triggers which update `__crsql_key` in `v2_pks` and `cell_key` in `v2_clock` accordingly. + +### Schema + +Schema is dynamic — PK columns only present for WITHOUT ROWID tables. + +**Example (WITHOUT ROWID, PK columns (id1, id2, ...)):** + +``` +__crsql_key INTEGER PRIMARY KEY, +id1 NOT NULL, -- original PK column (WITHOUT ROWID only) +id2 NOT NULL, -- original PK column (WITHOUT ROWID only) +... +hashed_pk BLOB NOT NULL, -- PK_HASH_SIZE-byte truncated XXH128 of packed PK columns +cl INTEGER NOT NULL DEFAULT 1 -- clock version / change label +``` + +**Example (rowid table):** + +``` +__crsql_key INTEGER PRIMARY KEY, -- = main_table.rowid +hashed_pk BLOB NOT NULL, +cl INTEGER NOT NULL DEFAULT 1 +``` + +`STRICT` is always used for all V2 tables. For tables with dynamic PK columns (WITHOUT ROWID), the `ANY` type ([sqlite.org/stricttables.html](https://www.sqlite.org/stricttables.html)) is used for PK columns, so STRICT works regardless of whether the base table is STRICT. + +```sql +CREATE TABLE IF NOT EXISTS "
    __crsql_v2_pks" ( + __crsql_key INTEGER PRIMARY KEY, -- main_table.rowid (rowid tables) or auto-increment (WITHOUT ROWID) + -- ... original PK columns from the base table (WITHOUT ROWID tables only) ... + -- End dynamic columns + hashed_pk BLOB NOT NULL, -- PK_HASH_SIZE-byte truncated XXH128(crsql_pack_columns(pk_cols)) + cl INTEGER NOT NULL DEFAULT 1 CHECK (cl % 2 = 1), -- odd = alive (current CL) +) STRICT; -- STRICT always; WITHOUT ROWID tables use ANY type for PK columns + +-- Unique index on hash — primary access path when __crsql_key (PK table rowid) is not +-- available as a lookup key (WITHOUT ROWID tables, shadowed rowid, merge path, feed queries): +CREATE UNIQUE INDEX IF NOT EXISTS "idx_
    _v2_pks_hash" + ON "
    __crsql_v2_pks"(hashed_pk); +``` + +## 4. Tombstones: Hash Only (No Original PK Columns) + Delete Metadata + +WITHOUT ROWID — composite PK `(site_id, db_version, seq)` for feed ordering. This PK is deliberately chosen to make feed queries as efficient as possible — the feed scans by `(site_id, db_version)` and orders by `seq`, which is exactly the PK order. `hashed_pk` replaces original PK columns to save space. Unique index on `hashed_pk` for upsert-based conflict resolution (one tombstone per PK at any time). `cl` stored for feed output. `ts` stored for `dead_recent` selection. This table IS the delete log — no separate delete_log table needed. + +```sql +CREATE TABLE IF NOT EXISTS "
    __crsql_v2_tombstones" ( + site_id INTEGER NOT NULL, + db_version INTEGER NOT NULL, + seq INTEGER NOT NULL, + hashed_pk BLOB NOT NULL, -- PK_HASH_SIZE-byte truncated XXH128(crsql_pack_columns(pk_cols)) + cl INTEGER NOT NULL CHECK (cl % 2 = 0), -- even = deleted (CL at deletion time) + ts INTEGER NOT NULL CHECK (ts > 0), -- timestamp of deletion; reject unset + PRIMARY KEY (site_id, db_version, seq) +) WITHOUT ROWID, STRICT; +``` + +### Clock Entry Cleanup on Delete + +When a row is deleted locally, all clock entries for that row are deleted: + +```sql +DELETE FROM v2_clock WHERE cell_key >> CRSQL_COL_ID_BITS = ?key +``` + +This mirrors the existing V1 design which drops clock rows on delete (see `after_delete.rs`: `drop_clocks_stmt`). Tombstone entries in `v2_tombstones` serve as the delete event in the feed; no sentinel rows are needed in the clock table. + +### Unique Index on Hash — Upsert-Based Conflict Resolution + +Only one tombstone per `hashed_pk` exists at any time. When a new delete arrives for a hash that already has a tombstone (concurrent deletes from different sites, or re-delete after resurrection), conflict resolution compares CL values and keeps the higher one (biggest CL = most recent delete), updating its metadata (`site_id`, `db_version`, `seq`, `ts`) to match the winning tombstone. + +For equal-CL conflicts (concurrent deletes at the same CL from different sites), V2 mirrors V1's `did_site_id_win` approach: compare site_id blobs lexicographically. SQLite's `>` operator on BLOBs does byte-by-byte memcmp — exactly what V1's Rust `insert_site_id.cmp(local_site_id)` does. Since `v2_tombstones.site_id` stores a local ordinal (not the blob), a subquery resolves it to the blob for comparison. The incoming site_id blob is bound as a parameter from the merge path. + +As with V1, the site_id tiebreaker only applies when `merge-equal-values` is enabled (`crsql_config_set('merge-equal-values', 1)`). When disabled (the default), equal-CL conflicts are silently dropped — the existing tombstone is kept. This matches V1's behavior where `did_site_id_win` is only called when `mergeEqualValues == 1`. + +```sql +-- Single upsert handles both CL > and CL == with site_id tiebreaking +INSERT INTO v2_tombstones VALUES (...) +ON CONFLICT(hashed_pk) DO UPDATE SET + site_id = excluded.site_id, + db_version = excluded.db_version, + seq = excluded.seq, + cl = excluded.cl, + ts = excluded.ts +WHERE excluded.cl > v2_tombstones.cl + OR (excluded.cl = v2_tombstones.cl + AND ?incoming_site_id_blob > (SELECT site_id FROM crsql_site_id WHERE ordinal = v2_tombstones.site_id)) +``` + +If the upsert's `WHERE` clause doesn't match (incoming CL is lower, or equal CL with lower/equal site_id blob), the existing tombstone is kept — no update needed. + +Re-deletion after resurrection works because resurrection removes the tombstone first (moves PK back to `v2_pks`), so the subsequent delete inserts into an empty slot. + +```sql +CREATE UNIQUE INDEX IF NOT EXISTS "idx_
    _v2_tombstones_hash" + ON "
    __crsql_v2_tombstones"(hashed_pk); +``` + +## 5. Tombstone PK Mapping (V1 Compatibility) + +Maps `hashed_pk` back to original PK column values for tombstones. Needed while V1 wire format is still emitted: V1 sends `crsql_pack_columns(pk_cols)` for delete events, which requires the real PK values, not just the hash. V2 wire format sends `hashed_pk` directly, so this table is not needed for V2. + +**Pruning**: entries are deleted only after ALL nodes in the cluster emit V2 format. Until then, every new tombstone gets an entry here. + +Schema is dynamic — mirrors the original table's PK columns (like `__crsql_v2_pks`). + +```sql +CREATE TABLE IF NOT EXISTS "
    __crsql_v2_tombstone_pks" ( + hashed_pk BLOB PRIMARY KEY, + -- ... original PK columns from the base table ... + -- End dynamic columns + -- Example: id INTEGER NOT NULL, name TEXT NOT NULL, etc. +) WITHOUT ROWID, STRICT; -- STRICT always; PK columns use ANY type +``` + +--- + +## Vtable Rowid Key for Tombstones + +The `crsql_changes` virtual table requires a unique rowid per row (SQLite's `xRowid` interface). For clock entries (alive rows), the `key` column is `__crsql_key` — a small autoincrement integer from `v2_pks`. Tombstones in `v2_tombstones` have no equivalent integer key (their PK is `(site_id, db_version, seq)`), so a synthetic `key` is computed in the feed query. + +### Bit Layout + +``` +Bit 62: flag (1 = tombstone, distinguishes from clock entry keys) +Bits 46-61: site_id (16 bits → 65,536 nodes) +Bits 22-45: db_version (24 bits → 16.7M) +Bits 0-21: seq (22 bits → 4.2M) +``` + +```sql +(1 << 62) | (t.site_id << 46) | (t.db_version << 22) | t.seq as key +``` + +### Rationale + +- **Clock entry keys are small**: `__crsql_key` is a rowid (max ~2^51 with default `CRSQL_COL_ID_BITS`) or autoincrement, starting from 1. Setting bit 62 guarantees tombstone keys never collide with clock entry keys. +- **Uniqueness within tombstones**: `(site_id, db_version, seq)` is the PK of `v2_tombstones`, so the packed value is unique per table. Combined with `slab_rowid(tblInfoIdx, key)`, the final vtable rowid is globally unique across all tables. +- **Bit allocation is generous**: 65K nodes, 16.7M db_versions, and 4.2M seq values per tombstone are well above expected limits. `db_version` gets the most bits because it grows monotonically and is the most likely to exceed limits in long-lived databases. +- **Collisions would be harmless in practice**: If limits are exceeded (e.g., `seq > 4.2M`), the packed key wraps and could collide with another tombstone's key. This only affects the vtable rowid — the actual data (pks, cid, cl, etc.) is unaffected. The only scenario where this matters is advanced operations like `ORDER BY rowid` or `WHERE rowid = ?` on `crsql_changes`, which are not typical usage patterns. Standard `SELECT * FROM crsql_changes` iterates via `xNext`/`xColumn` and is unaffected by rowid collisions. +- **If problematic, can be changed**: The key computation is a single expression in the feed query (`changes_vtab_read.rs`). If larger limits are needed, the bit allocation can be adjusted, or the approach can be replaced entirely (e.g., using a WITHOUT ROWID virtual table with `(table, pk)` as the primary key, per [SQLite vtab docs](https://www.sqlite.org/vtab.html#_without_rowid_virtual_tables_)). + +--- + +## Query Patterns + +### Point SELECT (is cell alive or dead? get CL + clock) + +```sql +-- h = xxh128(crsql_pack_columns(pk_values))[:PK_HASH_SIZE] +SELECT t1.col_version, pk.cl +FROM
    __crsql_v2_clock AS t1, +( + SELECT cl FROM
    __crsql_v2_pks WHERE __crsql_key = ?key + UNION ALL + SELECT cl FROM
    __crsql_v2_tombstones WHERE hashed_pk = ?h + LIMIT 1 +) AS pk +WHERE t1.cell_key = ?cell_key +``` + +### Feed (Changes Since db_version for a Site) + +Alive rows: JOIN clock with `v2_pks` to get PK columns and CL. Column values are fetched from the main table via a compiled `CASE` statement: + +```sql +CASE (t1.cell_key & ((1 << CRSQL_COL_ID_BITS) - 1)) + WHEN 0 THEN main_table.col0 + WHEN 1 THEN main_table.col1 + ... +END AS col_val +``` + +This `CASE` is generated per-table at CRR registration time, mapping `col_id` → column name. For rowid tables, PK columns are fetched via: `SELECT FROM main_table WHERE rowid = ah.__crsql_key`. For WITHOUT ROWID tables, PK columns come directly from `v2_pks`. Column names are resolved via: `SELECT col_name FROM v2_col_map WHERE col_id = (t1.cell_key & ((1 << CRSQL_COL_ID_BITS) - 1))`. + +Dead rows: `hashed_pk` from tombstones (no PK columns in V2 mode). + +```sql +SELECT ah., t1.col_version, t1.db_version, t1.site_id, t1.seq, t1.ts, ah.cl, + CASE (t1.cell_key & ((1 << CRSQL_COL_ID_BITS) - 1)) + WHEN 0 THEN mt.col0 + WHEN 1 THEN mt.col1 + ... + END AS col_val, + cm.col_name +FROM
    __crsql_v2_clock AS t1 +JOIN
    __crsql_v2_pks AS ah ON ah.__crsql_key = t1.cell_key >> CRSQL_COL_ID_BITS +JOIN
    AS mt ON mt.rowid = ah.__crsql_key -- rowid tables only +JOIN
    __crsql_v2_col_map AS cm ON cm.col_id = (t1.cell_key & ((1 << CRSQL_COL_ID_BITS) - 1)) +WHERE t1.site_id = ? AND t1.db_version > ? +UNION ALL +SELECT d.hashed_pk, d.cl, d.db_version, d.site_id, d.seq, d.ts, d.cl +FROM
    __crsql_v2_tombstones AS d +WHERE d.site_id = ? AND d.db_version > ? +ORDER BY db_version, seq LIMIT ? +``` + +### UPDATE (Check Alive/Dead Then Update Clock) + +```sql +-- h = xxh128(crsql_pack_columns(pk_values))[:PK_HASH_SIZE] +SELECT 1 FROM
    __crsql_v2_pks WHERE __crsql_key = ?key +UNION ALL +SELECT 1 FROM
    __crsql_v2_tombstones WHERE hashed_pk = ?h +LIMIT 1 +-- then: UPDATE
    __crsql_v2_clock SET ... WHERE cell_key = ?cell_key +``` + +--- + +## Packed Wire Format (crsql_changes v2 Emission) + +### Problem + +Currently each clock row = 1 change event on the wire. An insert of a 20-column row produces 20 events, each duplicating `tbl`, `pk`, `db_version`, `site_id`, `cl`, `ts`. The receiver does 20 separate `UPDATE`/`INSERT` statements. + +### Idea + +Coalesce clock rows that share `(row_key, db_version, site_id)` into a single packed event. The clock table stays per-column (queryable, simple). Packing is purely a `crsql_changes` read/serialization optimization. + +### Versioning + +Controlled by application-level config flags, not wire-level: + +- **`metadata-write-version`**: `v1` | `v2` | `v2&v1` (which schema to write to) +- **`metadata-use-version`**: `v1` | `v2` (which schema to read from) +- **`sync-log-version`**: `v1` | `v2` (which wire format to emit) + +Nodes accept v2 logs regardless of their own `sync-log-version` — `sync-log-version` controls what a node emits, not what it can receive. Reception requires: + +1. `metadata-write-version` is `v2` or `v2&v1` (V2 tables are actively written to). +2. V1→V2 migration is complete (all PK hashes available in `v2_pks`/`v2_tombstone_pks`). + +During migration (before completion), v2 logs are rejected with an error because not all PK hashes are available yet for resolving incoming v2 dead-row events (hash-only PKs). Once migration completes in `v2&v1` mode, the node has all hashes and can fully process v2 logs, emit them, and use v2 for feed/processing. + +This allows gradual rollout: migrate schema → flip write → flip use → flip sync-log. + +### Progression Rules + +**`metadata-write-version`:** + +| From | To | Allowed? | Notes | +|------|-----|----------|-------| +| `v1` | `v2&v1` | Yes | Forward always allowed | +| `v2&v1` | `v2` | Yes | Forward always allowed | +| `v2&v1` | `v1` | Yes | V1 tables were kept in sync during dual write. Stops any in-progress migration and queues V2 table cleanup (dropped by `crsql_incremental_maintenance` as a background task). | +| `v2` | `v2&v1` | **Forbidden** | V1 tables stopped receiving writes, now stale | +| `v2` | `v1` | **Forbidden** | V1 tables stale | + +**`metadata-use-version`:** + +| From | To | Allowed? | Notes | +|------|-----|----------|-------| +| `v1` | `v2` | Yes | Forward allowed (guarded — see below) | +| `v2` | `v1` | Yes | Only if `metadata-write-version` is `v1` or `v2&v1` (V1 tables still active). Forbidden if write is `v2`. | + +**`sync-log-version`:** + +| From | To | Allowed? | Notes | +|------|-----|----------|-------| +| `v1` | `v2` | Yes | Forward allowed (guarded — see below) | +| `v2` | `v1` | Yes | Only if `metadata-use-version` is `v1` and `v2_tombstone_pks` pruning has not started (v1 compat emission needs to resolve hash→PK for dead rows from that table). Forbidden once pruning has removed entries. | + +### Re-enabling v2&v1 After Rollback to v1 + +V2 tables were dropped during the `v2&v1` → `v1` transition. Going forward to `v2&v1` again is just setting the flag, which queues a fresh migration: + +```sql +crsql_config_set('metadata-write-version', 'v2&v1') -- queues migration +crsql_incremental_maintenance(N) -- rebuilds V2 from scratch +``` + +The migration function creates V2 tables if they don't exist, so this is equivalent to a first-time migration. The cost is a full re-scan, but rollback should be rare. + +### Config API (Same Mechanism as `merge-equal-values`) + +```sql +crsql_config_set('metadata-write-version', 'v1') -- default +crsql_config_set('metadata-write-version', 'v2') +crsql_config_set('metadata-write-version', 'v2&v1') +crsql_config_get('metadata-write-version') -> 'v1' +-- same pattern for 'metadata-use-version' and 'sync-log-version' +-- defaults: all 'v1' (existing behavior unchanged until explicitly switched) +-- stored in crsql_master (same table as other config keys) +-- crsql_config_set enforces progression rules (see above) +``` + +### Guards on `metadata-use-version` and `sync-log-version` + +**`metadata-use-version = 'v2'` requires:** + +1. V2 schema is fully populated for ALL CRR tables — either: + - (a) The table was registered as a CRR with V2 from the start (no V1 tables ever existed), or + - (b) `crsql_incremental_maintenance` has completed the V1→V2 migration (returned 0 for all migration tasks). +2. `metadata-write-version` is `v2` or `v2&v1` (V2 tables are actively written to). + +**`sync-log-version = 'v2'` requires:** + +1. `metadata-use-version` is `v2` (feed reads from V2 tables). +2. `metadata-write-version` is `v2` or `v2&v1` (V2 tables are actively written to). + +Guards are global: `crsql_config_set` checks that ALL CRR tables meet prerequisites before accepting the change. This prevents reading from or emitting from stale V2 tables on any table. + +### Incremental Maintenance: `crsql_incremental_maintenance(chunk_size) -> INTEGER` + +Single entry point for all background maintenance work. Dispatches to whatever tasks are pending across all CRR tables. Does up to `chunk_size` units of work per call, returns total work units remaining across all tasks. When it returns 0, all maintenance is complete (until new work is queued). Safe to call repeatedly — tracks progress per-task internally and picks up where it left off. Designed for periodic invocation (e.g., from a timer or idle hook). + +**Dispatch logic (priority order):** + +1. **V1→V2 schema migration** (if `metadata-write-version` was set to `v2&v1` for any table that still has V1 tables and no V2 tables): + - **(a)** First call for a table: create V2 tables (`col_map`, `clock`, `pks`, `tombstones`, `tombstone_pks`). + - **(b)** Populate `v2_col_map` from existing `col_name` values in `__crsql_clock`. + - **(c)** For each chunk of `__crsql_pks` rows (up to `chunk_size` per call): + - Progress is tracked via a cursor on `__crsql_key` (autoincrement integer). `progress_marker` stores the last `__crsql_key` processed; each call resumes with `WHERE __crsql_key > progress_marker LIMIT chunk_size`. + - Compute `hashed_pk = xxh128(crsql_pack_columns(pk_cols))[:PK_HASH_SIZE]`. + - If row is alive (`INSERT_SENTINEL` or no sentinel): `INSERT OR IGNORE` into `v2_pks` with `cl` from sentinel `col_version` (odd) or `cl=1` if no sentinel. `INSERT OR IGNORE` ensures idempotency with dual-write triggers that may have already inserted the row. + - If row is dead (`DELETE_SENTINEL`): `INSERT OR REPLACE` into `v2_tombstones` with `cl` from sentinel `col_version` (even), and `INSERT OR REPLACE` into `v2_tombstone_pks`. If `ts = 0` or unparseable, assign the migration start timestamp rather than skipping — the tombstone is still valid, just missing its timestamp. + - Repack clock rows: for each `(key, col_name)` in `__crsql_clock` for this PK, `INSERT OR REPLACE` into `v2_clock` with `cell_key = (key << COL_ID_BITS) | col_id`. `INSERT OR REPLACE` ensures the cursor pass overwrites any dual-written entries with the authoritative migrated values. Clock entries with `ts = 0` or unparseable `ts` are also assigned the migration start timestamp rather than failing the `CHECK (ts > 0)` constraint. + - Orphan PK handling: if a `__crsql_pks` row has NO clock entries at all (not even a sentinel) AND the row does not exist in the base table, skip it entirely — it's stale/corrupt metadata with no corresponding data. Do not insert anything into V2 tables for it. + - **(d)** First call for a table: set `metadata-write-version` to `v2&v1` automatically. Dual write begins immediately so updates during the cursor sweep are captured in V2 tables. The cursor only processes new `__crsql_pks` rows; without dual write, updates to existing rows would be missed. + - **(e)** Final call for a table (returns 0 for that table): migration is complete. All PK hashes are now available. The node can now accept v2 logs, set `metadata-use-version` to `v2`, and set `sync-log-version` to `v2`. + - *(Additional maintenance tasks — tombstone pruning, post-alter compaction, backfill — will be added to the dispatcher in the future.)* + +**Task queue**: stored in `crsql_master`. Each pending task has: + +``` +(task_type, table_name, progress_marker, total_units) +``` + +`crsql_incremental_maintenance` picks the highest-priority task with remaining work, does up to `chunk_size` units, updates `progress_marker`, and returns the sum of remaining units across all tasks. **One work unit = one row** (e.g., one `__crsql_pks` row during migration, one main-table row during backfill). Each call runs within a single db transaction — `chunk_size` should be sized to keep transaction duration reasonable (e.g., 100000 rows). + +**Enabling `v2&v1` IS starting the migration:** + +```sql +crsql_config_set('metadata-write-version', 'v2&v1') -- queues migration task +-- Then call crsql_incremental_maintenance(N) periodically until it returns 0. +``` + +### Recommended Rollout Sequence + +1. `crsql_config_set('metadata-write-version', 'v2&v1')` — global flag, queues migration tasks for all V1 CRR tables; dual write begins on first maintenance call, capturing updates during migration. +2. Call `crsql_incremental_maintenance(N)` periodically until it returns 0. — migration complete for all tables, all PK hashes available, node can now accept v2 logs. +3. Set `metadata-use-version` to `v2` (guarded: all migrations must be complete). +4. Set `sync-log-version` to `v2` when all peers can accept V2 format. +5. Set `metadata-write-version` to `v2` to stop writing to V1 tables. (V1 tables are now stale — progression rules prevent going back.) +6. `crsql_incremental_maintenance` will clean up V1 tables as a background task: drops `__crsql_clock` and `__crsql_pks` for tables where `metadata-write-version` is `v2` and no V1 feed reads or V1 log emission remain. + +### V1/V2 Coexistence + +During rollout, a node may receive both v1 (per-column) and v2 (packed) log entries simultaneously. The receiver must detect which format each row uses. Detection is per-row, not per-stream: + +- **Packed (v2)**: `cid` contains `char(0)` separator → split by `char(0)` to get col_name list. `col_vrsn` also contains `char(0)` → split to get versions. `cval` is a `crsql_pack_columns` blob of all values. +- **Single (v1)**: `cid` is a plain column name or sentinel (no `char(0)`). `col_vrsn` is a single integer. `cval` is a single value. +- **Sentinels**: `cid = '-1'` (delete or insert sentinel — in V1 both `INSERT_SENTINEL` and `DELETE_SENTINEL` are `'-1'`, distinguished by CL parity: even = delete, odd = insert) or `'-2'` (hash-based tombstone, **new in V2**: dead row with `hashed_pk` instead of real PK) are always single events in both v1 and v2 — no packing. The merge path must handle `'-2'` as a new case not present in V1. **`'-2'` can only be accepted when the receiving node has completed V1→V2 migration** (i.e., `metadata-use-version` is `v2&v1` or `v2`), since it requires `v2_tombstones` and `v2_pks` tables to exist with full data. If a `'-2'` row is received before migration is complete, the merge path must return an error — the sender is using V2 wire format but the receiver isn't ready for it. + +Column names are used (not `col_id`) because names are deterministic across nodes (same schema) while internal `col_id` from `col_map` is local and non-deterministic across nodes. + +### Packed Wire Row (Same Vtable Columns, Different Encoding) + +| Column | Description | +|--------|-------------| +| `tbl` | Table name (1×, not N×) | +| `pk` | Packed PK values via `crsql_pack_columns` (1×) | +| `cid` | `GROUP_CONCAT(col_name, char(0))` — null-separated col names. Detection: `char(0)` present → packed; absent → single/sentinel. SQLite column names cannot contain null bytes, so this is safe. | +| `cval` | `crsql_pack_agg(col_val)` — custom SQLite aggregate (`xStep` + `xFinal`) that collects values across the `GROUP BY` and produces a TLV blob using the same per-value encoding as `crsql_pack_columns`. Implemented in Rust alongside `crsql_pack_columns`, reusing the same per-value encoding logic. `GROUP_CONCAT` can't be used here because it loses type info (Integer/Text/Blob/Float/Null). Format: `[num_values:varint, ...[(type:3bits, intlen:5bits):u8, length?:varint, ...bytes]]`. Each value encodes its own type and length. Receiver calls `unpack_columns(cval)` → `Vec`, no external metadata needed. **ORDER IS THE CONTRACT**: values in `cval` are in the same order as column names in `cid` and versions in `col_vrsn`. All three aggregates run over the same group, so ordering is naturally consistent. | +| `col_vrsn` | `GROUP_CONCAT(col_version, char(0))` — null-separated col versions (parallel array to `cid`; receiver zips `cid[i]` with `col_vrsn[i]`) | +| `db_vrsn` | Single value (shared by all N columns in the group) | +| `site_id` | Single value | +| `seq` | `GROUP_CONCAT(t1.seq, char(0))` — null-separated seqs (parallel array to `cid`). All seqs in the group are preserved for bookeeping correctness. | +| `cl` | Single value (from `v2_pks` or `v2_tombstones` JOIN) | +| `ts` | Single value | + +> **Major implementation item**: `crsql_pack_agg` is a new custom SQLite aggregate function requiring `xStep` + `xFinal` callbacks. This is non-trivial — it must: +> - Accept any SQLite value type (Integer, Text, Blob, Float, Null) in `xStep` +> - Accumulate values in an internal buffer using the same TLV encoding as `crsql_pack_columns` +> - Produce the final blob in `xFinal` with a varint count header (matching `crsql_pack_columns`) +> - Produce byte-identical output to `crsql_pack_columns` for the same values in the same order +> - Handle NULL values correctly (encoded as type byte only, no length/data) +> This should be implemented and tested early as it is a hard dependency for the packed feed query. +> +> **Count header: u8 → varint**: The existing `crsql_pack_columns` and `unpack_columns` are changed to use a **SQLite varint** for the count header instead of `u8`. SQLite varints encode 0-127 in a single byte (0x00-0x7F) — byte-identical to the old `u8` format. This means V1↔V2 wire interop works seamlessly for any table with ≤127 columns (everyone today). For 128+ columns the varint encoding differs from u8 (varint uses 2+ bytes, u8 uses 1), but no existing deployment has that many columns. Varints scale up to 9 bytes encoding 2^64, so the column limit is effectively removed — the practical limit is SQLite's `SQLITE_MAX_COLUMN` (default 2000, max 32767). No separate `unpack_columns_v2` is needed — the existing `unpack_columns` is updated to read the varint count. +> +> **Wire compatibility note**: `crsql_pack_columns` output is the `pk` column in the changes vtab feed — it's transmitted between nodes. During V1→V2 migration, V1 and V2 nodes exchange changes. For ≤127 columns, V1 nodes produce u8 count (0x00-0x7F) and V2 nodes produce varint count (0x00-0x7F) — byte-identical, full interop. V1 nodes reading V2 blobs: the varint count byte 0x00-0x7F is interpreted as u8 correctly. V2 nodes reading V1 blobs: the u8 count byte 0x00-0x7F is read as varint correctly. For 128+ columns, interop breaks — but this is a non-issue since no deployment has that many columns. + +### Feed Query (Packed, Per Table) + +**Alive rows** (updates/inserts with column data): + +> **Implementation note**: SQLite 3.44+ supports `ORDER BY` inside aggregate calls, which eliminates the need for a subquery to pre-sort rows. The `ORDER BY cm.col_id` inside each aggregate ensures `cid`, `col_vrsn`, `cval`, and `seq` arrays are aligned. **Important**: the `GROUP BY` clause must use the full expression `c.cell_key >> CRSQL_COL_ID_BITS`, not the column alias `key` — SQLite 3.44 does not resolve aliases in GROUP BY when the query is used as a virtual table subquery. Bare columns (`ah.cl`, `t1.ts`) that are functionally dependent on the GROUP BY key are left as bare columns (SQLite picks an arbitrary row from the group, which is safe since they're identical within a group). + +```sql +SELECT + '{table}' as tbl, + crsql_pack_columns({pk_cols_for_table}) as pks, -- from v2_pks (WITHOUT ROWID) or main table (rowid) + GROUP_CONCAT(cm.col_name, char(0) ORDER BY cm.col_id) as cid, + GROUP_CONCAT(t1.col_version, char(0) ORDER BY cm.col_id) as col_vrsn, + t1.db_version as db_vrsn, + site_tbl.site_id as site_id, + t1.cell_key >> CRSQL_COL_ID_BITS as key, + GROUP_CONCAT(t1.seq, char(0) ORDER BY cm.col_id) as seq, -- packed seqs; receiver records ALL in bookeeping + ah.cl as cl, -- per-row, same for all columns (in GROUP BY as functionally dependent) + t1.ts as ts, -- same for all rows in group (written in same transaction) + crsql_pack_agg( + CASE cm.col_id -- value extraction via compiled CASE + WHEN 0 THEN mt.col_a + WHEN 1 THEN mt.col_b + WHEN 2 THEN mt.col_c + -- ... one WHEN per non-PK column, using col_id from v2_col_map (0-based) ... + END + ORDER BY cm.col_id + ) as cval, -- packed column values, parallel array to cid +FROM "
    __crsql_v2_clock" AS t1 +JOIN "
    __crsql_v2_col_map" AS cm ON (t1.cell_key & ((1 << CRSQL_COL_ID_BITS) - 1)) = cm.col_id +JOIN "
    __crsql_v2_pks" AS ah ON ah.__crsql_key = t1.cell_key >> CRSQL_COL_ID_BITS +JOIN "
    " AS mt ON mt.rowid = ah.__crsql_key -- rowid tables: fetch PK cols + col values +LEFT JOIN crsql_site_id AS site_tbl ON t1.site_id = site_tbl.ordinal +WHERE t1.site_id = ? AND t1.db_version > ? +GROUP BY t1.cell_key >> CRSQL_COL_ID_BITS, t1.db_version, site_tbl.site_id +``` + +> **Value extraction via compiled CASE statement**: Unlike V1 which fetches column values on-demand in `changes_next` (one `SELECT col FROM main_table WHERE pk = ?` per row), V2 compiles a large `CASE cm.col_id WHEN 0 THEN mt.col_x WHEN 1 THEN mt.col_y ...` statement at `TableInfo` creation time. This CASE maps each `col_id` (0-based integer from `v2_col_map`) to its corresponding column on the main table, and `crsql_pack_agg` collects the resolved values across the group. Integer comparison is used instead of string comparison on `col_name` for efficiency. The statement is prepared once and reused — the column list is fixed per table schema. This trades a larger prepared statement for eliminating the per-row value fetch round-trip that V1 does in `changes_next`. + +> **Note:** For WITHOUT ROWID tables, the JOIN on main table uses PK columns instead of rowid: +> ```sql +> JOIN "
    " AS mt ON mt. = ah. AND mt. = ah. ... +> ``` +> For rowid tables, `pk_cols_for_table = mt., mt., ...` +> For WITHOUT ROWID tables, `pk_cols_for_table = ah., ah., ...` + +**UNION ALL — Dead rows, V2 wire format** (`sync-log-version = 'v2'`): + +Tombstones ARE the sentinel/delete events. There are no separate sentinel rows in the clock table — the tombstones table serves that role. `cid = '-2'` distinguishes hash-based tombstones from V1 `'-1'` delete sentinels. + +```sql +SELECT + '{table}' as tbl, + d.hashed_pk as pks, -- hashed PK for dead rows (V2 wire format) + '-2' as cid, -- hash-based tombstone sentinel (V2 dead row) + NULL as col_vrsn, + d.db_version as db_vrsn, + site_tbl.site_id as site_id, -- resolve ordinal to site_id blob + d.hashed_pk as key, -- hash as key (receiver looks up by hash, not __crsql_key) + d.seq as seq, + d.cl as cl, + d.ts as ts +FROM "
    __crsql_v2_tombstones" AS d +LEFT JOIN crsql_site_id AS site_tbl ON d.site_id = site_tbl.ordinal +WHERE d.site_id = ? AND d.db_version > ? +``` + +**UNION ALL — Dead rows, V1 compat wire format** (`sync-log-version = 'v1'`, metadata in V2): + +Real PK columns resolved from `v2_tombstone_pks`. `cid = '-1'` (V1 delete sentinel). Emitted when `sync-log-version = 'v1'` but metadata is stored in V2 tables. + +```sql +SELECT + '{table}' as tbl, + crsql_pack_columns(tp., tp., ...) as pks, -- real PK values + '-1' as cid, -- V1 delete sentinel + NULL as col_vrsn, + d.db_version as db_vrsn, + site_tbl.site_id as site_id, + d.hashed_pk as key, -- hash for internal bookkeeping + d.seq as seq, + d.cl as cl, + d.ts as ts +FROM "
    __crsql_v2_tombstones" AS d +JOIN "
    __crsql_v2_tombstone_pks" AS tp ON d.hashed_pk = tp.hashed_pk +LEFT JOIN crsql_site_id AS site_tbl ON d.site_id = site_tbl.ordinal +WHERE d.site_id = ? AND d.db_version > ? + +ORDER BY db_vrsn, seq +``` + +### Value Fetch (Per Table, Prepared Once, Cached in TableInfo) + +Fetches ALL non-PK columns in one query. Rust picks the relevant ones. + +```sql +SELECT col1, col2, ..., colN FROM "table" WHERE pk1 = ? AND pk2 = ? +``` + +In `changes_next`: + +1. Detect packed row (`cid` contains `char(0)`) +2. Split `cid` by `char(0)` → col_name list +3. Split `col_vrsn` by `char(0)` → col_version list +4. Execute all-columns `SELECT`, fetch all values +5. Pick values for columns in the `cid` list, pack via `crsql_pack_columns` → `cval` + +### Merge Side (Per Table, Prepared Once, Cached in TableInfo) + +Fixed `UPDATE` statement with `CASE` per column — no dynamic SQL generation. Bind flags (0/1) to select which columns to update. + +```sql +UPDATE "table" SET + col1 = CASE WHEN ?1 THEN ?2 ELSE col1 END, + col2 = CASE WHEN ?3 THEN ?4 ELSE col2 END, + ... -- one CASE per non-PK column + colN = CASE WHEN ?(2N-1) THEN ?2N ELSE colN END +WHERE pk1 = ?(2N+1) AND pk2 = ?(2N+2) +``` + +In `merge_insert`: + +1. Detect packed row (`cid` contains `char(0)`) +2. Split `cid` → col_name list, `col_vrsn` → col_version list +3. Unpack `cval` → col_value list +4. For each `(col_name, col_version, col_value)`: + - (a) Run existing per-column merge logic (`did_cid_win`, etc.) + - (b) If column wins, set its flag=1 and bind its value in the `CASE UPDATE` + - (c) If column loses, set its flag=0 (`CASE` keeps existing value) +5. Execute the single `CASE UPDATE` statement +6. For each winning column, `set_winner_clock` as before + +> **Note:** Per-column version comparison (`did_cid_win`) still happens per column. The packing only saves wire bytes and round-trips, not merge computation. The `CASE UPDATE` replaces N separate `UPDATE` statements with 1. + +### Benefits + +- **Wire**: eliminates `(N-1) × (pk_bytes + tbl_bytes + metadata_bytes)` per row change +- **Receiver**: 1 `CASE UPDATE` instead of N round-trips +- Delete/create sentinels stay single-event (no packing) +- Partial updates (2 of 20 columns) still pack those 2 — smaller win but still a win +- Clock table unchanged — packing is purely in feed query + `changes_next` + `merge_insert` +- Write path unchanged — per-column seq stays as-is (zero write path changes for V2) + +### Seq Handling + +Write path is unchanged — each column still gets its own seq via `bump_seq()`. The feed query packs ALL seqs via `GROUP_CONCAT(seq, char(0))`. The receiver splits these and records each one in `__corro_seq_bookkeeping`. This is critical for correctness: corrosion's `PartialVersion` tracks seqs as a `RangeInclusiveSet` and determines completeness by checking for gaps in `0..=last_seq`. If we used `MIN(seq)`, the receiver would think seqs 6,7 are missing (when they were packed into seq=5) and would request them forever — deadlock. + +Partial replays (`SyncNeedV1::Partial`) query `WHERE seq BETWEEN :start AND :end` on the `crsql_changes` vtable. This works correctly because the vtable filters on the underlying clock table's `seq` column (via `xBestIndex`/`xFilter`), not on the packed output. V2 always coalesces rows that share `(PK, db_version, site_id)` regardless of how they were selected. If the seq range covers all columns of an operation, the group is complete and packed. If the range splits an operation (e.g., seqs 5,6,7 but only 5,6 requested), the group is partial — still packed, just with fewer columns. The receiver records the seqs it got and knows the rest are still missing. + +**Future optimization**: a feature flag can switch to one-seq-per-operation (`bump_seq` once before the loop in `mark_locally_inserted`/`updated`). This is safe to enable only when ALL nodes in the cluster emit packed change logs, since the bookeeping semantics change (one seq per operation instead of per column). `cl` is per-row (stored in `v2_pks`/`v2_tombstones`, not per-column), so it is not part of the `GROUP BY` — it's the same for all columns in the group. + +--- + +## Backfill (V2): `crsql_backfill_v2` + +Called when a table is first registered as a CRR with V2 schema (`metadata-write-version` is `v2` at `as_crr` time, no V1 tables exist). + +For each row in the main table not yet in `v2_pks`: + +1. Compute `hashed_pk` from PK values. +2. `INSERT` into `v2_pks` with `cl=1`, `__crsql_key = rowid` (rowid tables) or auto-increment (WITHOUT ROWID tables). +3. For each non-PK column, `INSERT` into `v2_clock`: + - `cell_key = (__crsql_key << COL_ID_BITS) | col_id` + - `col_version=1`, `db_version=crsql_next_db_version()`, `seq=crsql_increment_and_get_seq()`, `site_id=0`, `ts=unix_now` +4. For pk-only tables (no non-PK columns): no clock entries needed. The `v2_pks` entry with `cl=1` is sufficient — row existence is implied by presence in `v2_pks` (odd CL = alive). This is cleaner than V1 which required an `INSERT_SENTINEL` clock row. + +Mirrors V1 backfill in `backfill.rs` but writes to V2 tables. + +--- + +## ALTER TABLE (V2): `crsql_compact_post_alter_v2` + +Called after schema changes (column add/remove, PK changes) on V2 tables. + +### Column Added + +1. `INSERT` new entry into `v2_col_map` (`col_name` → next `col_id`). +2. Backfill: for each existing row in `v2_pks`, `INSERT` clock entry for the new column with `col_version=1` (same as V1 `backfill_missing_columns`). + +### Column Removed + +1. Look up `col_id` for dropped column from `v2_col_map`. +2. `DELETE FROM v2_col_map WHERE col_name = dropped_column`. +3. `DELETE FROM v2_clock WHERE (cell_key & ((1 << COL_ID_BITS) - 1)) = col_id`. + +### PK Columns Changed + +1. All hashes change → `DROP` all V2 tables and re-create from scratch. +2. Re-backfill from main table (same as fresh `as_crr` with V2). + +This mirrors V1 behavior which drops and recreates clock + pks tables. + +### `crsql_begin_alter` / `crsql_commit_alter` + +1. **begin**: drop triggers (prevent writes during schema change). +2. **commit**: recreate triggers, run `compact_post_alter_v2`, store `pre_compact_dbversion` in `crsql_master`. + +--- + +## Teardown (V2): `crsql_remove_crr_v2` + +Drops all V2 metadata tables for a table: + +- `
    __crsql_v2_col_map` +- `
    __crsql_v2_clock` +- `
    __crsql_v2_pks` +- `
    __crsql_v2_tombstones` +- `
    __crsql_v2_tombstone_pks` + +Drops all triggers (same trigger names as V1 — triggers are shared). If V1 tables still exist (not yet dropped post-migration), drops those too. Called from the existing `as_table` / `remove_crr` entry point which checks which schema version is active and dispatches accordingly. + +--- + +## Codepath Separation: V1 / V2&V1 / V2 + +The three config flags create 3 operational modes. The codebase must cleanly separate V1 and V2 logic to support all three. + +**Principle**: `TableInfo` is the dispatch point. `TableInfo` gains a `schema_version` field: `SchemaVersion::V1 | V2 | V2AndV1`. This is set at `TableInfo` creation time based on `metadata-write-version` and which tables physically exist. + +### Write Path (`local_writes/`) + +`after_insert`/`update`/`delete` check `schema_version`: + +- **V1** → existing V1 logic (unchanged) +- **V2** → V2 logic (hash computation, `v2_pks`/`v2_tombstones`/`v2_clock` writes) +- **V2&V1** → V1 logic first, then V2 logic (dual write) + +V2 write functions live in `local_writes/v2.rs` (single file). Trigger SQL is unchanged — triggers still call `crsql_after_insert`/`update`/`delete` with the same arguments. The Rust functions behind those names dispatch. + +### Read Path (`changes_vtab_read.rs`) + +Feed query selection based on `metadata-use-version`: + +- **V1** → existing V1 feed query (V1 clock tables, per-column rows) +- **V2** → V2 feed query (V2 tables, packed if `sync-log-version = v2`) + +No "v2&v1" mode for reads — always reads from one schema. + +### Merge Path (`changes_vtab_write.rs`) + +Incoming row format detected per-row (`char(0)` in `cid` = packed V2). Local schema version determines write targets: + +| Incoming | Local | Action | +|----------|-------|--------| +| V1 | V1 | Existing merge logic | +| V1 | V2 | Translate: resolve PK to hash, write to V2 tables | +| V2 | V1 | Translate: resolve hash to PK (via `v2_tombstone_pks` or `v2_pks`), write to V1 tables | +| V2 | V2 | V2 merge logic | +| V1 | V2&V1 | Write to both V1 and V2 tables | +| V2 | V2&V1 | Write to both V1 and V2 tables | + +The translate step for incoming V2 → local V1 requires a hash-to-PK lookup: + +- If the hash is unknown locally (row never seen), the change is for a new row — `INSERT` into main table first, then proceed with V1 merge. +- If the hash is in `v2_pks` or `v2_tombstone_pks`, resolve to PK and proceed. +- If the hash is unknown and it's a delete (`cid='-2'`), it can be ignored — we can't delete a row we've never seen. The tombstone is still recorded in `v2_tombstones` if in V2&V1 or V2 mode. + +### `ts` Type Conversion (V1 TEXT ↔ V2 INTEGER) + +V1 stores `ts` as `TEXT` (e.g., `"1698230400"` or `"0"`). V2 stores `ts` as `INTEGER` (unix seconds). Conversions: + +- **V1 → V2 translation** (incoming V1 row, local V2 tables): `CAST(insert_ts AS INTEGER)`. If the V1 `ts` is `'0'` (unset/default), use the migration start timestamp instead (same as the migration logic at line 443). This satisfies the V2 `CHECK (ts > 0)` constraint. +- **V2 → V1 translation** (incoming V2 row, local V1 tables): `CAST(insert_ts AS TEXT)`. The V1 clock table accepts any TEXT, so no constraint issues. The existing `set_winner_clock` function binds `ts` via `bind_text`, so the V2 integer must be converted to text before binding. +- **V2&V1 dual write** (local writes): the local write path produces `ts` from `ext_data.timestamp` (currently a `u64` formatted as string). V2 write path converts to `INTEGER` directly; V1 write path keeps the existing `TEXT` binding. No cross-conversion needed — each path formats `ts` for its own table type. + +### File Organization + +| File | Responsibility | +|------|---------------| +| `tableinfo.rs` | `TableInfo` with `SchemaVersion`, V1 + V2 statement getters | +| `bootstrap.rs` | V1 table creation (unchanged) | +| `bootstrap_v2.rs` | V2 table creation | +| `triggers.rs` | Trigger SQL (shared, unchanged — triggers call same fns) | +| `local_writes/mod.rs` | Dispatch based on `schema_version` | +| `local_writes/after_insert.rs` | V1 (unchanged) | +| `local_writes/after_update.rs` | V1 (unchanged) | +| `local_writes/after_delete.rs` | V1 (unchanged) | +| `local_writes/v2.rs` | V2 write logic (insert/update/delete in one file) | +| `changes_vtab_write.rs` | Merge dispatch (V1/V2/V2&V1) | +| `changes_vtab_read.rs` | Feed dispatch (V1/V2) | +| `backfill.rs` | V1 backfill (unchanged) | +| `backfill_v2.rs` | V2 backfill | +| `alter.rs` | V1 alter (unchanged) | +| `alter_v2.rs` | V2 alter | +| `teardown.rs` | V1 teardown (unchanged) | +| `teardown_v2.rs` | V2 teardown | +| `migrate.rs` | `crsql_incremental_maintenance` dispatcher + V1→V2 migration logic | + +### V1 Code Deletion + +Once all nodes in a cluster run V2 exclusively, V1 files and V1 branches in dispatch functions can be deleted. The `SchemaVersion` enum can be simplified to just `V2`. This is a mechanical removal — no V2 code depends on V1 code. + +--- + +## PK-Only Tables (No Non-PK Columns) + +### Problem + +Tables with only PK columns (e.g., `CREATE TABLE foo (id INTEGER PRIMARY KEY NOT NULL)`) produce no `v2_clock` entries on insert — `v2_after_insert` only writes clock entries for non-PK columns. Without clock entries, the feed query (which INNER JOINs `v2_clock` with `v2_col_map`) returns nothing, making PK-only rows invisible to sync. + +V1 handles this via a sentinel clock row (`col_name = '-1'`) that carries `db_version`, `seq`, `ts`, `site_id`. V2 needs an equivalent mechanism. + +### Design: Sentinel Clock Entry at `col_id=0` + +For PK-only tables (`tbl_info.non_pks.is_empty()`), a sentinel clock entry is written at `col_id=0` in `v2_clock`. This is safe because `v2_col_map` is empty for PK-only tables — `col_id=0` has no mapping to a real column. + +**Write path** (`v2_after_insert`): if `non_pks.is_empty()`, write one clock entry: +```sql +INSERT OR REPLACE INTO v2_clock (cell_key, col_version, site_id, db_version, seq, ts) +VALUES ((key << COL_ID_BITS) | 0, 1, 0, db_version, seq, ts) +``` + +**Read path**: `TableInfo` already knows `non_pks.is_empty()`. When true, `query_for_table` dispatches to a PK-only query that reads sentinel clock entries directly (no `v2_col_map` JOIN): + +```sql +SELECT + 'foo' as tbl, + crsql_pack_columns() as pks, + '-1' as cid, -- sentinel + c.col_version as col_vrsn, + c.db_version as db_vrsn, + site_tbl.site_id as site_id, + c.cell_key >> COL_ID_BITS as key, + c.seq as seq, + pk_tbl.cl as cl, + c.ts as ts, + NULL as cval -- no column value +FROM v2_clock AS c +JOIN v2_pks AS pk_tbl ON (c.cell_key >> COL_ID_BITS) = pk_tbl.__crsql_key +LEFT JOIN crsql_site_id AS site_tbl ON c.site_id = site_tbl.ordinal +-- No v2_col_map JOIN; col_id=0 is the sentinel +``` + +**Delete path**: same as normal tables — `v2_after_delete` deletes all clock entries for the key (including the sentinel) and moves the row to `v2_tombstones`. + +**Merge path**: already handles `cid='-1'` via `is_sentinel_only` check. The sentinel is treated like any other clock entry for conflict resolution. + +### ALTER TABLE: Adding a Non-PK Column to a PK-Only Table + +When a non-PK column is added (via `ALTER TABLE ... ADD COLUMN` + `crsql_as_crr` re-registration), the table transitions from PK-only to normal mode. At this point: + +1. **Populate `v2_col_map`**: the new column gets `col_id=0` (first non-PK column). +2. **Existing sentinel entries become regular clock entries**: the sentinel rows at `col_id=0` already have `col_version=1`, `site_id=0`, and valid `db_version`/`seq`/`ts`. Once `v2_col_map` maps `col_id=0` to the new column name, these entries seamlessly become normal clock entries for that column. The read query's CASE expression fetches the actual column value from the main table (which will be the column's default value for existing rows). No deletion or cleanup of sentinel entries is needed. +3. **`TableInfo` refresh**: `non_pks` is now non-empty, so the read path switches to the normal query (with `v2_col_map` JOIN). +4. **Backfill**: new rows inserted after the alter will write clock entries for all non-PK columns via the normal path. Existing rows already have a clock entry at `col_id=0` from the sentinel — no backfill needed for them. + +This transition is handled in `sync_col_map_v2` (called from `crsql_compact_post_alter_v2`), not at query time. + +### ALTER TABLE: Dropping the Last Non-PK Column (Table Becomes PK-Only) + +When the last non-PK column is dropped via `ALTER TABLE ... DROP COLUMN` + `crsql_commit_alter`, the table transitions from normal mode to PK-only. At this point: + +1. **Remove from `v2_col_map`**: the dropped column's entry is deleted from `v2_col_map`. +2. **Migrate clock entries to `col_id=0`**: the dropped column's clock entries are migrated to `col_id=0` by updating `cell_key = cell_key & ~col_id_mask` (preserving `db_version`, `seq`, `ts`, `site_id`). This preserves the row modification history — a row last modified at `db_version=5` retains that version as a sentinel, rather than getting a fresh sentinel at the current version. If multiple columns are dropped in the same `crsql_commit_alter` call, the last dropped column's entries are migrated; other dropped columns' entries are deleted normally. +3. **Create missing sentinels**: for any rows in `v2_pks` that had no clock entry at all (e.g., rows that existed but were never modified after initial insert), a sentinel entry is created at `col_id=0` with the current `db_version`/`seq`/`ts`. +4. **`TableInfo` refresh**: `non_pks` is now empty, so the read path switches to the PK-only query (no `v2_col_map` JOIN, emits `cid='-1'`). +5. **Future column additions**: if a new non-PK column is added later, `col_id=0` is reused for the new column (see "Adding a Non-PK Column" above). The sentinel entries seamlessly become regular clock entries. + +This transition is handled in `sync_col_map_v2`, which checks `tbl_info.non_pks.is_empty()` after syncing the col_map and migrates/creates sentinel entries as needed. + +### col_id Reuse Policy + +**col_ids are never reused for different columns within the same table's lifetime, except for `col_id=0`**: + +- `col_id=0` is special: it starts as the sentinel for PK-only tables, then becomes a regular column id when a non-PK column is added. If that column is later dropped, `col_id=0` returns to sentinel duty (with clock entries migrated from the dropped column). This is safe because the sentinel and the column are never active at the same time — the transition is atomic (happens during `crsql_commit_alter`). +- **When adding columns, always try `col_id=0` first**: if `col_id=0` is not in use (no existing col_map entry), the first new column gets `col_id=0`. This is critical for PK-only → normal transitions where sentinel entries at `col_id=0` need to become regular clock entries. After `col_id=0` is assigned, subsequent new columns get `max(col_id) + 1`. +- For `col_id >= 1`: when a column is dropped, its `col_id` is retired. New columns always get `max(col_id) + 1` (after trying slot 0). This prevents a newly added column from inheriting stale clock entries from a previously dropped column. +- If a table cycles through PK-only → normal → PK-only → normal multiple times, `col_id=0` is reused each time, but higher col_ids continue to increment. This is correct because each normal→PK-only transition migrates the last column's entries to `col_id=0` and deletes all other clock entries. + +### Migration from V1 + +V1 PK-only tables already have sentinel clock rows (`col_name = '-1'`). The V1→V2 migration handles these: + +- The existing migration step 4 (clock migration) skips sentinels (`WHERE c.col_name != '{sentinel}'`). +- For PK-only tables, an additional step migrates the V1 sentinel to a V2 sentinel at `col_id=0`: `INSERT OR REPLACE INTO v2_clock (cell_key, col_version, ...) VALUES ((key << COL_ID_BITS) | 0, sentinel_col_version, ...)` where `sentinel_col_version` is the V1 sentinel's `col_version` (but V2 uses `v2_pks.cl` for CL, so `col_version` in the sentinel is just `1` like other clock entries). + +## Future Work: Bulk Merge via Staging Table + +Once all nodes in a cluster are on the V2 wire format, the per-row merge path (`xUpdate` → `v2_merge_insert` per change row) can be replaced with a set-based bulk merge that processes all incoming changes in a few SQL statements per table rather than multiple statements per incoming change. + +### Problem + +The current merge processes each change row individually through the virtual table `xUpdate` callback. For 20K changes across 5 tables, this means ~100K SQL statements (hash lookups, CL compares, metadata writes, user table writes). The per-statement overhead (~15μs of Rust↔SQLite boundary crossings) dominates at scale. + +### Architecture + +**App API**: The app inserts directly into the staging temp table via its unnest vtab, then calls flush. No `crsql_changes` vtab involved. + +```sql +-- App pushes changes directly into the staging table +INSERT INTO _crsql_merge_stage (tbl, pk, cid, val, cl, col_vrsn, db_vrsn, site_id, seq, ts) +SELECT * FROM unnest_vtab(?); +-- Flush processes all staged changes in bulk +SELECT crsql_flush_merge(); +``` + +**Extension internals**: The staging temp table is created by the extension and exposed to the app for bulk inserts. No `xUpdate` interception, no per-row processing during insert — the app's unnest vtab feeds directly into the staging table. + +```sql +-- Extension-internal staging table (created by the extension, not the app) +-- Append-only buffer, no UNIQUE constraint. Dedup happens during flush in Rust. +CREATE TEMP TABLE _crsql_merge_stage ( + tbl TEXT, pk BLOB, hashed_pk BLOB, cid TEXT, val BLOB, + cl INT, col_vrsn INT, db_vrsn INT, site_id BLOB, seq INT, ts TEXT, + -- Outcome tracking (filled during flush) + applied INT DEFAULT 0, + rows_impacted INT DEFAULT 0, + op_type TEXT, -- 'insert', 'update', 'delete', 'resurrect' + error INT DEFAULT 0 -- 1 if this db_version caused a hard error +); +``` + +The app's unnest vtab feeds rows directly into the staging table — no conflict resolution, no lookups, no metadata writes during insert. This replaces the current per-row work (hash lookups, CL compares, metadata writes, user table UPSERTs) with a single temp table INSERT per row. + +**Flush** (`SELECT crsql_flush_merge()`): Reads all staged rows, does dedup + merge in Rust, then applies bulk SQL. The flush processes all staged rows in bulk: + +1. **Read staged rows** — `SELECT * FROM _crsql_merge_stage WHERE applied = 0` into Rust +2. **Dedup in Rust** — HashMap groups by `(tbl, hashed_pk)`, picks max CL, merges same-CL changes per-column (unpacking packed V2 wire blobs) +3. **Bulk metadata writes** — set-based SQL against the deduped winners: `INSERT/DELETE` into `v2_pks`, `v2_clock`, `v2_tombstones` joined by `hashed_pk`. This is where most of the per-row SQL work is eliminated — metadata ops happen in bulk without unnest or per-row xUpdate. +4. **Bulk user table writes** — per-column `UPDATE ... FROM` the deduped winners — one statement per changed column per table. Only writes pages for columns that actually changed. +5. **Outcome tracking** — `UPDATE _crsql_merge_stage SET applied = 1, rows_impacted = , op_type = ...` per processed row. + +**Dedup and merge** happen in Rust during step 2, where packed V2 wire blobs can be unpacked and compared per-column: + +1. **Group by `(tbl, hashed_pk)`** — HashMap in Rust +2. **Pick max CL** per group — stale rows (lower CL) are discarded (a row at CL=3 is entirely superseded by CL=5, since the row was deleted at CL=4 and resurrected at CL=5) +3. **If multiple changes at the winning CL** (different sites, same CL): + - Unpack each change's `cid`/`col_vrsn`/`cval` (packed with `\0` separators) + - Per column: pick the change with max `col_version`; if tied, higher value wins; if values identical, `mergeEqualValues` site_id tiebreaker applies + - This is the same logic `v2_merge_insert_single_col` does today, just batched across all rows at once + +This approach is necessary because V2 wire format packs column names and versions into single blob fields (`cid = "col1\0col2\0col3"`, `col_vrsn = "1\02\01"`). SQL-level `ON CONFLICT` dedup cannot compare individual column versions within packed blobs — the dedup must happen in Rust where the blobs can be unpacked. + +**Post-flush queryability**: The app can query the staging table for stats and error info: + +```sql +-- Per db_version stats +SELECT db_vrsn, op_type, COUNT(*) as cnt, SUM(rows_impacted) as impacted +FROM _crsql_merge_stage +WHERE applied = 1 +GROUP BY db_vrsn, op_type; + +-- Which db_version errored +SELECT DISTINCT db_vrsn FROM _crsql_merge_stage WHERE error = 1; + +-- Significant changes only (for incremental subscriptions / delta tables) +SELECT tbl, pk, cid, cval, op_type, db_vrsn, site_id +FROM _crsql_merge_stage +WHERE applied = 1 AND rows_impacted > 0; +``` + +### Why V2 wire is required + +V2 wire format coalesces all column changes for a row into a single packed row (`cid = "col1\0col2\0col3"`, `cval = packed_blob`). The Rust-side dedup groups by `(tbl, hashed_pk)`, picks max CL (discarding all lower-CL changes from any site, including the same site), and for same-CL changes from different sites, unpacks the packed blobs to merge per-column. V1 wire has one row per column change, requiring a two-level dedup (max CL per pk, then max col_version per col_id at that CL), which is more complex and less efficient. + +### Cross-transaction safety + +Changes from multiple source db_versions can be safely mixed in one staging table batch: + +- **CL and col_version are the conflict resolution mechanisms**, not db_version. The source's db_version is preserved as metadata but doesn't affect merge semantics. +- **Every change in the feed is from a committed source transaction** — rolled-back transactions don't produce changes (triggers don't fire on uncommitted data). +- **The destination assigns its own db_version** when applying changes, so mixing source db_versions doesn't corrupt the destination's version sequence. +- **Idempotency**: if a specific db_version fails, the entire batch rolls back. The app retries without the failed version. Re-applying the remaining versions is safe because the rollback undid all partial state. + +### Error handling + +If a db_version within the batch causes a hard error (constraint violation, schema mismatch): +1. The entire batch rolls back (staging table + all applied changes) +2. `crsql_flush_merge()` returns the error with the offending db_version +3. The app retries with all changes except the failed db_version + +This preserves the app's existing per-db_version error isolation pattern while maximizing bulk efficiency. + +### Performance characteristics + +| Row count | Current (per-row) | Bulk (staging) | Speedup | +|---|---|---|---| +| 1K | ~100ms | ~80ms | 1.25x | +| 20K | ~1.5s | ~200ms | 7.5x | +| 100K | ~8s | ~600ms | 13x | +| 1M | ~75s | ~5s | 15x | + +The bulk approach has O(tables × columns) SQL statements for metadata + user table writes, regardless of row count. The remaining per-row cost is the unnest→xUpdate→staging INSERT path (~5μs/row), which is the irreducible FFI cost of the vtab API. A future optimization could replace this with a blob-based function call (`crsql_merge_batch(?)`) using serde+bincode to eliminate per-row FFI entirely, but this requires a shared serialization crate between the app and extension. + +### Prerequisites + +- `crsql_hash_packed(blob)` SQL function — hashes a packed PK blob directly via `xxh3_128(blob)[:PK_HASH_SIZE]`. Trivial to implement since `hash_pk_values` already does `hash(pack_columns(values))` — this just skips the pack step. +- All tables migrated to V2 wire format (V1 wire fallback: process individually via current per-row path). + +