From 8c90340b8f4c9b5cc3906e123da59e8ec78f1098 Mon Sep 17 00:00:00 2001 From: russimicro Date: Sun, 26 Jul 2026 10:47:38 -0500 Subject: [PATCH 01/12] feat(adi): v2 dense leaf with front coding, compound keys, bulk build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reapplies the Russoft ADI v2 rework on top of v1.8.31. Upstream touched ADI only twice since our fork point (+47/-4), so the driver is taken from our tree and those two changes are re-applied on top: - CICHAR case-insensitive collation (9376af9): fold_for_compare_ + has_ci_component_, used by compare_keys_ and the branch descent. - IIndex::file_path() override for the management surface (9539cf3). What the v2 rework brings: a variable-length dense leaf whose entries are front-coded against the previous key (split chosen so BOTH halves stay under the page cap), compound keys, multilevel clear, and a bottom-up bulk build. clear_data() and build_bulk() move up to IIndex as virtuals with working defaults (erase-every-entry / per-record insert), so AdiIndex overrides them the same way CdxIndex does and callers dispatch polymorphically. Suite: 1177/1189, same 12 SQL-parser (7200) failures as the baseline — no regressions. (abi_remote_ordered_prefetch is flaky on its own: 1 of 3 solo runs fails on a byte-volume threshold, with and without this change.) Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 05f004f97fe1a0ac9772ba965ad0bfca46645cb1) (cherry picked from commit fcda3eb1303f20a6f297bdf27b1ae7029441420c) --- src/drivers/adi/adi_index.cpp | 1261 ++++++++++++++++++++++----------- src/drivers/adi/adi_index.h | 109 ++- src/drivers/index_trait.h | 27 + 3 files changed, 971 insertions(+), 426 deletions(-) diff --git a/src/drivers/adi/adi_index.cpp b/src/drivers/adi/adi_index.cpp index dcb9eccc..aa680618 100644 --- a/src/drivers/adi/adi_index.cpp +++ b/src/drivers/adi/adi_index.cpp @@ -52,14 +52,137 @@ void set_u32_be(std::uint8_t* p, std::uint32_t v) noexcept { } // Dense entry recno from a raw byte buffer (not a Page), given 0-based idx. +// v2 leaf entry (esz >= 4: recno[4 LE] + key[klen]) carries a 4-byte recno +// (supports >65535 records). Legacy entries are 2 or 3 bytes (recno 2B / 1B). std::uint32_t dense_recno_from_buf(const std::uint8_t* base, std::uint32_t idx, std::uint32_t esz) noexcept { const std::uint8_t* e = base + idx * esz; + if (esz >= 4) + return static_cast(e[0]) + | (static_cast(e[1]) << 8) + | (static_cast(e[2]) << 16) + | (static_cast(e[3]) << 24); if (esz >= 3) return static_cast(e[0]) | (static_cast(e[1]) << 8); return e[0]; } +// v2 dense-leaf entry key: klen opaque bytes right after the 4-byte recno. +// The full evaluated key lives in the leaf, so navigation/seek never re-read +// the ADT record (and compound/computed keys work without an evaluator). +std::string dense_entry_key_from_buf(const std::uint8_t* base, std::uint32_t idx, + std::uint32_t esz, + std::uint32_t klen) noexcept { + return std::string(reinterpret_cast(base + idx * esz + 4), klen); +} + +// ── Front-coding codec for the v2 dense leaf ───────────────────────────────── +// A v2 dense leaf stores VARIABLE-length entries, each front-coded against the +// PREVIOUS entry to elide the shared key prefix (SAP-style dup/trail), so the +// .adi reaches parity with ADS-SAP (~3x smaller than storing the full key per +// entry). On-disk entry, from ADI_DENSE_ENTRY_START, in key order: +// recno[4 LE] + dup[1] + suffix[(klen - dup) bytes] +// dup = number of leading bytes shared with the previous entry (0 = first) +// suffix = key[dup .. klen] +// Reconstruction: key = prev_key[0..dup] + suffix. The recno stays at offset 0 +// of every entry. dup is a single byte, so the shared prefix is capped at 254; +// a wider key just front-codes less (still correct). + +// Bytes of shared prefix between two klen-length keys, as written by the codec. +std::uint8_t fc_dup(const char* prev, const char* cur, + std::uint32_t klen) noexcept { + const std::uint32_t cap = klen < 254u ? klen : 254u; + std::uint32_t d = 0; + while (d < cap && prev[d] == cur[d]) ++d; + return static_cast(d); +} + +// Encode key-ordered (recno, key) entries into `body` (at most `cap` bytes). +// Each key must already be exactly klen bytes. Returns bytes written, or 0 if +// the set does not fit in cap (the caller then splits). An empty set writes 0. +std::size_t fc_encode_leaf( + const std::vector>& es, + std::uint32_t klen, std::uint8_t* body, std::size_t cap) noexcept { + std::size_t off = 0; + const char* prev = nullptr; + for (const auto& e : es) { + const char* k = e.second.data(); + std::uint8_t dup = prev ? fc_dup(prev, k, klen) : std::uint8_t{0}; + std::size_t suffix_len = static_cast(klen) - dup; + std::size_t need = 5u + suffix_len; + if (off + need > cap) return 0; + body[off] = static_cast( e.first & 0xFFu); + body[off + 1] = static_cast((e.first >> 8) & 0xFFu); + body[off + 2] = static_cast((e.first >> 16) & 0xFFu); + body[off + 3] = static_cast((e.first >> 24) & 0xFFu); + body[off + 4] = dup; + std::memcpy(body + off + 5, k + dup, suffix_len); + off += need; + prev = k; + } + return off; +} + +// Decode `count` front-coded entries from `body` into `out` (recno, full key). +void fc_decode_leaf(const std::uint8_t* body, std::uint16_t count, + std::uint32_t klen, + std::vector>& out) { + out.clear(); + out.reserve(count); // reserve up front: keeps the prev pointer below valid + const std::string* prev = nullptr; + std::size_t off = 0; + for (std::uint16_t i = 0; i < count; ++i) { + std::uint32_t recno = + static_cast(body[off]) + | (static_cast(body[off + 1]) << 8) + | (static_cast(body[off + 2]) << 16) + | (static_cast(body[off + 3]) << 24); + std::uint32_t dup = body[off + 4]; + if (dup > klen) dup = klen; // defensive against a corrupt dup byte + std::size_t suffix_len = static_cast(klen) - dup; + std::string key; + key.reserve(klen); + if (dup && prev) key.assign(*prev, 0, dup); + key.append(reinterpret_cast(body + off + 5), suffix_len); + if (key.size() < klen) key.append(klen - key.size(), ' '); + out.emplace_back(recno, std::move(key)); + prev = &out.back().second; + off += 5u + suffix_len; + } +} + +// Choose a split index in [1, n-1] so BOTH halves front-code within cap, as +// balanced as possible. Returns 0 only when no single-page split exists (one +// entry alone exceeds cap — a degenerate, far-too-wide key). +std::size_t fc_pick_split( + const std::vector>& es, + std::uint32_t klen, std::size_t cap) noexcept { + const std::size_t n = es.size(); + if (n < 2) return 0; + // The first entry of a leaf always costs 5+klen (dup=0); a later entry i + // costs 5 + (klen - dup(i-1,i)). size([lo,hi)) = first_cost + Σ inc(lo+1..hi-1). + const std::size_t first_cost = 5u + klen; + std::vector pref(n, 0); // pref[i] = Σ_{j=1..i} inc(j) + for (std::size_t i = 1; i < n; ++i) { + std::uint8_t dup = fc_dup(es[i - 1].second.data(), + es[i].second.data(), klen); + pref[i] = pref[i - 1] + (5u + (static_cast(klen) - dup)); + } + auto fits = [&](std::size_t lo, std::size_t hi) { + return first_cost + (pref[hi - 1] - pref[lo]) <= cap; + }; + const std::size_t mid = n / 2; + for (std::size_t d = 0; d < n; ++d) { + std::size_t up = mid + d; + if (up >= 1 && up <= n - 1 && fits(0, up) && fits(up, n)) return up; + if (mid >= d) { + std::size_t dn = mid - d; + if (dn >= 1 && dn <= n - 1 && fits(0, dn) && fits(dn, n)) return dn; + } + } + return 0; +} + platform::OpenMode map_open_mode(IndexOpenMode m) noexcept { if (m == IndexOpenMode::ReadOnly) return platform::OpenMode::ReadOnly; return platform::OpenMode::OpenExisting; @@ -134,6 +257,11 @@ std::uint32_t dense_entry_recno(const std::uint8_t* pg, int idx, std::uint32_t entry_sz) noexcept { const std::uint8_t* e = pg + ADI_DENSE_ENTRY_START + static_cast(idx) * entry_sz; + if (entry_sz >= 4) // v2: recno[4 LE] + key[klen] + return static_cast(e[0]) + | (static_cast(e[1]) << 8) + | (static_cast(e[2]) << 16) + | (static_cast(e[3]) << 24); if (entry_sz >= 3) return static_cast(e[0]) | (static_cast(e[1]) << 8); return e[0]; // 2-byte entry: recno in byte 0, byte 1 is key-flags @@ -324,11 +452,83 @@ std::vector parse_fmarker_all(const AdiIndex::Page& pg) { return result; } +// ── v2 (OpenADS-proprietary) per-tag metadata ──────────────────────────────── +// Stored in the per-tag header page (page XX). Gives tag identity by NAME and +// persists the key expression + FOR condition + full key length across reopen +// (the legacy format only stored an F-marker field number). Region lives at +// offsets 40..495, clear of the legacy control bytes (0..23) and footer +// (506/510). Pages without the magic are legacy and fall back to field identity. +constexpr std::uint16_t ADI_V2_MAGIC = 0xAD32; +constexpr std::size_t ADI_V2_OFF_MAGIC = 40; // u16 LE magic +constexpr std::size_t ADI_V2_OFF_KEYLEN = 42; // u16 LE full key length (klen) +constexpr std::size_t ADI_V2_OFF_FLAGS = 44; // u8: bit0 unique, bit1 descending +constexpr std::size_t ADI_V2_OFF_NLEN = 48; // u16 LE len(tag_name) +constexpr std::size_t ADI_V2_OFF_ELEN = 50; // u16 LE len(key_expr) +constexpr std::size_t ADI_V2_OFF_FLEN = 52; // u16 LE len(for_expr) +constexpr std::size_t ADI_V2_OFF_STRS = 64; // tag_name | key_expr | for_expr +constexpr std::size_t ADI_V2_STRS_MAX = 431; // 64..495 inclusive + +struct AdiV2Meta { + bool has = false; + std::string tag_name, key_expr, for_expr; + std::uint16_t key_len = 0; + bool unique = false, descending = false; +}; + +void write_adi_v2_meta(AdiIndex::Page& pg, + const AdiIndex::CreateParams& cp) noexcept { + // Only stamp the v2 region when the caller actually supplied v2 data (the + // ERP / ACE path sets key_len + tag_name). Legacy callers leave it absent so + // the page reads as a pure field-identity legacy tag. + if (cp.key_len == 0 && cp.tag_name.empty()) return; + std::string nm = cp.tag_name, ke = cp.key_expr, fe = cp.for_expr; + if (nm.size() > 63) nm.resize(63); + if (nm.size() + ke.size() + fe.size() > ADI_V2_STRS_MAX) { + if (ke.size() > 254) ke.resize(254); + std::size_t used = nm.size() + ke.size(); + std::size_t room = used < ADI_V2_STRS_MAX ? ADI_V2_STRS_MAX - used : 0; + if (fe.size() > room) fe.resize(room); + } + set_u16_le(pg.data() + ADI_V2_OFF_MAGIC, ADI_V2_MAGIC); + set_u16_le(pg.data() + ADI_V2_OFF_KEYLEN, cp.key_len); + std::uint8_t flags = 0; + if (cp.unique) flags |= 0x01u; + if (cp.descending) flags |= 0x02u; + pg[ADI_V2_OFF_FLAGS] = flags; + set_u16_le(pg.data() + ADI_V2_OFF_NLEN, static_cast(nm.size())); + set_u16_le(pg.data() + ADI_V2_OFF_ELEN, static_cast(ke.size())); + set_u16_le(pg.data() + ADI_V2_OFF_FLEN, static_cast(fe.size())); + std::size_t o = ADI_V2_OFF_STRS; + std::memcpy(pg.data() + o, nm.data(), nm.size()); o += nm.size(); + std::memcpy(pg.data() + o, ke.data(), ke.size()); o += ke.size(); + std::memcpy(pg.data() + o, fe.data(), fe.size()); +} + +AdiV2Meta read_adi_v2_meta(const AdiIndex::Page& pg) noexcept { + AdiV2Meta m; + if (u16_le(pg.data() + ADI_V2_OFF_MAGIC) != ADI_V2_MAGIC) return m; + std::size_t nl = u16_le(pg.data() + ADI_V2_OFF_NLEN); + std::size_t el = u16_le(pg.data() + ADI_V2_OFF_ELEN); + std::size_t fl = u16_le(pg.data() + ADI_V2_OFF_FLEN); + if (nl + el + fl > ADI_V2_STRS_MAX) return m; // corrupt → treat as legacy + m.has = true; + m.key_len = u16_le(pg.data() + ADI_V2_OFF_KEYLEN); + std::uint8_t flags = pg[ADI_V2_OFF_FLAGS]; + m.unique = (flags & 0x01u) != 0; + m.descending = (flags & 0x02u) != 0; + std::size_t o = ADI_V2_OFF_STRS; + m.tag_name.assign(reinterpret_cast(pg.data() + o), nl); o += nl; + m.key_expr.assign(reinterpret_cast(pg.data() + o), el); o += el; + m.for_expr.assign(reinterpret_cast(pg.data() + o), fl); + return m; +} + // One entry in the tag directory scan result. struct TagEntry { std::vector fnums; // 1-based field numbers (≥1 element) std::uint32_t root_pg; bool unique = false; // bit 0 of byte[14] in per-tag header page + AdiV2Meta v2; // v2 metadata (v2.has == false in legacy) }; // Scan tag directory (page 2) and return all tag entries. @@ -362,12 +562,18 @@ scan_tagdir(platform::File& adi_f) { auto fnums = parse_fmarker_all(fmk.value()); if (fnums.empty()) continue; - // Per-tag header is at page xx. Byte[14] bit 0 = unique flag. + // Per-tag header is at page xx. Byte[14] bit 0 = unique flag; the v2 + // region (if present) carries tag name / expr / FOR / klen. bool uniq = false; + AdiV2Meta v2; auto hdr_pg = read_one_page(adi_f, static_cast(xx)); - if (hdr_pg) uniq = (hdr_pg.value()[14] & 0x01u) != 0; + if (hdr_pg) { + uniq = (hdr_pg.value()[14] & 0x01u) != 0; + v2 = read_adi_v2_meta(hdr_pg.value()); + if (v2.has) uniq = v2.unique; + } - tags.push_back({std::move(fnums), root_pg, uniq}); + tags.push_back({std::move(fnums), root_pg, uniq, std::move(v2)}); } return tags; } @@ -389,14 +595,48 @@ util::Result AdiIndex::read_adi_page_(std::uint32_t page_no, // ── AdiIndex::load_dense_leaf_ ─────────────────────────────────────────────── +// Defined later in this file; needed by render_v2_leaf_ below. +void write_empty_dense_leaf_page(AdiIndex::Page& pg, std::uint16_t adt_type, + std::uint16_t fld_length) noexcept; + util::Result AdiIndex::load_dense_leaf_(std::uint32_t page_no) { - if (auto r = read_adi_page_(page_no, cur_page_); !r) return r; + Page pg{}; + if (auto r = read_adi_page_(page_no, pg); !r) return r; + adopt_leaf_page_(page_no, pg); + return {}; +} + +// ── AdiIndex::adopt_leaf_page_ ─────────────────────────────────────────────── + +void AdiIndex::adopt_leaf_page_(std::uint32_t page_no, const Page& pg) { + cur_page_ = pg; cur_pg_ = page_no; - cur_cnt_ = page_count(cur_page_.data()); - cur_lsib_ = page_lsib(cur_page_.data()); - cur_rsib_ = page_rsib(cur_page_.data()); + cur_cnt_ = page_count(pg.data()); + cur_lsib_ = page_lsib(pg.data()); + cur_rsib_ = page_rsib(pg.data()); cur_idx_ = -1; - return {}; + if (key_in_leaf_) + fc_decode_leaf(pg.data() + ADI_DENSE_ENTRY_START, cur_cnt_, + key_total_len_, leaf_entries_); + else + leaf_entries_.clear(); +} + +// ── AdiIndex::render_v2_leaf_ ──────────────────────────────────────────────── + +bool AdiIndex::render_v2_leaf_( + Page& pg, + const std::vector>& ents, + std::uint32_t lsib, std::uint32_t rsib) const { + // Header + sub-header (and a zeroed body so any unused tail stays clean). + write_empty_dense_leaf_page(pg, adt_type_, fld_length_); + set_u16_le(pg.data() + 2, static_cast(ents.size())); + set_u32_le(pg.data() + 4, lsib); + set_u32_le(pg.data() + 8, rsib); + std::size_t n = fc_encode_leaf(ents, key_total_len_, + pg.data() + ADI_DENSE_ENTRY_START, + ADI_PAGE_SIZE - ADI_DENSE_ENTRY_START); + return ents.empty() || n > 0; // n==0 with a non-empty run = page overflow } // ── AdiIndex::refresh_current_ ─────────────────────────────────────────────── @@ -408,6 +648,17 @@ util::Result AdiIndex::refresh_current_() { current_key_.clear(); return {}; } + if (key_in_leaf_) { + // v2: the decoded front-coded leaf is the source of truth. + if (static_cast(cur_idx_) >= leaf_entries_.size()) { + cur_recno_ = 0; + current_key_.clear(); + return {}; + } + cur_recno_ = leaf_entries_[cur_idx_].first; + current_key_ = leaf_entries_[cur_idx_].second; + return {}; + } cur_recno_ = dense_entry_recno(cur_page_.data(), cur_idx_, entry_size_); auto k = key_for_recno_(cur_recno_); if (!k) return k.error(); @@ -415,6 +666,61 @@ util::Result AdiIndex::refresh_current_() { return {}; } +// ── AdiIndex::entry_count ──────────────────────────────────────────────────── + +const std::vector& AdiIndex::ordered_recnos_cached() { + if (pos_cache_valid_) return ordered_recnos_; + ordered_recnos_.clear(); + pos_of_recno_.clear(); + // Descend to the leftmost dense leaf. + Page pg{}; + std::uint32_t cur = root_page_; + for (;;) { + if (!read_adi_page_(cur, pg)) { pos_cache_valid_ = true; return ordered_recnos_; } + if (is_dense_leaf(page_level(pg.data()))) break; + if (page_count(pg.data()) == 0) { pos_cache_valid_ = true; return ordered_recnos_; } + cur = branch_entry_page_(pg.data(), 0); + } + // Walk the dense-leaf chain left-to-right, collecting recnos in key order. + std::uint32_t guard = 0; + while (cur != ADI_INVALID_PAGE) { + if (!read_adi_page_(cur, pg)) break; + std::uint16_t cnt = page_count(pg.data()); + if (key_in_leaf_) { + // v2: front-coded leaf — decode to recover recnos in key order. + std::vector> ents; + fc_decode_leaf(pg.data() + ADI_DENSE_ENTRY_START, cnt, + key_total_len_, ents); + for (const auto& e : ents) { + pos_of_recno_[e.first] = + static_cast(ordered_recnos_.size()); + ordered_recnos_.push_back(e.first); + } + } else { + for (std::uint16_t i = 0; i < cnt; ++i) { + std::uint32_t rn = dense_entry_recno(pg.data(), i, entry_size_); + pos_of_recno_[rn] = + static_cast(ordered_recnos_.size()); + ordered_recnos_.push_back(rn); + } + } + cur = page_rsib(pg.data()); + if (++guard > (1u << 24)) break; // anti-loop on a corrupt rsib chain + } + pos_cache_valid_ = true; + return ordered_recnos_; +} + +std::uint32_t AdiIndex::pos_of_recno_cached(std::uint32_t recno) { + ordered_recnos_cached(); // ensure built + auto it = pos_of_recno_.find(recno); + return it == pos_of_recno_.end() ? 0xFFFFFFFFu : it->second; +} + +util::Result AdiIndex::entry_count() { + return static_cast(ordered_recnos_cached().size()); +} + // ── AdiIndex::branch_entry_page_ ──────────────────────────────────────────── std::uint32_t AdiIndex::branch_entry_page_(const std::uint8_t* pg, @@ -508,12 +814,8 @@ util::Result AdiIndex::navigate_leftmost_() { SeekOutcome o; o.hit = SeekHit::AfterEnd; return o; } if (is_dense_leaf(lv)) { - cur_page_ = pg; - cur_pg_ = cur; - cur_cnt_ = ct; - cur_lsib_ = page_lsib(pg.data()); - cur_rsib_ = page_rsib(pg.data()); - cur_idx_ = 0; + adopt_leaf_page_(cur, pg); + cur_idx_ = 0; if (auto r = refresh_current_(); !r) return r.error(); return make_positioned_(); } @@ -535,12 +837,8 @@ util::Result AdiIndex::navigate_rightmost_() { SeekOutcome o; o.hit = SeekHit::AfterEnd; return o; } if (is_dense_leaf(lv)) { - cur_page_ = pg; - cur_pg_ = cur; - cur_cnt_ = ct; - cur_lsib_ = page_lsib(pg.data()); - cur_rsib_ = page_rsib(pg.data()); - cur_idx_ = static_cast(ct) - 1; + adopt_leaf_page_(cur, pg); + cur_idx_ = static_cast(cur_cnt_) - 1; if (auto r = refresh_current_(); !r) return r.error(); return make_positioned_(); } @@ -560,7 +858,8 @@ util::Result AdiIndex::apply_tag_( const std::vector& fd_lengths, const std::vector& fd_names, std::uint32_t hlen, std::uint32_t rlen, - bool unique) + bool unique, + std::uint32_t v2_key_len) { if (fnums.empty()) return util::Error{5004, 0, "ADI tag has no field numbers", ""}; @@ -611,6 +910,19 @@ util::Result AdiIndex::apply_tag_( char_key_padded_len_ = 0; branch_entry_sz_ = ADI_TREE_ENTRY_SIZE; } + + // v2 leaf: opaque full key stored in the leaf (recno 4B + klen key). The key + // is the ACE-evaluated expression key, ordered by memcmp — reuse the char + // branch machinery (full key + 4-byte child page), drop the field-derived + // geometry. key_for_recno_ is no longer consulted for navigation. + if (v2_key_len > 0) { + key_in_leaf_ = true; + char_key_ = true; + key_total_len_ = v2_key_len; + entry_size_ = 4u + v2_key_len; + char_key_padded_len_ = (v2_key_len + 3u) & ~3u; + branch_entry_sz_ = char_key_padded_len_ + 8u; + } return {}; } @@ -646,8 +958,17 @@ util::Result AdiIndex::open(const std::string& path, IndexOpenMode mode) { lengths.push_back(fd.length); names.push_back(fd.name); } - return apply_tag_(tag.fnums, tag.root_pg, types, offsets, lengths, names, - hlen, rlen, tag.unique); + if (auto r = apply_tag_(tag.fnums, tag.root_pg, types, offsets, lengths, + names, hlen, rlen, tag.unique, + tag.v2.has ? tag.v2.key_len : 0u); !r) + return r; + if (tag.v2.has) { + tag_name_ = tag.v2.tag_name; + tag_expr_ = tag.v2.key_expr; + tag_cond_ = tag.v2.for_expr; + descending_ = tag.v2.descending; + } + return {}; } util::Result AdiIndex::open_named(const std::string& adi_path, @@ -691,13 +1012,28 @@ util::Result AdiIndex::open_named(const std::string& adi_path, } for (const auto& tag : tags.value()) { - if (tag.fnums.empty()) continue; - std::uint8_t fnum = tag.fnums[0]; - if (fnum == 0 || fnum > static_cast(fields.value().size())) continue; - const auto& fd = fields.value()[fnum - 1]; - if (!name_eq(fd.name, field_name)) continue; - return apply_tag_(tag.fnums, tag.root_pg, types, offsets, lengths, names, - hlen, rlen, tag.unique); + // v2: identity by tag NAME (the requested name is the tag name). + // Legacy: resolve the F-marker field's name. + bool match = false; + if (tag.v2.has && !tag.v2.tag_name.empty()) { + match = name_eq(tag.v2.tag_name, field_name); + } else if (!tag.fnums.empty()) { + std::uint8_t fnum = tag.fnums[0]; + if (fnum != 0 && fnum <= static_cast(fields.value().size())) + match = name_eq(fields.value()[fnum - 1].name, field_name); + } + if (!match) continue; + if (auto r = apply_tag_(tag.fnums, tag.root_pg, types, offsets, lengths, + names, hlen, rlen, tag.unique, + tag.v2.has ? tag.v2.key_len : 0u); !r) + return r; + if (tag.v2.has) { + tag_name_ = tag.v2.tag_name; + tag_expr_ = tag.v2.key_expr; + tag_cond_ = tag.v2.for_expr; + descending_ = tag.v2.descending; + } + return {}; } return util::Error{5004, 0, "ADI tag not found: " + field_name, adi_path}; } @@ -725,6 +1061,12 @@ AdiIndex::list_tags(const std::string& adi_path, const std::string& adt_path) { std::vector names; names.reserve(tags.value().size()); for (const auto& tag : tags.value()) { + // v2: identity by tag NAME (lets N tags share a field). Legacy: the + // resolved field name. + if (tag.v2.has && !tag.v2.tag_name.empty()) { + names.push_back(tag.v2.tag_name); + continue; + } if (tag.fnums.empty()) continue; std::uint8_t fnum = tag.fnums[0]; if (fnum == 0 || fnum > static_cast(fields.value().size())) continue; @@ -901,15 +1243,23 @@ util::Result AdiIndex::seek_key(const std::string& key, bool soft) if (auto r = load_dense_leaf_(dense_pg); !r) return r.error(); for (int i = 0; i < static_cast(cur_cnt_); ++i) { - std::uint32_t rno = dense_entry_recno(cur_page_.data(), i, entry_size_); - auto ck = key_for_recno_(rno); - if (!ck) return ck.error(); - int cmp = compare_keys_(ck.value(), nkey); + std::uint32_t rno; + std::string ckv; + if (key_in_leaf_) { + rno = leaf_entries_[i].first; + ckv = leaf_entries_[i].second; + } else { + rno = dense_entry_recno(cur_page_.data(), i, entry_size_); + auto ck = key_for_recno_(rno); + if (!ck) return ck.error(); + ckv = std::move(ck).value(); + } + int cmp = compare_keys_(ckv, nkey); if (cmp > 0) { if (soft) { cur_idx_ = i; cur_recno_ = rno; - current_key_ = std::move(ck).value(); + current_key_ = std::move(ckv); SeekOutcome o; o.hit = SeekHit::AfterKey; o.recno = cur_recno_; @@ -921,7 +1271,7 @@ util::Result AdiIndex::seek_key(const std::string& key, bool soft) if (cmp == 0) { cur_idx_ = i; cur_recno_ = rno; - current_key_ = std::move(ck).value(); + current_key_ = std::move(ckv); return make_positioned_(); } } @@ -979,6 +1329,18 @@ util::Result AdiIndex::alloc_page_() { void AdiIndex::build_dense_entry_(std::uint8_t* dst, std::uint32_t recno, const std::string& ikey) const noexcept { + if (key_in_leaf_) { + // v2: recno[4 LE] + key[key_total_len_] (the full evaluated key). + dst[0] = static_cast( recno & 0xFFu); + dst[1] = static_cast((recno >> 8) & 0xFFu); + dst[2] = static_cast((recno >> 16) & 0xFFu); + dst[3] = static_cast((recno >> 24) & 0xFFu); + std::size_t n = std::min(key_total_len_, ikey.size()); + std::memcpy(dst + 4, ikey.data(), n); + if (n < key_total_len_) + std::memset(dst + 4 + n, ' ', key_total_len_ - n); // pad char key + return; + } if (entry_size_ >= 3) { dst[0] = static_cast(recno); dst[1] = static_cast(recno >> 8); @@ -1234,6 +1596,97 @@ util::Result AdiIndex::insert(std::uint32_t recno, std::uint16_t leaf_lv = page_level(pg.data()); std::uint16_t leaf_cnt = page_count(pg.data()); + + // ── v2 front-coded leaf: decode → ordered insert → re-encode (or split) ── + if (key_in_leaf_) { + std::vector> ents; + fc_decode_leaf(pg.data() + ADI_DENSE_ENTRY_START, leaf_cnt, + key_total_len_, ents); + + // Insertion position by (key, recno) — the same order navigation uses. + std::size_t pos = 0; + while (pos < ents.size()) { + int cmp = compare_keys_(ents[pos].second, ikey); + if (cmp < 0 || (cmp == 0 && ents[pos].first < recno)) ++pos; + else break; + } + ents.insert(ents.begin() + static_cast(pos), {recno, ikey}); + + const std::uint32_t orig_lsib = page_lsib(pg.data()); + const std::uint32_t orig_rsib = page_rsib(pg.data()); + + // Fits in this page? Re-encode in place. + Page np{}; + if (render_v2_leaf_(np, ents, orig_lsib, orig_rsib)) { + if (auto r = write_adi_page_(cur, np); !r) return r; + if (cur_pg_ == cur) { + cur_page_ = np; + leaf_entries_ = std::move(ents); + cur_cnt_ = static_cast(leaf_entries_.size()); + if (cur_idx_ >= static_cast(pos)) ++cur_idx_; + } + return {}; + } + + // Overflow → split the run into two front-coded leaves. + const std::size_t cap = ADI_PAGE_SIZE - ADI_DENSE_ENTRY_START; + std::size_t mid = fc_pick_split(ents, key_total_len_, cap); + if (mid == 0) + return util::Error{5000, 0, "ADI v2 leaf: key too wide to split", ""}; + std::vector> + left(ents.begin(), ents.begin() + static_cast(mid)), + right(ents.begin() + static_cast(mid), ents.end()); + std::string left_max = left.back().second; + std::string right_max = right.back().second; + + if (path.empty()) { + // Root is the dense leaf: two fresh pages, root becomes a branch. + // Allocate+write left BEFORE allocating right (alloc grows the file). + auto lp = alloc_page_(); if (!lp) return lp.error(); + std::uint32_t left_pg = lp.value(); + Page lpg{}; + if (!render_v2_leaf_(lpg, left, orig_lsib, ADI_INVALID_PAGE)) + return util::Error{5000, 0, "ADI v2 split: left overflow", ""}; + if (auto r = write_adi_page_(left_pg, lpg); !r) return r; + auto rp = alloc_page_(); if (!rp) return rp.error(); + std::uint32_t right_pg = rp.value(); + set_u32_le(lpg.data() + 8, right_pg); // patch left.rsib + if (auto r = write_adi_page_(left_pg, lpg); !r) return r; + Page rpg{}; + if (!render_v2_leaf_(rpg, right, left_pg, orig_rsib)) + return util::Error{5000, 0, "ADI v2 split: right overflow", ""}; + if (auto r = write_adi_page_(right_pg, rpg); !r) return r; + if (orig_rsib != ADI_INVALID_PAGE) { + Page rsib{}; + if (auto r = read_adi_page_(orig_rsib, rsib); !r) return r; + set_u32_le(rsib.data() + 4, right_pg); + if (auto r = write_adi_page_(orig_rsib, rsib); !r) return r; + } + cur_pg_ = ADI_INVALID_PAGE; cur_idx_ = -1; cur_cnt_ = 0; + return promote_split_(path, left_pg, left_max, right_pg, right_max); + } + + // Non-root: left half stays in `cur`, right half goes to a new page. + auto rp = alloc_page_(); if (!rp) return rp.error(); + std::uint32_t right_pg = rp.value(); + Page lpg{}; + if (!render_v2_leaf_(lpg, left, orig_lsib, right_pg)) + return util::Error{5000, 0, "ADI v2 split: left overflow", ""}; + if (auto r = write_adi_page_(cur, lpg); !r) return r; + Page rpg{}; + if (!render_v2_leaf_(rpg, right, cur, orig_rsib)) + return util::Error{5000, 0, "ADI v2 split: right overflow", ""}; + if (auto r = write_adi_page_(right_pg, rpg); !r) return r; + if (orig_rsib != ADI_INVALID_PAGE) { + Page rsib{}; + if (auto r = read_adi_page_(orig_rsib, rsib); !r) return r; + set_u32_le(rsib.data() + 4, right_pg); + if (auto r = write_adi_page_(orig_rsib, rsib); !r) return r; + } + cur_pg_ = ADI_INVALID_PAGE; cur_idx_ = -1; cur_cnt_ = 0; + return promote_split_(path, cur, left_max, right_pg, right_max); + } + const std::uint32_t max_ents = (ADI_PAGE_SIZE - ADI_DENSE_ENTRY_START) / entry_size_; @@ -1244,9 +1697,17 @@ util::Result AdiIndex::insert(std::uint32_t recno, while (lo < hi) { int mid = (lo + hi) / 2; std::uint32_t mrec = dense_entry_recno(pg.data(), mid, entry_size_); - auto mk = key_for_recno_(mrec); - if (!mk) return mk.error(); - int cmp = compare_keys_(mk.value(), ikey); + std::string mkv; + if (key_in_leaf_) { + mkv = dense_entry_key_from_buf(pg.data() + ADI_DENSE_ENTRY_START, + static_cast(mid), + entry_size_, key_total_len_); + } else { + auto mk = key_for_recno_(mrec); + if (!mk) return mk.error(); + mkv = std::move(mk).value(); + } + int cmp = compare_keys_(mkv, ikey); if (cmp < 0 || (cmp == 0 && mrec < recno)) lo = mid + 1; else hi = mid; } @@ -1339,15 +1800,25 @@ util::Result AdiIndex::insert(std::uint32_t recno, cur_pg_ = ADI_INVALID_PAGE; cur_idx_ = -1; cur_cnt_ = 0; // Get max keys then rewrite root as branch. - auto left_max = key_for_recno_(dense_recno_from_buf( - combo.data(), lft_cnt - 1, entry_size_)); - if (!left_max) return left_max.error(); - auto right_max = key_for_recno_(dense_recno_from_buf( - combo.data(), total - 1, entry_size_)); - if (!right_max) return right_max.error(); + std::string left_max, right_max; + if (key_in_leaf_) { + left_max = dense_entry_key_from_buf(combo.data(), lft_cnt - 1, + entry_size_, key_total_len_); + right_max = dense_entry_key_from_buf(combo.data(), total - 1, + entry_size_, key_total_len_); + } else { + auto lm = key_for_recno_(dense_recno_from_buf( + combo.data(), lft_cnt - 1, entry_size_)); + if (!lm) return lm.error(); + auto rm = key_for_recno_(dense_recno_from_buf( + combo.data(), total - 1, entry_size_)); + if (!rm) return rm.error(); + left_max = std::move(lm).value(); + right_max = std::move(rm).value(); + } - return promote_split_(path, left_pg, left_max.value(), - right_pg, right_max.value()); + return promote_split_(path, left_pg, left_max, + right_pg, right_max); } // Non-root split: left half stays in cur, right half goes to new page. @@ -1393,15 +1864,25 @@ util::Result AdiIndex::insert(std::uint32_t recno, } } - auto left_max = key_for_recno_(dense_recno_from_buf( - combo.data(), lft_cnt - 1, entry_size_)); - if (!left_max) return left_max.error(); - auto right_max = key_for_recno_(dense_recno_from_buf( - combo.data(), total - 1, entry_size_)); - if (!right_max) return right_max.error(); + std::string left_max, right_max; + if (key_in_leaf_) { + left_max = dense_entry_key_from_buf(combo.data(), lft_cnt - 1, + entry_size_, key_total_len_); + right_max = dense_entry_key_from_buf(combo.data(), total - 1, + entry_size_, key_total_len_); + } else { + auto lm = key_for_recno_(dense_recno_from_buf( + combo.data(), lft_cnt - 1, entry_size_)); + if (!lm) return lm.error(); + auto rm = key_for_recno_(dense_recno_from_buf( + combo.data(), total - 1, entry_size_)); + if (!rm) return rm.error(); + left_max = std::move(lm).value(); + right_max = std::move(rm).value(); + } - return promote_split_(path, cur, left_max.value(), - right_pg, right_max.value()); + return promote_split_(path, cur, left_max, + right_pg, right_max); } // ── AdiIndex::erase ────────────────────────────────────────────────────────── @@ -1422,6 +1903,8 @@ util::Result AdiIndex::erase(std::uint32_t recno, const std::string& key) ikey.resize(8, '\0'); } + if (key_in_leaf_) return erase_v2_(recno, ikey); + // Seek to the correct dense leaf (soft=true: positions at or after key). auto sk = seek_key(ikey, /*soft=*/true); if (!sk) return sk.error(); @@ -1489,6 +1972,78 @@ util::Result AdiIndex::erase(std::uint32_t recno, const std::string& key) return util::Error{5044, 0, "ADI: key not found for erase", ""}; } +// ── AdiIndex::erase_v2_ ────────────────────────────────────────────────────── + +util::Result AdiIndex::erase_v2_(std::uint32_t recno, + const std::string& ikey) { + // Position at/after the key; seek_key decodes the owning leaf into + // leaf_entries_ and leaves cur_idx_ on the first entry >= ikey. + auto sk = seek_key(ikey, /*soft=*/true); + if (!sk) return sk.error(); + if (sk.value().hit == SeekHit::AfterEnd || !sk.value().positioned) + return util::Error{5044, 0, "ADI: key not found for erase", ""}; + + for (;;) { + if (cur_pg_ == ADI_INVALID_PAGE) break; + for (std::size_t i = static_cast(cur_idx_ < 0 ? 0 : cur_idx_); + i < leaf_entries_.size(); ++i) { + int cmp = compare_keys_(leaf_entries_[i].second, ikey); + if (cmp > 0) + return util::Error{5044, 0, "ADI: key not found for erase", ""}; + if (cmp == 0 && leaf_entries_[i].first == recno) { + const std::uint32_t write_pg = cur_pg_; + const std::uint32_t lsib = cur_lsib_; + const std::uint32_t rsib = cur_rsib_; + std::vector> ents = + leaf_entries_; + ents.erase(ents.begin() + static_cast(i)); + + if (ents.empty()) { + // Bypass the now-empty leaf in the sibling chain, then write + // it back empty (abandoned in place, like the legacy erase). + if (lsib != ADI_INVALID_PAGE) { + Page lp{}; + if (auto r = read_adi_page_(lsib, lp); !r) return r; + set_u32_le(lp.data() + 8, rsib); + if (auto r = write_adi_page_(lsib, lp); !r) return r; + } + if (rsib != ADI_INVALID_PAGE) { + Page rp{}; + if (auto r = read_adi_page_(rsib, rp); !r) return r; + set_u32_le(rp.data() + 4, lsib); + if (auto r = write_adi_page_(rsib, rp); !r) return r; + } + Page ep{}; + render_v2_leaf_(ep, ents, lsib, rsib); // count=0 + if (auto r = write_adi_page_(write_pg, ep); !r) return r; + cur_pg_ = ADI_INVALID_PAGE; + cur_idx_ = -1; + cur_cnt_ = 0; + leaf_entries_.clear(); + return {}; + } + + Page np{}; + if (!render_v2_leaf_(np, ents, lsib, rsib)) + return util::Error{5000, 0, "ADI v2 erase: render overflow", ""}; + if (auto r = write_adi_page_(write_pg, np); !r) return r; + cur_page_ = np; + leaf_entries_ = std::move(ents); + cur_cnt_ = static_cast(leaf_entries_.size()); + if (cur_idx_ > static_cast(i)) --cur_idx_; + if (cur_idx_ >= static_cast(cur_cnt_)) + cur_idx_ = static_cast(cur_cnt_) - 1; + return {}; + } + } + // Key may continue on the right sibling. + if (cur_rsib_ == ADI_INVALID_PAGE) break; + if (auto r = load_dense_leaf_(cur_rsib_); !r) return r.error(); + cur_idx_ = 0; + } + return util::Error{5044, 0, "ADI: key not found for erase", ""}; +} + // ── AdiIndex::flush ────────────────────────────────────────────────────────── util::Result AdiIndex::flush() { @@ -1574,6 +2129,9 @@ void write_adi_per_tag_header_page(AdiIndex::Page& pg, pg[17] = 0x04; pg[506] = 0x01; pg[510] = 0x03; + // v2 metadata (tag name / key expr / FOR / klen). Additive: legacy readers + // ignore offsets 40..495; v2 readers prefer it over the F-marker identity. + write_adi_v2_meta(pg, cp); } void write_fmarker_page(AdiIndex::Page& pg, std::uint8_t field_num) noexcept { @@ -1627,6 +2185,27 @@ util::Result AdiIndex::create(const std::string& adi_path, if (params.adt_hdr_len < 400 || params.adt_rec_len == 0) return util::Error{5004, 0, "ADI create: invalid ADT layout", ""}; + // Ensure the parent directory for the .ADI exists. In some PRG contexts + // (different cPatTem, temp copies, or path construction in _Indexar), + // the index bag path may point to a dir that wasn't explicitly created. + // This makes creation more robust (real ADS would also need the dir). + { + namespace fs = std::filesystem; + fs::path ap(adi_path); + fs::path par = ap.parent_path(); + if (!par.empty()) { + std::error_code ec; + fs::create_directories(par, ec); + // ignore ec; if it fails the subsequent open will report it + } + } + + // Remove any existing file (stale from previous failed attempt may be locked or partial). + { + std::error_code ec; + std::filesystem::remove(adi_path, ec); + } + auto fres = platform::File::open(adi_path, platform::OpenMode::CreateRW); if (!fres) return fres.error(); platform::File file = std::move(fres).value(); @@ -1673,38 +2252,61 @@ util::Result AdiIndex::create(const std::string& adi_path, // caller-supplied table path when present, else the structural default. std::string adt_p = params.adt_path.empty() ? adt_path_for(adi_path) : params.adt_path; - auto fa = platform::File::open(adt_p, platform::OpenMode::ReadOnly); - if (!fa) return fa.error(); - ix.adt_file_ = std::move(fa).value(); - - std::vector types(1, params.adt_type); - std::vector offsets(1, 0); - std::vector lengths(1, params.fld_length); - std::vector names(1, params.field_name); - std::uint32_t hlen = params.adt_hdr_len, rlen = params.adt_rec_len; - auto fields = read_adt_fields(ix.adt_file_, hlen, rlen); - if (!fields) return fields.error(); - types.clear(); - offsets.clear(); - lengths.clear(); - names.clear(); - for (const auto& fd : fields.value()) { + auto fa = platform::File::open(adt_p, platform::OpenMode::ReadOnly); + std::vector fields_vec; + if (fa) { + ix.adt_file_ = std::move(fa).value(); + auto fields = read_adt_fields(ix.adt_file_, hlen, rlen); + if (fields) { + fields_vec = std::move(fields).value(); + } + } + if (fields_vec.empty()) { + // Fallback when re-opening the ADT data file fails (e.g. share/lock + // issues with .DAT + ADS_ADT, or path casing). Use the info from + // CreateParams (populated from the already-open Table). + AdtFieldDesc fd; + fd.type = params.adt_type; + fd.offset = params.record_offset; + fd.length = params.fld_length; + fd.name = params.field_name; + fields_vec.push_back(fd); + } + + std::vector types; + std::vector offsets; + std::vector lengths; + std::vector names; + for (const auto& fd : fields_vec) { types.push_back(fd.type); offsets.push_back(fd.offset); lengths.push_back(fd.length); names.push_back(fd.name); } + // Always try to have adt_file_ open for later key_for_recno_ during insert and navigation. + if (!fa) { + auto fa2 = platform::File::open(adt_p, platform::OpenMode::ReadOnly); + if (fa2) { + ix.adt_file_ = std::move(fa2).value(); + } + } + std::vector fnums{params.field_num}; constexpr std::uint32_t kRootPage = 5; if (auto r = ix.apply_tag_(fnums, kRootPage, types, offsets, lengths, names, - params.adt_hdr_len, params.adt_rec_len, - params.unique); + hlen, rlen, + params.unique, params.key_len); !r) { return r.error(); } + // v2 identity / expression in memory (matches what was persisted on disk). + if (!params.tag_name.empty()) ix.tag_name_ = params.tag_name; + ix.tag_expr_ = params.key_expr; + ix.tag_cond_ = params.for_expr; + ix.descending_ = params.descending; return ix; } @@ -1744,21 +2346,25 @@ util::Result AdiIndex::add_tag(const std::string& adi_path, if (!fres) return fres.error(); platform::File file = std::move(fres).value(); - auto existing = list_tags(adi_path); + // Dedup by TAG NAME in v2 (lets N tags share a field, e.g. ORD1/ORD3/ORD4 + // all over field 0); legacy callers without a tag_name dedup by field name. + const std::string& dedup_key = + params.tag_name.empty() ? params.field_name : params.tag_name; + auto existing = list_tags(adi_path, params.adt_path); if (!existing) return existing.error(); for (const auto& tn : existing.value()) { - if (tn.size() != params.field_name.size()) continue; + if (tn.size() != dedup_key.size()) continue; bool eq = true; for (std::size_t i = 0; i < tn.size(); ++i) { if (std::tolower(static_cast(tn[i])) != - std::tolower(static_cast(params.field_name[i]))) { + std::tolower(static_cast(dedup_key[i]))) { eq = false; break; } } if (eq) { return util::Error{5044, 0, - "ADI already has a tag for field: " + params.field_name, ""}; + "ADI already has a tag: " + dedup_key, ""}; } } @@ -1860,358 +2466,217 @@ util::Result AdiIndex::add_tag(const std::string& adi_path, std::vector fnums{params.field_num}; if (auto r = ix.apply_tag_(fnums, root_pg, types, offsets, lengths, names, - hlen, rlen, params.unique); + hlen, rlen, params.unique, params.key_len); !r) { return r.error(); } + if (!params.tag_name.empty()) ix.tag_name_ = params.tag_name; + ix.tag_expr_ = params.key_expr; + ix.tag_cond_ = params.for_expr; + ix.descending_ = params.descending; return ix; } -// ── AdiIndex::clear_data ───────────────────────────────────────────────────── +// ── AdiIndex::build_bulk ───────────────────────────────────────────────────── -util::Result AdiIndex::clear_data() { +util::Result AdiIndex::build_bulk( + std::vector> keys) { if (mode_ == IndexOpenMode::ReadOnly) return util::Error{5000, 0, "ADI index is read-only", ""}; - Page pg{}; - if (auto r = read_adi_page_(root_page_, pg); !r) return r; - if (!is_dense_leaf(page_level(pg.data()))) - return util::Error{5000, 0, "ADI clear_data: root is not a dense leaf", ""}; - - set_u16_le(pg.data() + 2, 0); - cur_pg_ = root_page_; - cur_page_ = pg; - cur_cnt_ = 0; - cur_idx_ = -1; - cur_lsib_ = page_lsib(pg.data()); - cur_rsib_ = page_rsib(pg.data()); - cur_recno_ = 0; - current_key_.clear(); - return write_adi_page_(root_page_, pg); -} - -// ── ADI creation helpers ────────────────────────────────────────────────────── - -namespace { - -// Case-insensitive string comparison helper -bool ci_eq(const std::string& a, const std::string& b) { - if (a.size() != b.size()) return false; - for (std::size_t i = 0; i < a.size(); ++i) { - if (std::tolower(static_cast(a[i])) != - std::tolower(static_cast(b[i]))) return false; - } - return true; -} - -// Split a comma-separated expression into individual trimmed column names. -std::vector split_expr(const std::string& expr) { - std::vector parts; - std::size_t start = 0; - for (std::size_t i = 0; i <= expr.size(); ++i) { - if (i == expr.size() || expr[i] == ',') { - std::string s = expr.substr(start, i - start); - // trim whitespace - while (!s.empty() && std::isspace(static_cast(s.front()))) - s.erase(s.begin()); - while (!s.empty() && std::isspace(static_cast(s.back()))) - s.pop_back(); - if (!s.empty()) parts.push_back(std::move(s)); - start = i + 1; - } + if (!key_in_leaf_) { + // Legacy field-derived tag has no in-leaf key to bulk-pack; fall back to + // the per-record path (the empty tree was already prepared by the caller). + for (auto& kv : keys) + if (auto e = insert(kv.second, kv.first); !e) return e.error(); + return {}; } - return parts; -} -// Write one 512-byte ADI page. -util::Result write_page(platform::File& f, std::uint32_t page_no, - const AdiIndex::Page& pg) { - auto r = f.write_at(static_cast(page_no) * ADI_PAGE_SIZE, - pg.data(), pg.size()); - if (!r) return r.error(); - if (r.value() != ADI_PAGE_SIZE) - return util::Error{5000, 0, "short ADI page write in create", ""}; - return {}; -} + const std::uint32_t klen = key_total_len_; + // Sort by (key, recno): opaque memcmp on klen bytes, recno tie-break — the + // SAME order the per-record insert produces, so navigation is identical. + auto norm = [klen](const std::string& s) { + std::string k = s; + if (k.size() < klen) k.append(klen - k.size(), ' '); + else k.resize(klen); + return k; + }; + for (auto& kv : keys) kv.first = norm(kv.first); + std::sort(keys.begin(), keys.end(), + [](const std::pair& a, + const std::pair& b) { + if (a.first != b.first) return a.first < b.first; + return a.second < b.second; + }); + + // Seed the position cache directly from the sorted set (no re-walk needed). + invalidate_pos_cache(); + ordered_recnos_.reserve(keys.size()); + for (const auto& kv : keys) { + pos_of_recno_[kv.second] = static_cast(ordered_recnos_.size()); + ordered_recnos_.push_back(kv.second); + } + pos_cache_valid_ = true; -// Build the F-marker string for a list of 1-based field numbers. -std::string build_fmarker(const std::vector& fnums) { - std::string s; - for (std::size_t i = 0; i < fnums.size(); ++i) { - if (i > 0) s += ";F"; - else s += "F"; - s += std::to_string(fnums[i]); - } - return s; -} - -// Write the 3 pages for one ADI tag (per-tag header, F-marker, empty root leaf) -// starting at page hdr_pg. Returns nothing. -util::Result write_tag_pages(platform::File& f, - std::uint32_t hdr_pg, - const std::vector& fnums, - bool unique, - bool char_key) { - // Per-tag header page - AdiIndex::Page hdr{}; - hdr[14] = unique ? 0x01u : 0x00u; - if (auto r = write_page(f, hdr_pg, hdr); !r) return r; - - // F-marker page - AdiIndex::Page fmk{}; - std::string fm = build_fmarker(fnums); - std::memcpy(fmk.data(), fm.data(), std::min(fm.size(), static_cast(ADI_PAGE_SIZE - 1u))); - if (auto r = write_page(f, hdr_pg + 1, fmk); !r) return r; - - // Empty root dense leaf - AdiIndex::Page root{}; - std::uint16_t lv = char_key ? ADI_LVL_DENSE2 : ADI_LVL_DENSE; - set_u16_le(root.data(), lv); - set_u16_le(root.data() + 2, 0); - set_u32_le(root.data() + 4, ADI_INVALID_PAGE); - set_u32_le(root.data() + 8, ADI_INVALID_PAGE); - if (auto r = write_page(f, hdr_pg + 2, root); !r) return r; + if (keys.empty()) return clear_data(); - return {}; -} + const std::size_t cap = ADI_PAGE_SIZE - ADI_DENSE_ENTRY_START; -// Resolve expression (comma-separated column names) against ADT field list. -// Returns 1-based field numbers. -util::Result> -resolve_fnums(const std::vector& fields, - const std::string& expression) { - auto names = split_expr(expression); - if (names.empty()) - return util::Error{7200, 0, "empty index expression", expression}; - - std::vector fnums; - for (const auto& name : names) { - bool found = false; - for (std::uint32_t i = 0; i < fields.size(); ++i) { - if (ci_eq(fields[i].name, name)) { - fnums.push_back(static_cast(i + 1)); - found = true; - break; + // Partition the sorted keys into front-coded leaf runs: greedily extend a + // run while its encoded size stays within one page (always ≥1 entry/leaf). + // The per-leaf entry count is now VARIABLE — compression decides how many + // keys fit, so we can't slice by a fixed max_leaf any more. + std::vector> runs; // [lo, hi) + { + const std::size_t n = keys.size(); + std::size_t lo = 0; + while (lo < n) { + std::size_t sz = 5u + klen; // first entry of a leaf: dup=0 + std::size_t hi = lo + 1; + while (hi < n) { + std::uint8_t dup = fc_dup(keys[hi - 1].first.data(), + keys[hi].first.data(), klen); + std::size_t add = 5u + (static_cast(klen) - dup); + if (sz + add > cap) break; + sz += add; + ++hi; } + runs.push_back({lo, hi}); + lo = hi; } - if (!found) - return util::Error{7200, 0, "column not found in ADT: " + name, expression}; } - return fnums; -} - -} // anonymous namespace -// ── AdiIndex::create ───────────────────────────────────────────────────────── -// Creates a new .adi file with one tag. - -// static -util::Result -AdiIndex::create(const std::string& adi_path, - const std::string& adt_path, - const std::string& expression, - bool unique) { - // Open ADT and read field descriptors - auto fa = platform::File::open(adt_path, platform::OpenMode::ReadOnly); - if (!fa) return fa.error(); - platform::File adt_f = std::move(fa).value(); - std::uint32_t hlen = 0, rlen = 0; - auto fields_r = read_adt_fields(adt_f, hlen, rlen); - if (!fields_r) return fields_r.error(); - const auto& fields = fields_r.value(); - - // Resolve expression to field numbers - auto fnums_r = resolve_fnums(fields, expression); - if (!fnums_r) return fnums_r.error(); - const auto& fnums = fnums_r.value(); - - // Determine key type from first field - bool char_key = (fields[fnums[0] - 1].type == ADT_TYPE_CICHAR || - fields[fnums[0] - 1].type == ADT_TYPE_CHAR); - - // Create new ADI file - auto fi = platform::File::open(adi_path, platform::OpenMode::CreateRW); - if (!fi) return fi.error(); - platform::File adi_f = std::move(fi).value(); - - // Pages 0-1: zeros (file header placeholder) - AdiIndex::Page zero{}; - if (auto r = write_page(adi_f, 0, zero); !r) return r.error(); - if (auto r = write_page(adi_f, 1, zero); !r) return r.error(); - - // Page 2: tag directory — 1 tag, xx=3 (per-tag header at page 3) - AdiIndex::Page tagdir{}; - set_u16_le(tagdir.data(), ADI_LVL_TAGDIR); // level = 3 - set_u16_le(tagdir.data() + 2, 1); // count = 1 - set_u32_le(tagdir.data() + 4, ADI_INVALID_PAGE); // lsib - set_u32_le(tagdir.data() + 8, ADI_INVALID_PAGE); // rsib - tagdir[ADI_TAGDIR_ENTRY_START] = 3; // xx = 3 → hdr at pg 3 - if (auto r = write_page(adi_f, 2, tagdir); !r) return r.error(); - - // Pages 3-5: per-tag header, F-marker, empty root leaf - if (auto r = write_tag_pages(adi_f, 3, fnums, unique, char_key); !r) - return r.error(); - - if (auto s = adi_f.sync(); !s) return s.error(); - - // Build and return the AdiIndex - AdiIndex idx; - idx.mode_ = IndexOpenMode::Shared; - idx.adi_file_ = std::move(adi_f); - idx.adt_file_ = std::move(adt_f); - idx.adi_path_ = adi_path; + auto write_leaf_run = [&](std::uint32_t page_no, std::size_t lo, std::size_t hi, + std::uint32_t lsib, std::uint32_t rsib) + -> util::Result { + std::vector> ents; + ents.reserve(hi - lo); + for (std::size_t i = lo; i < hi; ++i) + ents.emplace_back(keys[i].second, keys[i].first); + Page pg{}; + if (!render_v2_leaf_(pg, ents, lsib, rsib)) + return util::Error{5000, 0, "ADI build_bulk: leaf run overflow", ""}; + if (auto w = write_adi_page_(page_no, pg); !w) return w.error(); + return keys[hi - 1].first; // max key of this leaf (already klen padded) + }; - std::vector types, offsets, lengths; - std::vector names; - for (const auto& fd : fields) { - types.push_back(fd.type); - offsets.push_back(fd.offset); - lengths.push_back(fd.length); - names.push_back(fd.name); + // Single dense leaf fits in the root. + if (runs.size() == 1) { + auto mk = write_leaf_run(root_page_, 0, keys.size(), + ADI_INVALID_PAGE, ADI_INVALID_PAGE); + if (!mk) return mk.error(); + cur_pg_ = ADI_INVALID_PAGE; cur_idx_ = -1; + return {}; } - if (auto r = idx.apply_tag_(fnums, 5, types, offsets, lengths, names, - hlen, rlen, unique); !r) - return r.error(); - - return idx; -} - -// ── AdiIndex::add_tag ──────────────────────────────────────────────────────── -// Adds a new tag to an existing .adi file. - -// static -util::Result -AdiIndex::add_tag(const std::string& adi_path, - const std::string& adt_path, - const std::string& expression, - bool unique) { - // Open ADT and read field descriptors - auto fa = platform::File::open(adt_path, platform::OpenMode::ReadOnly); - if (!fa) return fa.error(); - platform::File adt_f = std::move(fa).value(); - std::uint32_t hlen = 0, rlen = 0; - auto fields_r = read_adt_fields(adt_f, hlen, rlen); - if (!fields_r) return fields_r.error(); - const auto& fields = fields_r.value(); - - // Resolve expression to field numbers - auto fnums_r = resolve_fnums(fields, expression); - if (!fnums_r) return fnums_r.error(); - const auto& fnums = fnums_r.value(); - - bool char_key = (fields[fnums[0] - 1].type == ADT_TYPE_CICHAR || - fields[fnums[0] - 1].type == ADT_TYPE_CHAR); - // Open existing ADI file for read+write - auto fi = platform::File::open(adi_path, platform::OpenMode::OpenExisting); - if (!fi) return fi.error(); - platform::File adi_f = std::move(fi).value(); - - // Read tag directory (page 2) to find current tag count - AdiIndex::Page tagdir{}; + // Multi-level: build leaves in NEW pages (linked), then branch levels, with + // the TOP node written into root_page_ (the tag-directory-derived root, so a + // reopen finds it without a stored pointer). + struct Node { std::string max_key; std::uint32_t page; }; + std::vector level; { - auto got = adi_f.read_at(2 * ADI_PAGE_SIZE, tagdir.data(), tagdir.size()); - if (!got || got.value() < ADI_PAGE_SIZE) - return util::Error{6106, 0, "can't read ADI tag directory for add_tag", adi_path}; - } - std::uint16_t cur_count = u16_le(tagdir.data() + 2); - - // Each tag uses 3 pages (header, fmarker, root). After 6 pages of - // prefix (pages 0-2 = 3 header + 3 for first tag), subsequent tags - // start at page 3 + cur_count * 3. - std::uint32_t new_hdr_pg = 3u + static_cast(cur_count) * 3u; - std::uint32_t new_root_pg = new_hdr_pg + 2u; - if (new_hdr_pg > 255u) - return util::Error{7200, 0, "ADI tag count exceeds capacity", adi_path}; - - // Write new tag pages at end of file - if (auto r = write_tag_pages(adi_f, new_hdr_pg, fnums, unique, char_key); !r) - return r.error(); - - // Update tag directory: increment count, add entry - std::size_t entry_off = ADI_TAGDIR_ENTRY_START - + static_cast(cur_count) * ADI_TAGDIR_ENTRY_SIZE; - if (entry_off + 1 < ADI_PAGE_SIZE) { - tagdir[entry_off] = static_cast(new_hdr_pg); - } - set_u16_le(tagdir.data() + 2, cur_count + 1); - { - auto w = adi_f.write_at(2 * ADI_PAGE_SIZE, tagdir.data(), tagdir.size()); - if (!w) return w.error(); + const std::size_t nleaves = runs.size(); + std::vector pages(nleaves); + for (std::size_t i = 0; i < nleaves; ++i) { + auto p = alloc_page_(); if (!p) return p.error(); + pages[i] = p.value(); + } + level.reserve(nleaves); + for (std::size_t i = 0; i < nleaves; ++i) { + std::uint32_t lsib = (i == 0) ? ADI_INVALID_PAGE : pages[i - 1]; + std::uint32_t rsib = (i + 1 < nleaves) ? pages[i + 1] : ADI_INVALID_PAGE; + auto mk = write_leaf_run(pages[i], runs[i].first, runs[i].second, + lsib, rsib); + if (!mk) return mk.error(); + level.push_back({std::move(mk).value(), pages[i]}); + } } - if (auto s = adi_f.sync(); !s) return s.error(); + const std::uint32_t max_branch = + (ADI_PAGE_SIZE - ADI_TREE_ENTRY_START) / branch_entry_sz_; + if (max_branch < 2) + return util::Error{5000, 0, "ADI build_bulk: branch fanout < 2 (key too wide)", ""}; - // Build and return the AdiIndex - AdiIndex idx; - idx.mode_ = IndexOpenMode::Shared; - idx.adi_file_ = std::move(adi_f); - idx.adt_file_ = std::move(adt_f); - idx.adi_path_ = adi_path; - - std::vector types, offsets, lengths; - std::vector names; - for (const auto& fd : fields) { - types.push_back(fd.type); - offsets.push_back(fd.offset); - lengths.push_back(fd.length); - names.push_back(fd.name); - } - if (auto r = idx.apply_tag_(fnums, new_root_pg, types, offsets, lengths, names, - hlen, rlen, unique); !r) - return r.error(); - - return idx; -} + auto write_branch = [&](std::uint32_t page_no, const std::vector& lv, + std::size_t lo, std::size_t hi) + -> util::Result { + Page pg{}; + set_u16_le(pg.data(), ADI_LVL_BRANCH); + set_u16_le(pg.data() + 2, static_cast(hi - lo)); + set_u32_le(pg.data() + 4, ADI_INVALID_PAGE); + set_u32_le(pg.data() + 8, ADI_INVALID_PAGE); + for (std::size_t i = lo; i < hi; ++i) { + std::uint8_t* dst = pg.data() + ADI_TREE_ENTRY_START + + (i - lo) * branch_entry_sz_; + std::memset(dst, 0, branch_entry_sz_); // padded_key + cum[4]=0 + page[4] + const std::string& k = lv[i].max_key; + std::memcpy(dst, k.data(), std::min(klen, k.size())); + std::uint8_t* pp = dst + char_key_padded_len_ + 4; + std::uint32_t pno = lv[i].page; + pp[0] = static_cast( pno & 0xFFu); + pp[1] = static_cast((pno >> 8) & 0xFFu); + pp[2] = static_cast((pno >> 16) & 0xFFu); + pp[3] = static_cast((pno >> 24) & 0xFFu); + } + if (auto w = write_adi_page_(page_no, pg); !w) return w.error(); + return lv[hi - 1].max_key; + }; -const std::vector& AdiIndex::ordered_recnos_cached() { - if (pos_cache_valid_) return pos_recnos_; - - // Save cursor state. - const auto saved_pg = cur_pg_; - const auto saved_idx = cur_idx_; - const auto saved_cnt = cur_cnt_; - const auto saved_lsib = cur_lsib_; - const auto saved_rsib = cur_rsib_; - const auto saved_rn = cur_recno_; - const auto saved_key = current_key_; - const auto saved_page = cur_page_; - - pos_recnos_.clear(); - pos_map_.clear(); - - // Walk from first to last, collecting recnos in key order. Drive the - // walk through next() so EVERY entry of every dense leaf is collected - // (reading only entry 0 of each leaf under-counts multi-entry leaves). - auto first = navigate_leftmost_(); - if (first && first.value().positioned) { - std::uint32_t pos = 0; - for (;;) { - pos_recnos_.push_back(cur_recno_); - pos_map_[cur_recno_] = pos++; - auto n = next(); - if (!n || !n.value().positioned) break; + // Reduce branch levels until the top fits in one page → write it at root_page_. + while (level.size() > max_branch) { + std::vector next; + const std::size_t m = level.size(); + const std::size_t nbr = (m + max_branch - 1) / max_branch; + std::vector pages(nbr); + for (std::size_t i = 0; i < nbr; ++i) { + auto p = alloc_page_(); if (!p) return p.error(); + pages[i] = p.value(); } + next.reserve(nbr); + for (std::size_t i = 0; i < nbr; ++i) { + std::size_t lo = i * max_branch; + std::size_t hi = std::min(m, lo + max_branch); + auto mk = write_branch(pages[i], level, lo, hi); + if (!mk) return mk.error(); + next.push_back({std::move(mk).value(), pages[i]}); + } + level = std::move(next); } - - // Restore cursor state. - cur_pg_ = saved_pg; - cur_idx_ = saved_idx; - cur_cnt_ = saved_cnt; - cur_lsib_ = saved_lsib; - cur_rsib_ = saved_rsib; - cur_recno_ = saved_rn; - current_key_ = saved_key; - cur_page_ = saved_page; - - pos_cache_valid_ = true; - return pos_recnos_; + if (auto top = write_branch(root_page_, level, 0, level.size()); !top) + return top.error(); + cur_pg_ = ADI_INVALID_PAGE; cur_idx_ = -1; + return {}; } -std::uint32_t AdiIndex::pos_of_recno_cached(std::uint32_t recno) { - (void)ordered_recnos_cached(); - auto it = pos_map_.find(recno); - return it != pos_map_.end() ? it->second : 0xFFFFFFFFu; +// ── AdiIndex::clear_data ───────────────────────────────────────────────────── + +util::Result AdiIndex::clear_data() { + if (mode_ == IndexOpenMode::ReadOnly) + return util::Error{5000, 0, "ADI index is read-only", ""}; + invalidate_pos_cache(); // emptied + + // Reset the tag's root to a single EMPTY dense leaf, regardless of the + // current B-tree depth. For a large index (>1 level) the root page is a + // BRANCH, not a dense leaf — the previous code rejected that with + // "root is not a dense leaf" and aborted (ADSCDX/5000) when a CREATE INDEX + // overwrite landed on a multi-level tag (e.g. reindexing ESTAELEC, 441k + // recs). The root lives at a fixed page (fmk_pg+1) and promote_split_ keeps + // root_page_ on root splits, so overwriting it with an empty dense leaf is + // exactly the state a freshly created tag starts from; the caller's + // per-record insert loop then rebuilds the tree. The old branch/leaf pages + // are abandoned in the file (reclaimed on the next full REINDEX that + // recreates the bag) — same trade-off as a CDX clear/rebuild. + Page pg{}; + write_empty_dense_leaf_page(pg, adt_type_, fld_length_); // count=0, lsib/rsib=INVALID + cur_pg_ = root_page_; + cur_page_ = pg; + cur_cnt_ = 0; + cur_idx_ = -1; + cur_lsib_ = ADI_INVALID_PAGE; + cur_rsib_ = ADI_INVALID_PAGE; + cur_recno_ = 0; + current_key_.clear(); + return write_adi_page_(root_page_, pg); } } // namespace openads::drivers::adi diff --git a/src/drivers/adi/adi_index.h b/src/drivers/adi/adi_index.h index ff3510ed..53bb306a 100644 --- a/src/drivers/adi/adi_index.h +++ b/src/drivers/adi/adi_index.h @@ -94,9 +94,12 @@ class AdiIndex final : public IIndex { util::Result open(const std::string& path, IndexOpenMode mode) override; std::string name() const override { return tag_name_; } - std::string expression() const override { return tag_name_; } + std::string expression() const override { + return tag_expr_.empty() ? tag_name_ : tag_expr_; + } std::string file_path() const override { return adi_path_; } - bool descending() const override { return false; } + std::string condition() const override { return tag_cond_; } + bool descending() const override { return descending_; } bool unique() const override { return unique_; } std::uint16_t key_length() const override { return static_cast(key_total_len_); @@ -116,15 +119,23 @@ class AdiIndex final : public IIndex { const std::string& key) override; util::Result flush() override; - // Logical-position cache for O(1) scrollbar / OrdKeyNo / OrdKeyCount. - // Walks the B-tree ONCE (lazily) into an ordered recno list + a - // recno->position map, reused until the index is modified. + // Number of index entries (keys). For a conditional (FOR) tag this is fewer + // than the table's record count. O(1) after the first call (uses the + // logical-position cache below). + util::Result entry_count(); + + // Logical-position cache for the browse scrollbar math (mirrors CdxIndex): + // ordered_recnos_cached() is the recno list in key order; pos_of_recno_cached + // maps a recno to its 0-based position. Built lazily by walking the dense-leaf + // chain once (O(n)); then AdsGetKeyNum / GetRelKeyPos / SetRelKeyPos are O(1) + // per paint instead of an O(n) index walk (which froze large browses). + // Invalidated on insert / erase / clear_data / build_bulk. const std::vector& ordered_recnos_cached(); std::uint32_t pos_of_recno_cached(std::uint32_t recno); void invalidate_pos_cache() { pos_cache_valid_ = false; - pos_recnos_.clear(); - pos_map_.clear(); + ordered_recnos_.clear(); + pos_of_recno_.clear(); } // Parameters for writing a fresh single-tag .adi skeleton. @@ -136,6 +147,16 @@ class AdiIndex final : public IIndex { std::uint32_t adt_hdr_len = 0; // ADT header length (bytes 32..35) std::uint32_t adt_rec_len = 0; // ADT record length bool unique = false; + std::uint16_t record_offset = 0; // record offset of the (first) field (for fallback without re-opening ADT file) + // v2 (OpenADS-proprietary) tag metadata — persisted in the per-tag + // header so tag identity is by NAME and the key expression / FOR + // condition survive a reopen (the legacy format only stored a field + // number). key_len is the full evaluated-key length (ACE klen). + std::string tag_name; // tag name (e.g. "ORD1"); identity key in v2 + std::string key_expr; // index key expression (e.g. cA+cB / DTOS(d)) + std::string for_expr; // FOR condition (empty = unconditional) + std::uint16_t key_len = 0; // full key length (klen from ACE) + bool descending = false; // Full path of the ADT table this index belongs to. Required for a // NON-STRUCTURAL bag, whose .adi stem differs from the table's (the // `INDEX ON ... TAG ... TO ` form). When empty, the @@ -157,7 +178,15 @@ class AdiIndex final : public IIndex { // Wipe the B+tree for this tag (root dense leaf count → 0) so a // CREATE INDEX overwrite can rebuild from scratch. - util::Result clear_data(); + util::Result clear_data() override; + + // Bulk-load the (v2) tag from a key set in one bottom-up pass (sort → pack + // dense leaves → build branch levels), far faster than per-record insert on + // a full REINDEX. Only the v2 opaque-key leaf is supported; a legacy + // (field-derived) tag falls back to the per-record default. Call clear_data + // first / use on a fresh tag. + util::Result build_bulk( + std::vector> keys) override; // Multi-tag API (mirrors CdxIndex). adt_path is the owning table's path; // when empty the companion ADT is derived from the .adi stem (structural @@ -171,20 +200,6 @@ class AdiIndex final : public IIndex { const std::string& field_name, const std::string& adt_path = {}); - // Create a new ADI file with one tag (expression = comma-separated column names). - // On return the AdiIndex is positioned on that tag and ready for inserts. - static util::Result create(const std::string& adi_path, - const std::string& adt_path, - const std::string& expression, - bool unique); - - // Add a new tag to an existing ADI file. - // On return the AdiIndex is positioned on the new tag and ready for inserts. - static util::Result add_tag(const std::string& adi_path, - const std::string& adt_path, - const std::string& expression, - bool unique); - private: // Read / write a 512-byte page from/to the ADI file util::Result read_adi_page_ (std::uint32_t page_no, Page& buf); @@ -215,6 +230,23 @@ class AdiIndex final : public IIndex { // Load the dense leaf at page_no into cur_page_ and update cursor metadata util::Result load_dense_leaf_(std::uint32_t page_no); + // Adopt an already-read dense-leaf page as the cursor's current leaf: sets + // cur_pg_/cur_cnt_/cur_lsib_/cur_rsib_ and, for a v2 tag, decodes the + // front-coded entries into leaf_entries_ (legacy tags leave it empty). + void adopt_leaf_page_(std::uint32_t page_no, const Page& pg); + + // Render a v2 front-coded dense-leaf page (header + sub-header + entries) + // from a key-ordered run. Returns false if the run overflows one page + // (the caller must split first). + bool render_v2_leaf_( + Page& pg, + const std::vector>& ents, + std::uint32_t lsib, std::uint32_t rsib) const; + + // v2 (front-coded) erase: decode the owning leaf, drop (recno,key), + // re-encode (or unlink an emptied page). ikey must already be klen bytes. + util::Result erase_v2_(std::uint32_t recno, const std::string& ikey); + // Navigate to the first (leftmost) entry of the B-tree util::Result navigate_leftmost_(); @@ -236,7 +268,7 @@ class AdiIndex final : public IIndex { std::uint32_t branch_entry_page_(const std::uint8_t* pg, int idx) const noexcept; // Compare two keys. For numeric keys 8-byte memcmp; for char keys - // key_total_len_ bytes (memcmp; CICHAR case-insensitivity deferred). + // key_total_len_ bytes, with CICHAR components folded (see below). int compare_keys_(const std::string& a, const std::string& b) const noexcept; // CICHAR collation: return a comparison-normalized copy of a key with @@ -257,7 +289,8 @@ class AdiIndex final : public IIndex { const std::vector& fd_lengths, const std::vector& fd_names, std::uint32_t hlen, std::uint32_t rlen, - bool unique); + bool unique, + std::uint32_t v2_key_len = 0); // >0 → v2 leaf (recno4B + opaque key) // Open mode (set by open / open_named) @@ -269,7 +302,10 @@ class AdiIndex final : public IIndex { std::string adi_path_; // Tag metadata (primary / first-component field) - std::string tag_name_; // ADT field name of first component + std::string tag_name_; // v2: tag name; legacy: ADT field name + std::string tag_expr_; // v2: key expression (empty in legacy) + std::string tag_cond_; // v2: FOR condition (empty = unconditional) + bool descending_ = false; std::uint32_t root_page_ = 0; std::uint16_t adt_type_ = 0; // type of first-component field std::uint16_t fld_offset_ = 0; // offset of first-component field in ADT record @@ -293,8 +329,27 @@ class AdiIndex final : public IIndex { std::uint32_t adt_hdr_len_ = 0; std::uint32_t adt_rec_len_ = 0; + // v2 (OpenADS-proprietary) leaf: dense entries store [recno 4B][full key], + // so navigation/seek read the key from the leaf (no ADT re-read) and recno + // is 4 bytes. false = legacy field-derived leaf. + bool key_in_leaf_ = false; + + // Logical-position cache (recno-in-key-order + reverse map). entry_count() + // is its size. Invalidated via invalidate_pos_cache(). + std::vector ordered_recnos_; + std::unordered_map pos_of_recno_; + bool pos_cache_valid_ = false; + + // v2 front-coded dense leaf decoded into (recno, full key) pairs in key + // order — the in-memory image of the current leaf. Populated by + // adopt_leaf_page_ for v2 tags; empty for legacy field-derived leaves + // (which are read directly from cur_page_ with the fixed entry_size_). + std::vector> leaf_entries_; + // Dense-leaf cursor - std::uint32_t entry_size_ = 3; // dense_entry_size(fld_length_) + std::uint32_t entry_size_ = 3; // legacy: dense_entry_size(); v2 leaf is + // front-coded (variable) — entry_size_ is + // unused on the v2 path. std::uint32_t cur_pg_ = ADI_INVALID_PAGE; std::int32_t cur_idx_ = -1; std::uint16_t cur_cnt_ = 0; @@ -304,8 +359,6 @@ class AdiIndex final : public IIndex { std::string current_key_; Page cur_page_{}; - std::vector pos_recnos_; - std::unordered_map pos_map_; bool pos_cache_valid_ = false; }; diff --git a/src/drivers/index_trait.h b/src/drivers/index_trait.h index 94d6f8be..d46e2f23 100644 --- a/src/drivers/index_trait.h +++ b/src/drivers/index_trait.h @@ -86,6 +86,33 @@ class IIndex { virtual util::Result erase (std::uint32_t recno, const std::string& key) = 0; virtual util::Result flush() = 0; + + // Reset the index to empty so a caller (REINDEX / PACK) can rebuild it. + // Default: collect every entry then erase it (works for any IIndex). + // CdxIndex / AdiIndex override with an O(1)-ish structural reset. + virtual util::Result clear_data() { + std::vector> entries; + auto s = seek_first(); + while (s && s.value().positioned) { + entries.emplace_back(s.value().recno, current_key()); + s = next(); + } + for (auto& kv : entries) { + if (auto e = erase(kv.first, kv.second); !e) return e.error(); + } + return {}; + } + + // Bulk-load (key, recno) pairs into a freshly-cleared index. Default: + // per-record insert. CdxIndex / AdiIndex override with a bottom-up bulk + // build (~10x faster on a full REINDEX). Call clear_data() first. + virtual util::Result + build_bulk(std::vector> keys) { + for (auto& kv : keys) { + if (auto e = insert(kv.second, kv.first); !e) return e.error(); + } + return {}; + } }; } // namespace openads::drivers From 42ffa083dcef7a10ed48dd0d258a5088b47d5bf3 Mon Sep 17 00:00:00 2001 From: russimicro Date: Sun, 26 Jul 2026 10:50:05 -0500 Subject: [PATCH 02/12] perf(adi): take the bulk build path on CREATE INDEX / REINDEX The key-collection loop bulk-loaded only CDX; ADI fell back to per-record insertion. Now that clear_data/build_bulk are IIndex virtuals, collect for both and dispatch through idx_owner->build_bulk(), which reaches the v2 ADI dense-leaf bottom-up build. NTX keeps the incremental path via the default. Suite: 1177/1189, same 12 SQL-parser (7200) failures as the baseline. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 008c28b49d19cd27f2c92e8e141416dd77d70f89) (cherry picked from commit 39388566c1d33b807f23c0fba09d47540a7f4d8e) --- src/abi/ace_exports.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/abi/ace_exports.cpp b/src/abi/ace_exports.cpp index 1ce9dc15..f6e4d9bc 100644 --- a/src/abi/ace_exports.cpp +++ b/src/abi/ace_exports.cpp @@ -13456,11 +13456,12 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, // instead of record-by-record top-down insertion. Each page is encoded // once rather than decoded+re-encoded on every key, ~10x faster on a // full CREATE INDEX / REINDEX. NTX keeps the incremental path. - openads::drivers::cdx::CdxIndex* cdx_bulk = - is_cdx ? static_cast(idx_owner.get()) - : nullptr; + // CDX and the v2 ADI dense leaf both override IIndex::build_bulk, so the + // call dispatches to the right engine; NTX falls back to the per-record + // default and is left on the incremental path here. + const bool use_bulk = is_cdx || is_adi; std::vector> bulk_keys; - if (cdx_bulk) bulk_keys.reserve(rec_count); + if (use_bulk) bulk_keys.reserve(rec_count); for (std::uint32_t r = 1; r <= rec_count; ++r) { // Use direct driver read + bulk buffer load so the driver's // read-ahead cache is effective for this sequential scan. @@ -13496,14 +13497,14 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, if (!k) return fail(k.error()); kbytes = std::move(k).value(); } - if (cdx_bulk) { + if (use_bulk) { bulk_keys.emplace_back(std::move(kbytes), r); } else if (auto ins = idx_owner->insert(r, kbytes); !ins) { return fail(ins.error()); } } - if (cdx_bulk) { - if (auto b = cdx_bulk->build_bulk(std::move(bulk_keys)); !b) + if (use_bulk) { + if (auto b = idx_owner->build_bulk(std::move(bulk_keys)); !b) return fail(b.error()); } if (auto fl = idx_owner->flush(); !fl) return fail(fl.error()); From 53d12f64fdd9acab78dea10657501fbf52b5164a Mon Sep 17 00:00:00 2001 From: russimicro Date: Sun, 26 Jul 2026 11:15:11 -0500 Subject: [PATCH 03/12] fix(adi): recognise ADT tables kept as .DAT (Russoft ExtFile convention) The ERP stores ADT data in .DAT with a companion .ADI (ARC-CAJA ExtFile='.DAT'), opened via ADS_ADT. Both places that decided "is this an ADT table?" tested the extension only, so a .DAT table got the .cdx default bag on AdsCreateIndex61 (5000) and never auto-bound its .adi on open. - AdsCreateIndex61: fall back to dynamic_cast(t->driver()) when the extension is not .adt, so the structural bag defaults to .adi. - AdsOpenTable: auto-open the companion .adi for .dat as well as .adt (case-insensitive extension compare). Fixes abi_adi_dat_extension_path_test. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 477215095c3157a2a1db47d1788a28c846eff913) (cherry picked from commit 419ede3003bf16cdd535e41a7e40bf12c4e1b607) --- src/abi/ace_exports.cpp | 10 +- tests/unit/abi_adi_clear_multilevel_test.cpp | 114 +++++++ .../unit/abi_adi_dat_extension_path_test.cpp | 89 ++++++ tests/unit/abi_adi_estaelec_compound_test.cpp | 191 +++++++++++ tests/unit/abi_adi_frontcoding_size_test.cpp | 109 +++++++ tests/unit/abi_adi_native_estaelec_test.cpp | 208 ++++++++++++ tests/unit/abi_adi_reindex_bench_test.cpp | 297 ++++++++++++++++++ tests/unit/abi_sql_temp_browse_nav_test.cpp | 173 ++++++++++ 8 files changed, 1188 insertions(+), 3 deletions(-) create mode 100644 tests/unit/abi_adi_clear_multilevel_test.cpp create mode 100644 tests/unit/abi_adi_dat_extension_path_test.cpp create mode 100644 tests/unit/abi_adi_estaelec_compound_test.cpp create mode 100644 tests/unit/abi_adi_frontcoding_size_test.cpp create mode 100644 tests/unit/abi_adi_native_estaelec_test.cpp create mode 100644 tests/unit/abi_adi_reindex_bench_test.cpp create mode 100644 tests/unit/abi_sql_temp_browse_nav_test.cpp diff --git a/src/abi/ace_exports.cpp b/src/abi/ace_exports.cpp index f6e4d9bc..c1905cc6 100644 --- a/src/abi/ace_exports.cpp +++ b/src/abi/ace_exports.cpp @@ -7135,9 +7135,13 @@ UNSIGNED32 ENTRYPOINT AdsOpenTable(ADSHANDLE hConnect, } } // ADI auto-open: same convention for ADT tables — opening `.adt` - // auto-binds `.adi` if it exists, so every tag inside it becomes - // navigable without an explicit AdsOpenIndex call. - if (tp.extension() == ".adt" || tp.extension() == ".ADT") { + // (or `.dat`, the Russoft ERP convention of keeping ADT data in + // .DAT + .ADI per ARC-CAJA) auto-binds `.adi` if it exists, so every + // tag inside it becomes navigable without an explicit AdsOpenIndex call. + std::string tp_extl = tp.extension().string(); + for (auto& ch : tp_extl) + ch = static_cast(std::tolower(static_cast(ch))); + if (tp_extl == ".adt" || tp_extl == ".dat") { fs::path adi = tp; adi.replace_extension(".adi"); std::error_code ec; std::string adi_path = diff --git a/tests/unit/abi_adi_clear_multilevel_test.cpp b/tests/unit/abi_adi_clear_multilevel_test.cpp new file mode 100644 index 00000000..0ac6c7d4 --- /dev/null +++ b/tests/unit/abi_adi_clear_multilevel_test.cpp @@ -0,0 +1,114 @@ +// Repro for ADI clear_data on a MULTI-LEVEL index (large table). +// +// Re-creating an existing tag (CREATE INDEX overwrite, or the ERP reindex +// hitting the same field twice) takes the open_named + clear_data path in +// AdsCreateIndex61. clear_data used to require the tag's root page to be a +// dense leaf and aborted with ADSCDX/5000 "root is not a dense leaf" when the +// index was big enough to have a branch root (>1 B-tree level). This is what +// killed the reindex of ESTAELEC (441k records) at INDEX ON ... TAG ORD3. +// +// Here we build a tag over enough records to force a multi-level tree (a +// 512-byte dense leaf holds ~162 entries), then re-create the SAME tag so +// clear_data runs on the branch root. It must succeed and the rebuilt index +// must still walk every record in key order. + +#include "doctest.h" +#include "drivers/adi/adi_index.h" +#include "openads/ace.h" + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +std::string rtrim(std::string s) { + while (!s.empty() && s.back() == ' ') s.pop_back(); + return s; +} + +void append_key(ADSHANDLE hTable, const std::string& v) { + REQUIRE(AdsAppendRecord(hTable) == AE_SUCCESS); + UNSIGNED8 fld[] = "K"; + UNSIGNED8 val[16]{}; + std::memcpy(val, v.data(), v.size()); + REQUIRE(AdsSetString(hTable, fld, val, + static_cast(v.size())) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hTable) == AE_SUCCESS); +} + +} // namespace + +TEST_CASE("ADI clear_data: re-create tag on a multi-level (large) index") { + fs::path tmp = fs::temp_directory_path() / "openads_adi_clear_ml"; + { std::error_code ec; fs::create_directories(tmp, ec); } + { std::error_code ec; + fs::remove(tmp / "big.adt", ec); + fs::remove(tmp / "big.adi", ec); } + + UNSIGNED8 srv[260]{}; + std::memcpy(srv, tmp.string().c_str(), tmp.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + + UNSIGNED8 tbl[] = "big.adt"; + UNSIGNED8 flddef[] = "K,Character,8"; + ADSHANDLE hTable = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, + flddef, &hTable) == AE_SUCCESS); + + // 600 distinct keys → ~4 dense leaves → branch root (multi-level). + const int N = 600; + for (int i = 1; i <= N; ++i) { + char b[16]; + std::snprintf(b, sizeof(b), "%08d", i); + append_key(hTable, std::string(b)); + } + + UNSIGNED8 idxfile[] = "big.adi"; + UNSIGNED8 tag[] = "ORD1"; + UNSIGNED8 expr[] = "K"; + + // First build → multi-level tree. + ADSHANDLE hIdx1 = 0; + REQUIRE(AdsCreateIndex61(hTable, idxfile, tag, expr, + nullptr, nullptr, 0, 0, &hIdx1) == AE_SUCCESS); + + // Re-create the SAME tag → exists && have_tag → open_named + clear_data on + // a BRANCH root. Used to fail with ADSCDX/5000; must now succeed. + ADSHANDLE hIdx2 = 0; + REQUIRE(AdsCreateIndex61(hTable, idxfile, tag, expr, + nullptr, nullptr, 0, 0, &hIdx2) == AE_SUCCESS); + + // The rebuilt index must walk every record in ascending key order. + REQUIRE(AdsGotoTop(hTable) == AE_SUCCESS); + std::vector seen; + for (;;) { + UNSIGNED16 at_eof = 0; + REQUIRE(AdsAtEOF(hTable, &at_eof) == AE_SUCCESS); + if (at_eof) break; + UNSIGNED8 buf[32]{}; + UNSIGNED32 len = sizeof(buf); + REQUIRE(AdsGetString(hTable, (UNSIGNED8*)"K", buf, &len, 0) + == AE_SUCCESS); + seen.push_back(rtrim(std::string(reinterpret_cast(buf), len))); + REQUIRE(AdsSkip(hTable, 1) == AE_SUCCESS); + } + + REQUIRE(seen.size() == static_cast(N)); + CHECK(seen.front() == "00000001"); + CHECK(seen.back() == "00000600"); + bool ordered = true; + for (std::size_t i = 1; i < seen.size(); ++i) + if (seen[i] < seen[i - 1]) { ordered = false; break; } + CHECK(ordered); + + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + { std::error_code ec; fs::remove_all(tmp, ec); } +} diff --git a/tests/unit/abi_adi_dat_extension_path_test.cpp b/tests/unit/abi_adi_dat_extension_path_test.cpp new file mode 100644 index 00000000..696894ff --- /dev/null +++ b/tests/unit/abi_adi_dat_extension_path_test.cpp @@ -0,0 +1,89 @@ +// Repro for the Russoft ".DAT" companion-path bug in the ADI driver. +// +// The Russoft ERP keeps ADT-format data in files with a .DAT extension +// (ExtFile='.DAT') and a sibling .ADI compound index. The ADI driver used to +// derive the companion data path by blindly swapping the index extension to +// ".adt" (adt_path_for). For a .DAT table that file does not exist, so every +// implicit-path code path (list_tags / open_named / add_tag) failed with +// ERROR_FILE_NOT_FOUND, surfaced as ADSCDX/5103 "CreateFileA". +// +// The first tag survived because AdsCreateIndex61 passes the real table path +// (t->path()) into AdiIndex::create via CreateParams::adt_path. The SECOND tag +// took the `exists` branch, which called list_tags WITHOUT the data path and +// crashed. This test creates an ADT table on disk with a .DAT extension and +// adds a second (distinct-field) tag — it must succeed end to end. + +#include "doctest.h" +#include "drivers/adi/adi_index.h" +#include "openads/ace.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +TEST_CASE("ADI: second tag on an ADT table stored as .DAT (Russoft convention)") { + fs::path tmp = fs::temp_directory_path() / "openads_adi_dat_ext"; + { std::error_code ec; fs::create_directories(tmp, ec); } + { std::error_code ec; + fs::remove(tmp / "raddao.adt", ec); + fs::remove(tmp / "raddao.DAT", ec); + fs::remove(tmp / "raddao.adi", ec); } + + UNSIGNED8 srv[260]{}; + std::memcpy(srv, tmp.string().c_str(), tmp.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + + // Create an ADT-format table (AdsCreateTable forces a .adt extension). + UNSIGNED8 tbl[] = "raddao.adt"; + UNSIGNED8 flddef[] = "CCODIGOCON,Character,3;CDOCUMETRA,Character,8"; + ADSHANDLE hTable = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, + flddef, &hTable) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + + // Rename the data file to .DAT — the Russoft on-disk convention. + { std::error_code ec; + fs::rename(tmp / "raddao.adt", tmp / "raddao.DAT", ec); + REQUIRE(!ec); } + REQUIRE(fs::exists(tmp / "raddao.DAT")); + REQUIRE_FALSE(fs::exists(tmp / "raddao.adt")); + + // Re-open the .DAT as an ADT table. + UNSIGNED8 dat[] = "raddao.DAT"; + hTable = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, + 0, 0, 0, ADS_DEFAULT, &hTable) == AE_SUCCESS); + + UNSIGNED8 idxfile[] = "raddao.adi"; + + // First tag — exercises AdiIndex::create (passes t->path() explicitly, + // so it always worked). + ADSHANDLE hIdx1 = 0; + REQUIRE(AdsCreateIndex61(hTable, idxfile, (UNSIGNED8*)"ORD1", + (UNSIGNED8*)"CCODIGOCON", nullptr, nullptr, + 0, 0, &hIdx1) == AE_SUCCESS); + + // Second tag — used to take the `exists` branch and crash with 5103 + // because list_tags derived raddao.adt (absent). Must now succeed. + ADSHANDLE hIdx2 = 0; + REQUIRE(AdsCreateIndex61(hTable, idxfile, (UNSIGNED8*)"ORD2", + (UNSIGNED8*)"CDOCUMETRA", nullptr, nullptr, + 0, 0, &hIdx2) == AE_SUCCESS); + + // The .adi must now hold two distinct tags. list_tags is also called with + // only the .adi path here, which exercises the adt_path_for .DAT fallback. + auto tags = openads::drivers::adi::AdiIndex::list_tags( + (tmp / "raddao.adi").string()); + REQUIRE(tags); + CHECK(tags.value().size() == 2u); + + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + + { std::error_code ec; fs::remove_all(tmp, ec); } +} diff --git a/tests/unit/abi_adi_estaelec_compound_test.cpp b/tests/unit/abi_adi_estaelec_compound_test.cpp new file mode 100644 index 00000000..e65c6d22 --- /dev/null +++ b/tests/unit/abi_adi_estaelec_compound_test.cpp @@ -0,0 +1,191 @@ +// SPEC / ORACLE for ADS_ADT (.ADI) expression-index support — the ADT mirror of +// abi_cdx_estaelec_compound_test.cpp. This DEFINES "correct" for the ADI driver: +// the same ERP index patterns (UTILIDAD.PRG ESTAELEC) must work on an ADT table +// stored as .DAT with a .ADI bag, exactly as they do on CDX. +// +// ORD1 cCodigoCon+cDocumeTra (compound concat) +// ORD2 cCodigoCli (single field) +// ORD3 cPreFijTra+cDocumeTra (compound concat — must stay distinct +// from ORD1; today they collide on +// field 0 in the ADI driver) +// ORD4 DTOS(dFecTraTra) (computed) +// ORD5 DTOS(dFecTraTra) FOR cCorEnvEle != 'S' (computed + conditional) +// +// STATUS WHEN WRITTEN (2026-06-27): EXPECTED TO FAIL — the ADI driver indexes by +// field only (no expression / FOR / tag-name identity). This test is the target +// for the ADI expression-support work. As that lands, this should go green +// WITHOUT changing the CDX equivalent's expectations. + +#include "doctest.h" +#include "openads/ace.h" + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +// Sets an env var for the test's lifetime and clears it on exit (leak-proof +// across doctest's shared process, even if a REQUIRE throws). +struct EnvGuard { + const char* name_; + EnvGuard(const char* n, const char* v) : name_(n) { +#ifdef _WIN32 + _putenv_s(n, v); +#else + setenv(n, v, 1); +#endif + } + ~EnvGuard() { +#ifdef _WIN32 + _putenv_s(name_, ""); +#else + unsetenv(name_); +#endif + } +}; + +void set_c(ADSHANDLE h, const char* field, const char* val) { + UNSIGNED8 f[32]{}; std::strncpy(reinterpret_cast(f), field, 31); + UNSIGNED8 v[32]{}; std::strncpy(reinterpret_cast(v), val, 31); + REQUIRE(AdsSetString(h, f, v, + static_cast(std::strlen(val))) == AE_SUCCESS); +} + +UNSIGNED32 recno(ADSHANDLE h) { + UNSIGNED32 rn = 0; + REQUIRE(AdsGetRecordNum(h, 0, &rn) == AE_SUCCESS); + return rn; +} + +ADSHANDLE make_tag(ADSHANDLE hTable, const char* bag, const char* tag, + const char* expr, const char* cond) { + UNSIGNED8 b[260]{}; std::strncpy(reinterpret_cast(b), bag, 259); + UNSIGNED8 t[64]{}; std::strncpy(reinterpret_cast(t), tag, 63); + UNSIGNED8 e[128]{}; std::strncpy(reinterpret_cast(e), expr, 127); + UNSIGNED8 c[128]{}; + UNSIGNED8* cp = nullptr; + if (cond) { std::strncpy(reinterpret_cast(c), cond, 127); cp = c; } + ADSHANDLE h = 0; + UNSIGNED32 rc = AdsCreateIndex61(hTable, b, t, e, cp, nullptr, 0, 0, &h); + REQUIRE(rc == AE_SUCCESS); + return h; +} + +} // namespace + +// Validates the EXPERIMENTAL "CDX-over-ADT" reroute (env OPENADS_ADT_CDX_INDEX=1): +// an ADT table stored as .DAT with a .ADI bag, indexed via the CdxIndex engine, +// must handle the ERP's ESTAELEC index patterns (compound/computed/FOR) exactly +// as CDX does. The EnvGuard turns the reroute on only for this test. +TEST_CASE("ADI->CDX reroute handles ESTAELEC compound/computed/conditional tags") { + EnvGuard _cdx_adt("OPENADS_ADT_CDX_INDEX", "1"); + fs::path dir = fs::temp_directory_path() / "openads_adi_estaelec"; + std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); + + UNSIGNED8 srv[260]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + + // ADT table; rename to .DAT (Russoft convention) and reopen as ADS_ADT. + UNSIGNED8 tbl[] = "estaelec.adt"; + UNSIGNED8 def[] = "CCODIGOCON,Character,3;CDOCUMETRA,Character,8;" + "CCODIGOCLI,Character,10;CPREFIJTRA,Character,4;" + "DFECTRATRA,Date,8;CCORENVELE,Character,1"; + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, def, &hT) + == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + fs::rename(dir / "estaelec.adt", dir / "estaelec.DAT", ec); + REQUIRE(!ec); + UNSIGNED8 dat[] = "estaelec.DAT"; + hT = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + == AE_SUCCESS); + + struct Row { const char* con; const char* doc; const char* cli; + const char* pre; const char* fec; const char* cor; }; + const Row rows[4] = { + {"001", "00000010", "CLIENTE001", "FA01", "20260103", "S"}, // rec1 + {"002", "00000020", "CLIENTE002", "FA02", "20260101", "N"}, // rec2 + {"001", "00000030", "CLIENTE003", "FA03", "20260102", "S"}, // rec3 + {"003", "00000005", "CLIENTE004", "FA01", "20260104", "N"}, // rec4 + }; + for (const auto& r : rows) { + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + set_c(hT, "CCODIGOCON", r.con); + set_c(hT, "CDOCUMETRA", r.doc); + set_c(hT, "CCODIGOCLI", r.cli); + set_c(hT, "CPREFIJTRA", r.pre); + set_c(hT, "DFECTRATRA", r.fec); + set_c(hT, "CCORENVELE", r.cor); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); + } + + std::string bags = (dir / "estaelec.adi").string(); + ADSHANDLE o1 = make_tag(hT, bags.c_str(), "ORD1", "CCODIGOCON+CDOCUMETRA", nullptr); + ADSHANDLE o2 = make_tag(hT, bags.c_str(), "ORD2", "CCODIGOCLI", nullptr); + ADSHANDLE o3 = make_tag(hT, bags.c_str(), "ORD3", "CPREFIJTRA+CDOCUMETRA", nullptr); + ADSHANDLE o4 = make_tag(hT, bags.c_str(), "ORD4", "DTOS(DFECTRATRA)", nullptr); + ADSHANDLE o5 = make_tag(hT, bags.c_str(), "ORD5", "DTOS(DFECTRATRA)", "CCORENVELE != 'S'"); + (void)o2; + + // 5 distinct tags coexist (today: collapse on field 0). + UNSIGNED16 nidx = 0; + REQUIRE(AdsGetNumIndexes(hT, &nidx) == AE_SUCCESS); + CHECK(nidx == 5); + + // ORD1 (con+doc) top = rec1 ; ORD3 (pre+doc) top = rec4 — distinct compound orders. + REQUIRE(AdsSetIndexOrderByHandle(hT, o1) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 1u); + REQUIRE(AdsSetIndexOrderByHandle(hT, o3) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 4u); + + // ORD4 DTOS(date): earliest is rec2. + REQUIRE(AdsSetIndexOrderByHandle(hT, o4) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 2u); + + // ORD5 conditional: only rec2 and rec4 (cor != 'S'). + UNSIGNED32 kc = 0; + REQUIRE(AdsGetKeyCount(o5, 0, &kc) == AE_SUCCESS); + CHECK(kc == 2u); + + // Exact seek on compound ORD1 -> rec3. + REQUIRE(AdsSetIndexOrderByHandle(hT, o1) == AE_SUCCESS); + UNSIGNED8 key[] = "00100000030"; + UNSIGNED16 found = 0; + REQUIRE(AdsSeek(o1, key, 11, ADS_STRINGKEY, 0, &found) == AE_SUCCESS); + CHECK(found != 0); + CHECK(recno(hT) == 3u); + + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + + // Reopen: table auto-open binds the .adi bag through the CDX reroute + // (AdsOpenIndex path). The 5 orders must persist and stay distinct, addressed + // by tag NAME (handles are gone after close). + hT = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + == AE_SUCCESS); + UNSIGNED16 nidx2 = 0; + REQUIRE(AdsGetNumIndexes(hT, &nidx2) == AE_SUCCESS); + CHECK(nidx2 == 5); + REQUIRE(AdsSetIndexOrder(hT, (UNSIGNED8*)"ORD3") == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 4u); + REQUIRE(AdsSetIndexOrder(hT, (UNSIGNED8*)"ORD1") == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 1u); + + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/abi_adi_frontcoding_size_test.cpp b/tests/unit/abi_adi_frontcoding_size_test.cpp new file mode 100644 index 00000000..030b7d00 --- /dev/null +++ b/tests/unit/abi_adi_frontcoding_size_test.cpp @@ -0,0 +1,109 @@ +// Front-coding size oracle: a v2 char-key .adi must store its dense leaves +// front-coded (dup/trail), so an index over keys with a long shared prefix +// occupies FAR fewer leaf pages than the uncompressed "full key per entry" +// layout would — parity with ADS-SAP (~3x smaller). The build must also stay +// fully navigable (every key visited once, ascending) so the size win is not +// bought with corruption. +#include "doctest.h" +#include "openads/ace.h" + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { +std::string trim_sp(std::string s) { + while (!s.empty() && s.back() == ' ') s.pop_back(); + return s; +} +} // namespace + +TEST_CASE("ADI: front-coded leaf shrinks an index over high-prefix keys") { + fs::path tmp = fs::temp_directory_path() / "openads_adi_frontcoding"; + { std::error_code ec; fs::create_directories(tmp, ec); + fs::remove(tmp / "fc.adt", ec); fs::remove(tmp / "fc.adi", ec); } + + UNSIGNED8 srv[260]{}; + std::memcpy(srv, tmp.string().c_str(), tmp.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + + // Key width 20; the first 16 bytes are constant across every row, so the + // shared prefix is large and front-coding is highly effective. + const std::uint32_t KLEN = 20; + UNSIGNED8 tbl[] = "fc.adt"; + UNSIGNED8 flddef[] = "Code,Character,20"; + ADSHANDLE hTable = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, + flddef, &hTable) == AE_SUCCESS); + + const int N = 4000; + for (int i = 0; i < N; ++i) { + char code[32]; + std::snprintf(code, sizeof(code), "CONSTANTPREFIX__%04d", i); // 16 + 4 + REQUIRE(AdsAppendRecord(hTable) == AE_SUCCESS); + REQUIRE(AdsSetString(hTable, (UNSIGNED8*)"Code", (UNSIGNED8*)code, KLEN) + == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hTable) == AE_SUCCESS); + } + + UNSIGNED8 idxfile[] = "fc.adi"; + ADSHANDLE hIdx = 0; + REQUIRE(AdsCreateIndex61(hTable, idxfile, (UNSIGNED8*)"CODE", + (UNSIGNED8*)"Code", nullptr, nullptr, 0, 0, &hIdx) + == AE_SUCCESS); + + // Uncompressed leaf layout would store recno(4)+key(KLEN) per entry, packing + // floor(488/(4+KLEN)) entries per 512-byte leaf page. Front-coding must beat + // that leaf-page footprint by a wide margin. + std::error_code ec; + const std::uintmax_t fsize = fs::file_size(tmp / "fc.adi", ec); + REQUIRE(!ec); + const std::uint32_t fixed_entry = 4u + KLEN; // 24 + const std::uint32_t fixed_per_leaf = (512u - 24u) / fixed_entry; // 20 + const std::uint32_t fixed_leaves = (N + fixed_per_leaf - 1) / fixed_per_leaf; + const std::uintmax_t fixed_leaf_bytes = std::uintmax_t(fixed_leaves) * 512u; + + // The whole front-coded file (header + branches + leaves) must fit in less + // than the uncompressed LEAF pages alone — i.e. a clear, structural win. + CHECK(fsize < fixed_leaf_bytes); + // And it should be a big win, not a marginal one: < half the fixed leaves. + CHECK(fsize * 2u < fixed_leaf_bytes); + + // Correctness: a full ordered walk visits every key exactly once, ascending. + REQUIRE(AdsGotoTop(hTable) == AE_SUCCESS); + int walked = 0; + std::string prev; + for (;;) { + UNSIGNED16 eof = 0; + REQUIRE(AdsAtEOF(hTable, &eof) == AE_SUCCESS); + if (eof) break; + UNSIGNED8 buf[32]{}; UNSIGNED32 len = sizeof(buf); + REQUIRE(AdsGetString(hTable, (UNSIGNED8*)"Code", buf, &len, 0) + == AE_SUCCESS); + std::string cur = trim_sp(std::string(reinterpret_cast(buf), len)); + CHECK(cur > prev); + prev = cur; + ++walked; + REQUIRE(AdsSkip(hTable, 1) == AE_SUCCESS); + } + CHECK(walked == N); + + // Seek a few scattered keys -> all found. + for (int key : {0, 1, 1999, 2500, N - 1}) { + char code[32]; + std::snprintf(code, sizeof(code), "CONSTANTPREFIX__%04d", key); + UNSIGNED16 found = 0; + REQUIRE(AdsSeek(hIdx, (UNSIGNED8*)code, KLEN, ADS_STRINGKEY, 0, &found) + == AE_SUCCESS); + CHECK(found != 0); + } + + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); +} diff --git a/tests/unit/abi_adi_native_estaelec_test.cpp b/tests/unit/abi_adi_native_estaelec_test.cpp new file mode 100644 index 00000000..cc83f042 --- /dev/null +++ b/tests/unit/abi_adi_native_estaelec_test.cpp @@ -0,0 +1,208 @@ +// NATIVE-ADI ORACLE for the ESTAELEC index patterns (the ERP's real index set), +// WITHOUT the CDX reroute (OPENADS_ADT_CDX_INDEX is NOT set). This pins what the +// genuine OpenADS AdiIndex driver must do once reworked: +// +// ORD1 cCodigoCon+cDocumeTra (compound concat) +// ORD2 cCodigoCli (single field) +// ORD3 cPreFijTra+cDocumeTra (compound concat — distinct from ORD1) +// ORD4 DTOS(dFecTraTra) (computed) +// ORD5 DTOS(dFecTraTra) FOR cCorEnvEle != 'S' (computed + conditional) +// +// STATUS WHEN WRITTEN (2026-06-27): EXPECTED TO FAIL — the native AdiIndex indexes +// by field only (no expression / FOR / tag-name identity, recno too narrow). This +// is the target oracle for the AdiIndex rework. As each gap lands the matching +// CHECK goes green; when ALL pass, drop the should_fail decorator. The CDX mirror +// (abi_cdx_estaelec_compound_test.cpp) proves the ERP pattern is correct on CDX. +#include "doctest.h" +#include "openads/ace.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { +void set_c(ADSHANDLE h, const char* field, const char* val) { + UNSIGNED8 f[32]{}; std::strncpy(reinterpret_cast(f), field, 31); + UNSIGNED8 v[32]{}; std::strncpy(reinterpret_cast(v), val, 31); + REQUIRE(AdsSetString(h, f, v, + static_cast(std::strlen(val))) == AE_SUCCESS); +} +UNSIGNED32 recno(ADSHANDLE h) { + UNSIGNED32 rn = 0; + REQUIRE(AdsGetRecordNum(h, 0, &rn) == AE_SUCCESS); + return rn; +} +ADSHANDLE make_tag(ADSHANDLE hTable, const char* bag, const char* tag, + const char* expr, const char* cond) { + UNSIGNED8 b[260]{}; std::strncpy(reinterpret_cast(b), bag, 259); + UNSIGNED8 t[64]{}; std::strncpy(reinterpret_cast(t), tag, 63); + UNSIGNED8 e[128]{}; std::strncpy(reinterpret_cast(e), expr, 127); + UNSIGNED8 c[128]{}; + UNSIGNED8* cp = nullptr; + if (cond) { std::strncpy(reinterpret_cast(c), cond, 127); cp = c; } + ADSHANDLE h = 0; + UNSIGNED32 rc = AdsCreateIndex61(hTable, b, t, e, cp, nullptr, 0, 0, &h); + REQUIRE(rc == AE_SUCCESS); + return h; +} +} // namespace + +TEST_CASE("native AdiIndex handles ESTAELEC compound/computed/conditional tags") { + // NO EnvGuard: exercise the genuine AdiIndex path (not the CDX reroute). + fs::path dir = fs::temp_directory_path() / "openads_adi_native_estaelec"; + std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); + + UNSIGNED8 srv[260]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + + UNSIGNED8 tbl[] = "estaelec.adt"; + UNSIGNED8 def[] = "CCODIGOCON,Character,3;CDOCUMETRA,Character,8;" + "CCODIGOCLI,Character,10;CPREFIJTRA,Character,4;" + "DFECTRATRA,Date,8;CCORENVELE,Character,1"; + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, def, &hT) + == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + fs::rename(dir / "estaelec.adt", dir / "estaelec.DAT", ec); + REQUIRE(!ec); + UNSIGNED8 dat[] = "estaelec.DAT"; + hT = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + == AE_SUCCESS); + + struct Row { const char* con; const char* doc; const char* cli; + const char* pre; const char* fec; const char* cor; }; + const Row rows[4] = { + {"001", "00000010", "CLIENTE001", "FA01", "20260103", "S"}, // rec1 + {"002", "00000020", "CLIENTE002", "FA02", "20260101", "N"}, // rec2 + {"001", "00000030", "CLIENTE003", "FA03", "20260102", "S"}, // rec3 + {"003", "00000005", "CLIENTE004", "FA01", "20260104", "N"}, // rec4 + }; + for (const auto& r : rows) { + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + set_c(hT, "CCODIGOCON", r.con); + set_c(hT, "CDOCUMETRA", r.doc); + set_c(hT, "CCODIGOCLI", r.cli); + set_c(hT, "CPREFIJTRA", r.pre); + set_c(hT, "DFECTRATRA", r.fec); + set_c(hT, "CCORENVELE", r.cor); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); + } + + std::string bags = (dir / "estaelec.adi").string(); + ADSHANDLE o1 = make_tag(hT, bags.c_str(), "ORD1", "CCODIGOCON+CDOCUMETRA", nullptr); + ADSHANDLE o2 = make_tag(hT, bags.c_str(), "ORD2", "CCODIGOCLI", nullptr); + ADSHANDLE o3 = make_tag(hT, bags.c_str(), "ORD3", "CPREFIJTRA+CDOCUMETRA", nullptr); + ADSHANDLE o4 = make_tag(hT, bags.c_str(), "ORD4", "DTOS(DFECTRATRA)", nullptr); + ADSHANDLE o5 = make_tag(hT, bags.c_str(), "ORD5", "DTOS(DFECTRATRA)", "CCORENVELE != 'S'"); + (void)o2; + + UNSIGNED16 nidx = 0; + REQUIRE(AdsGetNumIndexes(hT, &nidx) == AE_SUCCESS); + CHECK(nidx == 5); + + REQUIRE(AdsSetIndexOrderByHandle(hT, o1) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 1u); + REQUIRE(AdsSetIndexOrderByHandle(hT, o3) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 4u); + + REQUIRE(AdsSetIndexOrderByHandle(hT, o4) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 2u); + + UNSIGNED32 kc = 0; + REQUIRE(AdsGetKeyCount(o5, 0, &kc) == AE_SUCCESS); + CHECK(kc == 2u); + + REQUIRE(AdsSetIndexOrderByHandle(hT, o1) == AE_SUCCESS); + UNSIGNED8 key[] = "00100000030"; + UNSIGNED16 found = 0; + REQUIRE(AdsSeek(o1, key, 11, ADS_STRINGKEY, 0, &found) == AE_SUCCESS); + CHECK(found != 0); + CHECK(recno(hT) == 3u); + + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + + // Reopen: auto-open must rebind the 5 distinct tags BY NAME. + hT = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + == AE_SUCCESS); + UNSIGNED16 nidx2 = 0; + REQUIRE(AdsGetNumIndexes(hT, &nidx2) == AE_SUCCESS); + CHECK(nidx2 == 5); + REQUIRE(AdsSetIndexOrder(hT, (UNSIGNED8*)"ORD3") == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 4u); + REQUIRE(AdsSetIndexOrder(hT, (UNSIGNED8*)"ORD1") == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(recno(hT) == 1u); + + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + fs::remove_all(dir, ec); +} + +// Tag ORDINALS must follow creation order (ordinal 1 = first created), matching +// CDX/DBFCDX, because the ERP's xxBrowse column-click ordering (ordenaColumna) +// does DBSETORDER(n)/ORDNAME(n) by tag NUMBER. SAP's .adi prepends new tags +// (reversing ordinals) — add_tag must APPEND so DBSETORDER(1) picks the FIRST +// created tag, not the last. +TEST_CASE("native AdiIndex tag ordinals follow creation order (xxBrowse DBSETORDER)") { + fs::path dir = fs::temp_directory_path() / "openads_adi_ordinal"; + std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); + UNSIGNED8 srv[260]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + UNSIGNED8 tbl[] = "ord.adt"; + UNSIGNED8 def[] = "CCODE,Character,4;CNAME,Character,10;CCITY,Character,8"; + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, def, &hT) + == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + fs::rename(dir / "ord.adt", dir / "ord.DAT", ec); REQUIRE(!ec); + UNSIGNED8 dat[] = "ord.DAT"; + hT = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + == AE_SUCCESS); + for (int i = 1; i <= 3; ++i) { + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + set_c(hT, "CCODE", ("C" + std::to_string(i)).c_str()); + set_c(hT, "CNAME", ("N" + std::to_string(i)).c_str()); + set_c(hT, "CCITY", ("Y" + std::to_string(i)).c_str()); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); + } + std::string bag = (dir / "ord.adi").string(); + make_tag(hT, bag.c_str(), "TAGCODE", "CCODE", nullptr); // ordinal 1 + make_tag(hT, bag.c_str(), "TAGNAME", "CNAME", nullptr); // ordinal 2 + make_tag(hT, bag.c_str(), "TAGCITY", "CCITY", nullptr); // ordinal 3 + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + + hT = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + == AE_SUCCESS); + UNSIGNED8 b[260]{}; std::memcpy(b, bag.c_str(), bag.size()); + ADSHANDLE arr[16] = {0}; UNSIGNED16 alen = 16; + REQUIRE(AdsOpenIndex(hT, b, arr, &alen) == AE_SUCCESS); + REQUIRE(alen == 3); + auto iname = [](ADSHANDLE h) { + UNSIGNED8 nm[64]{}; UNSIGNED16 nl = 64; + AdsGetIndexName(h, nm, &nl); + return std::string(reinterpret_cast(nm)); + }; + CHECK(iname(arr[0]) == "TAGCODE"); // DBSETORDER(1) -> first created + CHECK(iname(arr[1]) == "TAGNAME"); // DBSETORDER(2) + CHECK(iname(arr[2]) == "TAGCITY"); // DBSETORDER(3) + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/abi_adi_reindex_bench_test.cpp b/tests/unit/abi_adi_reindex_bench_test.cpp new file mode 100644 index 00000000..9e53e4fc --- /dev/null +++ b/tests/unit/abi_adi_reindex_bench_test.cpp @@ -0,0 +1,297 @@ +// MANUAL BENCHMARK (skipped by default; run with --no-skip --test-case="BENCH*"). +// Measures where ADT reindex time goes, to decide whether a single-scan, +// all-tags-per-pass reindex is worth changing the (CDX-shared) reindex path. +// +// Model: Table::reindex scans the whole table ONCE PER TAG. For T tags that is +// T * (scan+decode) + T * (eval+sort+build). +// A single-scan reindex would be +// 1 * (scan+decode) + T * (eval+sort+build), +// i.e. it saves (T-1) * (scan+decode). So the decisive number is the cost of +// one full scan+decode (Tscan) relative to the whole reindex. +#include "doctest.h" +#include "openads/ace.h" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using clk = std::chrono::steady_clock; + +namespace { +double ms_since(clk::time_point t0) { + return std::chrono::duration(clk::now() - t0).count(); +} +void set_c(ADSHANDLE h, const char* field, const char* val) { + UNSIGNED8 f[32]{}; std::strncpy(reinterpret_cast(f), field, 31); + UNSIGNED8 v[32]{}; std::strncpy(reinterpret_cast(v), val, 31); + AdsSetString(h, f, v, static_cast(std::strlen(val))); +} +} // namespace + +TEST_CASE("BENCH: ADT reindex scan-per-tag cost" * doctest::skip()) { + const int N = 200000; // ~half of ESTAELEC; extrapolate linearly + const int NTAGS = 5; + + fs::path dir = fs::temp_directory_path() / "openads_reindex_bench"; + std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); + + UNSIGNED8 srv[260]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + + // ESTAELEC-like schema (same fields the native oracle uses). + UNSIGNED8 tbl[] = "bench.adt"; + UNSIGNED8 def[] = "CCODIGOCON,Character,3;CDOCUMETRA,Character,8;" + "CCODIGOCLI,Character,10;CPREFIJTRA,Character,4;" + "DFECTRATRA,Date,8;CCORENVELE,Character,1"; + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, def, &hT) + == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + fs::rename(dir / "bench.adt", dir / "bench.DAT", ec); REQUIRE(!ec); + UNSIGNED8 dat[] = "bench.DAT"; + hT = 0; + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + == AE_SUCCESS); + + // ── Populate ───────────────────────────────────────────────────────────── + auto t0 = clk::now(); + for (int i = 0; i < N; ++i) { + char con[8], doc[16], cli[16], pre[8], fec[16], cor[2]; + std::snprintf(con, sizeof(con), "%03d", i % 1000); + std::snprintf(doc, sizeof(doc), "%08d", i); + std::snprintf(cli, sizeof(cli), "CLIENTE%03d", i % 1000); + std::snprintf(pre, sizeof(pre), "FA%02d", i % 100); + std::snprintf(fec, sizeof(fec), "202601%02d", (i % 28) + 1); + cor[0] = (i % 2) ? 'S' : 'N'; cor[1] = 0; + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + set_c(hT, "CCODIGOCON", con); + set_c(hT, "CDOCUMETRA", doc); + set_c(hT, "CCODIGOCLI", cli); + set_c(hT, "CPREFIJTRA", pre); + set_c(hT, "DFECTRATRA", fec); + set_c(hT, "CCORENVELE", cor); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); + } + double t_pop = ms_since(t0); + + // ── Tscan: one full physical scan+decode (no index active yet) ────────── + t0 = clk::now(); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + long scanned = 0; + for (;;) { + UNSIGNED16 eof = 0; + AdsAtEOF(hT, &eof); + if (eof) break; + UNSIGNED8 b1[16]{}, b2[16]{}; UNSIGNED32 l1 = sizeof(b1), l2 = sizeof(b2); + AdsGetString(hT, (UNSIGNED8*)"CCODIGOCON", b1, &l1, 0); + AdsGetString(hT, (UNSIGNED8*)"CDOCUMETRA", b2, &l2, 0); + ++scanned; + AdsSkip(hT, 1); + } + double t_scan = ms_since(t0); + + // ── Build the 5 tags (each AdsCreateIndex61 = its own full scan) ───────── + std::string bag = (dir / "bench.adi").string(); + struct Tag { const char* name; const char* expr; const char* cond; }; + const Tag tags[NTAGS] = { + {"ORD1", "CCODIGOCON+CDOCUMETRA", nullptr}, + {"ORD2", "CCODIGOCLI", nullptr}, + {"ORD3", "CPREFIJTRA+CDOCUMETRA", nullptr}, + {"ORD4", "DTOS(DFECTRATRA)", nullptr}, + {"ORD5", "DTOS(DFECTRATRA)", "CCORENVELE != 'S'"}, + }; + double t_create_total = 0; + for (const auto& tg : tags) { + UNSIGNED8 b[260]{}; std::strncpy(reinterpret_cast(b), bag.c_str(), 259); + UNSIGNED8 t[64]{}; std::strncpy(reinterpret_cast(t), tg.name, 63); + UNSIGNED8 e[128]{}; std::strncpy(reinterpret_cast(e), tg.expr, 127); + UNSIGNED8 c[128]{}; UNSIGNED8* cp = nullptr; + if (tg.cond) { std::strncpy(reinterpret_cast(c), tg.cond, 127); cp = c; } + ADSHANDLE h = 0; + auto tc = clk::now(); + REQUIRE(AdsCreateIndex61(hT, b, t, e, cp, nullptr, 0, 0, &h) == AE_SUCCESS); + t_create_total += ms_since(tc); + } + + // ── T_reindex: the real Table::reindex (scans the table once PER tag) ──── + t0 = clk::now(); + REQUIRE(AdsReindex(hT) == AE_SUCCESS); + double t_reindex = ms_since(t0); + + // ── Report + projection ────────────────────────────────────────────────── + double saved = (NTAGS - 1) * t_scan; // single-scan saves (T-1) scans + double proj = t_reindex - saved; + std::fprintf(stderr, + "\n===== REINDEX BENCH (N=%d rows, %d tags) =====\n" + " populate : %8.0f ms\n" + " Tscan (1 pass) : %8.0f ms (%.1f %% of reindex)\n" + " create 5 tags total: %8.0f ms\n" + " AdsReindex (N-scan): %8.0f ms\n" + " -- projection --\n" + " single-scan saves : %8.0f ms ((T-1) x Tscan)\n" + " projected 1-scan : %8.0f ms (%.0f %% of current)\n" + " extrapolated x2.2 (->441k): reindex %.1f s -> 1-scan %.1f s\n" + "================================================\n", + N, NTAGS, t_pop, t_scan, 100.0 * t_scan / t_reindex, + t_create_total, t_reindex, saved, proj, 100.0 * proj / t_reindex, + t_reindex * 2.2 / 1000.0, proj * 2.2 / 1000.0); + std::fflush(stderr); + + CHECK(scanned == N); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + fs::remove_all(dir, ec); +} + +// Where does the ~1 ms/record write cost come from? Breaks the ADT write path +// into append / set-fields / rewrite / index-maintenance / delete / PACK so we +// can see which one dominates the GUI reindex (4 min vs SAP 2 min). +TEST_CASE("BENCH: ADT write-path breakdown" * doctest::skip()) { + const int N = 30000; + + auto make_table = [&](ADSHANDLE hConn, const char* stem) -> ADSHANDLE { + UNSIGNED8 tbl[64]; std::snprintf(reinterpret_cast(tbl), 64, "%s.adt", stem); + UNSIGNED8 def[] = "CCODIGOCON,Character,3;CDOCUMETRA,Character,8;" + "CCODIGOCLI,Character,10;CPREFIJTRA,Character,4;" + "DFECTRATRA,Date,8;CCORENVELE,Character,1"; + ADSHANDLE h = 0; + REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0, def, &h) + == AE_SUCCESS); + return h; + }; + auto fill = [&](ADSHANDLE h, int i) { + char con[8], doc[16], cli[16], pre[8], fec[16], cor[2]; + std::snprintf(con, sizeof(con), "%03d", i % 1000); + std::snprintf(doc, sizeof(doc), "%08d", i); + std::snprintf(cli, sizeof(cli), "CLIENTE%03d", i % 1000); + std::snprintf(pre, sizeof(pre), "FA%02d", i % 100); + std::snprintf(fec, sizeof(fec), "202601%02d", (i % 28) + 1); + cor[0] = (i % 2) ? 'S' : 'N'; cor[1] = 0; + set_c(h, "CCODIGOCON", con); set_c(h, "CDOCUMETRA", doc); + set_c(h, "CCODIGOCLI", cli); set_c(h, "CPREFIJTRA", pre); + set_c(h, "DFECTRATRA", fec); set_c(h, "CCORENVELE", cor); + }; + + fs::path dir = fs::temp_directory_path() / "openads_writebench"; + std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); + UNSIGNED8 srv[260]{}; std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) == AE_SUCCESS); + + // (1) append-only: AdsAppendRecord + AdsWriteRecord, no field sets. + ADSHANDLE a = make_table(hConn, "appendonly"); + auto t0 = clk::now(); + for (int i = 0; i < N; ++i) { + REQUIRE(AdsAppendRecord(a) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(a) == AE_SUCCESS); + } + double t_append = ms_since(t0); + + // (2) full populate: append + 6 set-fields + write. + ADSHANDLE b = make_table(hConn, "populate"); + t0 = clk::now(); + for (int i = 0; i < N; ++i) { + REQUIRE(AdsAppendRecord(b) == AE_SUCCESS); + fill(b, i); + REQUIRE(AdsWriteRecord(b) == AE_SUCCESS); + } + double t_pop = ms_since(t0); + + // (3) rewrite one field in place, NO index bound. + t0 = clk::now(); + REQUIRE(AdsGotoTop(b) == AE_SUCCESS); + for (int i = 0; i < N; ++i) { + set_c(b, "CCODIGOCLI", "REWRITTEN0"); + REQUIRE(AdsWriteRecord(b) == AE_SUCCESS); + AdsSkip(b, 1); + } + double t_rewrite_noidx = ms_since(t0); + + // (4) delete half + PACK (the DELETE-ALL-FOR + copy-down the GUI reindex does). + t0 = clk::now(); + REQUIRE(AdsGotoTop(b) == AE_SUCCESS); + for (int i = 0; i < N; ++i) { + if (i % 2 == 0) AdsDeleteRecord(b); + AdsSkip(b, 1); + } + double t_delete = ms_since(t0); + t0 = clk::now(); + REQUIRE(AdsPackTable(b) == AE_SUCCESS); + double t_pack = ms_since(t0); + REQUIRE(AdsCloseTable(b) == AE_SUCCESS); + + // (5) rewrite one INDEXED field WITH 5 tags bound (incremental maintenance). + ADSHANDLE c = make_table(hConn, "withidx"); + for (int i = 0; i < N; ++i) { + REQUIRE(AdsAppendRecord(c) == AE_SUCCESS); fill(c, i); + REQUIRE(AdsWriteRecord(c) == AE_SUCCESS); + } + std::string bag = (dir / "withidx.adi").string(); + const char* exprs[5] = {"CCODIGOCON+CDOCUMETRA","CCODIGOCLI","CPREFIJTRA+CDOCUMETRA", + "DTOS(DFECTRATRA)","DTOS(DFECTRATRA)"}; + const char* names[5] = {"ORD1","ORD2","ORD3","ORD4","ORD5"}; + for (int k = 0; k < 5; ++k) { + UNSIGNED8 bb[260]{}; std::strncpy(reinterpret_cast(bb), bag.c_str(), 259); + UNSIGNED8 tt[64]{}; std::strncpy(reinterpret_cast(tt), names[k], 63); + UNSIGNED8 ee[128]{}; std::strncpy(reinterpret_cast(ee), exprs[k], 127); + ADSHANDLE h = 0; + REQUIRE(AdsCreateIndex61(c, bb, tt, ee, nullptr, nullptr, 0, 0, &h) == AE_SUCCESS); + } + t0 = clk::now(); + REQUIRE(AdsGotoTop(c) == AE_SUCCESS); + for (int i = 0; i < N; ++i) { + char doc[16]; std::snprintf(doc, sizeof(doc), "%08d", i + N); // change a key field + set_c(c, "CDOCUMETRA", doc); + REQUIRE(AdsWriteRecord(c) == AE_SUCCESS); + AdsSkip(c, 1); + } + double t_rewrite_idx = ms_since(t0); + REQUIRE(AdsCloseTable(c) == AE_SUCCESS); + + // (6) append+write with DEFERRED FLUSH (no per-record fsync) + 1 final sync. + ADSHANDLE d = make_table(hConn, "deferred"); + AdsSetDeferredFlush(d, 1); + t0 = clk::now(); + for (int i = 0; i < N; ++i) { + REQUIRE(AdsAppendRecord(d) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(d) == AE_SUCCESS); + } + AdsFlushFileBuffers(d); + double t_append_deferred = ms_since(t0); + REQUIRE(AdsCloseTable(d) == AE_SUCCESS); + + auto us = [&](double ms_total) { return 1000.0 * ms_total / N; }; + std::fprintf(stderr, + "\n===== ADT WRITE-PATH BREAKDOWN (N=%d) =====\n" + " (1) append+write : %8.0f ms (%6.1f us/rec)\n" + " (6) append+write DEFERRED : %8.0f ms (%6.1f us/rec) [no per-record fsync]\n" + " (2) populate (6 fields) : %8.0f ms (%6.1f us/rec)\n" + " -> set 6 fields : %8.0f ms (%6.1f us/rec)\n" + " (3) rewrite, no index : %8.0f ms (%6.1f us/rec)\n" + " (5) rewrite, 5 indexes : %8.0f ms (%6.1f us/rec)\n" + " -> index maintenance : %8.0f ms (%6.1f us/rec) [5 tags: erase+insert]\n" + " (4) delete half : %8.0f ms (%6.1f us/rec)\n" + " PACK (copy-down N/2) : %8.0f ms (%6.1f us/surv)\n" + "===========================================\n", + N, + t_append, us(t_append), + t_append_deferred, us(t_append_deferred), + t_pop, us(t_pop), + t_pop - t_append, us(t_pop - t_append), + t_rewrite_noidx, us(t_rewrite_noidx), + t_rewrite_idx, us(t_rewrite_idx), + t_rewrite_idx - t_rewrite_noidx, us(t_rewrite_idx - t_rewrite_noidx), + t_delete, us(t_delete), + t_pack, 1000.0 * t_pack / (N / 2)); + std::fflush(stderr); + + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/abi_sql_temp_browse_nav_test.cpp b/tests/unit/abi_sql_temp_browse_nav_test.cpp new file mode 100644 index 00000000..2ff039a5 --- /dev/null +++ b/tests/unit/abi_sql_temp_browse_nav_test.cpp @@ -0,0 +1,173 @@ +#include "doctest.h" +#include "openads/ace.h" + +#include +#include +#include +#include +#include +#include +#include + +// A single-table SELECT ... ORDER BY (and DISTINCT / LIMIT) returns a +// MATERIALISED static cursor (ADS_CDX semantics): its own temp table, isolated +// from the source, with its own recnos 1..N in result order. The ERP browses +// these in a TXBrowse (e.g. BuscaRegistro: "Select * From [articulo.dat] WHERE +// UPPER(cnombreart) LIKE 'X%' ORDER BY cnombreart") and then runs INDEX ON / +// DBSETORDER on the result — which must hit the TEMP, never the production +// table's official .cdx. This test pins that the materialised cursor browses +// cleanly: GOTO-then-SKIP walks in result order, OrdKeyNo / KeyCount / RelKeyPos +// and bookmark round-trip all agree (recno == position on the static cursor). + +namespace fs = std::filesystem; + +namespace { + +fs::path stage_dbf(const fs::path& dir) { + fs::create_directories(dir); + auto p = dir / "data.dbf"; + fs::remove(p); + std::vector file; + auto push = [&](const void* d, std::size_t n) { + const auto* b = static_cast(d); + file.insert(file.end(), b, b + n); + }; + std::array hdr{}; + hdr[0] = 0x03; + hdr[4] = 5; // record count + hdr[8] = 32 + 32 + 1; // header length + hdr[10] = 1 + 4; // record length + push(hdr.data(), hdr.size()); + std::array fd{}; + std::strncpy(reinterpret_cast(fd.data()), "TAG", 11); + fd[11] = 'C'; fd[16] = 4; + push(fd.data(), fd.size()); + file.push_back(0x0D); + auto rec = [&](const char* s) { + file.push_back(' '); // not-deleted flag + for (int i = 0; i < 4; ++i) + file.push_back(i < (int)std::strlen(s) + ? static_cast(s[i]) : ' '); + }; + // recno: 1=CCCC 2=AAAA 3=DDDD 4=BBBB 5=EEEE (sorted order != recno order) + rec("CCCC"); rec("AAAA"); rec("DDDD"); rec("BBBB"); rec("EEEE"); + file.push_back(0x1A); + std::ofstream(p, std::ios::binary).write( + reinterpret_cast(file.data()), + static_cast(file.size())); + return p; +} + +std::string field(ADSHANDLE h, const char* f) { + UNSIGNED8 buf[32] = {0}; + UNSIGNED32 cap = sizeof(buf); + UNSIGNED8 fld[16] = {0}; + std::memcpy(fld, f, std::strlen(f) + 1); + if (AdsGetField(h, fld, buf, &cap, 0) != 0) return {}; + std::string s(reinterpret_cast(buf), cap); + while (!s.empty() && s.back() == ' ') s.pop_back(); + return s; +} + +} // namespace + +TEST_CASE("SQL single-table ORDER BY result: browse nav stays in sync") { + auto dir = fs::temp_directory_path() / "openads_sql_temp_browse_nav"; + std::error_code ec; + fs::remove_all(dir, ec); + stage_dbf(dir); + + UNSIGNED8 srv[256]; + std::memcpy(srv, dir.string().c_str(), dir.string().size() + 1); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, + nullptr, nullptr, 0, &hConn) == 0); + ADSHANDLE hStmt = 0; + REQUIRE(AdsCreateSQLStatement(hConn, &hStmt) == 0); + + // WHERE drops AAAA (recno 2). Sorted visible set: BBBB,CCCC,DDDD,EEEE. + UNSIGNED8 sql[200] = + "SELECT * FROM data.dbf WHERE TAG >= 'BBBB' ORDER BY TAG"; + ADSHANDLE hCur = 0; + REQUIRE(AdsExecuteSQLDirect(hStmt, sql, &hCur) == 0); + + // The ORDER BY result is a materialised static cursor (ADS_CDX semantics), + // so it has its OWN recnos 1..N in result order — not the source recnos. + const UNSIGNED32 exp_recno[4] = {1, 2, 3, 4}; + const char* exp_val[4] = {"BBBB", "CCCC", "DDDD", "EEEE"}; + + // Counts reflect the visible (filtered+sorted) set, not the full table. + UNSIGNED32 rcount = 0; + REQUIRE(AdsGetRecordCount(hCur, 0, &rcount) == 0); + CHECK(rcount == 4u); + UNSIGNED32 kcount = 0; + REQUIRE(AdsGetKeyCount(hCur, 0, &kcount) == 0); + CHECK(kcount == 4u); + + // Walk top->EOF: sorted values in order; OrdKeyNo = 1..N; recno = source. + REQUIRE(AdsGotoTop(hCur) == 0); + for (int i = 0; i < 4; ++i) { + UNSIGNED32 rn = 0, kn = 0; + REQUIRE(AdsGetRecordNum(hCur, 0, &rn) == 0); + REQUIRE(AdsGetKeyNum(hCur, 0, &kn) == 0); + CHECK(rn == exp_recno[i]); + CHECK(kn == static_cast(i + 1)); + CHECK(field(hCur, "TAG") == exp_val[i]); + if (i < 3) REQUIRE(AdsSkip(hCur, 1) == 0); + } + UNSIGNED16 eof = 0; + REQUIRE(AdsSkip(hCur, 1) == 0); + REQUIRE(AdsAtEOF(hCur, &eof) == 0); + CHECK(eof == 1); + + // THE BUG: land on a row by recno (bookmark restore), then SKIP must + // advance in SORTED order and read the right row — not jump to a physical + // neighbour. OrdKeyNo at the landed row is the sorted position, not recno. + for (int i = 0; i < 3; ++i) { + REQUIRE(AdsGotoRecord(hCur, exp_recno[i]) == 0); + UNSIGNED32 kn = 0; + REQUIRE(AdsGetKeyNum(hCur, 0, &kn) == 0); + CHECK(kn == static_cast(i + 1)); + CHECK(field(hCur, "TAG") == exp_val[i]); + REQUIRE(AdsSkip(hCur, 1) == 0); + UNSIGNED32 rn = 0; + REQUIRE(AdsGetRecordNum(hCur, 0, &rn) == 0); + CHECK(rn == exp_recno[i + 1]); + CHECK(field(hCur, "TAG") == exp_val[i + 1]); + } + + // RelKeyPos: first sorted row -> 0.0, last -> 1.0 (scrollbar thumb). + REQUIRE(AdsGotoRecord(hCur, exp_recno[0]) == 0); + double p0 = -1.0; + REQUIRE(AdsGetRelKeyPos(hCur, &p0) == 0); + CHECK(p0 == doctest::Approx(0.0)); + REQUIRE(AdsGotoRecord(hCur, exp_recno[3]) == 0); + double p3 = -1.0; + REQUIRE(AdsGetRelKeyPos(hCur, &p3) == 0); + CHECK(p3 == doctest::Approx(1.0)); + + // SetRelKeyPos (thumb drag): fraction -> sorted position, then SKIP keeps + // walking the sequence. + REQUIRE(AdsSetRelKeyPos(hCur, 0.0) == 0); + CHECK(field(hCur, "TAG") == "BBBB"); + REQUIRE(AdsSetRelKeyPos(hCur, 1.0) == 0); + CHECK(field(hCur, "TAG") == "EEEE"); + + // Bookmark round-trip on a middle row (DDDD, sorted pos 3 / source recno 3). + REQUIRE(AdsGotoRecord(hCur, 3) == 0); + UNSIGNED8 bm[16] = {0}; + UNSIGNED32 bmlen = sizeof(bm); + REQUIRE(AdsGetBookmark60(hCur, bm, &bmlen) == 0); + REQUIRE(AdsGotoTop(hCur) == 0); + REQUIRE(AdsGotoBookmark60(hCur, bm, bmlen) == 0); + UNSIGNED32 rn = 0, kn = 0; + REQUIRE(AdsGetRecordNum(hCur, 0, &rn) == 0); + REQUIRE(AdsGetKeyNum(hCur, 0, &kn) == 0); + CHECK(rn == 3u); + CHECK(kn == 3u); + CHECK(field(hCur, "TAG") == "DDDD"); + + REQUIRE(AdsCloseSQLStatement(hStmt) == 0); + REQUIRE(AdsDisconnect(hConn) == 0); + fs::remove_all(dir, ec); +} From 606e8f5f3871f9bef77f207882e1c1503020e8ae Mon Sep 17 00:00:00 2001 From: russimicro Date: Sun, 26 Jul 2026 11:15:11 -0500 Subject: [PATCH 04/12] test: register the Russoft ADI/CDX/PACK regression tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings over the 9 tests written against our June tree and registers them in tests/CMakeLists.txt. They pin ERP behaviour that upstream has no coverage for, so they double as the checklist for what is still missing from this integration branch. Green now: abi_adi_clear_multilevel, abi_adi_dat_extension_path, abi_adi_frontcoding_size (abi_adi_reindex_bench is skip-by-default). RED — remaining gaps, all in ace_exports/engine logic not yet ported: abi_adi_estaelec_compound (ADI->CDX reroute, 5000) abi_adi_native_estaelec (2 cases: compound/computed/FOR tags, and tag ordinals in creation order, 5000) abi_cdx_estaelec_compound (navigation lands on recno 1) abi_stale_index_walk (AdsGotoTop 5000 after PACK) abi_sql_temp_browse_nav (ORDER BY result browse out of sync) Suite: 1181/1199 — the 12 pre-existing SQL-parser (7200) failures plus these 6. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 4a9a27a15a933bef1e3397579141528a1281841c) (cherry picked from commit 3901f4f1d106da76b9ca4a1e2bcd1cef423cdf9b) --- tests/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d1fbb3ec..26bc897f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -242,9 +242,18 @@ add_executable(openads_unit_tests unit/abi_adi_multilevel_build_test.cpp unit/abi_adt_scope_validation_test.cpp unit/abi_adi_smoke_test.cpp + unit/abi_adi_clear_multilevel_test.cpp + unit/abi_adi_dat_extension_path_test.cpp + unit/abi_adi_estaelec_compound_test.cpp + unit/abi_adi_frontcoding_size_test.cpp unit/abi_adi_keycount_test.cpp + unit/abi_adi_native_estaelec_test.cpp + unit/abi_adi_reindex_bench_test.cpp unit/abi_adi_tagdir_order_test.cpp unit/abi_adi_tagdir_wide_page_test.cpp + unit/abi_cdx_estaelec_compound_test.cpp + unit/abi_sql_temp_browse_nav_test.cpp + unit/abi_stale_index_walk_test.cpp unit/abi_adt_sql_test.cpp unit/abi_adt_dat_extension_sql_test.cpp unit/openads_sql_c_test.cpp From 347c4c93615ad863292cbdfc7b2de87238a110e0 Mon Sep 17 00:00:00 2001 From: russimicro Date: Sun, 26 Jul 2026 11:29:39 -0500 Subject: [PATCH 05/12] feat(adi): create v2 tags for compound/computed/FOR expressions on ADT Upstream's ADI create path accepts only a bare field name ('ADI index expression must be a bare field name'), so the ERP's ESTAELEC tag set (cCodigoCon+cDocumeTra, DTOS(dFecTraTra), FOR cCorEnvEle != 'S') could not be built on an ADT table at all. Ported from our tree: - AdsCreateIndex61 falls back to the first field for tag metadata when the expression is not a bare field, and creates a v2 tag (identity by NAME, with key expression / FOR condition / full key length persisted in the per-tag header). A bare numeric or date field keeps the legacy numeric ADI leaf so existing packed-key seeks still work. - Overwriting an existing tag matches on the v2 tag name (falling back to the field name for legacy bags) and clears its tree instead of failing. - ADI->CDX reroute: an ADT table's .adi bag can be routed through the CdxIndex engine via OPENADS_ADT_CDX_INDEX=1, and is routed automatically whenever the bag on disk already carries the CDX 'RCHB' signature, so a reroute-written bag opens correctly even without the env flag. AdsOpenIndex mirrors the same routing. Fixes abi_adi_native_estaelec 'tag ordinals follow creation order'. Suite 1184/1199: the 12 pre-existing SQL-parser failures plus 3 still-red Russoft tests (FOR-clause key count on a native ADI tag; index count after reopening a rerouted bag; SQL ORDER BY cursor recno semantics). Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 1f4f310276d92af8c541dfb1f06bee540992a89c) (cherry picked from commit c65b8bde3176d0f267168b128e597e5fdfc2c756) --- src/abi/ace_exports.cpp | 109 ++++++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 21 deletions(-) diff --git a/src/abi/ace_exports.cpp b/src/abi/ace_exports.cpp index c1905cc6..a94e0f57 100644 --- a/src/abi/ace_exports.cpp +++ b/src/abi/ace_exports.cpp @@ -12426,6 +12426,34 @@ make_index_for(const std::string& path) { } // extern "C++" +namespace { +// Route an ADT table's .adi bag through the CdxIndex engine (full evaluated +// key + 32-bit recno), so compound / computed / FOR tags work over ADT the +// same way they do over DBF. The file is then CDX-format even though it is +// named .adi — only enable when those .ADI files are NOT interchanged with +// real Advantage. Gate: env OPENADS_ADT_CDX_INDEX=1. +bool adt_cdx_index_enabled() { + const char* e = std::getenv("OPENADS_ADT_CDX_INDEX"); + return e != nullptr && e[0] == '1'; +} + +// A .adi bag written by the CdxIndex reroute carries the Harbour CDX structure +// signature "RCHB" at byte offset 20 (see cdx_index.cpp); a native AdiIndex bag +// does not. Detecting it lets the OPEN path pick the right engine REGARDLESS of +// OPENADS_ADT_CDX_INDEX. Without this, a .adi built CDX-format (reroute on at +// reindex) but opened with the flag absent routes to the native AdiIndex +// reader, which cannot parse it -> 5004 -> the table opens with 0 indexes and +// the first DBSETORDER/SEEK fails. +bool adi_bag_is_cdx_format(const std::string& path) { + std::ifstream f(path, std::ios::binary); + if (!f) return false; + char hdr[24] = {0}; + f.read(hdr, sizeof(hdr)); + if (f.gcount() < 24) return false; + return hdr[20] == 'R' && hdr[21] == 'C' && hdr[22] == 'H' && hdr[23] == 'B'; +} +} // namespace + // Compare two filesystem paths for the "is this the same on-disk // file?" question. Falls back to a case-insensitive lexical compare // when canonical resolution fails (e.g. file doesn't exist yet). @@ -12643,7 +12671,15 @@ UNSIGNED32 ENTRYPOINT AdsOpenIndex(ADSHANDLE hTable, UNSIGNED8* pucName, // reopening takes the CDX path instead of misreading it as NTX. is_cdx = file_has_cdx_signature(path); } - if (is_cdx) { + // Open an ADT table's .adi through the CdxIndex engine when the env opt-in + // is set OR the bag on disk is already CDX-format (reroute-written). The + // format check makes OPEN robust to a missing env flag; it mirrors the + // create-side routing in AdsCreateIndex61. + const bool is_adt_tbl = + (dynamic_cast(t->driver()) != nullptr); + const bool adt_to_cdx = is_adi && is_adt_tbl && + (adt_cdx_index_enabled() || adi_bag_is_cdx_format(path)); + if (is_cdx || adt_to_cdx) { auto r = openads::drivers::cdx::CdxIndex::list_tags(path); if (!r) return fail(r.error()); tags = std::move(r).value(); @@ -13305,50 +13341,81 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, exists = (is_cdx || is_adi) && fs::exists(p, ec); } - if (is_adi && is_adt_table) { - // ADT tables use a single .adi bag; each tag indexes one field. + // ADT table + .adi bag -> route to the CdxIndex engine when the env opt-in + // is set OR the bag already exists in CDX format (so adding a tag to a + // reroute-built bag stays CDX even with the flag absent — never mix formats + // in one bag). A fresh bag (the ERP erases the .adi before reindex) has no + // file, so the env flag decides the format to write. + const bool adt_to_cdx = is_adi && is_adt_table && + (adt_cdx_index_enabled() || + (std::filesystem::exists(p) && adi_bag_is_cdx_format(p.string()))); + + if (is_adi && is_adt_table && !adt_to_cdx) { + // ADT tables use a single .adi bag. Prefer bare field, but allow + // compound expressions (Russoft INDEX ON fld1+fld2 for .ADI) by + // falling back to first field for tag metadata. Keys are built from + // the full evaluated expr at population time. const std::string bare = openads::engine::strip_alias_qualifiers(expr); std::int32_t fidx = t->field_index(bare); + const bool is_compound = (fidx < 0); // computed / multi-field expression if (fidx < 0) { - return fail(openads::AE_COLUMN_NOT_FOUND, - "ADI index expression must be a bare field name"); + fidx = 0; // fallback for compound expr } if (fidx + 1 > 255) { return fail(openads::AE_INTERNAL_ERROR, "ADI index does not support field numbers greater than 255"); } const auto& fd = t->field_descriptor(static_cast(fidx)); + const std::uint16_t adt_t = static_cast( + static_cast(fd.raw_type)); + const bool is_char_field = + adt_t == openads::drivers::adi::ADT_TYPE_CHAR || + adt_t == openads::drivers::adi::ADT_TYPE_CICHAR; + // The v2 opaque-key leaf (full key stored, memcmp-ordered) is correct for + // char fields and ANY computed/compound expression — exactly what the ERP + // uses (STR()/DTOS()/concat → character keys). A BARE numeric/date field + // keeps the legacy numeric ADI leaf (sign-flipped float keys) so the + // existing packed-key seeks keep working. + const bool use_v2 = is_char_field || is_compound; openads::drivers::adi::AdiIndex::CreateParams cp{}; cp.field_num = static_cast(fidx + 1); cp.field_name = fd.name; - cp.adt_type = static_cast( - static_cast(fd.raw_type)); + cp.adt_type = adt_t; cp.fld_length = fd.length; + cp.record_offset = fd.record_offset; cp.adt_hdr_len = t->driver()->header_length(); cp.adt_rec_len = t->driver()->record_length(); cp.unique = unique; - // Pass the real table path so a non-structural bag (its .adi stem - // differs from the table's) opens the correct companion ADT instead - // of deriving ".adt" from the index file name. - cp.adt_path = t->path(); - - const bool is_char_key = - cp.adt_type == openads::drivers::adi::ADT_TYPE_CHAR || - cp.adt_type == openads::drivers::adi::ADT_TYPE_CICHAR; - klen = is_char_key ? fd.length : 8; + cp.adt_path = t->path(); // important for .DAT + ADS_ADT case (not .adt) + if (use_v2) { + // identity by NAME + persisted expression/FOR + full opaque key + // (klen stays the full evaluated-expression length; the build loop + // below evaluates the whole expression). + cp.tag_name = tag; + cp.key_expr = expr; + cp.for_expr = for_expr; + cp.key_len = static_cast(klen); + cp.descending = descend; + } else { + klen = 8; // bare numeric/date → legacy numeric ADI leaf geometry + } if (exists) { + // v2 identity is the TAG name (list_tags returns tag names now); + // legacy callers without a tag_name fall back to the field name. + const std::string ident = + cp.tag_name.empty() ? fd.name : cp.tag_name; auto tags = openads::drivers::adi::AdiIndex::list_tags( p.string(), t->path()); bool have_tag = false; if (tags) { for (const auto& tn : tags.value()) { - if (tn.size() == fd.name.size()) { + if (tn.size() == ident.size()) { bool eq = true; for (std::size_t i = 0; i < tn.size(); ++i) { if (std::tolower(static_cast(tn[i])) != std::tolower(static_cast( - fd.name[i]))) { + ident[i]))) { eq = false; break; } @@ -13361,7 +13428,7 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, openads::drivers::adi::AdiIndex existing; auto reopen = existing.open_named( p.string(), openads::drivers::IndexOpenMode::Shared, - fd.name, t->path()); + ident, t->path()); if (!reopen) return fail(reopen.error()); if (auto cl = existing.clear_data(); !cl) return fail(cl.error()); idx_owner = std::make_unique< @@ -13380,7 +13447,7 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, idx_owner = std::make_unique( std::move(created).value()); } - } else if (is_cdx && exists) { + } else if ((is_cdx || adt_to_cdx) && exists) { // Harbour rddads / Clipper semantics: re-creating an // existing tag is a silent overwrite, not an error. If the // tag already exists, open it and clear its B+tree so the @@ -13420,7 +13487,7 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, idx_owner = std::make_unique( std::move(added).value()); } - } else if (is_cdx) { + } else if (is_cdx || adt_to_cdx) { auto created = openads::drivers::cdx::CdxIndex::create( p.string(), tag, expr, klen, unique, descend, for_expr); if (!created) return fail(created.error()); From 62370b724e6847ceca5d3a574f7e5613ae71b8dd Mon Sep 17 00:00:00 2001 From: russimicro Date: Sun, 26 Jul 2026 11:46:53 -0500 Subject: [PATCH 06/12] fix(adi): key count on a FOR tag, and open a rerouted bag through CdxIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ADI paths still assumed CDX-or-nothing: - AdsGetKeyCount / AdsGetRecordCount(index handle) special-cased CdxIndex and otherwise fell through to the table's record_count(). A native ADI v2 tag with a FOR clause therefore reported every row (4) instead of its matching subset (2), which is what rddads' OrdKeyCount drives xBrowse position math from. Both now count the ADI index walk, honouring SET DELETED. - AdsOpenIndex listed a rerouted bag's tags through CdxIndex but then opened each tag with the native AdiIndex reader, which cannot parse a CDX-format .adi — the call failed and the auto-open on AdsOpenTable swallowed it, so a reopened table showed 0 indexes and the first AdsSetIndexOrder returned 5000. The per-tag open now follows the same adt_to_cdx routing as the listing. Fixes abi_adi_native_estaelec and abi_adi_estaelec_compound. Suite 1186/1199: the 12 pre-existing SQL-parser (7200) failures plus abi_sql_temp_browse_nav, which is a deliberate semantic difference (ORDER BY cursor recno numbering), not a port gap. Co-Authored-By: Claude Opus 5 (1M context) (cherry picked from commit 2e845acdffe950a79afaba1c207c667473057c8a) (cherry picked from commit c82b422476fcbb1e4117e01ee745a02756092d83) --- src/abi/ace_exports.cpp | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/abi/ace_exports.cpp b/src/abi/ace_exports.cpp index a94e0f57..908a214c 100644 --- a/src/abi/ace_exports.cpp +++ b/src/abi/ace_exports.cpp @@ -10150,6 +10150,21 @@ UNSIGNED32 ENTRYPOINT AdsGetRecordCount(ADSHANDLE hTable, UNSIGNED16 bFilterOpti cdx->ordered_recnos_cached().size()); return ok(); } + if (auto* adi = + dynamic_cast(idx)) { + // Native ADI tag: same rule as CDX — count the index walk, not + // the table, so a FOR-clause tag reports its matching subset. + const bool hide_del = t && !t->show_deleted_records(); + const std::uint32_t saved_rn = t ? t->recno() : 0u; + std::uint32_t n = 0; + for (std::uint32_t rn : adi->ordered_recnos_cached()) { + if (!hide_del) { ++n; continue; } + if (t->goto_record(rn) && !t->is_deleted()) ++n; + } + *pulRecordCount = n; + if (hide_del && saved_rn != 0) (void)t->goto_record(saved_rn); + return ok(); + } } // M10.31 / M10.32 — when SQL has materialised a traversal sequence // (DISTINCT / LIMIT / OFFSET / ORDER BY), report that sequence's @@ -12726,7 +12741,10 @@ UNSIGNED32 ENTRYPOINT AdsOpenIndex(ADSHANDLE hTable, UNSIGNED8* pucName, for (const auto& name : tags) { if (count >= cap) break; std::unique_ptr sub; - if (is_adi) { + // A rerouted bag was listed through CdxIndex above, so it must be + // opened through CdxIndex too — handing a CDX-format .adi to the + // native AdiIndex reader fails and drops every tag on the floor. + if (is_adi && !adt_to_cdx) { auto idx = std::make_unique(); if (auto r = idx->open_named(path, openads::drivers::IndexOpenMode::Shared, @@ -35223,6 +35241,23 @@ UNSIGNED32 ENTRYPOINT AdsGetKeyCount(ADSHANDLE hIndex, UNSIGNED16 /*usFilter*/, } return ok(); } + if (auto* adi = + dynamic_cast(ord->index())) { + // Same reasoning for a native ADI tag: a v2 tag with a FOR clause + // holds only the matching rows, so falling through to the table's + // record_count() reported every row and broke OrdKeyCount on a + // conditional order (the ERP's ORD5 FOR cCorEnvEle != 'S'). + const bool hide_del = !t->show_deleted_records(); + const std::uint32_t saved_rn = t->recno(); + std::uint32_t n = 0; + for (std::uint32_t rn : adi->ordered_recnos_cached()) { + if (!hide_del) { ++n; continue; } + if (t->goto_record(rn) && !t->is_deleted()) ++n; + } + *pulCount = n; + if (hide_del && saved_rn != 0) (void)t->goto_record(saved_rn); + return ok(); + } } *pulCount = t->record_count(); return ok(); From 44a3800b83db44b6c946628fddd997cb5b3a38ec Mon Sep 17 00:00:00 2001 From: russimicro Date: Wed, 5 Aug 2026 10:00:03 -0500 Subject: [PATCH 07/12] fix(abi): AdsGetRecordCount over an ADI tag counts the index walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With an ADI order active the count now comes from the index walk (through count_live_recnos, the helper the CDX branch already uses) instead of the table's physical record count. Without it a conditional (FOR) tag reports every row in the table rather than the subset it indexes — the rule AdsGetKeyCount already applies. It must not walk the table per call: rddads answers OrdKeyCount() through this entry point and FWH's TXBrowse asks for it hundreds of times while opening a screen. Co-Authored-By: Claude Opus 5 (1M context) --- src/abi/ace_exports.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/abi/ace_exports.cpp b/src/abi/ace_exports.cpp index 908a214c..ddfb2278 100644 --- a/src/abi/ace_exports.cpp +++ b/src/abi/ace_exports.cpp @@ -10154,15 +10154,17 @@ UNSIGNED32 ENTRYPOINT AdsGetRecordCount(ADSHANDLE hTable, UNSIGNED16 bFilterOpti dynamic_cast(idx)) { // Native ADI tag: same rule as CDX — count the index walk, not // the table, so a FOR-clause tag reports its matching subset. + // + // Y se cuenta con count_live_recnos, igual que CDX: mirar el + // borrado con goto_record() invalida la cache de lectura en cada + // vuelta, o sea un bloque leido por registro y sin reuso entre + // llamadas. rddads pide OrdKeyCount por esta puerta y un TXBrowse + // la golpea cientos de veces al abrir: con goto_record eran ~9 s + // de pantalla sobre 34.595 filas. const bool hide_del = t && !t->show_deleted_records(); - const std::uint32_t saved_rn = t ? t->recno() : 0u; - std::uint32_t n = 0; - for (std::uint32_t rn : adi->ordered_recnos_cached()) { - if (!hide_del) { ++n; continue; } - if (t->goto_record(rn) && !t->is_deleted()) ++n; - } - *pulRecordCount = n; - if (hide_del && saved_rn != 0) (void)t->goto_record(saved_rn); + *pulRecordCount = hide_del + ? count_live_recnos(t, adi->ordered_recnos_cached()) + : static_cast(adi->ordered_recnos_cached().size()); return ok(); } } From 0be640239c3d396ddbc82c25ec0a5de5e94d43cc Mon Sep 17 00:00:00 2001 From: russimicro Date: Wed, 5 Aug 2026 09:58:55 -0500 Subject: [PATCH 08/12] feat(adi): make the v2 tag layout opt-in via OPENADS_ADI_V2 With the switch off, AdsCreateIndex61 behaves exactly as it did before: a bare field tag takes the legacy layout, and a compound / computed expression is rejected rather than silently written into a tag header that has nowhere to keep it. With the switch on, the v2 layout is used for character and expression keys. The two test cases that exercise v2 turn it on for their own duration and put it back afterwards, so the rest of the suite keeps running against the legacy layout. Co-Authored-By: Claude Opus 5 (1M context) --- src/abi/ace_exports.cpp | 25 +++++++++++++++++++- src/drivers/adi/adi_index.cpp | 1 + src/drivers/adi/adi_index.h | 2 -- tests/unit/abi_adi_frontcoding_size_test.cpp | 12 ++++++++++ tests/unit/abi_adi_native_estaelec_test.cpp | 13 ++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/abi/ace_exports.cpp b/src/abi/ace_exports.cpp index ddfb2278..34332067 100644 --- a/src/abi/ace_exports.cpp +++ b/src/abi/ace_exports.cpp @@ -9967,6 +9967,17 @@ SIGNED32 to_julian(int y, int m, int d) { // of the same 34,595-row table: 14 ms where key order happened to follow recno, // 492 ms where it did not. The count does not depend on the order, so read the // records in the order the file stores them. +// Opt-in switch for the ADI v2 tag layout (compound / computed / FOR tags, +// dense leaf with front coding). OFF by default, so a deployment that does not +// ask for it keeps the legacy single-field bag byte for byte. Read on every +// call — index creation is not a hot path — so a test or an application can turn +// it on for one process without a restart. +bool adi_v2_enabled() noexcept { + const char* e = std::getenv("OPENADS_ADI_V2"); + if (e == nullptr || *e == 0) return false; + return !(e[0] == '0' && e[1] == 0); +} + std::uint32_t count_live_recnos(openads::engine::Table* t, const std::vector& walk) { std::vector by_recno(walk); @@ -13396,7 +13407,19 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, // uses (STR()/DTOS()/concat → character keys). A BARE numeric/date field // keeps the legacy numeric ADI leaf (sign-flipped float keys) so the // existing packed-key seeks keep working. - const bool use_v2 = is_char_field || is_compound; + // v2 is OPT-IN. With the switch off this call behaves exactly as it did + // before: a bare field tag takes the legacy layout, and an expression + // the legacy format cannot represent is rejected the way it was + // rejected before (there is nowhere in a legacy tag header to keep the + // expression, so accepting it silently would produce a bag whose tag + // says one thing and whose keys say another). + const bool v2_on = adi_v2_enabled(); + if (!v2_on && is_compound) { + return fail(openads::AE_COLUMN_NOT_FOUND, + "ADI: a compound / computed index expression needs the " + "v2 tag layout (set OPENADS_ADI_V2=1)"); + } + const bool use_v2 = v2_on && (is_char_field || is_compound); openads::drivers::adi::AdiIndex::CreateParams cp{}; cp.field_num = static_cast(fidx + 1); cp.field_name = fd.name; diff --git a/src/drivers/adi/adi_index.cpp b/src/drivers/adi/adi_index.cpp index aa680618..a226d6b0 100644 --- a/src/drivers/adi/adi_index.cpp +++ b/src/drivers/adi/adi_index.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/src/drivers/adi/adi_index.h b/src/drivers/adi/adi_index.h index 53bb306a..26b2a47f 100644 --- a/src/drivers/adi/adi_index.h +++ b/src/drivers/adi/adi_index.h @@ -358,8 +358,6 @@ class AdiIndex final : public IIndex { std::uint32_t cur_recno_ = 0; std::string current_key_; Page cur_page_{}; - - bool pos_cache_valid_ = false; }; } // namespace openads::drivers::adi diff --git a/tests/unit/abi_adi_frontcoding_size_test.cpp b/tests/unit/abi_adi_frontcoding_size_test.cpp index 030b7d00..9208e40e 100644 --- a/tests/unit/abi_adi_frontcoding_size_test.cpp +++ b/tests/unit/abi_adi_frontcoding_size_test.cpp @@ -4,6 +4,7 @@ // layout would — parity with ADS-SAP (~3x smaller). The build must also stay // fully navigable (every key visited once, ascending) so the size win is not // bought with corruption. +#include #include "doctest.h" #include "openads/ace.h" @@ -13,6 +14,16 @@ #include #include +namespace { +// The v2 tag layout is opt-in (OPENADS_ADI_V2). These cases exercise it, so +// they turn it on for their own duration and put it back afterwards — the rest +// of the suite must keep running against the legacy layout. +struct AdiV2Scope { + AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", "1"); } + ~AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", ""); } +}; +} // namespace + namespace fs = std::filesystem; namespace { @@ -23,6 +34,7 @@ std::string trim_sp(std::string s) { } // namespace TEST_CASE("ADI: front-coded leaf shrinks an index over high-prefix keys") { + AdiV2Scope _v2; fs::path tmp = fs::temp_directory_path() / "openads_adi_frontcoding"; { std::error_code ec; fs::create_directories(tmp, ec); fs::remove(tmp / "fc.adt", ec); fs::remove(tmp / "fc.adi", ec); } diff --git a/tests/unit/abi_adi_native_estaelec_test.cpp b/tests/unit/abi_adi_native_estaelec_test.cpp index cc83f042..4adeae38 100644 --- a/tests/unit/abi_adi_native_estaelec_test.cpp +++ b/tests/unit/abi_adi_native_estaelec_test.cpp @@ -13,6 +13,7 @@ // is the target oracle for the AdiIndex rework. As each gap lands the matching // CHECK goes green; when ALL pass, drop the should_fail decorator. The CDX mirror // (abi_cdx_estaelec_compound_test.cpp) proves the ERP pattern is correct on CDX. +#include #include "doctest.h" #include "openads/ace.h" @@ -21,6 +22,16 @@ #include #include +namespace { +// The v2 tag layout is opt-in (OPENADS_ADI_V2). These cases exercise it, so +// they turn it on for their own duration and put it back afterwards — the rest +// of the suite must keep running against the legacy layout. +struct AdiV2Scope { + AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", "1"); } + ~AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", ""); } +}; +} // namespace + namespace fs = std::filesystem; namespace { @@ -51,6 +62,7 @@ ADSHANDLE make_tag(ADSHANDLE hTable, const char* bag, const char* tag, } // namespace TEST_CASE("native AdiIndex handles ESTAELEC compound/computed/conditional tags") { + AdiV2Scope _v2; // NO EnvGuard: exercise the genuine AdiIndex path (not the CDX reroute). fs::path dir = fs::temp_directory_path() / "openads_adi_native_estaelec"; std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); @@ -156,6 +168,7 @@ TEST_CASE("native AdiIndex handles ESTAELEC compound/computed/conditional tags") // (reversing ordinals) — add_tag must APPEND so DBSETORDER(1) picks the FIRST // created tag, not the last. TEST_CASE("native AdiIndex tag ordinals follow creation order (xxBrowse DBSETORDER)") { + AdiV2Scope _v2; fs::path dir = fs::temp_directory_path() / "openads_adi_ordinal"; std::error_code ec; fs::remove_all(dir, ec); fs::create_directories(dir); UNSIGNED8 srv[260]{}; From 75dc37d14f791a270578e433e848404e12583a12 Mon Sep 17 00:00:00 2001 From: Antonio Linares Date: Wed, 5 Aug 2026 17:20:21 +0200 Subject: [PATCH 09/12] fix(cdx): mark clear_data/build_bulk as override (clang -Werror) IIndex now declares virtual clear_data() and build_bulk(); CdxIndex already implements both but lacked the override keyword, so -Winconsistent-missing-override failed the CI build under clang (PR #165). --- src/drivers/cdx/cdx_index.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/cdx/cdx_index.h b/src/drivers/cdx/cdx_index.h index 4f88ad8d..1f2d1e38 100644 --- a/src/drivers/cdx/cdx_index.h +++ b/src/drivers/cdx/cdx_index.h @@ -77,7 +77,7 @@ class CdxIndex final : public IIndex { // root) so a CREATE-INDEX-with-existing-tag can rebuild from // scratch on top of an old layout. Old leaves stay on disk // (page leak); a future M(cdx-compact) milestone can reclaim. - util::Result clear_data(); + util::Result clear_data() override; // Overwrite the unique / descend bits in the on-disk sub-tag // header. Used when CREATE INDEX overwrites an existing tag @@ -111,7 +111,7 @@ class CdxIndex final : public IIndex { // decodes + re-encodes a leaf on every key (~10x slower). Call on a // fresh (root_page_ == 0) or clear_data()'d tag, then flush(). util::Result - build_bulk(std::vector> keys); + build_bulk(std::vector> keys) override; // Logical-position cache for O(1) scrollbar / OrdKeyNo / OrdKeyCount. // Walks the index ONCE (lazily) into an ordered recno list + a From 850655ffa78c23bf83a3e95c41610718ff1048ea Mon Sep 17 00:00:00 2001 From: Antonio Linares Date: Wed, 5 Aug 2026 17:23:18 +0200 Subject: [PATCH 10/12] fix(adi): cast leaf_entries_ indexes to size_t (clang -Wsign-conversion) cur_idx_ and the dense-leaf loop counter are signed; indexing the vector with them trips -Wsign-conversion under -Werror on clang CI. --- src/drivers/adi/adi_index.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/drivers/adi/adi_index.cpp b/src/drivers/adi/adi_index.cpp index a226d6b0..a4136955 100644 --- a/src/drivers/adi/adi_index.cpp +++ b/src/drivers/adi/adi_index.cpp @@ -656,8 +656,8 @@ util::Result AdiIndex::refresh_current_() { current_key_.clear(); return {}; } - cur_recno_ = leaf_entries_[cur_idx_].first; - current_key_ = leaf_entries_[cur_idx_].second; + cur_recno_ = leaf_entries_[static_cast(cur_idx_)].first; + current_key_ = leaf_entries_[static_cast(cur_idx_)].second; return {}; } cur_recno_ = dense_entry_recno(cur_page_.data(), cur_idx_, entry_size_); @@ -1247,8 +1247,8 @@ util::Result AdiIndex::seek_key(const std::string& key, bool soft) std::uint32_t rno; std::string ckv; if (key_in_leaf_) { - rno = leaf_entries_[i].first; - ckv = leaf_entries_[i].second; + rno = leaf_entries_[static_cast(i)].first; + ckv = leaf_entries_[static_cast(i)].second; } else { rno = dense_entry_recno(cur_page_.data(), i, entry_size_); auto ck = key_for_recno_(rno); From 61265bdd048b944e075f4c726e8b0e616a1353e8 Mon Sep 17 00:00:00 2001 From: Antonio Linares Date: Wed, 5 Aug 2026 17:28:13 +0200 Subject: [PATCH 11/12] fix(test): portable OPENADS_ADI_V2 env toggle (setenv on non-Windows) _putenv_s is MSVC-only; the two v2-gated ADI tests used it unconditionally and failed the clang CI build. Mirror the EnvGuard pattern already used by abi_adi_estaelec_compound_test. --- tests/unit/abi_adi_frontcoding_size_test.cpp | 16 ++++++++++++++-- tests/unit/abi_adi_native_estaelec_test.cpp | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/unit/abi_adi_frontcoding_size_test.cpp b/tests/unit/abi_adi_frontcoding_size_test.cpp index 9208e40e..b384295d 100644 --- a/tests/unit/abi_adi_frontcoding_size_test.cpp +++ b/tests/unit/abi_adi_frontcoding_size_test.cpp @@ -19,8 +19,20 @@ namespace { // they turn it on for their own duration and put it back afterwards — the rest // of the suite must keep running against the legacy layout. struct AdiV2Scope { - AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", "1"); } - ~AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", ""); } + AdiV2Scope() { +#ifdef _WIN32 + _putenv_s("OPENADS_ADI_V2", "1"); +#else + setenv("OPENADS_ADI_V2", "1", 1); +#endif + } + ~AdiV2Scope() { +#ifdef _WIN32 + _putenv_s("OPENADS_ADI_V2", ""); +#else + unsetenv("OPENADS_ADI_V2"); +#endif + } }; } // namespace diff --git a/tests/unit/abi_adi_native_estaelec_test.cpp b/tests/unit/abi_adi_native_estaelec_test.cpp index 4adeae38..6e8b9bcd 100644 --- a/tests/unit/abi_adi_native_estaelec_test.cpp +++ b/tests/unit/abi_adi_native_estaelec_test.cpp @@ -27,8 +27,20 @@ namespace { // they turn it on for their own duration and put it back afterwards — the rest // of the suite must keep running against the legacy layout. struct AdiV2Scope { - AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", "1"); } - ~AdiV2Scope() { _putenv_s("OPENADS_ADI_V2", ""); } + AdiV2Scope() { +#ifdef _WIN32 + _putenv_s("OPENADS_ADI_V2", "1"); +#else + setenv("OPENADS_ADI_V2", "1", 1); +#endif + } + ~AdiV2Scope() { +#ifdef _WIN32 + _putenv_s("OPENADS_ADI_V2", ""); +#else + unsetenv("OPENADS_ADI_V2"); +#endif + } }; } // namespace From 0e8ef471555028a55afa2595f6ee0525955088be Mon Sep 17 00:00:00 2001 From: Antonio Linares Date: Wed, 5 Aug 2026 17:35:30 +0200 Subject: [PATCH 12/12] fix(test): open .DAT ADT tables exclusive in ESTAELEC ADI tests Reopen used ADS_DEFAULT (shared). Shared GoHot requires RLock/FLock for AdsWriteRecord; AdsAppendRecord auto-lock was not enough on this path and the suite returned 5035. Open exclusive for write/setup, matching the pattern already used by abi_pritpal_lock_test. --- tests/unit/abi_adi_estaelec_compound_test.cpp | 8 ++++++-- tests/unit/abi_adi_native_estaelec_test.cpp | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/unit/abi_adi_estaelec_compound_test.cpp b/tests/unit/abi_adi_estaelec_compound_test.cpp index e65c6d22..5216b2b2 100644 --- a/tests/unit/abi_adi_estaelec_compound_test.cpp +++ b/tests/unit/abi_adi_estaelec_compound_test.cpp @@ -106,7 +106,9 @@ TEST_CASE("ADI->CDX reroute handles ESTAELEC compound/computed/conditional tags" REQUIRE(!ec); UNSIGNED8 dat[] = "estaelec.DAT"; hT = 0; - REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_EXCLUSIVE, &hT) == AE_SUCCESS); struct Row { const char* con; const char* doc; const char* cli; @@ -173,7 +175,9 @@ TEST_CASE("ADI->CDX reroute handles ESTAELEC compound/computed/conditional tags" // (AdsOpenIndex path). The 5 orders must persist and stay distinct, addressed // by tag NAME (handles are gone after close). hT = 0; - REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_EXCLUSIVE, &hT) == AE_SUCCESS); UNSIGNED16 nidx2 = 0; REQUIRE(AdsGetNumIndexes(hT, &nidx2) == AE_SUCCESS); diff --git a/tests/unit/abi_adi_native_estaelec_test.cpp b/tests/unit/abi_adi_native_estaelec_test.cpp index 6e8b9bcd..633b0b10 100644 --- a/tests/unit/abi_adi_native_estaelec_test.cpp +++ b/tests/unit/abi_adi_native_estaelec_test.cpp @@ -97,7 +97,9 @@ TEST_CASE("native AdiIndex handles ESTAELEC compound/computed/conditional tags") REQUIRE(!ec); UNSIGNED8 dat[] = "estaelec.DAT"; hT = 0; - REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_EXCLUSIVE, &hT) == AE_SUCCESS); struct Row { const char* con; const char* doc; const char* cli; @@ -157,7 +159,9 @@ TEST_CASE("native AdiIndex handles ESTAELEC compound/computed/conditional tags") // Reopen: auto-open must rebind the 5 distinct tags BY NAME. hT = 0; - REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_EXCLUSIVE, &hT) == AE_SUCCESS); UNSIGNED16 nidx2 = 0; REQUIRE(AdsGetNumIndexes(hT, &nidx2) == AE_SUCCESS); @@ -197,7 +201,9 @@ TEST_CASE("native AdiIndex tag ordinals follow creation order (xxBrowse DBSETORD fs::rename(dir / "ord.adt", dir / "ord.DAT", ec); REQUIRE(!ec); UNSIGNED8 dat[] = "ord.DAT"; hT = 0; - REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_EXCLUSIVE, &hT) == AE_SUCCESS); for (int i = 1; i <= 3; ++i) { REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); @@ -213,7 +219,9 @@ TEST_CASE("native AdiIndex tag ordinals follow creation order (xxBrowse DBSETORD REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); hT = 0; - REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, 0, 0, 0, ADS_DEFAULT, &hT) + REQUIRE(AdsOpenTable(hConn, dat, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_EXCLUSIVE, &hT) == AE_SUCCESS); UNSIGNED8 b[260]{}; std::memcpy(b, bag.c_str(), bag.size()); ADSHANDLE arr[16] = {0}; UNSIGNED16 alen = 16;