feat(adi): v2 tags — compound / computed / FOR expressions on ADT, opt-in

src/drivers/adi/adi_index.h

@@ -94,9 +94,12 @@ class AdiIndex final : public IIndex {
     util::Result<void> open(const std::string& path, IndexOpenMode mode) override;
 
     std::string    name()       const override { return tag_name_; }
-    std::string    expression() const override { return tag_name_; }
+    std::string    expression() const override {
+        return tag_expr_.empty() ? tag_name_ : tag_expr_;
+    }
     std::string    file_path()  const override { return adi_path_; }
-    bool           descending() const override { return false; }
+    std::string    condition()  const override { return tag_cond_; }
+    bool           descending() const override { return descending_; }
     bool           unique()     const override { return unique_; }
     std::uint16_t  key_length() const override {
         return static_cast<std::uint16_t>(key_total_len_);
@@ -116,15 +119,23 @@ class AdiIndex final : public IIndex {
                                const std::string& key) override;
     util::Result<void> flush() override;
 
-    // Logical-position cache for O(1) scrollbar / OrdKeyNo / OrdKeyCount.
-    // Walks the B-tree ONCE (lazily) into an ordered recno list + a
-    // recno->position map, reused until the index is modified.
+    // Number of index entries (keys). For a conditional (FOR) tag this is fewer
+    // than the table's record count. O(1) after the first call (uses the
+    // logical-position cache below).
+    util::Result<std::uint32_t> entry_count();
+
+    // Logical-position cache for the browse scrollbar math (mirrors CdxIndex):
+    // ordered_recnos_cached() is the recno list in key order; pos_of_recno_cached
+    // maps a recno to its 0-based position. Built lazily by walking the dense-leaf
+    // chain once (O(n)); then AdsGetKeyNum / GetRelKeyPos / SetRelKeyPos are O(1)
+    // per paint instead of an O(n) index walk (which froze large browses).
+    // Invalidated on insert / erase / clear_data / build_bulk.
     const std::vector<std::uint32_t>& ordered_recnos_cached();
     std::uint32_t pos_of_recno_cached(std::uint32_t recno);
     void invalidate_pos_cache() {
         pos_cache_valid_ = false;
-        pos_recnos_.clear();
-        pos_map_.clear();
+        ordered_recnos_.clear();
+        pos_of_recno_.clear();
     }
 
     // Parameters for writing a fresh single-tag .adi skeleton.
@@ -136,6 +147,16 @@ class AdiIndex final : public IIndex {
         std::uint32_t adt_hdr_len = 0;   // ADT header length (bytes 32..35)
         std::uint32_t adt_rec_len = 0;   // ADT record length
         bool          unique      = false;
+        std::uint16_t record_offset = 0; // record offset of the (first) field (for fallback without re-opening ADT file)
+        // v2 (OpenADS-proprietary) tag metadata — persisted in the per-tag
+        // header so tag identity is by NAME and the key expression / FOR
+        // condition survive a reopen (the legacy format only stored a field
+        // number). key_len is the full evaluated-key length (ACE klen).
+        std::string   tag_name;          // tag name (e.g. "ORD1"); identity key in v2
+        std::string   key_expr;          // index key expression (e.g. cA+cB / DTOS(d))
+        std::string   for_expr;          // FOR condition (empty = unconditional)
+        std::uint16_t key_len     = 0;   // full key length (klen from ACE)
+        bool          descending  = false;
         // Full path of the ADT table this index belongs to. Required for a
         // NON-STRUCTURAL bag, whose .adi stem differs from the table's (the
         // `INDEX ON ... TAG ... TO <other path>` form). When empty, the
@@ -157,7 +178,15 @@ class AdiIndex final : public IIndex {
 
     // Wipe the B+tree for this tag (root dense leaf count → 0) so a
     // CREATE INDEX overwrite can rebuild from scratch.
-    util::Result<void> clear_data();
+    util::Result<void> clear_data() override;
+
+    // Bulk-load the (v2) tag from a key set in one bottom-up pass (sort → pack
+    // dense leaves → build branch levels), far faster than per-record insert on
+    // a full REINDEX. Only the v2 opaque-key leaf is supported; a legacy
+    // (field-derived) tag falls back to the per-record default. Call clear_data
+    // first / use on a fresh tag.
+    util::Result<void> build_bulk(
+        std::vector<std::pair<std::string, std::uint32_t>> keys) override;
 
     // Multi-tag API (mirrors CdxIndex). adt_path is the owning table's path;
     // when empty the companion ADT is derived from the .adi stem (structural
@@ -171,20 +200,6 @@ class AdiIndex final : public IIndex {
                                   const std::string&  field_name,
                                   const std::string&  adt_path = {});
 
-    // Create a new ADI file with one tag (expression = comma-separated column names).
-    // On return the AdiIndex is positioned on that tag and ready for inserts.
-    static util::Result<AdiIndex> create(const std::string& adi_path,
-                                         const std::string& adt_path,
-                                         const std::string& expression,
-                                         bool               unique);
-
-    // Add a new tag to an existing ADI file.
-    // On return the AdiIndex is positioned on the new tag and ready for inserts.
-    static util::Result<AdiIndex> add_tag(const std::string& adi_path,
-                                           const std::string& adt_path,
-                                           const std::string& expression,
-                                           bool               unique);
-
 private:
     // Read / write a 512-byte page from/to the ADI file
     util::Result<void> read_adi_page_ (std::uint32_t page_no, Page& buf);
@@ -215,6 +230,23 @@ class AdiIndex final : public IIndex {
     // Load the dense leaf at page_no into cur_page_ and update cursor metadata
     util::Result<void> load_dense_leaf_(std::uint32_t page_no);
 
+    // Adopt an already-read dense-leaf page as the cursor's current leaf: sets
+    // cur_pg_/cur_cnt_/cur_lsib_/cur_rsib_ and, for a v2 tag, decodes the
+    // front-coded entries into leaf_entries_ (legacy tags leave it empty).
+    void adopt_leaf_page_(std::uint32_t page_no, const Page& pg);
+
+    // Render a v2 front-coded dense-leaf page (header + sub-header + entries)
+    // from a key-ordered run. Returns false if the run overflows one page
+    // (the caller must split first).
+    bool render_v2_leaf_(
+        Page& pg,
+        const std::vector<std::pair<std::uint32_t, std::string>>& ents,
+        std::uint32_t lsib, std::uint32_t rsib) const;
+
+    // v2 (front-coded) erase: decode the owning leaf, drop (recno,key),
+    // re-encode (or unlink an emptied page). ikey must already be klen bytes.
+    util::Result<void> erase_v2_(std::uint32_t recno, const std::string& ikey);
+
     // Navigate to the first (leftmost) entry of the B-tree
     util::Result<SeekOutcome> navigate_leftmost_();
 
@@ -236,7 +268,7 @@ class AdiIndex final : public IIndex {
     std::uint32_t branch_entry_page_(const std::uint8_t* pg, int idx) const noexcept;
 
     // Compare two keys.  For numeric keys 8-byte memcmp; for char keys
-    // key_total_len_ bytes (memcmp; CICHAR case-insensitivity deferred).
+    // key_total_len_ bytes, with CICHAR components folded (see below).
     int compare_keys_(const std::string& a, const std::string& b) const noexcept;
 
     // CICHAR collation: return a comparison-normalized copy of a key with
@@ -257,7 +289,8 @@ class AdiIndex final : public IIndex {
         const std::vector<std::uint16_t>& fd_lengths,
         const std::vector<std::string>&   fd_names,
         std::uint32_t hlen, std::uint32_t rlen,
-        bool unique);
+        bool unique,
+        std::uint32_t v2_key_len = 0);  // >0 → v2 leaf (recno4B + opaque key)
 
 
     // Open mode (set by open / open_named)
@@ -269,7 +302,10 @@ class AdiIndex final : public IIndex {
     std::string     adi_path_;
 
     // Tag metadata (primary / first-component field)
-    std::string     tag_name_;        // ADT field name of first component
+    std::string     tag_name_;        // v2: tag name; legacy: ADT field name
+    std::string     tag_expr_;        // v2: key expression (empty in legacy)
+    std::string     tag_cond_;        // v2: FOR condition (empty = unconditional)
+    bool            descending_ = false;
     std::uint32_t   root_page_  = 0;
     std::uint16_t   adt_type_   = 0;  // type of first-component field
     std::uint16_t   fld_offset_ = 0;  // offset of first-component field in ADT record
@@ -293,8 +329,27 @@ class AdiIndex final : public IIndex {
     std::uint32_t   adt_hdr_len_ = 0;
     std::uint32_t   adt_rec_len_ = 0;
 
+    // v2 (OpenADS-proprietary) leaf: dense entries store [recno 4B][full key],
+    // so navigation/seek read the key from the leaf (no ADT re-read) and recno
+    // is 4 bytes. false = legacy field-derived leaf.
+    bool            key_in_leaf_ = false;
+
+    // Logical-position cache (recno-in-key-order + reverse map). entry_count()
+    // is its size. Invalidated via invalidate_pos_cache().
+    std::vector<std::uint32_t>                          ordered_recnos_;
+    std::unordered_map<std::uint32_t, std::uint32_t>    pos_of_recno_;
+    bool                                                pos_cache_valid_ = false;
+
+    // v2 front-coded dense leaf decoded into (recno, full key) pairs in key
+    // order — the in-memory image of the current leaf. Populated by
+    // adopt_leaf_page_ for v2 tags; empty for legacy field-derived leaves
+    // (which are read directly from cur_page_ with the fixed entry_size_).
+    std::vector<std::pair<std::uint32_t, std::string>> leaf_entries_;
+
     // Dense-leaf cursor
-    std::uint32_t   entry_size_ = 3;  // dense_entry_size(fld_length_)
+    std::uint32_t   entry_size_ = 3;  // legacy: dense_entry_size(); v2 leaf is
+                                      // front-coded (variable) — entry_size_ is
+                                      // unused on the v2 path.
     std::uint32_t   cur_pg_    = ADI_INVALID_PAGE;
     std::int32_t    cur_idx_   = -1;
     std::uint16_t   cur_cnt_   = 0;
@@ -303,10 +358,6 @@ class AdiIndex final : public IIndex {
     std::uint32_t   cur_recno_ = 0;
     std::string     current_key_;
     Page            cur_page_{};
-
-    std::vector<std::uint32_t>                       pos_recnos_;
-    std::unordered_map<std::uint32_t, std::uint32_t> pos_map_;
-    bool                                             pos_cache_valid_ = false;
 };
 
 } // namespace openads::drivers::adi

src/drivers/cdx/cdx_index.h

@@ -77,7 +77,7 @@ class CdxIndex final : public IIndex {
     // root) so a CREATE-INDEX-with-existing-tag can rebuild from
     // scratch on top of an old layout. Old leaves stay on disk
     // (page leak); a future M(cdx-compact) milestone can reclaim.
-    util::Result<void> clear_data();
+    util::Result<void> clear_data() override;
 
     // Overwrite the unique / descend bits in the on-disk sub-tag
     // header. Used when CREATE INDEX overwrites an existing tag
@@ -111,7 +111,7 @@ class CdxIndex final : public IIndex {
     // decodes + re-encodes a leaf on every key (~10x slower). Call on a
     // fresh (root_page_ == 0) or clear_data()'d tag, then flush().
     util::Result<void>
-        build_bulk(std::vector<std::pair<std::string, std::uint32_t>> keys);
+        build_bulk(std::vector<std::pair<std::string, std::uint32_t>> keys) override;
 
     // Logical-position cache for O(1) scrollbar / OrdKeyNo / OrdKeyCount.
     // Walks the index ONCE (lazily) into an ordered recno list + a

src/drivers/index_trait.h

@@ -86,6 +86,33 @@ class IIndex {
     virtual util::Result<void> erase (std::uint32_t recno,
                                       const std::string& key) = 0;
     virtual util::Result<void> flush() = 0;
+
+    // Reset the index to empty so a caller (REINDEX / PACK) can rebuild it.
+    // Default: collect every entry then erase it (works for any IIndex).
+    // CdxIndex / AdiIndex override with an O(1)-ish structural reset.
+    virtual util::Result<void> clear_data() {
+        std::vector<std::pair<std::uint32_t, std::string>> entries;
+        auto s = seek_first();
+        while (s && s.value().positioned) {
+            entries.emplace_back(s.value().recno, current_key());
+            s = next();
+        }
+        for (auto& kv : entries) {
+            if (auto e = erase(kv.first, kv.second); !e) return e.error();
+        }
+        return {};
+    }
+
+    // Bulk-load (key, recno) pairs into a freshly-cleared index. Default:
+    // per-record insert. CdxIndex / AdiIndex override with a bottom-up bulk
+    // build (~10x faster on a full REINDEX). Call clear_data() first.
+    virtual util::Result<void>
+    build_bulk(std::vector<std::pair<std::string, std::uint32_t>> keys) {
+        for (auto& kv : keys) {
+            if (auto e = insert(kv.second, kv.first); !e) return e.error();
+        }
+        return {};
+    }
 };
 
 } // namespace openads::drivers

tests/CMakeLists.txt

@@ -243,10 +243,19 @@ add_executable(openads_unit_tests
     unit/abi_adi_multilevel_build_test.cpp
     unit/abi_adt_scope_validation_test.cpp
     unit/abi_adi_smoke_test.cpp
+    unit/abi_adi_clear_multilevel_test.cpp
+    unit/abi_adi_dat_extension_path_test.cpp
+    unit/abi_adi_estaelec_compound_test.cpp
+    unit/abi_adi_frontcoding_size_test.cpp
     unit/abi_adi_keycount_test.cpp
+    unit/abi_adi_native_estaelec_test.cpp
+    unit/abi_adi_reindex_bench_test.cpp
     unit/abi_keycount_cache_test.cpp
     unit/abi_adi_tagdir_order_test.cpp
     unit/abi_adi_tagdir_wide_page_test.cpp
+    unit/abi_cdx_estaelec_compound_test.cpp
+    unit/abi_sql_temp_browse_nav_test.cpp
+    unit/abi_stale_index_walk_test.cpp
     unit/abi_adt_sql_test.cpp
     unit/abi_adt_dat_extension_sql_test.cpp
     unit/openads_sql_c_test.cpp

tests/unit/abi_adi_clear_multilevel_test.cpp

@@ -0,0 +1,114 @@
+// Repro for ADI clear_data on a MULTI-LEVEL index (large table).
+//
+// Re-creating an existing tag (CREATE INDEX overwrite, or the ERP reindex
+// hitting the same field twice) takes the open_named + clear_data path in
+// AdsCreateIndex61. clear_data used to require the tag's root page to be a
+// dense leaf and aborted with ADSCDX/5000 "root is not a dense leaf" when the
+// index was big enough to have a branch root (>1 B-tree level). This is what
+// killed the reindex of ESTAELEC (441k records) at INDEX ON ... TAG ORD3.
+//
+// Here we build a tag over enough records to force a multi-level tree (a
+// 512-byte dense leaf holds ~162 entries), then re-create the SAME tag so
+// clear_data runs on the branch root. It must succeed and the rebuilt index
+// must still walk every record in key order.
+
+#include "doctest.h"
+#include "drivers/adi/adi_index.h"
+#include "openads/ace.h"
+
+#include <cstdint>
+#include <cstring>
+#include <filesystem>
+#include <string>
+#include <vector>
+
+namespace fs = std::filesystem;
+
+namespace {
+
+std::string rtrim(std::string s) {
+    while (!s.empty() && s.back() == ' ') s.pop_back();
+    return s;
+}
+
+void append_key(ADSHANDLE hTable, const std::string& v) {
+    REQUIRE(AdsAppendRecord(hTable) == AE_SUCCESS);
+    UNSIGNED8 fld[] = "K";
+    UNSIGNED8 val[16]{};
+    std::memcpy(val, v.data(), v.size());
+    REQUIRE(AdsSetString(hTable, fld, val,
+                         static_cast<UNSIGNED32>(v.size())) == AE_SUCCESS);
+    REQUIRE(AdsWriteRecord(hTable) == AE_SUCCESS);
+}
+
+} // namespace
+
+TEST_CASE("ADI clear_data: re-create tag on a multi-level (large) index") {
+    fs::path tmp = fs::temp_directory_path() / "openads_adi_clear_ml";
+    { std::error_code ec; fs::create_directories(tmp, ec); }
+    { std::error_code ec;
+      fs::remove(tmp / "big.adt", ec);
+      fs::remove(tmp / "big.adi", ec); }
+
+    UNSIGNED8 srv[260]{};
+    std::memcpy(srv, tmp.string().c_str(), tmp.string().size());
+    ADSHANDLE hConn = 0;
+    REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn)
+            == AE_SUCCESS);
+
+    UNSIGNED8 tbl[]    = "big.adt";
+    UNSIGNED8 flddef[] = "K,Character,8";
+    ADSHANDLE hTable   = 0;
+    REQUIRE(AdsCreateTable(hConn, tbl, nullptr, ADS_ADT, ADS_ANSI, 0, 0, 0,
+                           flddef, &hTable) == AE_SUCCESS);
+
+    // 600 distinct keys → ~4 dense leaves → branch root (multi-level).
+    const int N = 600;
+    for (int i = 1; i <= N; ++i) {
+        char b[16];
+        std::snprintf(b, sizeof(b), "%08d", i);
+        append_key(hTable, std::string(b));
+    }
+
+    UNSIGNED8 idxfile[] = "big.adi";
+    UNSIGNED8 tag[]      = "ORD1";
+    UNSIGNED8 expr[]     = "K";
+
+    // First build → multi-level tree.
+    ADSHANDLE hIdx1 = 0;
+    REQUIRE(AdsCreateIndex61(hTable, idxfile, tag, expr,
+                             nullptr, nullptr, 0, 0, &hIdx1) == AE_SUCCESS);
+
+    // Re-create the SAME tag → exists && have_tag → open_named + clear_data on
+    // a BRANCH root. Used to fail with ADSCDX/5000; must now succeed.
+    ADSHANDLE hIdx2 = 0;
+    REQUIRE(AdsCreateIndex61(hTable, idxfile, tag, expr,
+                             nullptr, nullptr, 0, 0, &hIdx2) == AE_SUCCESS);
+
+    // The rebuilt index must walk every record in ascending key order.
+    REQUIRE(AdsGotoTop(hTable) == AE_SUCCESS);
+    std::vector<std::string> seen;
+    for (;;) {
+        UNSIGNED16 at_eof = 0;
+        REQUIRE(AdsAtEOF(hTable, &at_eof) == AE_SUCCESS);
+        if (at_eof) break;
+        UNSIGNED8 buf[32]{};
+        UNSIGNED32 len = sizeof(buf);
+        REQUIRE(AdsGetString(hTable, (UNSIGNED8*)"K", buf, &len, 0)
+                == AE_SUCCESS);
+        seen.push_back(rtrim(std::string(reinterpret_cast<char*>(buf), len)));
+        REQUIRE(AdsSkip(hTable, 1) == AE_SUCCESS);
+    }
+
+    REQUIRE(seen.size() == static_cast<std::size_t>(N));
+    CHECK(seen.front() == "00000001");
+    CHECK(seen.back()  == "00000600");
+    bool ordered = true;
+    for (std::size_t i = 1; i < seen.size(); ++i)
+        if (seen[i] < seen[i - 1]) { ordered = false; break; }
+    CHECK(ordered);
+
+    REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS);
+    REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS);
+    { std::error_code ec; fs::remove_all(tmp, ec); }
+}