perf(pack): single-shot trailing truncate + sequential bulk scan

src/drivers/adt/adt_driver.cpp

@@ -367,6 +367,27 @@ util::Result<void> AdtDriver::zap() {
     return file_.sync();
 }
 
+util::Result<bool> AdtDriver::truncate_to(std::uint32_t recno) {
+    if (mode_ == DriverOpenMode::ReadOnly) {
+        return util::Error{5000, 0, "table opened read-only", ""};
+    }
+    // Hold the header region while we refresh the count and shrink the file.
+    auto lk = acquire_with_retry_(file_, 0, 400);
+    if (!lk) return lk.error();
+    if (auto rh = refresh_record_count_(); !rh) return rh.error();
+    if (recno > rec_count_) return false;   // can't grow; nothing trailing to drop
+    rec_count_ = recno;
+    if (auto r = rewrite_header_(); !r) return r.error();
+    // ADT keeps no 0x1A EOF marker (unlike DBF): file size is exactly
+    // hdr_len_ + rec_count_*rec_len_, so shrink to that and a later reopen's
+    // physical count matches the header.
+    std::uint64_t end_off = static_cast<std::uint64_t>(hdr_len_) +
+                            static_cast<std::uint64_t>(rec_count_) *
+                            static_cast<std::uint64_t>(rec_len_);
+    if (auto tr = file_.truncate(end_off); !tr) return tr.error();
+    return true;
+}
+
 util::Result<std::uint32_t>
 AdtDriver::bump_autoinc(std::uint16_t field_index) {
     if (field_index >= fields_.size()) {

src/drivers/adt/adt_driver.h

@@ -55,6 +55,10 @@ class AdtDriver final : public IDriver {
     util::Result<void> flush() override;
     util::Result<void> zap()   override;
 
+    // Bulk trailing truncation used by Table::pack(). ADT counterpart
+    // of CdxDriver::truncate_to; ADT keeps no trailing EOF byte (see zap()).
+    util::Result<bool> truncate_to(std::uint32_t recno) override;
+
     util::Result<std::uint32_t>
         bump_autoinc(std::uint16_t field_index) override;
 

src/drivers/cdx/cdx_driver.cpp

@@ -374,6 +374,31 @@ util::Result<bool> CdxDriver::truncate_trailing(std::uint32_t recno) {
     return true;
 }
 
+util::Result<bool> CdxDriver::truncate_to(std::uint32_t recno) {
+    if (mode_ == DriverOpenMode::ReadOnly) {
+        return util::Error{5000, 0, "table opened read-only", ""};
+    }
+    invalidate_read_cache_();
+    auto lk = acquire_with_retry_(file_, 0, 32);
+    if (!lk) return lk.error();
+    if (auto rh = refresh_record_count_(); !rh) return rh.error();
+    if (recno > rec_count_) return false;   // can't grow; nothing trailing
+    rec_count_ = recno;
+    if (auto r = rewrite_header_(); !r) return r.error();
+    std::uint64_t eof_off = static_cast<std::uint64_t>(hdr_len_) +
+                            static_cast<std::uint64_t>(rec_count_) *
+                            static_cast<std::uint64_t>(rec_len_);
+    std::uint8_t eof = 0x1A;
+    if (auto w = file_.write_at(eof_off, &eof, 1); !w) return w.error();
+    // Physically shrink the file to match the new logical count (the old
+    // zap + re-append PACK path did this implicitly). Without it the .dbf
+    // keeps the stale trailing records on disk; a later physical-order
+    // DBEVAL / reopen that derives the count from file size then walks past
+    // rec_count_ -> ADSCDX/5000 "record number out of range".
+    if (auto tr = file_.truncate(eof_off + 1); !tr) return tr.error();
+    return true;
+}
+
 util::Result<std::uint32_t>
 CdxDriver::bump_autoinc(std::uint16_t field_index) {
     if (field_index >= fields_.size()) {

src/drivers/cdx/cdx_driver.h

@@ -33,6 +33,7 @@ class CdxDriver final : public IDriver {
     util::Result<void> flush() override;
     util::Result<void> zap()   override;
     util::Result<bool> truncate_trailing(std::uint32_t recno) override;
+    util::Result<bool> truncate_to(std::uint32_t recno) override;
 
     util::Result<std::uint32_t>
         bump_autoinc(std::uint16_t field_index) override;

src/drivers/driver_trait.h

@@ -63,6 +63,15 @@ class IDriver {
         return false;
     }
 
+    // Drop every record above `recno` in one shot (keep 1..recno), updating
+    // the header count + EOF marker. Used by PACK after the survivors have
+    // been copied down to 1..dst, so it avoids the read-all + zap + re-append
+    // pass. Default rejects with false so drivers that don't implement it fall
+    // back to the legacy save/zap/re-append path.
+    virtual util::Result<bool> truncate_to(std::uint32_t /*recno*/) {
+        return false;
+    }
+
     // VFP autoinc bump (M10.11). Returns the value to use for the
     // pending append (the field's current `autoinc_next`), advances
     // the in-memory counter by `autoinc_step`, and persists the new

src/engine/table.cpp

@@ -1381,7 +1381,12 @@ util::Result<void> Table::pack() {
     std::uint32_t dst = 0;
     std::uint32_t total = driver_->record_count();
     for (std::uint32_t src = 1; src <= total; ++src) {
-        if (auto g = goto_record(src); !g) return g.error();
+        // Sequential scan: read straight from the driver (read-ahead friendly)
+        // instead of goto_record(), which would reseek the active order on
+        // every record. Nothing here consults the order cursor.
+        auto raw = driver_->read_record_raw(src);
+        if (!raw) return raw.error();
+        load_record_for_bulk_scan(std::move(raw.value()), src);
         if (is_deleted()) continue;
         ++dst;
         if (dst != src) {
@@ -1391,26 +1396,37 @@ util::Result<void> Table::pack() {
             }
         }
     }
-    // 2) Truncate the on-disk record count to `dst` by saving the
-    //    survivors, zapping the driver (DBF-only — does NOT touch
-    //    bound indexes), and re-appending. Pack matches Clipper's
-    //    semantics: indexes are left stale, the caller must REINDEX.
-    std::vector<std::vector<std::uint8_t>> survivors;
-    survivors.reserve(dst);
-    for (std::uint32_t i = 1; i <= dst; ++i) {
-        auto rec = driver_->read_record_raw(i);
-        if (!rec) return rec.error();
-        survivors.push_back(std::move(rec).value());
-    }
-    if (auto r = driver_->zap(); !r) return r.error();
-    for (auto& buf : survivors) {
-        auto a = driver_->append_record_raw(buf.data(), buf.size());
-        if (!a) return a.error();
+    // 2) Drop the trailing stale rows. The copy-down above already placed the
+    //    survivors at recnos 1..dst, so this is a single header/EOF rewrite +
+    //    physical file truncate via truncate_to() — avoiding the read-all + zap
+    //    + re-append pass (a full extra I/O pass, with a per-record flush, that
+    //    dominated PACK on large tables). Drivers without truncate_to fall back
+    //    to that legacy path. Pack matches Clipper's semantics: indexes are
+    //    left stale, the caller must REINDEX.
+    bool truncated = (dst >= total);   // nothing removed → records unchanged
+    if (!truncated) {
+        auto t = driver_->truncate_to(dst);
+        if (!t) return t.error();
+        truncated = t.value();
+    }
+    if (!truncated) {
+        std::vector<std::vector<std::uint8_t>> survivors;
+        survivors.reserve(dst);
+        for (std::uint32_t i = 1; i <= dst; ++i) {
+            auto rec = driver_->read_record_raw(i);
+            if (!rec) return rec.error();
+            survivors.push_back(std::move(rec).value());
+        }
+        if (auto r = driver_->zap(); !r) return r.error();
+        for (auto& buf : survivors) {
+            auto a = driver_->append_record_raw(buf.data(), buf.size());
+            if (!a) return a.error();
+        }
     }
     record_buf_.assign(driver_->record_length(), 0);
     // Clipper / DBFCDX semantics: PACK rebuilds the controlled indexes so a
     // post-PACK index walk never references a recno beyond the compacted
-    // record count. (zap() above intentionally leaves bound indexes stale —
+    // record count. (the truncate/zap above intentionally leaves bound indexes stale —
     // without this rebuild, dbGoTop+dbSkip over a stale tag walks onto a
     // dropped recno and raises ADSCDX/5000 "record number out of range".)
     // Only needed when records were actually removed: if nothing was deleted