perf: stream analyze statistics with KLL sketch

.github/workflows/ci.yml

@@ -15,67 +15,71 @@ jobs:
   # 1
   check:
     name: Rust project check
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Use preinstalled tools
-        run: |
-          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
-          echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
 
       - name: Run cargo test
         run: make test
   # 2
   fmt:
     name: Rust fmt
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Use preinstalled tools
-        run: |
-          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
-          echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v2
 
       - name: Run cargo fmt
         run: make fmt
   # 3
   e2e:
     name: Rust e2e sqllogictest
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Use preinstalled tools
-        run: |
-          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
-          echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
 
       - name: Run sqllogictest suite
         run: make test-slt
   # 4
   wasm-tests:
     name: Wasm cargo tests
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-unknown-unknown
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - uses: Swatinem/rust-cache@v2
 
-      - name: Use preinstalled tools
-        run: |
-          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
-          echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
+      - uses: taiki-e/install-action@wasm-pack
 
       - name: Run wasm-bindgen tests (wasm32 target)
         run: make test-wasm
   # 5
   wasm-examples:
     name: Wasm examples (nodejs)
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: wasm32-unknown-unknown
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - uses: Swatinem/rust-cache@v2
 
-      - name: Use preinstalled tools
-        run: |
-          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
-          echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
+      - uses: taiki-e/install-action@wasm-pack
 
       - name: Build wasm package
         run: make wasm-build
@@ -85,28 +89,25 @@ jobs:
   # 6
   native-examples:
     name: Native examples
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Use preinstalled tools
-        run: |
-          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
-          echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
 
       - name: Run native examples
         run: make native-examples
   # 7
   python-tests:
     name: Python bindings tests
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Use preinstalled tools
-        run: |
-          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
-          echo "$HOME/actions-runner/externals/node20/bin" >> "$GITHUB_PATH"
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: Swatinem/rust-cache@v2
 
       - name: Run python binding tests
-        run: make test-python
+        run: PYO3_PYTHON="$(which python)" make test-python

Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name          = "kite_sql"
-version       = "0.3.0"
+version       = "0.3.1"
 edition       = "2021"
 build         = "build.rs"
 authors       = ["Kould <kould2333@gmail.com>", "Xwg <loloxwg@gmail.com>"]

src/execution/ddl/create_index.rs

@@ -140,7 +140,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for CreateIndex {
             with_projection_tmp_value(arena, None, &column_exprs, |arena, value| {
                 let mut state = arena.local_state(plan_arena);
                 let (transaction, table_codec) = state.transaction_codec_mut();
-                let index = Index::new(index_id, value, ty);
+                let index = Index::new(index_id, &value, ty);
                 transaction.add_index(table_codec, table_name.as_ref(), index, &tuple_pk)
             })?;
         }

src/execution/dml/analyze.rs

@@ -20,7 +20,7 @@ use crate::execution::{
 };
 use crate::expression::ScalarExpression;
 use crate::iter_ext::Itertools;
-use crate::optimizer::core::histogram::HistogramBuilder;
+use crate::optimizer::core::histogram::{HistogramBuilder, ANALYZE_STATISTICS_RELATIVE_ERROR};
 use crate::optimizer::core::statistics_meta::StatisticsMeta;
 use crate::planner::operator::analyze::AnalyzeOperator;
 use crate::planner::LogicalPlan;
@@ -105,7 +105,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Analyze {
                     Ok(State {
                         index_id: index.id,
                         exprs: index.column_exprs(table, plan_arena)?,
-                        builder: HistogramBuilder::new(index, None),
+                        builder: HistogramBuilder::new(index, ANALYZE_STATISTICS_RELATIVE_ERROR)?,
                         histogram_buckets: self.histogram_buckets,
                     })
                 })

src/execution/dml/delete.rs

@@ -102,7 +102,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Delete {
                         transaction.del_index(
                             table_codec,
                             &self.table_name,
-                            &Index::new(*index_id, value, *index_ty),
+                            &Index::new(*index_id, &value, *index_ty),
                             tuple_id,
                         )
                     })?;

src/execution/dml/insert.rs

@@ -170,7 +170,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Insert {
                     with_projection_tmp_value(arena, Some(&tuple), exprs, |arena, value| {
                         let mut state = arena.local_state(plan_arena);
                         let (transaction, table_codec) = state.transaction_codec_mut();
-                        let index = Index::new(index_meta.id, value, index_meta.ty);
+                        let index = Index::new(index_meta.id, &value, index_meta.ty);
                         transaction.add_index(table_codec, &self.table_name, index, tuple_id)
                     })?;
                 }

src/execution/dml/update.rs

@@ -124,7 +124,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Update {
                     with_projection_tmp_value(arena, Some(&tuple), exprs, |arena, value| {
                         let mut state = arena.local_state(plan_arena);
                         let (transaction, table_codec) = state.transaction_codec_mut();
-                        let index = Index::new(index_meta.id, value, index_meta.ty);
+                        let index = Index::new(index_meta.id, &value, index_meta.ty);
                         transaction.del_index(table_codec, &self.table_name, &index, &old_pk)
                     })?;
                 }
@@ -152,7 +152,7 @@ impl<'a, T: Transaction + 'a> ExecutorNode<'a, T> for Update {
                     with_projection_tmp_value(arena, Some(&tuple), exprs, |arena, value| {
                         let mut state = arena.local_state(plan_arena);
                         let (transaction, table_codec) = state.transaction_codec_mut();
-                        let index = Index::new(index_meta.id, value, index_meta.ty);
+                        let index = Index::new(index_meta.id, &value, index_meta.ty);
                         transaction.add_index(table_codec, &self.table_name, index, new_pk)
                     })?;
                 }

src/execution/dql/sort.rs

@@ -65,11 +65,6 @@ impl<'a, T> NullableVec<'a, T> {
         unsafe { self.0[offset].assume_init_read() }
     }
 
-    #[inline]
-    pub(crate) fn get(&self, offset: usize) -> &T {
-        unsafe { self.0[offset].assume_init_ref() }
-    }
-
     #[inline]
     pub(crate) fn len(&self) -> usize {
         self.0.len()

src/execution/mod.rs

@@ -413,7 +413,7 @@ pub(crate) fn with_projection_tmp_value<'a, T: Transaction + 'a>(
     arena: &mut ExecArena<'a, T>,
     tuple: Option<&dyn TupleLike>,
     exprs: &[ScalarExpression],
-    f: impl FnOnce(&mut ExecArena<'a, T>, &DataValue) -> Result<(), DatabaseError>,
+    f: impl FnOnce(&mut ExecArena<'a, T>, DataValue) -> Result<(), DatabaseError>,
 ) -> Result<(), DatabaseError> {
     arena.with_projection_tmp(|arena, projection_tmp| {
         {
@@ -424,16 +424,16 @@ pub(crate) fn with_projection_tmp_value<'a, T: Transaction + 'a>(
             }
         }
 
-        if projection_tmp.len() > 1 {
-            let value = DataValue::Tuple(std::mem::take(projection_tmp), false);
-            let ret = f(arena, &value);
-            let DataValue::Tuple(values, _) = value else {
-                unreachable!()
-            };
-            *projection_tmp = values;
-            ret?;
-        } else if let Some(value) = projection_tmp.first() {
-            f(arena, value)?;
+        match projection_tmp.len() {
+            0 => {}
+            1 => {
+                let value = projection_tmp.pop().expect("projection has one value");
+                f(arena, value)?;
+            }
+            _ => {
+                let value = DataValue::Tuple(std::mem::take(projection_tmp), false);
+                f(arena, value)?;
+            }
         }
         Ok(())
     })