Fix memory safety issues

rust/perspective-js/test/js/aggregates.spec.js

@@ -164,6 +164,52 @@ const std = (nums) => {
             table.delete();
         });
 
+        test("`first` aggregate with a hidden sort column does not crash", async function () {
+            const table = await perspective.table([
+                { g: "x", v: 1 },
+                { g: "x", v: 2 },
+                { g: "y", v: 3 },
+            ]);
+            const view = await table.view({
+                group_by: ["g"],
+                columns: ["g"],
+                aggregates: { v: "first" },
+                sort: [["v", "desc"]],
+            });
+
+            const result = await view.to_json();
+            const paths = result.map((r) => r.__ROW_PATH__);
+            expect(paths.length).toEqual(3);
+            expect(paths).toContainEqual([]);
+            expect(paths).toContainEqual(["x"]);
+            expect(paths).toContainEqual(["y"]);
+            await view.delete();
+            await table.delete();
+        });
+
+        test("`max by` with a non-visible by-column", async function () {
+            const table = await perspective.table([
+                { g: "x", v: 1, w: 10 },
+                { g: "x", v: 2, w: 20 },
+                { g: "y", v: 3, w: 5 },
+            ]);
+            const view = await table.view({
+                group_by: ["g"],
+                columns: ["v"],
+                aggregates: { v: ["max by", ["w"]] },
+            });
+
+            const result = await view.to_json();
+            // `max by` picks the `v` at the row with the maximum `w`.
+            expect(result).toEqual([
+                { __ROW_PATH__: [], v: 2 },
+                { __ROW_PATH__: ["x"], v: 2 },
+                { __ROW_PATH__: ["y"], v: 3 },
+            ]);
+            await view.delete();
+            await table.delete();
+        });
+
         test("Aggregates are not in columns are ignored", async function () {
             const table = await perspective.table(data);
             const view = await table.view({

rust/perspective-js/test/js/constructors.spec.js

@@ -732,6 +732,78 @@ function validate_typed_array(typed_array, column_data) {
                 table.delete();
             }
         });
+
+        test("Table constructor pads short trailing columns with null", async function () {
+            const table = await perspective.table({
+                overflow: [1, 2, 3, 4, 5, 6, 7, 8, 9],
+                short: [1],
+            });
+            const view = await table.view();
+            const result = await view.to_columns();
+            expect(result).toEqual({
+                overflow: [1, 2, 3, 4, 5, 6, 7, 8, 9],
+                short: [1, null, null, null, null, null, null, null, null],
+            });
+            await view.delete();
+            await table.delete();
+        });
+
+        test("Table constructor pads short leading columns with null", async function () {
+            const table = await perspective.table({
+                short: [1],
+                overflow: [1, 2, 3, 4, 5, 6, 7, 8, 9],
+            });
+            const view = await table.view();
+            const result = await view.to_columns();
+            expect(result).toEqual({
+                short: [1, null, null, null, null, null, null, null, null],
+                overflow: [1, 2, 3, 4, 5, 6, 7, 8, 9],
+            });
+            await view.delete();
+            await table.delete();
+        });
+
+        test("Table constructor handles columns of equal length", async function () {
+            const table = await perspective.table({
+                a: [1, 2, 3],
+                b: ["x", "y", "z"],
+            });
+            const view = await table.view();
+            const result = await view.to_columns();
+            expect(result).toEqual({
+                a: [1, 2, 3],
+                b: ["x", "y", "z"],
+            });
+            await view.delete();
+            await table.delete();
+        });
+
+        test("Table update pads short columns with null", async function () {
+            const table = await perspective.table({
+                a: "integer",
+                b: "integer",
+            });
+            await table.update({
+                a: [1, 2, 3, 4, 5],
+                b: [1],
+            });
+            const view = await table.view();
+            const result = await view.to_columns();
+            expect(result).toEqual({
+                a: [1, 2, 3, 4, 5],
+                b: [1, null, null, null, null],
+            });
+
+            // The table must remain usable for subsequent updates.
+            await table.update({ a: [10], b: [20] });
+            const result2 = await view.to_columns();
+            expect(result2).toEqual({
+                a: [1, 2, 3, 4, 5, 10],
+                b: [1, null, null, null, null, 20],
+            });
+            await view.delete();
+            await table.delete();
+        });
     });
 
     test.describe("Formatters", function () {
@@ -918,6 +990,20 @@ function validate_typed_array(typed_array, column_data) {
             view.delete();
             table.delete();
         });
+
+        test("Handles many objects without newline separators", async function () {
+            const expected = [];
+            for (let i = 0; i < 64; i++) {
+                expected.push({ a: `row-${i}` });
+            }
+            const data = expected.map(JSON.stringify).join("");
+            const table = await perspective.table(data, { format: "ndjson" });
+            const view = await table.view();
+            expect(await table.size()).toEqual(64);
+            expect(await view.to_json()).toEqual(expected);
+            await view.delete();
+            await table.delete();
+        });
     });
 
     test.describe("Constructors", function () {

rust/perspective-js/test/js/constructors/arrow.spec.ts

@@ -162,5 +162,30 @@ test.describe("Arrow", function () {
                 __ROW_PATH__: [[], [null], [""], ["AAAA"]],
             });
         });
+
+        test("Loads a time32 (millisecond) column without overflow", async function () {
+            const expected = Array.from({ length: 64 }, (_, i) => i);
+            const tableData = arrow.tableFromArrays({
+                t: arrow.vectorFromArray(expected, new arrow.TimeMillisecond()),
+            });
+
+            const table = await perspective.table(arrow.tableToIPC(tableData));
+            const view = await table.view();
+            expect(await table.size()).toEqual(64);
+            expect(await view.to_columns()).toEqual({ t: expected });
+            await view.delete();
+            await table.delete();
+        });
+    });
+
+    test.describe("Malformed input", function () {
+        test("Rejects an Arrow with a row count that exceeds 32 bits", async function () {
+            const bytes = new Uint8Array(
+                fs.readFileSync(`${__dirname}/../../arrow/bad_row_count.arrow`),
+            );
+            await expect(perspective.table(bytes)).rejects.toThrow(
+                /row count exceeds maximum supported size/,
+            );
+        });
     });
 });

rust/perspective-js/test/js/expressions/vectors.spec.js

@@ -117,5 +117,38 @@ import * as common from "./common.js";
             await view.delete();
             await table.delete();
         });
+
+        test("`norm3` with a vector shorter than 3 is rejected", async () => {
+            const table = await perspective.table(common.int_float_data);
+            await expect(
+                table.view({
+                    expressions: { a: `var v[2] := {1, 2}; norm3(v)` },
+                }),
+            ).rejects.toThrow();
+            await table.delete();
+        });
+
+        test("`diff3` with vectors shorter than 3 is rejected", async () => {
+            const table = await perspective.table(common.int_float_data);
+            await expect(
+                table.view({
+                    expressions: {
+                        a: `var x[2] := {1, 2}; var y[2] := {3, 4}; var o[2]; diff3(x, y, o)`,
+                    },
+                }),
+            ).rejects.toThrow();
+            await table.delete();
+        });
+
+        test("`norm3` with a 3-element vector still computes", async () => {
+            const table = await perspective.table(common.int_float_data);
+            const view = await table.view({
+                expressions: { a: `var v[3] := {3, 4, 0}; norm3(v)` },
+            });
+            const result = await view.to_columns();
+            expect(result["a"]).toEqual(Array(4).fill(5)); // sqrt(9+16+0)
+            await view.delete();
+            await table.delete();
+        });
     });
 })(perspective);

rust/perspective-server/cpp/perspective/CMakeLists.txt

@@ -186,6 +186,9 @@ string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LOWER)
 if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
     set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Red}Building DEBUG${ColorReset}")
     add_definitions(-DPSP_DEBUG)
+    add_definitions(-DPSP_STORAGE_VERIFY)
+    add_definitions(-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST)
+    add_definitions(-D_GLIBCXX_ASSERTIONS)
 else()
     set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building RELEASE${ColorReset}")
 endif()

rust/perspective-server/cpp/perspective/src/cpp/aggregate.cpp

@@ -168,10 +168,9 @@ t_aggregate::init() {
         case AGGTYPE_COUNT: {
             switch (m_icolumns[0]->get_dtype()) {
                 case DTYPE_STR: {
-                    build_aggregate<t_aggimpl_count<
-                        std::uint64_t,
-                        std::uint64_t,
-                        std::uint64_t>>();
+                    build_aggregate<
+                        t_aggimpl_count<t_uindex, std::uint64_t, std::uint64_t>>(
+                    );
                 } break;
                 case DTYPE_TIME:
                 case DTYPE_INT64: {

rust/perspective-server/cpp/perspective/src/cpp/arrow_loader.cpp

@@ -19,6 +19,7 @@
 #include <arrow/type_fwd.h>
 #include <cstdint>
 #include <exception>
+#include <limits>
 #include <memory>
 #include <mutex>
 #include <perspective/arrow_loader.h>
@@ -211,6 +212,11 @@ ArrowLoader::initialize(const std::uint8_t* ptr, const uint32_t length) {
         load_stream(ptr, length, m_table);
     }
 
+    const auto validation = m_table->ValidateFull();
+    if (!validation.ok()) {
+        PSP_COMPLAIN_AND_ABORT(validation.ToString());
+    }
+
     std::shared_ptr<arrow::Schema> schema = m_table->schema();
     std::vector<std::shared_ptr<arrow::Field>> fields = schema->fields();
 
@@ -782,9 +788,9 @@ copy_array(
         case arrow::Time32Type::type_id: {
             auto scol = std::static_pointer_cast<arrow::Time32Array>(src);
             std::memcpy(
-                dest->get_nth<std::uint64_t>(offset),
+                dest->get_nth<std::uint32_t>(offset),
                 (void*)scol->raw_values(),
-                len * 8
+                len * 4
             );
         } break;
         // case arrow::Type {
@@ -972,7 +978,16 @@ ArrowLoader::fill_column(
 
 std::uint32_t
 ArrowLoader::row_count() const {
-    return m_table->num_rows();
+    const std::int64_t n = m_table->num_rows();
+    if (n < 0
+        || static_cast<std::uint64_t>(n)
+            > std::numeric_limits<std::uint32_t>::max()) {
+        PSP_COMPLAIN_AND_ABORT(
+            "Arrow table row count exceeds maximum supported size"
+        );
+    }
+
+    return static_cast<std::uint32_t>(n);
 }
 
 std::vector<std::string>

rust/perspective-server/cpp/perspective/src/cpp/column.cpp

@@ -341,6 +341,7 @@ t_column::size() const {
 
 void
 t_column::set_size(t_uindex size) {
+    reserve(size);
 #ifdef PSP_COLUMN_VERIFY
     PSP_VERBOSE_ASSERT(
         size * get_dtype_size(m_dtype) <= m_data->capacity(),

rust/perspective-server/cpp/perspective/src/cpp/computed_function.cpp

@@ -1885,6 +1885,11 @@ diff3::operator()(t_parameter_list parameters) {
     t_vector_view v2(parameters[1]);
     t_vector_view out(parameters[2]);
 
+    if (v1.size() < 3 || v2.size() < 3 || out.size() < 3) {
+        rval.m_status = STATUS_CLEAR;
+        return rval;
+    }
+
     t_tscalar o1;
     o1.set(v1[0] - v2[0]);
 
@@ -1912,6 +1917,10 @@ norm3::operator()(t_parameter_list parameters) {
     rval.clear();
     rval.m_type = DTYPE_FLOAT64;
     t_vector_view v1(parameters[0]);
+    if (v1.size() < 3) {
+        rval.m_status = STATUS_CLEAR;
+        return rval;
+    }
     double a = v1[0].to_double();
     double b = v1[1].to_double();
     double c = v1[2].to_double();
@@ -1935,6 +1944,11 @@ cross_product3::operator()(t_parameter_list parameters) {
     t_vector_view v2(parameters[1]);
     t_vector_view out(parameters[2]);
 
+    if (v1.size() < 3 || v2.size() < 3 || out.size() < 3) {
+        rval.m_status = STATUS_CLEAR;
+        return rval;
+    }
+
     // a2 * b3 - a3 * b2
     t_tscalar o1;
     o1.set(v1[1] * v2[2] - v1[2] * v2[1]);
@@ -1969,6 +1983,11 @@ dot_product3::operator()(t_parameter_list parameters) {
     t_vector_view v1(parameters[0]);
     t_vector_view v2(parameters[1]);
 
+    if (v1.size() < 3 || v2.size() < 3) {
+        rval.m_status = STATUS_CLEAR;
+        return rval;
+    }
+
     rval.set(v1[0] * v2[0] + v1[1] * v2[1] + v1[2] * v2[2]);
     return rval;
 }