Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/unit-tests-internal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
}
echo "Pre-pulling Docker images with authenticated session..."
pull postgres:14
pull clickhouse/clickhouse-server:25.4-alpine
pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251
pull redis:7.2
pull testcontainers/ryuk:0.14.0
pull electricsql/electric:1.2.4
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/unit-tests-packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
}
echo "Pre-pulling Docker images with authenticated session..."
pull postgres:14
pull clickhouse/clickhouse-server:25.4-alpine
pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251
pull redis:7.2
pull testcontainers/ryuk:0.14.0
pull electricsql/electric:1.2.4
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/unit-tests-webapp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
}
echo "Pre-pulling Docker images with authenticated session..."
pull postgres:14
pull clickhouse/clickhouse-server:25.4-alpine
pull clickhouse/clickhouse-server:26.2.19.43-alpine@sha256:c6ad6a7eb2fb5999df3adfb8b69a0c7222c68fa9b8f6b04a088564ebbc959251
pull redis:7.2
pull testcontainers/ryuk:0.14.0
pull electricsql/electric:1.2.4
Expand Down
6 changes: 6 additions & 0 deletions .server-changes/logs-search-memory-and-pagination.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
area: webapp
type: fix
---

Keep logs search within bounded ClickHouse memory when browsing long time ranges, and fix pagination that could skip or duplicate entries sharing a timestamp.
28 changes: 28 additions & 0 deletions apps/webapp/app/env.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1642,6 +1642,34 @@ const EnvironmentSchema = z
CLICKHOUSE_LOGS_LIST_MAX_THREADS: z.coerce.number().int().default(2),
CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ: z.coerce.number().int().default(10_000_000),
CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME: z.coerce.number().int().default(120),
// Bound read-in-order memory on object-storage reads: each part opens a per-column read
// stream, and the default ~1 MiB+ S3 buffers dominate peak memory. These two byte sizes
// cap the per-stream buffers and exist on every supported ClickHouse, so they are always on.
CLICKHOUSE_LOGS_LIST_PREFETCH_BUFFER_SIZE: z.coerce.number().int().nonnegative().default(262_144),
CLICKHOUSE_LOGS_LIST_MAX_READ_BUFFER_SIZE: z.coerce.number().int().nonnegative().default(262_144),
// The decisive lever on Cloud SharedMergeTree, but it only exists on newer ClickHouse and
// is a no-op on local-disk MergeTree, so it is opt-in: unset means it is never sent (safe on
// any self-hosted version). Set to 0 on object-storage deployments to get the memory win.
CLICKHOUSE_LOGS_LIST_FILESYSTEM_CACHE_PREFER_BIGGER_BUFFER_SIZE: z.coerce
.number()
.int()
.nonnegative()
.optional(),

// Logs list pagination tuning (page sizing + recent-first probe windows).
LOGS_LIST_DEFAULT_PAGE_SIZE: z.coerce.number().int().positive().default(50),
LOGS_LIST_MAX_PAGE_SIZE: z.coerce.number().int().positive().default(100),
// Days back from the page ceiling to probe before widening to the full requested window,
// comma-separated. Empty disables narrowing (a single full-window query).
LOGS_LIST_RECENT_FIRST_PROBE_DAYS: z
.string()
.default("1,7")
.transform((s) =>
s
.split(",")
.map((v) => Number(v.trim()))
.filter((n) => Number.isFinite(n) && n > 0)
),

// Query feature flag
QUERY_FEATURE_ENABLED: z.string().default("1"),
Expand Down
Loading