feat: canister logs extensions

docs/guides/canister-management/logs.md

@@ -102,6 +102,8 @@ icp canister logs <canister-name> -e ic --since-index 100 --until-index 200
 
 Timestamp and index filters cannot be combined with `--follow`.
 
+Filtering also changes which records are dropped when the selected logs exceed the response size. An unfiltered read trims the oldest records, so it returns the most recent logs (the output ends with the newest record). A filtered read trims the newest records instead, so it returns the oldest records that match the filter (the output starts with the oldest matching record), which lets you page forward through a range by advancing the start of the filter.
+
 To output logs as JSON for programmatic processing:
 
 ```bash

docs/guides/canister-management/settings.mdx

@@ -197,8 +197,6 @@ settings:
 
 ### Log memory limit
 
-{/* Needs human verification: log_memory_limit is exposed by icp-cli but is absent from the canonical ic.did: verify whether this is a management canister setting or an icp-cli layer setting */}
-
 Maximum memory for storing canister logs. Oldest logs are purged when usage exceeds this value.
 
 | Property | Value |

docs/references/ic-interface-spec/abstract-behavior.md

@@ -515,6 +515,7 @@ S = {
   certified_data: CanisterId ↦ Blob;
   canister_history: CanisterId ↦ CanisterHistory;
   canister_log_visibility: CanisterId ↦ CanisterLogVisibility;
+  canister_log_memory_limit: CanisterId ↦ Nat;
   canister_snapshot_visibility: CanisterId ↦ CanisterSnapshotVisibility;
   canister_logs: CanisterId ↦ [CanisterLog];
   query_stats: CanisterId ↦ [QueryStats];
@@ -624,6 +625,7 @@ The initial state of the IC is
   certified_data = ();
   canister_history = ();
   canister_log_visibility = ();
+  canister_log_memory_limit = ();
   canister_snapshot_visibility = ();
   canister_logs = ();
   query_stats = ();
@@ -1704,6 +1706,11 @@ if A.settings.log_visibility is not null:
 else:
   New_canister_log_visibility = Controllers
 
+if A.settings.log_memory_limit is not null:
+  New_canister_log_memory_limit = A.settings.log_memory_limit
+else:
+  New_canister_log_memory_limit = 4096
+
 if A.settings.snapshot_visibility is not null:
   New_canister_snapshot_visibility = A.settings.snapshot_visibility
 else:
@@ -1735,6 +1742,7 @@ S' = S with
     query_stats[Canister_id] = []
     canister_history[Canister_id] = New_canister_history
     canister_log_visibility[Canister_id] = New_canister_log_visibility
+    canister_log_memory_limit[Canister_id] = New_canister_log_memory_limit
     canister_snapshot_visibility[Canister_id] = New_canister_snapshot_visibility
     canister_logs[Canister_id] = []
     messages = Older_messages · Younger_messages ·
@@ -1871,6 +1879,8 @@ S' = S with
     canister_version[A.canister_id] = S.canister_version[A.canister_id] + 1
     if A.settings.log_visibility is not null:
       canister_log_visibility[A.canister_id] = A.settings.log_visibility
+    if A.settings.log_memory_limit is not null:
+      canister_log_memory_limit[A.canister_id] = A.settings.log_memory_limit
     if A.settings.snapshot_visibility is not null:
       canister_snapshot_visibility[A.canister_id] = A.settings.snapshot_visibility
     messages = Older_messages · Younger_messages ·
@@ -2896,7 +2906,8 @@ S with
     certified_data[A.canister_id] = (deleted)
     canister_history[A.canister_id] = (deleted)
     canister_log_visibility[A.canister_id] = (deleted)
-   canister_snapshot_visibility[A.canister_id] = (deleted)
+    canister_log_memory_limit[A.canister_id] = (deleted)
+    canister_snapshot_visibility[A.canister_id] = (deleted)
     canister_logs[A.canister_id] = (deleted)
     query_stats[A.canister_id] = (deleted)
     chunk_store[A.canister_id] = (deleted)
@@ -3129,6 +3140,11 @@ if A.settings.log_visibility is not null:
 else:
   New_canister_log_visibility = Controllers
 
+if A.settings.log_memory_limit is not null:
+  New_canister_log_memory_limit = A.settings.log_memory_limit
+else:
+  New_canister_log_memory_limit = 4096
+
 if A.settings.snapshot_visibility is not null:
   New_canister_snapshot_visibility = A.settings.snapshot_visibility
 else:
@@ -3158,6 +3174,7 @@ S' = S with
     certified_data[Canister_id] = ""
     canister_history[Canister_id] = New_canister_history
     canister_log_visibility[Canister_id] = New_canister_log_visibility
+    canister_log_memory_limit[Canister_id] = New_canister_log_memory_limit
     canister_snapshot_visibility[Canister_id] = New_canister_snapshot_visibility
     canister_logs[Canister_id] = []
     query_stats[CanisterId] = []
@@ -3799,6 +3816,113 @@ S with
 
 ```
 
+#### IC Management Canister: Canister logs {#ic-mgmt-canister-fetch-canister-logs}
+
+Given a state `S`, `Canister_id`, `filter`, and `Sender`, we define
+
+```html
+
+canister_logs(S, Canister_id, filter) =
+  let Selected_logs = the sublist of S.canister_logs[Canister_id] (preserving order) such that
+    if filter = by_idx Range:
+      Log ∈ Selected_logs ⟺ Log ∈ S.canister_logs[Canister_id] ∧ Range.start <= Log.idx ∧ Log.idx < Range.end
+    else if filter = by_timestamp_nanos Range:
+      Log ∈ Selected_logs ⟺ Log ∈ S.canister_logs[Canister_id] ∧ Range.start <= Log.timestamp_nanos ∧ Log.timestamp_nanos < Range.end
+    else:
+      Log ∈ Selected_logs ⟺ Log ∈ S.canister_logs[Canister_id]
+  in
+    if filter = null:
+      the longest suffix Younger_logs of Selected_logs
+        such that canister_log_memory_usage(Younger_logs) ≤ max_response_size
+    else:
+      the longest prefix Older_logs of Selected_logs
+        such that canister_log_memory_usage(Older_logs) ≤ max_response_size
+max_response_size = <implementation-specific>
+is_sender_authorized(S, Canister_id, Sender) =
+  (S[Canister_id].canister_log_visibility = Public)
+  or
+  (S[Canister_id].canister_log_visibility = Controllers and Sender in S[Canister_id].controllers)
+  or
+  (S[Canister_id].canister_log_visibility = AllowedViewers Principals and (Sender in S[Canister_id].controllers or Sender in Principals))
+
+```
+
+When the selected logs do not all fit within a single response, they are trimmed to fit and the direction of trimming depends on whether a filter is provided.
+An unfiltered read trims the oldest log records (keeps the longest suffix), so the response ends with the newest log record.
+A filtered read trims the newest log records (keeps the longest prefix), so the response starts with the oldest log record satisfying the filter.
+Thus an unfiltered read surfaces the most recent activity, while a filtered read can page forward through logs starting from the beginning of the requested range.
+
+Conditions
+
+```html
+
+S.messages = Older_messages · CallMessage M · Younger_messages
+(M.queue = Unordered) or (∀ CallMessage M' | FuncMessage M' ∈ Older_messages. M'.queue ≠ M.queue)
+M.callee = ic_principal
+M.method_name = 'fetch_canister_logs'
+M.arg = candid(A)
+is_sender_authorized(S, A.canister_id, M.caller)
+
+```
+
+State after
+
+```html
+
+S with
+    messages = Older_messages · Younger_messages ·
+      ResponseMessage {
+        origin = M.origin
+        response = candid(canister_logs(S, A.canister_id, A.filter))
+        refunded_cycles = M.transferred_cycles
+      }
+
+```
+
+The IC method `fetch_canister_logs` can also be invoked via management canister query calls.
+They are calls to `/api/v3/canister/<ECID>/query`
+with CBOR content `Q` such that `Q.canister_id = ic_principal`.
+
+Submitted request to `/api/v3/canister/<ECID>/query`
+
+```html
+
+E : Envelope
+
+```
+
+Conditions
+
+```html
+
+E.content = CanisterQuery Q
+Q.canister_id = ic_principal
+Q.method_name = 'fetch_canister_logs'
+|Q.nonce| <= 32
+is_effective_canister_id(E.content, ECID)
+S.system_time <= Q.ingress_expiry or Q.sender = anonymous_id
+Q.arg = candid(A)
+A.canister_id ∈ verify_envelope(E, Q.sender, S.system_time)
+is_sender_authorized(S, A.canister_id, Q.sender)
+
+```
+
+Query response `R`:
+
+```html
+
+{status: "replied"; reply: {arg: candid(canister_logs(S, A.canister_id, A.filter))}, signatures: Sigs}
+
+```
+
+where the query `Q`, the response `R`, and a certificate `Cert` that is obtained by requesting the path `/subnet` in a **separate** read state request to `/api/v3/canister/<ECID>/read_state` satisfy the following:
+
+```html
+
+verify_response(Q, R, Cert) ∧ lookup(["time"], Cert) = Found S.system_time // or "recent enough"
+
+```
+
 #### Callback invocation
 
 When an inter-canister call has been responded to, we can queue the call to the callback.
@@ -4016,6 +4140,7 @@ S with
     global_timer[CanisterId] = 0
     compute_allocation[Canister_id] = 0
     memory_allocation[Canister_id] = 0
+    log_memory_limit[Canister_id] = 0
 
     messages = S.messages ·
       [ ResponseMessage {
@@ -4137,6 +4262,8 @@ S with
   canister_history[Canister_id] = (deleted)
   canister_log_visibility[New_canister_id] = S.canister_log_visibility[Canister_id]
   canister_log_visibility[Canister_id] = (deleted)
+  canister_log_memory_limit[New_canister_id] = S.canister_log_memory_limit[Canister_id]
+  canister_log_memory_limit[Canister_id] = (deleted)
   canister_snapshot_visibility[New_canister_id] = S.canister_snapshot_visibility[Canister_id]
   canister_snapshot_visibility[Canister_id] = (deleted)
   canister_logs[New_canister_id] = S.canister_logs[Canister_id]
@@ -4300,16 +4427,19 @@ S with
 
 ```
 
-#### Trimming canister logs
+#### Purging canister logs
+
+Oldest canister logs are purged if the total memory used for canister logs exceeds the value `log_memory_limit` in canister settings.
 
-Canister logs can be trimmed if their total length exceeds 4KiB.
+The (unspecified) function `canister_log_memory_usage(logs)` models the total memory used by `logs`.
 
 Conditions
 
 ```html
 
 S.canister_logs[CanisterId] = Older_logs · Newer_logs
-SUM { |l| | l <- Older_logs } > 4KiB
+canister_log_memory_usage(Older_logs · Newer_logs) > S.canister_log_memory_limit[CanisterId]
+canister_log_memory_usage(Newer_logs) ≤ S.canister_log_memory_limit[CanisterId]
 
 ```
 
@@ -4322,60 +4452,6 @@ S with
 
 ```
 
-#### IC Management Canister: Canister logs (query call) {#ic-mgmt-canister-fetch-canister-logs}
-
-This section specifies management canister query calls.
-They are calls to `/api/v3/canister/<ECID>/query`
-with CBOR content `Q` such that `Q.canister_id = ic_principal`.
-
-The management canister offers the method `fetch_canister_logs`
-that can be called as a query call and
-returns logs of a requested canister.
-
-Submitted request to `/api/v3/canister/<ECID>/query`
-
-```html
-
-E : Envelope
-
-```
-
-Conditions
-
-```html
-
-E.content = CanisterQuery Q
-Q.canister_id = ic_principal
-Q.method_name = 'fetch_canister_logs'
-|Q.nonce| <= 32
-is_effective_canister_id(E.content, ECID)
-S.system_time <= Q.ingress_expiry or Q.sender = anonymous_id
-Q.arg = candid(A)
-A.canister_id ∈ verify_envelope(E, Q.sender, S.system_time)
-(S[A.canister_id].canister_log_visibility = Public)
-  or
-  (S[A.canister_id].canister_log_visibility = Controllers and Q.sender in S[A.canister_id].controllers)
-  or
-  (S[A.canister_id].canister_log_visibility = AllowedViewers Principals and (Q.sender in S[A.canister_id].controllers or Q.sender in Principals))
-
-```
-
-Query response `R`:
-
-```html
-
-{status: "replied"; reply: {arg: candid(S.canister_logs[A.canister_id])}, signatures: Sigs}
-
-```
-
-where the query `Q`, the response `R`, and a certificate `Cert` that is obtained by requesting the path `/subnet` in a **separate** read state request to `/api/v3/canister/<ECID>/read_state` satisfy the following:
-
-```html
-
-verify_response(Q, R, Cert) ∧ lookup(["time"], Cert) = Found S.system_time // or "recent enough"
-
-```
-
 #### IC Management Canister: List canisters (query call) {#ic-mgmt-canister-list-canisters}
 
 This section specifies the `list_canisters` management canister query call.

docs/references/ic-interface-spec/management-canister.md

@@ -99,6 +99,13 @@ The optional `settings` parameter can be used to set the following settings:
 
     Default value: `controllers`.
 
+-   `log_memory_limit` (`nat`)
+
+    Must be at most `2097152` (`2 MiB`) and indicates the maximum amount of memory used for canister logs.
+    Oldest canister logs are purged if the total memory used for canister logs exceeds this value.
+
+    Default value: `4096`.
+
 -   `snapshot_visibility` (`snapshot_visibility`)
 
     Controls who can access the canister's snapshots through the following endpoints of the management canister:
@@ -304,6 +311,8 @@ Only the controllers of the canister or the canister itself or subnet admins can
 
     * `snapshots_size`: Represents the memory consumed by all snapshots that belong to this canister.
 
+    * `log_memory_store_size`: Represents the memory used by canister logs of the canister.
+
 All sizes are expressed in bytes.
 
 ### IC method `canister_metrics` {#ic-canister_metrics}
@@ -921,12 +930,12 @@ A snapshot may be deleted only by the controllers of the canister that the snaps
 
 ### IC method `fetch_canister_logs` {#ic-fetch_canister_logs}
 
-This method can only be called by external users via non-replicated (query) calls, i.e., it cannot be called by canisters, cannot be called via replicated calls, and cannot be called from composite query calls.
+This method can only be called by external users via non-replicated (query) calls or by canisters (via replicated calls), i.e., it cannot be called by external users via replicated (update) calls and it cannot be called from composite query calls.
 
 Given a canister ID as input, this method returns a vector of logs of that canister including its trap messages.
 The canister logs are *not* collected in canister methods running in non-replicated mode (NRQ, TQ, CQ, CRy, CRt, CC, and F modes, as defined in [Overview of imports](./canister-interface.md#system-api-imports)) and the canister logs are *purged* when the canister is reinstalled or uninstalled.
-The total size of all returned logs does not exceed 4KiB.
-If new logs are added resulting in exceeding the maximum total log size of 4KiB, the oldest logs will be removed.
+The total size of all returned logs does not exceed an implementation-defined constant chosen so as not to exceed the maximum response size.
+Oldest canister logs are purged if the total memory used for canister logs exceeds the value `log_memory_limit` in canister settings.
 Logs persist across canister upgrades and they are deleted if the canister is reinstalled or uninstalled.
 
 The log visibility is defined in the `log_visibility` field of `canister_settings` and can be one of the following variants:
@@ -941,6 +950,14 @@ A single log is a record with the following fields:
 - `timestamp_nanos` (`nat64`): the timestamp as nanoseconds since 1970-01-01 at which the log was recorded;
 - `content` (`blob`): the actual content of the log;
 
+To filter canister logs, an optional filter can be provided and have one of the following variants:
+- `by_idx` (`record { start : nat64; end : nat64 }`): only logs are returned whose `idx` is within the provided range (`start` is inclusive, but `end` is exclusive);
+- `by_timestamp_nanos` (`record { start : nat64; end : nat64 }`): only logs are returned whose `timestamp_nanos` is within the provided range (`start` is inclusive, but `end` is exclusive).
+
+When the logs selected for the response do not all fit within a single response, they are trimmed to fit, and the direction of trimming differs between filtered and unfiltered reads:
+- An **unfiltered** read trims the **oldest** log records, so the response ends with the newest log record. This surfaces the most recent activity.
+- A **filtered** read trims the **newest** log records, so the response starts with the oldest log record satisfying the filter. This lets a filtered read page forward through logs starting from the beginning of the requested range.
+
 :::warning
 
 The response of a query comes from a single replica, and is therefore not appropriate for security-sensitive applications.

public/references/ic.did

@@ -26,6 +26,7 @@ type canister_settings = record {
     freezing_threshold : opt nat;
     reserved_cycles_limit : opt nat;
     log_visibility : opt log_visibility;
+    log_memory_limit : opt nat;
     snapshot_visibility : opt snapshot_visibility;
     wasm_memory_limit : opt nat;
     wasm_memory_threshold : opt nat;
@@ -39,6 +40,7 @@ type definite_canister_settings = record {
     freezing_threshold : nat;
     reserved_cycles_limit : nat;
     log_visibility : log_visibility;
+    log_memory_limit : nat;
     snapshot_visibility : snapshot_visibility;
     wasm_memory_limit : nat;
     wasm_memory_threshold : nat;
@@ -300,6 +302,7 @@ type canister_status_result = record {
         canister_history_size : nat;
         wasm_chunk_store_size : nat;
         snapshots_size : nat;
+        log_memory_store_size : nat;
     };
     cycles : nat;
     reserved_cycles : nat;
@@ -499,6 +502,10 @@ type delete_canister_snapshot_args = record {
 
 type fetch_canister_logs_args = record {
     canister_id : canister_id;
+    filter : opt variant {
+        by_idx : record { start : nat64; end : nat64 };
+        by_timestamp_nanos : record { start : nat64; end : nat64 };
+    }
 };
 
 type canister_log_record = record {