From 4be8ffa318e8a74a0a8170b0987ca7701fd93660 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Wed, 8 Apr 2026 11:17:25 +0800 Subject: [PATCH 01/18] add turn detection protobufs --- .../agent/livekit_agent_turn_detector.proto | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 protobufs/agent/livekit_agent_turn_detector.proto diff --git a/protobufs/agent/livekit_agent_turn_detector.proto b/protobufs/agent/livekit_agent_turn_detector.proto new file mode 100644 index 000000000..de17b08ae --- /dev/null +++ b/protobufs/agent/livekit_agent_turn_detector.proto @@ -0,0 +1,142 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto3"; + +package livekit.agent; + +option go_package = "github.com/livekit/protocol/livekit/agent"; +option csharp_namespace = "LiveKit.Proto"; +option ruby_package = "LiveKit::Proto"; +option optimize_for = SPEED; + +import "agent/livekit_agent_session.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +enum TdAudioEncoding { + TD_AUDIO_ENCODING_OPUS = 0; +} + +message TdSessionSettings { + uint32 sample_rate = 1; + TdAudioEncoding encoding = 2; +} + +message TdInferenceStats { + google.protobuf.Duration e2e_latency = 1; + google.protobuf.Duration preprocessing_duration = 2; + google.protobuf.Duration inference_duration = 3; +} + +message TdError { + string message = 1; + // error code follows the HTTP status code convention + // 4xx for client errors + // 5xx for server errors + uint32 code = 2; +} + +// --- Client -> Server --- + +message TdSessionCreate { + TdSessionSettings settings = 1; +} + +message TdInputAudio { + bytes audio = 1; + google.protobuf.Timestamp created_at = 2; +} + +message TdInputChatContext { + repeated ChatMessage messages = 1; +} + +message TdSessionFlush {} + +message TdSessionClose {} + +message TdInferenceStart { + string request_id = 1; +} + +message TdInferenceStop { + string request_id = 1; +} + +message TdClientMessage { + oneof message { + TdSessionCreate session_create = 1; + TdInputAudio input_audio = 2; + TdInputChatContext input_chat_context = 3; + TdSessionFlush session_flush = 4; + TdSessionClose session_close = 5; + TdInferenceStart inference_start = 6; + TdInferenceStop inference_stop = 7; + } + google.protobuf.Timestamp created_at = 8; +} + +// --- Server -> Model --- + +message TdInferenceRequest { + bytes audio = 1; + string assistant_text = 2; + TdAudioEncoding encoding = 3; + uint32 sample_rate = 4; +} + +// --- Model -> Server --- + +message TdInferenceResponse { + float probability = 1; + TdInferenceStats stats = 2; +} + +// --- Server -> Client --- + +message TdSessionCreated {} + +message TdProcessingStats { + google.protobuf.Timestamp earliest_client_created_at = 1; + google.protobuf.Timestamp latest_client_created_at = 2; + // server-side E2E latency + google.protobuf.Duration e2e_latency = 3; + // stats including model-side E2E latency + TdInferenceStats inference_stats = 4; +} + +message TdEouPrediction { + float probability = 1; + TdProcessingStats processing_stats = 2; +} + +message TdInferenceStarted {} + +message TdInferenceStopped {} + +message TdSessionClosed {} + +message TdServerMessage { + oneof message { + TdSessionCreated session_created = 1; + TdInferenceStarted inference_started = 2; + TdInferenceStopped inference_stopped = 3; + TdEouPrediction eou_prediction = 4; + TdSessionClosed session_closed = 5; + TdError error = 6; + } + optional string request_id = 7; + google.protobuf.Timestamp server_created_at = 8; + optional google.protobuf.Timestamp client_created_at = 9; +} From 073003beb279edf537c0210405617b289d78148f Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Thu, 9 Apr 2026 16:50:30 +0800 Subject: [PATCH 02/18] Create many-seas-fry.md --- .changeset/many-seas-fry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/many-seas-fry.md diff --git a/.changeset/many-seas-fry.md b/.changeset/many-seas-fry.md new file mode 100644 index 000000000..a31f49c77 --- /dev/null +++ b/.changeset/many-seas-fry.md @@ -0,0 +1,5 @@ +--- +"@fake-scope/fake-pkg": patch +--- + +Add turn detection protobufs From d8da1d27ef6723c303a2e5d843ab5dc6c70d8f5d Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Thu, 9 Apr 2026 22:21:47 +0800 Subject: [PATCH 03/18] add num_samples in each audio frame --- protobufs/agent/livekit_agent_turn_detector.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protobufs/agent/livekit_agent_turn_detector.proto b/protobufs/agent/livekit_agent_turn_detector.proto index de17b08ae..ac4c08cca 100644 --- a/protobufs/agent/livekit_agent_turn_detector.proto +++ b/protobufs/agent/livekit_agent_turn_detector.proto @@ -56,6 +56,8 @@ message TdSessionCreate { message TdInputAudio { bytes audio = 1; google.protobuf.Timestamp created_at = 2; + // used for buffer size calculation + uint32 num_samples = 3; } message TdInputChatContext { From 8cc28aa1ee57739e5f803b9393bff61e53f4bbee Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Fri, 10 Apr 2026 15:02:03 +0800 Subject: [PATCH 04/18] add barge in --- protobufs/agent/livekit_agent_inference.proto | 190 ++++++++++++++++++ .../agent/livekit_agent_turn_detector.proto | 144 ------------- 2 files changed, 190 insertions(+), 144 deletions(-) create mode 100644 protobufs/agent/livekit_agent_inference.proto delete mode 100644 protobufs/agent/livekit_agent_turn_detector.proto diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto new file mode 100644 index 000000000..1b27fe37e --- /dev/null +++ b/protobufs/agent/livekit_agent_inference.proto @@ -0,0 +1,190 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto3"; + +package livekit.agent; + +option go_package = "github.com/livekit/protocol/livekit/agent"; +option csharp_namespace = "LiveKit.Proto"; +option ruby_package = "LiveKit::Proto"; +option optimize_for = SPEED; + +import "agent/livekit_agent_session.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +// --- Shared Types --- + +enum AudioEncoding { + AUDIO_ENCODING_PCM_S16LE = 0; + AUDIO_ENCODING_OPUS = 1; +} + +message SessionSettings { + uint32 sample_rate = 1; + AudioEncoding encoding = 2; + oneof type_settings { + TdSettings td_settings = 3; + BiSettings bi_settings = 4; + } +} + +message InferenceStats { + google.protobuf.Duration e2e_latency = 1; + google.protobuf.Duration preprocessing_duration = 2; + google.protobuf.Duration inference_duration = 3; +} + +message Error { + string message = 1; + // error code follows the HTTP status code convention + // 4xx for client errors + // 5xx for server errors + uint32 code = 2; +} + +message ProcessingStats { + google.protobuf.Timestamp earliest_client_created_at = 1; + google.protobuf.Timestamp latest_client_created_at = 2; + google.protobuf.Duration e2e_latency = 3; + InferenceStats inference_stats = 4; +} + +// --- Turn Detection Settings --- + +message TdSettings { + float detection_interval = 1; +} + +// --- Barge-in Settings --- + +message BiSettings { + float threshold = 1; + uint32 min_frames = 2; + float max_audio_duration = 3; + float audio_prefix_duration = 4; + float detection_interval = 5; +} + +// --- Client -> Server --- + +message SessionCreate { + SessionSettings settings = 1; +} + +message InputAudio { + bytes audio = 1; + google.protobuf.Timestamp created_at = 2; + uint32 num_samples = 3; +} + +message TdInputChatContext { + repeated ChatMessage messages = 1; +} + +message SessionFlush {} + +message SessionClose {} + +message InferenceStart { + string request_id = 1; +} + +message InferenceStop { + string request_id = 1; +} + +message BufferStart {} + +message BufferStop {} + +message ClientMessage { + google.protobuf.Timestamp created_at = 1; + oneof message { + SessionCreate session_create = 2; + InputAudio input_audio = 3; + SessionFlush session_flush = 4; + SessionClose session_close = 5; + InferenceStart inference_start = 6; + InferenceStop inference_stop = 7; + BufferStart buffer_start = 8; + BufferStop buffer_stop = 9; + // only for turn detection + TdInputChatContext td_input_chat_context = 10; + } +} + +// --- Server -> Model --- + +message TdInferenceRequest { + bytes audio = 1; + string assistant_text = 2; + AudioEncoding encoding = 3; + uint32 sample_rate = 4; +} + +message TdInferenceResponse { + float probability = 1; + InferenceStats stats = 2; +} + +message BiInferenceRequest { + bytes audio = 1; + AudioEncoding encoding = 2; + uint32 sample_rate = 3; +} + +message BiInferenceResponse { + bool is_bargein = 1; + repeated float probabilities = 2; + InferenceStats stats = 3; +} + +// --- Server -> Client --- + +message SessionCreated {} + +message InferenceStarted {} + +message InferenceStopped {} + +message SessionClosed {} + +message TdPrediction { + float probability = 1; + ProcessingStats processing_stats = 2; +} + +message BiPrediction { + bool is_bargein = 1; + repeated float probabilities = 2; + ProcessingStats processing_stats = 3; + int64 created_at = 4; + float prediction_duration = 5; +} + +message ServerMessage { + google.protobuf.Timestamp server_created_at = 1; + optional string request_id = 2; + optional google.protobuf.Timestamp client_created_at = 3; + oneof message { + SessionCreated session_created = 4; + InferenceStarted inference_started = 5; + InferenceStopped inference_stopped = 6; + SessionClosed session_closed = 7; + Error error = 8; + TdPrediction td_prediction = 9; + BiPrediction bi_prediction = 10; + } +} diff --git a/protobufs/agent/livekit_agent_turn_detector.proto b/protobufs/agent/livekit_agent_turn_detector.proto deleted file mode 100644 index ac4c08cca..000000000 --- a/protobufs/agent/livekit_agent_turn_detector.proto +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -syntax = "proto3"; - -package livekit.agent; - -option go_package = "github.com/livekit/protocol/livekit/agent"; -option csharp_namespace = "LiveKit.Proto"; -option ruby_package = "LiveKit::Proto"; -option optimize_for = SPEED; - -import "agent/livekit_agent_session.proto"; -import "google/protobuf/duration.proto"; -import "google/protobuf/timestamp.proto"; - -enum TdAudioEncoding { - TD_AUDIO_ENCODING_OPUS = 0; -} - -message TdSessionSettings { - uint32 sample_rate = 1; - TdAudioEncoding encoding = 2; -} - -message TdInferenceStats { - google.protobuf.Duration e2e_latency = 1; - google.protobuf.Duration preprocessing_duration = 2; - google.protobuf.Duration inference_duration = 3; -} - -message TdError { - string message = 1; - // error code follows the HTTP status code convention - // 4xx for client errors - // 5xx for server errors - uint32 code = 2; -} - -// --- Client -> Server --- - -message TdSessionCreate { - TdSessionSettings settings = 1; -} - -message TdInputAudio { - bytes audio = 1; - google.protobuf.Timestamp created_at = 2; - // used for buffer size calculation - uint32 num_samples = 3; -} - -message TdInputChatContext { - repeated ChatMessage messages = 1; -} - -message TdSessionFlush {} - -message TdSessionClose {} - -message TdInferenceStart { - string request_id = 1; -} - -message TdInferenceStop { - string request_id = 1; -} - -message TdClientMessage { - oneof message { - TdSessionCreate session_create = 1; - TdInputAudio input_audio = 2; - TdInputChatContext input_chat_context = 3; - TdSessionFlush session_flush = 4; - TdSessionClose session_close = 5; - TdInferenceStart inference_start = 6; - TdInferenceStop inference_stop = 7; - } - google.protobuf.Timestamp created_at = 8; -} - -// --- Server -> Model --- - -message TdInferenceRequest { - bytes audio = 1; - string assistant_text = 2; - TdAudioEncoding encoding = 3; - uint32 sample_rate = 4; -} - -// --- Model -> Server --- - -message TdInferenceResponse { - float probability = 1; - TdInferenceStats stats = 2; -} - -// --- Server -> Client --- - -message TdSessionCreated {} - -message TdProcessingStats { - google.protobuf.Timestamp earliest_client_created_at = 1; - google.protobuf.Timestamp latest_client_created_at = 2; - // server-side E2E latency - google.protobuf.Duration e2e_latency = 3; - // stats including model-side E2E latency - TdInferenceStats inference_stats = 4; -} - -message TdEouPrediction { - float probability = 1; - TdProcessingStats processing_stats = 2; -} - -message TdInferenceStarted {} - -message TdInferenceStopped {} - -message TdSessionClosed {} - -message TdServerMessage { - oneof message { - TdSessionCreated session_created = 1; - TdInferenceStarted inference_started = 2; - TdInferenceStopped inference_stopped = 3; - TdEouPrediction eou_prediction = 4; - TdSessionClosed session_closed = 5; - TdError error = 6; - } - optional string request_id = 7; - google.protobuf.Timestamp server_created_at = 8; - optional google.protobuf.Timestamp client_created_at = 9; -} From f4d080fb25a80260772f656f4656f94b4df16b46 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Fri, 10 Apr 2026 15:16:16 +0800 Subject: [PATCH 05/18] add request, response, and prediction --- protobufs/agent/livekit_agent_inference.proto | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index 1b27fe37e..aa0479036 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -134,23 +134,38 @@ message TdInferenceRequest { uint32 sample_rate = 4; } -message TdInferenceResponse { - float probability = 1; - InferenceStats stats = 2; -} - message BiInferenceRequest { bytes audio = 1; AudioEncoding encoding = 2; uint32 sample_rate = 3; } +message InferenceRequest { + oneof request { + TdInferenceRequest td_inference_request = 1; + BiInferenceRequest bi_inference_request = 2; + } +} + + +message TdInferenceResponse { + float probability = 1; + InferenceStats stats = 2; +} + message BiInferenceResponse { bool is_bargein = 1; repeated float probabilities = 2; InferenceStats stats = 3; } +message InferenceResponse { + oneof response { + TdInferenceResponse td_inference_response = 1; + BiInferenceResponse bi_inference_response = 2; + } +} + // --- Server -> Client --- message SessionCreated {} @@ -174,6 +189,13 @@ message BiPrediction { float prediction_duration = 5; } +message Prediction { + oneof prediction { + TdPrediction td_prediction = 1; + BiPrediction bi_prediction = 2; + } +} + message ServerMessage { google.protobuf.Timestamp server_created_at = 1; optional string request_id = 2; @@ -184,7 +206,6 @@ message ServerMessage { InferenceStopped inference_stopped = 6; SessionClosed session_closed = 7; Error error = 8; - TdPrediction td_prediction = 9; - BiPrediction bi_prediction = 10; + Prediction prediction = 9; } } From 4124981591d6793d9370b82ac70b96686d3d572e Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Mon, 20 Apr 2026 11:01:11 +0800 Subject: [PATCH 06/18] rename for eou and interruption --- protobufs/agent/livekit_agent_inference.proto | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index aa0479036..09439d591 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -20,9 +20,9 @@ option csharp_namespace = "LiveKit.Proto"; option ruby_package = "LiveKit::Proto"; option optimize_for = SPEED; -import "agent/livekit_agent_session.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; +import "agent/livekit_agent_session.proto"; // --- Shared Types --- @@ -35,17 +35,11 @@ message SessionSettings { uint32 sample_rate = 1; AudioEncoding encoding = 2; oneof type_settings { - TdSettings td_settings = 3; - BiSettings bi_settings = 4; + EouSettings eou_settings = 3; + InterruptionSettings interruption_settings = 4; } } -message InferenceStats { - google.protobuf.Duration e2e_latency = 1; - google.protobuf.Duration preprocessing_duration = 2; - google.protobuf.Duration inference_duration = 3; -} - message Error { string message = 1; // error code follows the HTTP status code convention @@ -54,22 +48,15 @@ message Error { uint32 code = 2; } -message ProcessingStats { - google.protobuf.Timestamp earliest_client_created_at = 1; - google.protobuf.Timestamp latest_client_created_at = 2; - google.protobuf.Duration e2e_latency = 3; - InferenceStats inference_stats = 4; -} - -// --- Turn Detection Settings --- +// --- End of Turn (EOU) Settings --- -message TdSettings { +message EouSettings { float detection_interval = 1; } -// --- Barge-in Settings --- +// --- Interruption Settings --- -message BiSettings { +message InterruptionSettings { float threshold = 1; uint32 min_frames = 2; float max_audio_duration = 3; @@ -89,7 +76,7 @@ message InputAudio { uint32 num_samples = 3; } -message TdInputChatContext { +message EouInputChatContext { repeated ChatMessage messages = 1; } @@ -120,21 +107,21 @@ message ClientMessage { InferenceStop inference_stop = 7; BufferStart buffer_start = 8; BufferStop buffer_stop = 9; - // only for turn detection - TdInputChatContext td_input_chat_context = 10; + // only for end of turn + EouInputChatContext eou_input_chat_context = 10; } } // --- Server -> Model --- -message TdInferenceRequest { +message EouInferenceRequest { bytes audio = 1; string assistant_text = 2; AudioEncoding encoding = 3; uint32 sample_rate = 4; } -message BiInferenceRequest { +message InterruptionInferenceRequest { bytes audio = 1; AudioEncoding encoding = 2; uint32 sample_rate = 3; @@ -142,18 +129,31 @@ message BiInferenceRequest { message InferenceRequest { oneof request { - TdInferenceRequest td_inference_request = 1; - BiInferenceRequest bi_inference_request = 2; + EouInferenceRequest eou_inference_request = 1; + InterruptionInferenceRequest interruption_inference_request = 2; } } +message InferenceStats { + google.protobuf.Duration e2e_latency = 1; + google.protobuf.Duration preprocessing_duration = 2; + google.protobuf.Duration inference_duration = 3; +} + +message ProcessingStats { + google.protobuf.Timestamp earliest_client_created_at = 1; + google.protobuf.Timestamp latest_client_created_at = 2; + google.protobuf.Duration e2e_latency = 3; + InferenceStats inference_stats = 4; +} + -message TdInferenceResponse { +message EouInferenceResponse { float probability = 1; InferenceStats stats = 2; } -message BiInferenceResponse { +message InterruptionInferenceResponse { bool is_bargein = 1; repeated float probabilities = 2; InferenceStats stats = 3; @@ -161,8 +161,8 @@ message BiInferenceResponse { message InferenceResponse { oneof response { - TdInferenceResponse td_inference_response = 1; - BiInferenceResponse bi_inference_response = 2; + EouInferenceResponse eou_inference_response = 1; + InterruptionInferenceResponse interruption_inference_response = 2; } } @@ -176,12 +176,12 @@ message InferenceStopped {} message SessionClosed {} -message TdPrediction { +message EouPrediction { float probability = 1; ProcessingStats processing_stats = 2; } -message BiPrediction { +message InterruptionPrediction { bool is_bargein = 1; repeated float probabilities = 2; ProcessingStats processing_stats = 3; @@ -191,8 +191,8 @@ message BiPrediction { message Prediction { oneof prediction { - TdPrediction td_prediction = 1; - BiPrediction bi_prediction = 2; + EouPrediction eou_prediction = 1; + InterruptionPrediction interruption_prediction = 2; } } From ea0d27e94a2cc81856866a2a01b5869068dda595 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Mon, 20 Apr 2026 11:15:45 +0800 Subject: [PATCH 07/18] refactor --- protobufs/agent/livekit_agent_inference.proto | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index 09439d591..01e869602 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + syntax = "proto3"; package livekit.agent; @@ -20,9 +21,9 @@ option csharp_namespace = "LiveKit.Proto"; option ruby_package = "LiveKit::Proto"; option optimize_for = SPEED; +import "agent/livekit_agent_session.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; -import "agent/livekit_agent_session.proto"; // --- Shared Types --- @@ -51,17 +52,19 @@ message Error { // --- End of Turn (EOU) Settings --- message EouSettings { - float detection_interval = 1; + google.protobuf.Duration detection_interval = 1; } // --- Interruption Settings --- message InterruptionSettings { + // detection threshold in range [0.0, 1.0]; higher values are less sensitive float threshold = 1; + // minimum number of frames with probability greater than threshold to trigger an interruption uint32 min_frames = 2; - float max_audio_duration = 3; - float audio_prefix_duration = 4; - float detection_interval = 5; + google.protobuf.Duration max_audio_duration = 3; + google.protobuf.Duration audio_prefix_duration = 4; + google.protobuf.Duration detection_interval = 5; } // --- Client -> Server --- @@ -92,6 +95,7 @@ message InferenceStop { string request_id = 1; } +// audio buffer sentinel messages message BufferStart {} message BufferStop {} @@ -135,6 +139,7 @@ message InferenceRequest { } message InferenceStats { + // server-side e2e latency (server input to server output) google.protobuf.Duration e2e_latency = 1; google.protobuf.Duration preprocessing_duration = 2; google.protobuf.Duration inference_duration = 3; @@ -143,6 +148,7 @@ message InferenceStats { message ProcessingStats { google.protobuf.Timestamp earliest_client_created_at = 1; google.protobuf.Timestamp latest_client_created_at = 2; + // client-side e2e latency (client send to client receive) google.protobuf.Duration e2e_latency = 3; InferenceStats inference_stats = 4; } @@ -155,6 +161,7 @@ message EouInferenceResponse { message InterruptionInferenceResponse { bool is_bargein = 1; + // per frame probabilities repeated float probabilities = 2; InferenceStats stats = 3; } @@ -185,8 +192,8 @@ message InterruptionPrediction { bool is_bargein = 1; repeated float probabilities = 2; ProcessingStats processing_stats = 3; - int64 created_at = 4; - float prediction_duration = 5; + google.protobuf.Timestamp created_at = 4; + google.protobuf.Duration prediction_duration = 5; } message Prediction { @@ -199,6 +206,7 @@ message Prediction { message ServerMessage { google.protobuf.Timestamp server_created_at = 1; optional string request_id = 2; + // echoes the client-side created_at timestamp optional google.protobuf.Timestamp client_created_at = 3; oneof message { SessionCreated session_created = 4; From 9ebcbd5c3d2817a1e4c594b9921fb4788ffb99dc Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Mon, 20 Apr 2026 11:19:39 +0800 Subject: [PATCH 08/18] rename --- protobufs/agent/livekit_agent_inference.proto | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index 01e869602..fae017193 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -36,7 +36,7 @@ message SessionSettings { uint32 sample_rate = 1; AudioEncoding encoding = 2; oneof type_settings { - EouSettings eou_settings = 3; + EotSettings eot_settings = 3; InterruptionSettings interruption_settings = 4; } } @@ -49,9 +49,9 @@ message Error { uint32 code = 2; } -// --- End of Turn (EOU) Settings --- +// --- End of Turn (EOT) Settings --- -message EouSettings { +message EotSettings { google.protobuf.Duration detection_interval = 1; } @@ -79,7 +79,7 @@ message InputAudio { uint32 num_samples = 3; } -message EouInputChatContext { +message EotInputChatContext { repeated ChatMessage messages = 1; } @@ -112,13 +112,13 @@ message ClientMessage { BufferStart buffer_start = 8; BufferStop buffer_stop = 9; // only for end of turn - EouInputChatContext eou_input_chat_context = 10; + EotInputChatContext eot_input_chat_context = 10; } } // --- Server -> Model --- -message EouInferenceRequest { +message EotInferenceRequest { bytes audio = 1; string assistant_text = 2; AudioEncoding encoding = 3; @@ -133,7 +133,7 @@ message InterruptionInferenceRequest { message InferenceRequest { oneof request { - EouInferenceRequest eou_inference_request = 1; + EotInferenceRequest eot_inference_request = 1; InterruptionInferenceRequest interruption_inference_request = 2; } } @@ -154,7 +154,7 @@ message ProcessingStats { } -message EouInferenceResponse { +message EotInferenceResponse { float probability = 1; InferenceStats stats = 2; } @@ -168,7 +168,7 @@ message InterruptionInferenceResponse { message InferenceResponse { oneof response { - EouInferenceResponse eou_inference_response = 1; + EotInferenceResponse eot_inference_response = 1; InterruptionInferenceResponse interruption_inference_response = 2; } } @@ -183,7 +183,7 @@ message InferenceStopped {} message SessionClosed {} -message EouPrediction { +message EotPrediction { float probability = 1; ProcessingStats processing_stats = 2; } @@ -198,7 +198,7 @@ message InterruptionPrediction { message Prediction { oneof prediction { - EouPrediction eou_prediction = 1; + EotPrediction eot_prediction = 1; InterruptionPrediction interruption_prediction = 2; } } From 68c05a633019d663cf54418e7f8143b205beca08 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Mon, 20 Apr 2026 11:33:29 +0800 Subject: [PATCH 09/18] rename bargein to interruption --- protobufs/agent/livekit_agent_inference.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index fae017193..843fdbd0a 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -189,7 +189,7 @@ message EotPrediction { } message InterruptionPrediction { - bool is_bargein = 1; + bool is_interruption = 1; repeated float probabilities = 2; ProcessingStats processing_stats = 3; google.protobuf.Timestamp created_at = 4; From a233c45ca8748275bea25a41542761e8f3893151 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Mon, 20 Apr 2026 11:34:23 +0800 Subject: [PATCH 10/18] rename bargein to interruption --- protobufs/agent/livekit_agent_inference.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index 843fdbd0a..fd6ac2ee9 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -160,7 +160,7 @@ message EotInferenceResponse { } message InterruptionInferenceResponse { - bool is_bargein = 1; + bool is_interruption = 1; // per frame probabilities repeated float probabilities = 2; InferenceStats stats = 3; From 720f1f575d751fb16f2f5d8de6d7ec6584063013 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Tue, 21 Apr 2026 16:48:37 +0800 Subject: [PATCH 11/18] address comments --- protobufs/agent/livekit_agent_inference.proto | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index fd6ac2ee9..dedd2e694 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -41,7 +41,7 @@ message SessionSettings { } } -message Error { +message InferenceError { string message = 1; // error code follows the HTTP status code convention // 4xx for client errors @@ -196,13 +196,6 @@ message InterruptionPrediction { google.protobuf.Duration prediction_duration = 5; } -message Prediction { - oneof prediction { - EotPrediction eot_prediction = 1; - InterruptionPrediction interruption_prediction = 2; - } -} - message ServerMessage { google.protobuf.Timestamp server_created_at = 1; optional string request_id = 2; @@ -213,7 +206,8 @@ message ServerMessage { InferenceStarted inference_started = 5; InferenceStopped inference_stopped = 6; SessionClosed session_closed = 7; - Error error = 8; - Prediction prediction = 9; + InferenceError error = 8; + EotPrediction eot_prediction = 9; + InterruptionPrediction interruption_prediction = 10; } } From a84e436abc1b8d2e70e19277304e449b702e05fe Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Tue, 21 Apr 2026 17:07:13 +0800 Subject: [PATCH 12/18] add file to magefile --- magefile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/magefile.go b/magefile.go index d0744f7f9..b52889270 100644 --- a/magefile.go +++ b/magefile.go @@ -60,6 +60,7 @@ func Proto() error { agentProtoFiles := []string{ "agent/livekit_agent_session.proto", "agent/livekit_agent_dev.proto", + "agent/livekit_agent_inference.proto", } protoFiles := []string{ From 9670cbd69d5f786c45a34e770f290164e33c3afe Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 09:08:00 +0000 Subject: [PATCH 13/18] generated protobuf --- livekit/agent/livekit_agent_inference.pb.go | 2234 +++++++++++++++++++ 1 file changed, 2234 insertions(+) create mode 100644 livekit/agent/livekit_agent_inference.pb.go diff --git a/livekit/agent/livekit_agent_inference.pb.go b/livekit/agent/livekit_agent_inference.pb.go new file mode 100644 index 000000000..fc4e390ac --- /dev/null +++ b/livekit/agent/livekit_agent_inference.pb.go @@ -0,0 +1,2234 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v4.23.4 +// source: agent/livekit_agent_inference.proto + +package agent + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type AudioEncoding int32 + +const ( + AudioEncoding_AUDIO_ENCODING_PCM_S16LE AudioEncoding = 0 + AudioEncoding_AUDIO_ENCODING_OPUS AudioEncoding = 1 +) + +// Enum value maps for AudioEncoding. +var ( + AudioEncoding_name = map[int32]string{ + 0: "AUDIO_ENCODING_PCM_S16LE", + 1: "AUDIO_ENCODING_OPUS", + } + AudioEncoding_value = map[string]int32{ + "AUDIO_ENCODING_PCM_S16LE": 0, + "AUDIO_ENCODING_OPUS": 1, + } +) + +func (x AudioEncoding) Enum() *AudioEncoding { + p := new(AudioEncoding) + *p = x + return p +} + +func (x AudioEncoding) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AudioEncoding) Descriptor() protoreflect.EnumDescriptor { + return file_agent_livekit_agent_inference_proto_enumTypes[0].Descriptor() +} + +func (AudioEncoding) Type() protoreflect.EnumType { + return &file_agent_livekit_agent_inference_proto_enumTypes[0] +} + +func (x AudioEncoding) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AudioEncoding.Descriptor instead. +func (AudioEncoding) EnumDescriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{0} +} + +type SessionSettings struct { + state protoimpl.MessageState `protogen:"open.v1"` + SampleRate uint32 `protobuf:"varint,1,opt,name=sample_rate,json=sampleRate,proto3" json:"sample_rate,omitempty"` + Encoding AudioEncoding `protobuf:"varint,2,opt,name=encoding,proto3,enum=livekit.agent.AudioEncoding" json:"encoding,omitempty"` + // Types that are valid to be assigned to TypeSettings: + // + // *SessionSettings_EotSettings + // *SessionSettings_InterruptionSettings + TypeSettings isSessionSettings_TypeSettings `protobuf_oneof:"type_settings"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionSettings) Reset() { + *x = SessionSettings{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionSettings) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionSettings) ProtoMessage() {} + +func (x *SessionSettings) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionSettings.ProtoReflect.Descriptor instead. +func (*SessionSettings) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{0} +} + +func (x *SessionSettings) GetSampleRate() uint32 { + if x != nil { + return x.SampleRate + } + return 0 +} + +func (x *SessionSettings) GetEncoding() AudioEncoding { + if x != nil { + return x.Encoding + } + return AudioEncoding_AUDIO_ENCODING_PCM_S16LE +} + +func (x *SessionSettings) GetTypeSettings() isSessionSettings_TypeSettings { + if x != nil { + return x.TypeSettings + } + return nil +} + +func (x *SessionSettings) GetEotSettings() *EotSettings { + if x != nil { + if x, ok := x.TypeSettings.(*SessionSettings_EotSettings); ok { + return x.EotSettings + } + } + return nil +} + +func (x *SessionSettings) GetInterruptionSettings() *InterruptionSettings { + if x != nil { + if x, ok := x.TypeSettings.(*SessionSettings_InterruptionSettings); ok { + return x.InterruptionSettings + } + } + return nil +} + +type isSessionSettings_TypeSettings interface { + isSessionSettings_TypeSettings() +} + +type SessionSettings_EotSettings struct { + EotSettings *EotSettings `protobuf:"bytes,3,opt,name=eot_settings,json=eotSettings,proto3,oneof"` +} + +type SessionSettings_InterruptionSettings struct { + InterruptionSettings *InterruptionSettings `protobuf:"bytes,4,opt,name=interruption_settings,json=interruptionSettings,proto3,oneof"` +} + +func (*SessionSettings_EotSettings) isSessionSettings_TypeSettings() {} + +func (*SessionSettings_InterruptionSettings) isSessionSettings_TypeSettings() {} + +type InferenceError struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + // error code follows the HTTP status code convention + // 4xx for client errors + // 5xx for server errors + Code uint32 `protobuf:"varint,2,opt,name=code,proto3" json:"code,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceError) Reset() { + *x = InferenceError{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceError) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceError) ProtoMessage() {} + +func (x *InferenceError) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceError.ProtoReflect.Descriptor instead. +func (*InferenceError) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{1} +} + +func (x *InferenceError) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *InferenceError) GetCode() uint32 { + if x != nil { + return x.Code + } + return 0 +} + +type EotSettings struct { + state protoimpl.MessageState `protogen:"open.v1"` + DetectionInterval *durationpb.Duration `protobuf:"bytes,1,opt,name=detection_interval,json=detectionInterval,proto3" json:"detection_interval,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EotSettings) Reset() { + *x = EotSettings{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EotSettings) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EotSettings) ProtoMessage() {} + +func (x *EotSettings) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EotSettings.ProtoReflect.Descriptor instead. +func (*EotSettings) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{2} +} + +func (x *EotSettings) GetDetectionInterval() *durationpb.Duration { + if x != nil { + return x.DetectionInterval + } + return nil +} + +type InterruptionSettings struct { + state protoimpl.MessageState `protogen:"open.v1"` + // detection threshold in range [0.0, 1.0]; higher values are less sensitive + Threshold float32 `protobuf:"fixed32,1,opt,name=threshold,proto3" json:"threshold,omitempty"` + // minimum number of frames with probability greater than threshold to trigger an interruption + MinFrames uint32 `protobuf:"varint,2,opt,name=min_frames,json=minFrames,proto3" json:"min_frames,omitempty"` + MaxAudioDuration *durationpb.Duration `protobuf:"bytes,3,opt,name=max_audio_duration,json=maxAudioDuration,proto3" json:"max_audio_duration,omitempty"` + AudioPrefixDuration *durationpb.Duration `protobuf:"bytes,4,opt,name=audio_prefix_duration,json=audioPrefixDuration,proto3" json:"audio_prefix_duration,omitempty"` + DetectionInterval *durationpb.Duration `protobuf:"bytes,5,opt,name=detection_interval,json=detectionInterval,proto3" json:"detection_interval,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InterruptionSettings) Reset() { + *x = InterruptionSettings{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InterruptionSettings) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InterruptionSettings) ProtoMessage() {} + +func (x *InterruptionSettings) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InterruptionSettings.ProtoReflect.Descriptor instead. +func (*InterruptionSettings) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{3} +} + +func (x *InterruptionSettings) GetThreshold() float32 { + if x != nil { + return x.Threshold + } + return 0 +} + +func (x *InterruptionSettings) GetMinFrames() uint32 { + if x != nil { + return x.MinFrames + } + return 0 +} + +func (x *InterruptionSettings) GetMaxAudioDuration() *durationpb.Duration { + if x != nil { + return x.MaxAudioDuration + } + return nil +} + +func (x *InterruptionSettings) GetAudioPrefixDuration() *durationpb.Duration { + if x != nil { + return x.AudioPrefixDuration + } + return nil +} + +func (x *InterruptionSettings) GetDetectionInterval() *durationpb.Duration { + if x != nil { + return x.DetectionInterval + } + return nil +} + +type SessionCreate struct { + state protoimpl.MessageState `protogen:"open.v1"` + Settings *SessionSettings `protobuf:"bytes,1,opt,name=settings,proto3" json:"settings,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionCreate) Reset() { + *x = SessionCreate{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionCreate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionCreate) ProtoMessage() {} + +func (x *SessionCreate) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionCreate.ProtoReflect.Descriptor instead. +func (*SessionCreate) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{4} +} + +func (x *SessionCreate) GetSettings() *SessionSettings { + if x != nil { + return x.Settings + } + return nil +} + +type InputAudio struct { + state protoimpl.MessageState `protogen:"open.v1"` + Audio []byte `protobuf:"bytes,1,opt,name=audio,proto3" json:"audio,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + NumSamples uint32 `protobuf:"varint,3,opt,name=num_samples,json=numSamples,proto3" json:"num_samples,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InputAudio) Reset() { + *x = InputAudio{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InputAudio) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InputAudio) ProtoMessage() {} + +func (x *InputAudio) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InputAudio.ProtoReflect.Descriptor instead. +func (*InputAudio) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{5} +} + +func (x *InputAudio) GetAudio() []byte { + if x != nil { + return x.Audio + } + return nil +} + +func (x *InputAudio) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *InputAudio) GetNumSamples() uint32 { + if x != nil { + return x.NumSamples + } + return 0 +} + +type EotInputChatContext struct { + state protoimpl.MessageState `protogen:"open.v1"` + Messages []*ChatMessage `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EotInputChatContext) Reset() { + *x = EotInputChatContext{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EotInputChatContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EotInputChatContext) ProtoMessage() {} + +func (x *EotInputChatContext) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EotInputChatContext.ProtoReflect.Descriptor instead. +func (*EotInputChatContext) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{6} +} + +func (x *EotInputChatContext) GetMessages() []*ChatMessage { + if x != nil { + return x.Messages + } + return nil +} + +type SessionFlush struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionFlush) Reset() { + *x = SessionFlush{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionFlush) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionFlush) ProtoMessage() {} + +func (x *SessionFlush) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionFlush.ProtoReflect.Descriptor instead. +func (*SessionFlush) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{7} +} + +type SessionClose struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionClose) Reset() { + *x = SessionClose{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionClose) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionClose) ProtoMessage() {} + +func (x *SessionClose) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionClose.ProtoReflect.Descriptor instead. +func (*SessionClose) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{8} +} + +type InferenceStart struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceStart) Reset() { + *x = InferenceStart{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceStart) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceStart) ProtoMessage() {} + +func (x *InferenceStart) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceStart.ProtoReflect.Descriptor instead. +func (*InferenceStart) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{9} +} + +func (x *InferenceStart) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +type InferenceStop struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceStop) Reset() { + *x = InferenceStop{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceStop) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceStop) ProtoMessage() {} + +func (x *InferenceStop) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceStop.ProtoReflect.Descriptor instead. +func (*InferenceStop) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{10} +} + +func (x *InferenceStop) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +// audio buffer sentinel messages +type BufferStart struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BufferStart) Reset() { + *x = BufferStart{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BufferStart) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BufferStart) ProtoMessage() {} + +func (x *BufferStart) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BufferStart.ProtoReflect.Descriptor instead. +func (*BufferStart) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{11} +} + +type BufferStop struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BufferStop) Reset() { + *x = BufferStop{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BufferStop) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BufferStop) ProtoMessage() {} + +func (x *BufferStop) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BufferStop.ProtoReflect.Descriptor instead. +func (*BufferStop) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{12} +} + +type ClientMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + // Types that are valid to be assigned to Message: + // + // *ClientMessage_SessionCreate + // *ClientMessage_InputAudio + // *ClientMessage_SessionFlush + // *ClientMessage_SessionClose + // *ClientMessage_InferenceStart + // *ClientMessage_InferenceStop + // *ClientMessage_BufferStart + // *ClientMessage_BufferStop + // *ClientMessage_EotInputChatContext + Message isClientMessage_Message `protobuf_oneof:"message"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClientMessage) Reset() { + *x = ClientMessage{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClientMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientMessage) ProtoMessage() {} + +func (x *ClientMessage) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientMessage.ProtoReflect.Descriptor instead. +func (*ClientMessage) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{13} +} + +func (x *ClientMessage) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *ClientMessage) GetMessage() isClientMessage_Message { + if x != nil { + return x.Message + } + return nil +} + +func (x *ClientMessage) GetSessionCreate() *SessionCreate { + if x != nil { + if x, ok := x.Message.(*ClientMessage_SessionCreate); ok { + return x.SessionCreate + } + } + return nil +} + +func (x *ClientMessage) GetInputAudio() *InputAudio { + if x != nil { + if x, ok := x.Message.(*ClientMessage_InputAudio); ok { + return x.InputAudio + } + } + return nil +} + +func (x *ClientMessage) GetSessionFlush() *SessionFlush { + if x != nil { + if x, ok := x.Message.(*ClientMessage_SessionFlush); ok { + return x.SessionFlush + } + } + return nil +} + +func (x *ClientMessage) GetSessionClose() *SessionClose { + if x != nil { + if x, ok := x.Message.(*ClientMessage_SessionClose); ok { + return x.SessionClose + } + } + return nil +} + +func (x *ClientMessage) GetInferenceStart() *InferenceStart { + if x != nil { + if x, ok := x.Message.(*ClientMessage_InferenceStart); ok { + return x.InferenceStart + } + } + return nil +} + +func (x *ClientMessage) GetInferenceStop() *InferenceStop { + if x != nil { + if x, ok := x.Message.(*ClientMessage_InferenceStop); ok { + return x.InferenceStop + } + } + return nil +} + +func (x *ClientMessage) GetBufferStart() *BufferStart { + if x != nil { + if x, ok := x.Message.(*ClientMessage_BufferStart); ok { + return x.BufferStart + } + } + return nil +} + +func (x *ClientMessage) GetBufferStop() *BufferStop { + if x != nil { + if x, ok := x.Message.(*ClientMessage_BufferStop); ok { + return x.BufferStop + } + } + return nil +} + +func (x *ClientMessage) GetEotInputChatContext() *EotInputChatContext { + if x != nil { + if x, ok := x.Message.(*ClientMessage_EotInputChatContext); ok { + return x.EotInputChatContext + } + } + return nil +} + +type isClientMessage_Message interface { + isClientMessage_Message() +} + +type ClientMessage_SessionCreate struct { + SessionCreate *SessionCreate `protobuf:"bytes,2,opt,name=session_create,json=sessionCreate,proto3,oneof"` +} + +type ClientMessage_InputAudio struct { + InputAudio *InputAudio `protobuf:"bytes,3,opt,name=input_audio,json=inputAudio,proto3,oneof"` +} + +type ClientMessage_SessionFlush struct { + SessionFlush *SessionFlush `protobuf:"bytes,4,opt,name=session_flush,json=sessionFlush,proto3,oneof"` +} + +type ClientMessage_SessionClose struct { + SessionClose *SessionClose `protobuf:"bytes,5,opt,name=session_close,json=sessionClose,proto3,oneof"` +} + +type ClientMessage_InferenceStart struct { + InferenceStart *InferenceStart `protobuf:"bytes,6,opt,name=inference_start,json=inferenceStart,proto3,oneof"` +} + +type ClientMessage_InferenceStop struct { + InferenceStop *InferenceStop `protobuf:"bytes,7,opt,name=inference_stop,json=inferenceStop,proto3,oneof"` +} + +type ClientMessage_BufferStart struct { + BufferStart *BufferStart `protobuf:"bytes,8,opt,name=buffer_start,json=bufferStart,proto3,oneof"` +} + +type ClientMessage_BufferStop struct { + BufferStop *BufferStop `protobuf:"bytes,9,opt,name=buffer_stop,json=bufferStop,proto3,oneof"` +} + +type ClientMessage_EotInputChatContext struct { + // only for end of turn + EotInputChatContext *EotInputChatContext `protobuf:"bytes,10,opt,name=eot_input_chat_context,json=eotInputChatContext,proto3,oneof"` +} + +func (*ClientMessage_SessionCreate) isClientMessage_Message() {} + +func (*ClientMessage_InputAudio) isClientMessage_Message() {} + +func (*ClientMessage_SessionFlush) isClientMessage_Message() {} + +func (*ClientMessage_SessionClose) isClientMessage_Message() {} + +func (*ClientMessage_InferenceStart) isClientMessage_Message() {} + +func (*ClientMessage_InferenceStop) isClientMessage_Message() {} + +func (*ClientMessage_BufferStart) isClientMessage_Message() {} + +func (*ClientMessage_BufferStop) isClientMessage_Message() {} + +func (*ClientMessage_EotInputChatContext) isClientMessage_Message() {} + +type EotInferenceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Audio []byte `protobuf:"bytes,1,opt,name=audio,proto3" json:"audio,omitempty"` + AssistantText string `protobuf:"bytes,2,opt,name=assistant_text,json=assistantText,proto3" json:"assistant_text,omitempty"` + Encoding AudioEncoding `protobuf:"varint,3,opt,name=encoding,proto3,enum=livekit.agent.AudioEncoding" json:"encoding,omitempty"` + SampleRate uint32 `protobuf:"varint,4,opt,name=sample_rate,json=sampleRate,proto3" json:"sample_rate,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EotInferenceRequest) Reset() { + *x = EotInferenceRequest{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EotInferenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EotInferenceRequest) ProtoMessage() {} + +func (x *EotInferenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EotInferenceRequest.ProtoReflect.Descriptor instead. +func (*EotInferenceRequest) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{14} +} + +func (x *EotInferenceRequest) GetAudio() []byte { + if x != nil { + return x.Audio + } + return nil +} + +func (x *EotInferenceRequest) GetAssistantText() string { + if x != nil { + return x.AssistantText + } + return "" +} + +func (x *EotInferenceRequest) GetEncoding() AudioEncoding { + if x != nil { + return x.Encoding + } + return AudioEncoding_AUDIO_ENCODING_PCM_S16LE +} + +func (x *EotInferenceRequest) GetSampleRate() uint32 { + if x != nil { + return x.SampleRate + } + return 0 +} + +type InterruptionInferenceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Audio []byte `protobuf:"bytes,1,opt,name=audio,proto3" json:"audio,omitempty"` + Encoding AudioEncoding `protobuf:"varint,2,opt,name=encoding,proto3,enum=livekit.agent.AudioEncoding" json:"encoding,omitempty"` + SampleRate uint32 `protobuf:"varint,3,opt,name=sample_rate,json=sampleRate,proto3" json:"sample_rate,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InterruptionInferenceRequest) Reset() { + *x = InterruptionInferenceRequest{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InterruptionInferenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InterruptionInferenceRequest) ProtoMessage() {} + +func (x *InterruptionInferenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InterruptionInferenceRequest.ProtoReflect.Descriptor instead. +func (*InterruptionInferenceRequest) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{15} +} + +func (x *InterruptionInferenceRequest) GetAudio() []byte { + if x != nil { + return x.Audio + } + return nil +} + +func (x *InterruptionInferenceRequest) GetEncoding() AudioEncoding { + if x != nil { + return x.Encoding + } + return AudioEncoding_AUDIO_ENCODING_PCM_S16LE +} + +func (x *InterruptionInferenceRequest) GetSampleRate() uint32 { + if x != nil { + return x.SampleRate + } + return 0 +} + +type InferenceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Request: + // + // *InferenceRequest_EotInferenceRequest + // *InferenceRequest_InterruptionInferenceRequest + Request isInferenceRequest_Request `protobuf_oneof:"request"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceRequest) Reset() { + *x = InferenceRequest{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceRequest) ProtoMessage() {} + +func (x *InferenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceRequest.ProtoReflect.Descriptor instead. +func (*InferenceRequest) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{16} +} + +func (x *InferenceRequest) GetRequest() isInferenceRequest_Request { + if x != nil { + return x.Request + } + return nil +} + +func (x *InferenceRequest) GetEotInferenceRequest() *EotInferenceRequest { + if x != nil { + if x, ok := x.Request.(*InferenceRequest_EotInferenceRequest); ok { + return x.EotInferenceRequest + } + } + return nil +} + +func (x *InferenceRequest) GetInterruptionInferenceRequest() *InterruptionInferenceRequest { + if x != nil { + if x, ok := x.Request.(*InferenceRequest_InterruptionInferenceRequest); ok { + return x.InterruptionInferenceRequest + } + } + return nil +} + +type isInferenceRequest_Request interface { + isInferenceRequest_Request() +} + +type InferenceRequest_EotInferenceRequest struct { + EotInferenceRequest *EotInferenceRequest `protobuf:"bytes,1,opt,name=eot_inference_request,json=eotInferenceRequest,proto3,oneof"` +} + +type InferenceRequest_InterruptionInferenceRequest struct { + InterruptionInferenceRequest *InterruptionInferenceRequest `protobuf:"bytes,2,opt,name=interruption_inference_request,json=interruptionInferenceRequest,proto3,oneof"` +} + +func (*InferenceRequest_EotInferenceRequest) isInferenceRequest_Request() {} + +func (*InferenceRequest_InterruptionInferenceRequest) isInferenceRequest_Request() {} + +type InferenceStats struct { + state protoimpl.MessageState `protogen:"open.v1"` + // server-side e2e latency (server input to server output) + E2ELatency *durationpb.Duration `protobuf:"bytes,1,opt,name=e2e_latency,json=e2eLatency,proto3" json:"e2e_latency,omitempty"` + PreprocessingDuration *durationpb.Duration `protobuf:"bytes,2,opt,name=preprocessing_duration,json=preprocessingDuration,proto3" json:"preprocessing_duration,omitempty"` + InferenceDuration *durationpb.Duration `protobuf:"bytes,3,opt,name=inference_duration,json=inferenceDuration,proto3" json:"inference_duration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceStats) Reset() { + *x = InferenceStats{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceStats) ProtoMessage() {} + +func (x *InferenceStats) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceStats.ProtoReflect.Descriptor instead. +func (*InferenceStats) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{17} +} + +func (x *InferenceStats) GetE2ELatency() *durationpb.Duration { + if x != nil { + return x.E2ELatency + } + return nil +} + +func (x *InferenceStats) GetPreprocessingDuration() *durationpb.Duration { + if x != nil { + return x.PreprocessingDuration + } + return nil +} + +func (x *InferenceStats) GetInferenceDuration() *durationpb.Duration { + if x != nil { + return x.InferenceDuration + } + return nil +} + +type ProcessingStats struct { + state protoimpl.MessageState `protogen:"open.v1"` + EarliestClientCreatedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=earliest_client_created_at,json=earliestClientCreatedAt,proto3" json:"earliest_client_created_at,omitempty"` + LatestClientCreatedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=latest_client_created_at,json=latestClientCreatedAt,proto3" json:"latest_client_created_at,omitempty"` + // client-side e2e latency (client send to client receive) + E2ELatency *durationpb.Duration `protobuf:"bytes,3,opt,name=e2e_latency,json=e2eLatency,proto3" json:"e2e_latency,omitempty"` + InferenceStats *InferenceStats `protobuf:"bytes,4,opt,name=inference_stats,json=inferenceStats,proto3" json:"inference_stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProcessingStats) Reset() { + *x = ProcessingStats{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProcessingStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProcessingStats) ProtoMessage() {} + +func (x *ProcessingStats) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProcessingStats.ProtoReflect.Descriptor instead. +func (*ProcessingStats) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{18} +} + +func (x *ProcessingStats) GetEarliestClientCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.EarliestClientCreatedAt + } + return nil +} + +func (x *ProcessingStats) GetLatestClientCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.LatestClientCreatedAt + } + return nil +} + +func (x *ProcessingStats) GetE2ELatency() *durationpb.Duration { + if x != nil { + return x.E2ELatency + } + return nil +} + +func (x *ProcessingStats) GetInferenceStats() *InferenceStats { + if x != nil { + return x.InferenceStats + } + return nil +} + +type EotInferenceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Probability float32 `protobuf:"fixed32,1,opt,name=probability,proto3" json:"probability,omitempty"` + Stats *InferenceStats `protobuf:"bytes,2,opt,name=stats,proto3" json:"stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EotInferenceResponse) Reset() { + *x = EotInferenceResponse{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EotInferenceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EotInferenceResponse) ProtoMessage() {} + +func (x *EotInferenceResponse) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EotInferenceResponse.ProtoReflect.Descriptor instead. +func (*EotInferenceResponse) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{19} +} + +func (x *EotInferenceResponse) GetProbability() float32 { + if x != nil { + return x.Probability + } + return 0 +} + +func (x *EotInferenceResponse) GetStats() *InferenceStats { + if x != nil { + return x.Stats + } + return nil +} + +type InterruptionInferenceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + IsInterruption bool `protobuf:"varint,1,opt,name=is_interruption,json=isInterruption,proto3" json:"is_interruption,omitempty"` + // per frame probabilities + Probabilities []float32 `protobuf:"fixed32,2,rep,packed,name=probabilities,proto3" json:"probabilities,omitempty"` + Stats *InferenceStats `protobuf:"bytes,3,opt,name=stats,proto3" json:"stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InterruptionInferenceResponse) Reset() { + *x = InterruptionInferenceResponse{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InterruptionInferenceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InterruptionInferenceResponse) ProtoMessage() {} + +func (x *InterruptionInferenceResponse) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InterruptionInferenceResponse.ProtoReflect.Descriptor instead. +func (*InterruptionInferenceResponse) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{20} +} + +func (x *InterruptionInferenceResponse) GetIsInterruption() bool { + if x != nil { + return x.IsInterruption + } + return false +} + +func (x *InterruptionInferenceResponse) GetProbabilities() []float32 { + if x != nil { + return x.Probabilities + } + return nil +} + +func (x *InterruptionInferenceResponse) GetStats() *InferenceStats { + if x != nil { + return x.Stats + } + return nil +} + +type InferenceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Response: + // + // *InferenceResponse_EotInferenceResponse + // *InferenceResponse_InterruptionInferenceResponse + Response isInferenceResponse_Response `protobuf_oneof:"response"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceResponse) Reset() { + *x = InferenceResponse{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceResponse) ProtoMessage() {} + +func (x *InferenceResponse) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceResponse.ProtoReflect.Descriptor instead. +func (*InferenceResponse) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{21} +} + +func (x *InferenceResponse) GetResponse() isInferenceResponse_Response { + if x != nil { + return x.Response + } + return nil +} + +func (x *InferenceResponse) GetEotInferenceResponse() *EotInferenceResponse { + if x != nil { + if x, ok := x.Response.(*InferenceResponse_EotInferenceResponse); ok { + return x.EotInferenceResponse + } + } + return nil +} + +func (x *InferenceResponse) GetInterruptionInferenceResponse() *InterruptionInferenceResponse { + if x != nil { + if x, ok := x.Response.(*InferenceResponse_InterruptionInferenceResponse); ok { + return x.InterruptionInferenceResponse + } + } + return nil +} + +type isInferenceResponse_Response interface { + isInferenceResponse_Response() +} + +type InferenceResponse_EotInferenceResponse struct { + EotInferenceResponse *EotInferenceResponse `protobuf:"bytes,1,opt,name=eot_inference_response,json=eotInferenceResponse,proto3,oneof"` +} + +type InferenceResponse_InterruptionInferenceResponse struct { + InterruptionInferenceResponse *InterruptionInferenceResponse `protobuf:"bytes,2,opt,name=interruption_inference_response,json=interruptionInferenceResponse,proto3,oneof"` +} + +func (*InferenceResponse_EotInferenceResponse) isInferenceResponse_Response() {} + +func (*InferenceResponse_InterruptionInferenceResponse) isInferenceResponse_Response() {} + +type SessionCreated struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionCreated) Reset() { + *x = SessionCreated{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionCreated) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionCreated) ProtoMessage() {} + +func (x *SessionCreated) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionCreated.ProtoReflect.Descriptor instead. +func (*SessionCreated) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{22} +} + +type InferenceStarted struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceStarted) Reset() { + *x = InferenceStarted{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceStarted) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceStarted) ProtoMessage() {} + +func (x *InferenceStarted) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceStarted.ProtoReflect.Descriptor instead. +func (*InferenceStarted) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{23} +} + +type InferenceStopped struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InferenceStopped) Reset() { + *x = InferenceStopped{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InferenceStopped) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InferenceStopped) ProtoMessage() {} + +func (x *InferenceStopped) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InferenceStopped.ProtoReflect.Descriptor instead. +func (*InferenceStopped) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{24} +} + +type SessionClosed struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SessionClosed) Reset() { + *x = SessionClosed{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SessionClosed) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SessionClosed) ProtoMessage() {} + +func (x *SessionClosed) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SessionClosed.ProtoReflect.Descriptor instead. +func (*SessionClosed) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{25} +} + +type EotPrediction struct { + state protoimpl.MessageState `protogen:"open.v1"` + Probability float32 `protobuf:"fixed32,1,opt,name=probability,proto3" json:"probability,omitempty"` + ProcessingStats *ProcessingStats `protobuf:"bytes,2,opt,name=processing_stats,json=processingStats,proto3" json:"processing_stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EotPrediction) Reset() { + *x = EotPrediction{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EotPrediction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EotPrediction) ProtoMessage() {} + +func (x *EotPrediction) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EotPrediction.ProtoReflect.Descriptor instead. +func (*EotPrediction) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{26} +} + +func (x *EotPrediction) GetProbability() float32 { + if x != nil { + return x.Probability + } + return 0 +} + +func (x *EotPrediction) GetProcessingStats() *ProcessingStats { + if x != nil { + return x.ProcessingStats + } + return nil +} + +type InterruptionPrediction struct { + state protoimpl.MessageState `protogen:"open.v1"` + IsInterruption bool `protobuf:"varint,1,opt,name=is_interruption,json=isInterruption,proto3" json:"is_interruption,omitempty"` + Probabilities []float32 `protobuf:"fixed32,2,rep,packed,name=probabilities,proto3" json:"probabilities,omitempty"` + ProcessingStats *ProcessingStats `protobuf:"bytes,3,opt,name=processing_stats,json=processingStats,proto3" json:"processing_stats,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + PredictionDuration *durationpb.Duration `protobuf:"bytes,5,opt,name=prediction_duration,json=predictionDuration,proto3" json:"prediction_duration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InterruptionPrediction) Reset() { + *x = InterruptionPrediction{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InterruptionPrediction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InterruptionPrediction) ProtoMessage() {} + +func (x *InterruptionPrediction) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InterruptionPrediction.ProtoReflect.Descriptor instead. +func (*InterruptionPrediction) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{27} +} + +func (x *InterruptionPrediction) GetIsInterruption() bool { + if x != nil { + return x.IsInterruption + } + return false +} + +func (x *InterruptionPrediction) GetProbabilities() []float32 { + if x != nil { + return x.Probabilities + } + return nil +} + +func (x *InterruptionPrediction) GetProcessingStats() *ProcessingStats { + if x != nil { + return x.ProcessingStats + } + return nil +} + +func (x *InterruptionPrediction) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *InterruptionPrediction) GetPredictionDuration() *durationpb.Duration { + if x != nil { + return x.PredictionDuration + } + return nil +} + +type ServerMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + ServerCreatedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=server_created_at,json=serverCreatedAt,proto3" json:"server_created_at,omitempty"` + RequestId *string `protobuf:"bytes,2,opt,name=request_id,json=requestId,proto3,oneof" json:"request_id,omitempty"` + // echoes the client-side created_at timestamp + ClientCreatedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=client_created_at,json=clientCreatedAt,proto3,oneof" json:"client_created_at,omitempty"` + // Types that are valid to be assigned to Message: + // + // *ServerMessage_SessionCreated + // *ServerMessage_InferenceStarted + // *ServerMessage_InferenceStopped + // *ServerMessage_SessionClosed + // *ServerMessage_Error + // *ServerMessage_EotPrediction + // *ServerMessage_InterruptionPrediction + Message isServerMessage_Message `protobuf_oneof:"message"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServerMessage) Reset() { + *x = ServerMessage{} + mi := &file_agent_livekit_agent_inference_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServerMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerMessage) ProtoMessage() {} + +func (x *ServerMessage) ProtoReflect() protoreflect.Message { + mi := &file_agent_livekit_agent_inference_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerMessage.ProtoReflect.Descriptor instead. +func (*ServerMessage) Descriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{28} +} + +func (x *ServerMessage) GetServerCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.ServerCreatedAt + } + return nil +} + +func (x *ServerMessage) GetRequestId() string { + if x != nil && x.RequestId != nil { + return *x.RequestId + } + return "" +} + +func (x *ServerMessage) GetClientCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.ClientCreatedAt + } + return nil +} + +func (x *ServerMessage) GetMessage() isServerMessage_Message { + if x != nil { + return x.Message + } + return nil +} + +func (x *ServerMessage) GetSessionCreated() *SessionCreated { + if x != nil { + if x, ok := x.Message.(*ServerMessage_SessionCreated); ok { + return x.SessionCreated + } + } + return nil +} + +func (x *ServerMessage) GetInferenceStarted() *InferenceStarted { + if x != nil { + if x, ok := x.Message.(*ServerMessage_InferenceStarted); ok { + return x.InferenceStarted + } + } + return nil +} + +func (x *ServerMessage) GetInferenceStopped() *InferenceStopped { + if x != nil { + if x, ok := x.Message.(*ServerMessage_InferenceStopped); ok { + return x.InferenceStopped + } + } + return nil +} + +func (x *ServerMessage) GetSessionClosed() *SessionClosed { + if x != nil { + if x, ok := x.Message.(*ServerMessage_SessionClosed); ok { + return x.SessionClosed + } + } + return nil +} + +func (x *ServerMessage) GetError() *InferenceError { + if x != nil { + if x, ok := x.Message.(*ServerMessage_Error); ok { + return x.Error + } + } + return nil +} + +func (x *ServerMessage) GetEotPrediction() *EotPrediction { + if x != nil { + if x, ok := x.Message.(*ServerMessage_EotPrediction); ok { + return x.EotPrediction + } + } + return nil +} + +func (x *ServerMessage) GetInterruptionPrediction() *InterruptionPrediction { + if x != nil { + if x, ok := x.Message.(*ServerMessage_InterruptionPrediction); ok { + return x.InterruptionPrediction + } + } + return nil +} + +type isServerMessage_Message interface { + isServerMessage_Message() +} + +type ServerMessage_SessionCreated struct { + SessionCreated *SessionCreated `protobuf:"bytes,4,opt,name=session_created,json=sessionCreated,proto3,oneof"` +} + +type ServerMessage_InferenceStarted struct { + InferenceStarted *InferenceStarted `protobuf:"bytes,5,opt,name=inference_started,json=inferenceStarted,proto3,oneof"` +} + +type ServerMessage_InferenceStopped struct { + InferenceStopped *InferenceStopped `protobuf:"bytes,6,opt,name=inference_stopped,json=inferenceStopped,proto3,oneof"` +} + +type ServerMessage_SessionClosed struct { + SessionClosed *SessionClosed `protobuf:"bytes,7,opt,name=session_closed,json=sessionClosed,proto3,oneof"` +} + +type ServerMessage_Error struct { + Error *InferenceError `protobuf:"bytes,8,opt,name=error,proto3,oneof"` +} + +type ServerMessage_EotPrediction struct { + EotPrediction *EotPrediction `protobuf:"bytes,9,opt,name=eot_prediction,json=eotPrediction,proto3,oneof"` +} + +type ServerMessage_InterruptionPrediction struct { + InterruptionPrediction *InterruptionPrediction `protobuf:"bytes,10,opt,name=interruption_prediction,json=interruptionPrediction,proto3,oneof"` +} + +func (*ServerMessage_SessionCreated) isServerMessage_Message() {} + +func (*ServerMessage_InferenceStarted) isServerMessage_Message() {} + +func (*ServerMessage_InferenceStopped) isServerMessage_Message() {} + +func (*ServerMessage_SessionClosed) isServerMessage_Message() {} + +func (*ServerMessage_Error) isServerMessage_Message() {} + +func (*ServerMessage_EotPrediction) isServerMessage_Message() {} + +func (*ServerMessage_InterruptionPrediction) isServerMessage_Message() {} + +var File_agent_livekit_agent_inference_proto protoreflect.FileDescriptor + +const file_agent_livekit_agent_inference_proto_rawDesc = "" + + "\n" + + "#agent/livekit_agent_inference.proto\x12\rlivekit.agent\x1a!agent/livekit_agent_session.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x9a\x02\n" + + "\x0fSessionSettings\x12\x1f\n" + + "\vsample_rate\x18\x01 \x01(\rR\n" + + "sampleRate\x128\n" + + "\bencoding\x18\x02 \x01(\x0e2\x1c.livekit.agent.AudioEncodingR\bencoding\x12?\n" + + "\feot_settings\x18\x03 \x01(\v2\x1a.livekit.agent.EotSettingsH\x00R\veotSettings\x12Z\n" + + "\x15interruption_settings\x18\x04 \x01(\v2#.livekit.agent.InterruptionSettingsH\x00R\x14interruptionSettingsB\x0f\n" + + "\rtype_settings\">\n" + + "\x0eInferenceError\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage\x12\x12\n" + + "\x04code\x18\x02 \x01(\rR\x04code\"W\n" + + "\vEotSettings\x12H\n" + + "\x12detection_interval\x18\x01 \x01(\v2\x19.google.protobuf.DurationR\x11detectionInterval\"\xb5\x02\n" + + "\x14InterruptionSettings\x12\x1c\n" + + "\tthreshold\x18\x01 \x01(\x02R\tthreshold\x12\x1d\n" + + "\n" + + "min_frames\x18\x02 \x01(\rR\tminFrames\x12G\n" + + "\x12max_audio_duration\x18\x03 \x01(\v2\x19.google.protobuf.DurationR\x10maxAudioDuration\x12M\n" + + "\x15audio_prefix_duration\x18\x04 \x01(\v2\x19.google.protobuf.DurationR\x13audioPrefixDuration\x12H\n" + + "\x12detection_interval\x18\x05 \x01(\v2\x19.google.protobuf.DurationR\x11detectionInterval\"K\n" + + "\rSessionCreate\x12:\n" + + "\bsettings\x18\x01 \x01(\v2\x1e.livekit.agent.SessionSettingsR\bsettings\"~\n" + + "\n" + + "InputAudio\x12\x14\n" + + "\x05audio\x18\x01 \x01(\fR\x05audio\x129\n" + + "\n" + + "created_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x1f\n" + + "\vnum_samples\x18\x03 \x01(\rR\n" + + "numSamples\"M\n" + + "\x13EotInputChatContext\x126\n" + + "\bmessages\x18\x01 \x03(\v2\x1a.livekit.agent.ChatMessageR\bmessages\"\x0e\n" + + "\fSessionFlush\"\x0e\n" + + "\fSessionClose\"/\n" + + "\x0eInferenceStart\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\".\n" + + "\rInferenceStop\x12\x1d\n" + + "\n" + + "request_id\x18\x01 \x01(\tR\trequestId\"\r\n" + + "\vBufferStart\"\f\n" + + "\n" + + "BufferStop\"\xcd\x05\n" + + "\rClientMessage\x129\n" + + "\n" + + "created_at\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12E\n" + + "\x0esession_create\x18\x02 \x01(\v2\x1c.livekit.agent.SessionCreateH\x00R\rsessionCreate\x12<\n" + + "\vinput_audio\x18\x03 \x01(\v2\x19.livekit.agent.InputAudioH\x00R\n" + + "inputAudio\x12B\n" + + "\rsession_flush\x18\x04 \x01(\v2\x1b.livekit.agent.SessionFlushH\x00R\fsessionFlush\x12B\n" + + "\rsession_close\x18\x05 \x01(\v2\x1b.livekit.agent.SessionCloseH\x00R\fsessionClose\x12H\n" + + "\x0finference_start\x18\x06 \x01(\v2\x1d.livekit.agent.InferenceStartH\x00R\x0einferenceStart\x12E\n" + + "\x0einference_stop\x18\a \x01(\v2\x1c.livekit.agent.InferenceStopH\x00R\rinferenceStop\x12?\n" + + "\fbuffer_start\x18\b \x01(\v2\x1a.livekit.agent.BufferStartH\x00R\vbufferStart\x12<\n" + + "\vbuffer_stop\x18\t \x01(\v2\x19.livekit.agent.BufferStopH\x00R\n" + + "bufferStop\x12Y\n" + + "\x16eot_input_chat_context\x18\n" + + " \x01(\v2\".livekit.agent.EotInputChatContextH\x00R\x13eotInputChatContextB\t\n" + + "\amessage\"\xad\x01\n" + + "\x13EotInferenceRequest\x12\x14\n" + + "\x05audio\x18\x01 \x01(\fR\x05audio\x12%\n" + + "\x0eassistant_text\x18\x02 \x01(\tR\rassistantText\x128\n" + + "\bencoding\x18\x03 \x01(\x0e2\x1c.livekit.agent.AudioEncodingR\bencoding\x12\x1f\n" + + "\vsample_rate\x18\x04 \x01(\rR\n" + + "sampleRate\"\x8f\x01\n" + + "\x1cInterruptionInferenceRequest\x12\x14\n" + + "\x05audio\x18\x01 \x01(\fR\x05audio\x128\n" + + "\bencoding\x18\x02 \x01(\x0e2\x1c.livekit.agent.AudioEncodingR\bencoding\x12\x1f\n" + + "\vsample_rate\x18\x03 \x01(\rR\n" + + "sampleRate\"\xec\x01\n" + + "\x10InferenceRequest\x12X\n" + + "\x15eot_inference_request\x18\x01 \x01(\v2\".livekit.agent.EotInferenceRequestH\x00R\x13eotInferenceRequest\x12s\n" + + "\x1einterruption_inference_request\x18\x02 \x01(\v2+.livekit.agent.InterruptionInferenceRequestH\x00R\x1cinterruptionInferenceRequestB\t\n" + + "\arequest\"\xe8\x01\n" + + "\x0eInferenceStats\x12:\n" + + "\ve2e_latency\x18\x01 \x01(\v2\x19.google.protobuf.DurationR\n" + + "e2eLatency\x12P\n" + + "\x16preprocessing_duration\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\x15preprocessingDuration\x12H\n" + + "\x12inference_duration\x18\x03 \x01(\v2\x19.google.protobuf.DurationR\x11inferenceDuration\"\xc3\x02\n" + + "\x0fProcessingStats\x12W\n" + + "\x1aearliest_client_created_at\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x17earliestClientCreatedAt\x12S\n" + + "\x18latest_client_created_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x15latestClientCreatedAt\x12:\n" + + "\ve2e_latency\x18\x03 \x01(\v2\x19.google.protobuf.DurationR\n" + + "e2eLatency\x12F\n" + + "\x0finference_stats\x18\x04 \x01(\v2\x1d.livekit.agent.InferenceStatsR\x0einferenceStats\"m\n" + + "\x14EotInferenceResponse\x12 \n" + + "\vprobability\x18\x01 \x01(\x02R\vprobability\x123\n" + + "\x05stats\x18\x02 \x01(\v2\x1d.livekit.agent.InferenceStatsR\x05stats\"\xa3\x01\n" + + "\x1dInterruptionInferenceResponse\x12'\n" + + "\x0fis_interruption\x18\x01 \x01(\bR\x0eisInterruption\x12$\n" + + "\rprobabilities\x18\x02 \x03(\x02R\rprobabilities\x123\n" + + "\x05stats\x18\x03 \x01(\v2\x1d.livekit.agent.InferenceStatsR\x05stats\"\xf4\x01\n" + + "\x11InferenceResponse\x12[\n" + + "\x16eot_inference_response\x18\x01 \x01(\v2#.livekit.agent.EotInferenceResponseH\x00R\x14eotInferenceResponse\x12v\n" + + "\x1finterruption_inference_response\x18\x02 \x01(\v2,.livekit.agent.InterruptionInferenceResponseH\x00R\x1dinterruptionInferenceResponseB\n" + + "\n" + + "\bresponse\"\x10\n" + + "\x0eSessionCreated\"\x12\n" + + "\x10InferenceStarted\"\x12\n" + + "\x10InferenceStopped\"\x0f\n" + + "\rSessionClosed\"|\n" + + "\rEotPrediction\x12 \n" + + "\vprobability\x18\x01 \x01(\x02R\vprobability\x12I\n" + + "\x10processing_stats\x18\x02 \x01(\v2\x1e.livekit.agent.ProcessingStatsR\x0fprocessingStats\"\xb9\x02\n" + + "\x16InterruptionPrediction\x12'\n" + + "\x0fis_interruption\x18\x01 \x01(\bR\x0eisInterruption\x12$\n" + + "\rprobabilities\x18\x02 \x03(\x02R\rprobabilities\x12I\n" + + "\x10processing_stats\x18\x03 \x01(\v2\x1e.livekit.agent.ProcessingStatsR\x0fprocessingStats\x129\n" + + "\n" + + "created_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12J\n" + + "\x13prediction_duration\x18\x05 \x01(\v2\x19.google.protobuf.DurationR\x12predictionDuration\"\x89\x06\n" + + "\rServerMessage\x12F\n" + + "\x11server_created_at\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x0fserverCreatedAt\x12\"\n" + + "\n" + + "request_id\x18\x02 \x01(\tH\x01R\trequestId\x88\x01\x01\x12K\n" + + "\x11client_created_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampH\x02R\x0fclientCreatedAt\x88\x01\x01\x12H\n" + + "\x0fsession_created\x18\x04 \x01(\v2\x1d.livekit.agent.SessionCreatedH\x00R\x0esessionCreated\x12N\n" + + "\x11inference_started\x18\x05 \x01(\v2\x1f.livekit.agent.InferenceStartedH\x00R\x10inferenceStarted\x12N\n" + + "\x11inference_stopped\x18\x06 \x01(\v2\x1f.livekit.agent.InferenceStoppedH\x00R\x10inferenceStopped\x12E\n" + + "\x0esession_closed\x18\a \x01(\v2\x1c.livekit.agent.SessionClosedH\x00R\rsessionClosed\x125\n" + + "\x05error\x18\b \x01(\v2\x1d.livekit.agent.InferenceErrorH\x00R\x05error\x12E\n" + + "\x0eeot_prediction\x18\t \x01(\v2\x1c.livekit.agent.EotPredictionH\x00R\reotPrediction\x12`\n" + + "\x17interruption_prediction\x18\n" + + " \x01(\v2%.livekit.agent.InterruptionPredictionH\x00R\x16interruptionPredictionB\t\n" + + "\amessageB\r\n" + + "\v_request_idB\x14\n" + + "\x12_client_created_at*F\n" + + "\rAudioEncoding\x12\x1c\n" + + "\x18AUDIO_ENCODING_PCM_S16LE\x10\x00\x12\x17\n" + + "\x13AUDIO_ENCODING_OPUS\x10\x01BNH\x01Z)github.com/livekit/protocol/livekit/agent\xaa\x02\rLiveKit.Proto\xea\x02\x0eLiveKit::Protob\x06proto3" + +var ( + file_agent_livekit_agent_inference_proto_rawDescOnce sync.Once + file_agent_livekit_agent_inference_proto_rawDescData []byte +) + +func file_agent_livekit_agent_inference_proto_rawDescGZIP() []byte { + file_agent_livekit_agent_inference_proto_rawDescOnce.Do(func() { + file_agent_livekit_agent_inference_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_agent_livekit_agent_inference_proto_rawDesc), len(file_agent_livekit_agent_inference_proto_rawDesc))) + }) + return file_agent_livekit_agent_inference_proto_rawDescData +} + +var file_agent_livekit_agent_inference_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_agent_livekit_agent_inference_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_agent_livekit_agent_inference_proto_goTypes = []any{ + (AudioEncoding)(0), // 0: livekit.agent.AudioEncoding + (*SessionSettings)(nil), // 1: livekit.agent.SessionSettings + (*InferenceError)(nil), // 2: livekit.agent.InferenceError + (*EotSettings)(nil), // 3: livekit.agent.EotSettings + (*InterruptionSettings)(nil), // 4: livekit.agent.InterruptionSettings + (*SessionCreate)(nil), // 5: livekit.agent.SessionCreate + (*InputAudio)(nil), // 6: livekit.agent.InputAudio + (*EotInputChatContext)(nil), // 7: livekit.agent.EotInputChatContext + (*SessionFlush)(nil), // 8: livekit.agent.SessionFlush + (*SessionClose)(nil), // 9: livekit.agent.SessionClose + (*InferenceStart)(nil), // 10: livekit.agent.InferenceStart + (*InferenceStop)(nil), // 11: livekit.agent.InferenceStop + (*BufferStart)(nil), // 12: livekit.agent.BufferStart + (*BufferStop)(nil), // 13: livekit.agent.BufferStop + (*ClientMessage)(nil), // 14: livekit.agent.ClientMessage + (*EotInferenceRequest)(nil), // 15: livekit.agent.EotInferenceRequest + (*InterruptionInferenceRequest)(nil), // 16: livekit.agent.InterruptionInferenceRequest + (*InferenceRequest)(nil), // 17: livekit.agent.InferenceRequest + (*InferenceStats)(nil), // 18: livekit.agent.InferenceStats + (*ProcessingStats)(nil), // 19: livekit.agent.ProcessingStats + (*EotInferenceResponse)(nil), // 20: livekit.agent.EotInferenceResponse + (*InterruptionInferenceResponse)(nil), // 21: livekit.agent.InterruptionInferenceResponse + (*InferenceResponse)(nil), // 22: livekit.agent.InferenceResponse + (*SessionCreated)(nil), // 23: livekit.agent.SessionCreated + (*InferenceStarted)(nil), // 24: livekit.agent.InferenceStarted + (*InferenceStopped)(nil), // 25: livekit.agent.InferenceStopped + (*SessionClosed)(nil), // 26: livekit.agent.SessionClosed + (*EotPrediction)(nil), // 27: livekit.agent.EotPrediction + (*InterruptionPrediction)(nil), // 28: livekit.agent.InterruptionPrediction + (*ServerMessage)(nil), // 29: livekit.agent.ServerMessage + (*durationpb.Duration)(nil), // 30: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 31: google.protobuf.Timestamp + (*ChatMessage)(nil), // 32: livekit.agent.ChatMessage +} +var file_agent_livekit_agent_inference_proto_depIdxs = []int32{ + 0, // 0: livekit.agent.SessionSettings.encoding:type_name -> livekit.agent.AudioEncoding + 3, // 1: livekit.agent.SessionSettings.eot_settings:type_name -> livekit.agent.EotSettings + 4, // 2: livekit.agent.SessionSettings.interruption_settings:type_name -> livekit.agent.InterruptionSettings + 30, // 3: livekit.agent.EotSettings.detection_interval:type_name -> google.protobuf.Duration + 30, // 4: livekit.agent.InterruptionSettings.max_audio_duration:type_name -> google.protobuf.Duration + 30, // 5: livekit.agent.InterruptionSettings.audio_prefix_duration:type_name -> google.protobuf.Duration + 30, // 6: livekit.agent.InterruptionSettings.detection_interval:type_name -> google.protobuf.Duration + 1, // 7: livekit.agent.SessionCreate.settings:type_name -> livekit.agent.SessionSettings + 31, // 8: livekit.agent.InputAudio.created_at:type_name -> google.protobuf.Timestamp + 32, // 9: livekit.agent.EotInputChatContext.messages:type_name -> livekit.agent.ChatMessage + 31, // 10: livekit.agent.ClientMessage.created_at:type_name -> google.protobuf.Timestamp + 5, // 11: livekit.agent.ClientMessage.session_create:type_name -> livekit.agent.SessionCreate + 6, // 12: livekit.agent.ClientMessage.input_audio:type_name -> livekit.agent.InputAudio + 8, // 13: livekit.agent.ClientMessage.session_flush:type_name -> livekit.agent.SessionFlush + 9, // 14: livekit.agent.ClientMessage.session_close:type_name -> livekit.agent.SessionClose + 10, // 15: livekit.agent.ClientMessage.inference_start:type_name -> livekit.agent.InferenceStart + 11, // 16: livekit.agent.ClientMessage.inference_stop:type_name -> livekit.agent.InferenceStop + 12, // 17: livekit.agent.ClientMessage.buffer_start:type_name -> livekit.agent.BufferStart + 13, // 18: livekit.agent.ClientMessage.buffer_stop:type_name -> livekit.agent.BufferStop + 7, // 19: livekit.agent.ClientMessage.eot_input_chat_context:type_name -> livekit.agent.EotInputChatContext + 0, // 20: livekit.agent.EotInferenceRequest.encoding:type_name -> livekit.agent.AudioEncoding + 0, // 21: livekit.agent.InterruptionInferenceRequest.encoding:type_name -> livekit.agent.AudioEncoding + 15, // 22: livekit.agent.InferenceRequest.eot_inference_request:type_name -> livekit.agent.EotInferenceRequest + 16, // 23: livekit.agent.InferenceRequest.interruption_inference_request:type_name -> livekit.agent.InterruptionInferenceRequest + 30, // 24: livekit.agent.InferenceStats.e2e_latency:type_name -> google.protobuf.Duration + 30, // 25: livekit.agent.InferenceStats.preprocessing_duration:type_name -> google.protobuf.Duration + 30, // 26: livekit.agent.InferenceStats.inference_duration:type_name -> google.protobuf.Duration + 31, // 27: livekit.agent.ProcessingStats.earliest_client_created_at:type_name -> google.protobuf.Timestamp + 31, // 28: livekit.agent.ProcessingStats.latest_client_created_at:type_name -> google.protobuf.Timestamp + 30, // 29: livekit.agent.ProcessingStats.e2e_latency:type_name -> google.protobuf.Duration + 18, // 30: livekit.agent.ProcessingStats.inference_stats:type_name -> livekit.agent.InferenceStats + 18, // 31: livekit.agent.EotInferenceResponse.stats:type_name -> livekit.agent.InferenceStats + 18, // 32: livekit.agent.InterruptionInferenceResponse.stats:type_name -> livekit.agent.InferenceStats + 20, // 33: livekit.agent.InferenceResponse.eot_inference_response:type_name -> livekit.agent.EotInferenceResponse + 21, // 34: livekit.agent.InferenceResponse.interruption_inference_response:type_name -> livekit.agent.InterruptionInferenceResponse + 19, // 35: livekit.agent.EotPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats + 19, // 36: livekit.agent.InterruptionPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats + 31, // 37: livekit.agent.InterruptionPrediction.created_at:type_name -> google.protobuf.Timestamp + 30, // 38: livekit.agent.InterruptionPrediction.prediction_duration:type_name -> google.protobuf.Duration + 31, // 39: livekit.agent.ServerMessage.server_created_at:type_name -> google.protobuf.Timestamp + 31, // 40: livekit.agent.ServerMessage.client_created_at:type_name -> google.protobuf.Timestamp + 23, // 41: livekit.agent.ServerMessage.session_created:type_name -> livekit.agent.SessionCreated + 24, // 42: livekit.agent.ServerMessage.inference_started:type_name -> livekit.agent.InferenceStarted + 25, // 43: livekit.agent.ServerMessage.inference_stopped:type_name -> livekit.agent.InferenceStopped + 26, // 44: livekit.agent.ServerMessage.session_closed:type_name -> livekit.agent.SessionClosed + 2, // 45: livekit.agent.ServerMessage.error:type_name -> livekit.agent.InferenceError + 27, // 46: livekit.agent.ServerMessage.eot_prediction:type_name -> livekit.agent.EotPrediction + 28, // 47: livekit.agent.ServerMessage.interruption_prediction:type_name -> livekit.agent.InterruptionPrediction + 48, // [48:48] is the sub-list for method output_type + 48, // [48:48] is the sub-list for method input_type + 48, // [48:48] is the sub-list for extension type_name + 48, // [48:48] is the sub-list for extension extendee + 0, // [0:48] is the sub-list for field type_name +} + +func init() { file_agent_livekit_agent_inference_proto_init() } +func file_agent_livekit_agent_inference_proto_init() { + if File_agent_livekit_agent_inference_proto != nil { + return + } + file_agent_livekit_agent_session_proto_init() + file_agent_livekit_agent_inference_proto_msgTypes[0].OneofWrappers = []any{ + (*SessionSettings_EotSettings)(nil), + (*SessionSettings_InterruptionSettings)(nil), + } + file_agent_livekit_agent_inference_proto_msgTypes[13].OneofWrappers = []any{ + (*ClientMessage_SessionCreate)(nil), + (*ClientMessage_InputAudio)(nil), + (*ClientMessage_SessionFlush)(nil), + (*ClientMessage_SessionClose)(nil), + (*ClientMessage_InferenceStart)(nil), + (*ClientMessage_InferenceStop)(nil), + (*ClientMessage_BufferStart)(nil), + (*ClientMessage_BufferStop)(nil), + (*ClientMessage_EotInputChatContext)(nil), + } + file_agent_livekit_agent_inference_proto_msgTypes[16].OneofWrappers = []any{ + (*InferenceRequest_EotInferenceRequest)(nil), + (*InferenceRequest_InterruptionInferenceRequest)(nil), + } + file_agent_livekit_agent_inference_proto_msgTypes[21].OneofWrappers = []any{ + (*InferenceResponse_EotInferenceResponse)(nil), + (*InferenceResponse_InterruptionInferenceResponse)(nil), + } + file_agent_livekit_agent_inference_proto_msgTypes[28].OneofWrappers = []any{ + (*ServerMessage_SessionCreated)(nil), + (*ServerMessage_InferenceStarted)(nil), + (*ServerMessage_InferenceStopped)(nil), + (*ServerMessage_SessionClosed)(nil), + (*ServerMessage_Error)(nil), + (*ServerMessage_EotPrediction)(nil), + (*ServerMessage_InterruptionPrediction)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_agent_livekit_agent_inference_proto_rawDesc), len(file_agent_livekit_agent_inference_proto_rawDesc)), + NumEnums: 1, + NumMessages: 29, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_agent_livekit_agent_inference_proto_goTypes, + DependencyIndexes: file_agent_livekit_agent_inference_proto_depIdxs, + EnumInfos: file_agent_livekit_agent_inference_proto_enumTypes, + MessageInfos: file_agent_livekit_agent_inference_proto_msgTypes, + }.Build() + File_agent_livekit_agent_inference_proto = out.File + file_agent_livekit_agent_inference_proto_goTypes = nil + file_agent_livekit_agent_inference_proto_depIdxs = nil +} From 628f3b07bc0fbce25b40569681942059f19bea3e Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Wed, 22 Apr 2026 11:22:59 +0800 Subject: [PATCH 14/18] add backend field --- protobufs/agent/livekit_agent_inference.proto | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index dedd2e694..8536e708e 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -184,8 +184,14 @@ message InferenceStopped {} message SessionClosed {} message EotPrediction { + enum EotBackend { + EOT_BACKEND_UNKNOWN = 0; + EOT_BACKEND_MULTIMODAL = 1; + EOT_BACKEND_TEXT = 2; + } float probability = 1; ProcessingStats processing_stats = 2; + EotBackend backend = 3; } message InterruptionPrediction { From 47d6736ff9134e53428635e4eff9083b5da7af0b Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 03:24:32 +0000 Subject: [PATCH 15/18] generated protobuf --- livekit/agent/livekit_agent_inference.pb.go | 243 +++++++++++++------- 1 file changed, 154 insertions(+), 89 deletions(-) diff --git a/livekit/agent/livekit_agent_inference.pb.go b/livekit/agent/livekit_agent_inference.pb.go index fc4e390ac..3c350e753 100644 --- a/livekit/agent/livekit_agent_inference.pb.go +++ b/livekit/agent/livekit_agent_inference.pb.go @@ -83,6 +83,55 @@ func (AudioEncoding) EnumDescriptor() ([]byte, []int) { return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{0} } +type EotPrediction_EotBackend int32 + +const ( + EotPrediction_EOT_BACKEND_UNKNOWN EotPrediction_EotBackend = 0 + EotPrediction_EOT_BACKEND_MULTIMODAL EotPrediction_EotBackend = 1 + EotPrediction_EOT_BACKEND_TEXT EotPrediction_EotBackend = 2 +) + +// Enum value maps for EotPrediction_EotBackend. +var ( + EotPrediction_EotBackend_name = map[int32]string{ + 0: "EOT_BACKEND_UNKNOWN", + 1: "EOT_BACKEND_MULTIMODAL", + 2: "EOT_BACKEND_TEXT", + } + EotPrediction_EotBackend_value = map[string]int32{ + "EOT_BACKEND_UNKNOWN": 0, + "EOT_BACKEND_MULTIMODAL": 1, + "EOT_BACKEND_TEXT": 2, + } +) + +func (x EotPrediction_EotBackend) Enum() *EotPrediction_EotBackend { + p := new(EotPrediction_EotBackend) + *p = x + return p +} + +func (x EotPrediction_EotBackend) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EotPrediction_EotBackend) Descriptor() protoreflect.EnumDescriptor { + return file_agent_livekit_agent_inference_proto_enumTypes[1].Descriptor() +} + +func (EotPrediction_EotBackend) Type() protoreflect.EnumType { + return &file_agent_livekit_agent_inference_proto_enumTypes[1] +} + +func (x EotPrediction_EotBackend) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EotPrediction_EotBackend.Descriptor instead. +func (EotPrediction_EotBackend) EnumDescriptor() ([]byte, []int) { + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{26, 0} +} + type SessionSettings struct { state protoimpl.MessageState `protogen:"open.v1"` SampleRate uint32 `protobuf:"varint,1,opt,name=sample_rate,json=sampleRate,proto3" json:"sample_rate,omitempty"` @@ -1622,9 +1671,10 @@ func (*SessionClosed) Descriptor() ([]byte, []int) { } type EotPrediction struct { - state protoimpl.MessageState `protogen:"open.v1"` - Probability float32 `protobuf:"fixed32,1,opt,name=probability,proto3" json:"probability,omitempty"` - ProcessingStats *ProcessingStats `protobuf:"bytes,2,opt,name=processing_stats,json=processingStats,proto3" json:"processing_stats,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Probability float32 `protobuf:"fixed32,1,opt,name=probability,proto3" json:"probability,omitempty"` + ProcessingStats *ProcessingStats `protobuf:"bytes,2,opt,name=processing_stats,json=processingStats,proto3" json:"processing_stats,omitempty"` + Backend EotPrediction_EotBackend `protobuf:"varint,3,opt,name=backend,proto3,enum=livekit.agent.EotPrediction_EotBackend" json:"backend,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1673,6 +1723,13 @@ func (x *EotPrediction) GetProcessingStats() *ProcessingStats { return nil } +func (x *EotPrediction) GetBackend() EotPrediction_EotBackend { + if x != nil { + return x.Backend + } + return EotPrediction_EOT_BACKEND_UNKNOWN +} + type InterruptionPrediction struct { state protoimpl.MessageState `protogen:"open.v1"` IsInterruption bool `protobuf:"varint,1,opt,name=is_interruption,json=isInterruption,proto3" json:"is_interruption,omitempty"` @@ -2039,10 +2096,16 @@ const file_agent_livekit_agent_inference_proto_rawDesc = "" + "\x0eSessionCreated\"\x12\n" + "\x10InferenceStarted\"\x12\n" + "\x10InferenceStopped\"\x0f\n" + - "\rSessionClosed\"|\n" + + "\rSessionClosed\"\x98\x02\n" + "\rEotPrediction\x12 \n" + "\vprobability\x18\x01 \x01(\x02R\vprobability\x12I\n" + - "\x10processing_stats\x18\x02 \x01(\v2\x1e.livekit.agent.ProcessingStatsR\x0fprocessingStats\"\xb9\x02\n" + + "\x10processing_stats\x18\x02 \x01(\v2\x1e.livekit.agent.ProcessingStatsR\x0fprocessingStats\x12A\n" + + "\abackend\x18\x03 \x01(\x0e2'.livekit.agent.EotPrediction.EotBackendR\abackend\"W\n" + + "\n" + + "EotBackend\x12\x17\n" + + "\x13EOT_BACKEND_UNKNOWN\x10\x00\x12\x1a\n" + + "\x16EOT_BACKEND_MULTIMODAL\x10\x01\x12\x14\n" + + "\x10EOT_BACKEND_TEXT\x10\x02\"\xb9\x02\n" + "\x16InterruptionPrediction\x12'\n" + "\x0fis_interruption\x18\x01 \x01(\bR\x0eisInterruption\x12$\n" + "\rprobabilities\x18\x02 \x03(\x02R\rprobabilities\x12I\n" + @@ -2082,97 +2145,99 @@ func file_agent_livekit_agent_inference_proto_rawDescGZIP() []byte { return file_agent_livekit_agent_inference_proto_rawDescData } -var file_agent_livekit_agent_inference_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_agent_livekit_agent_inference_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_agent_livekit_agent_inference_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_agent_livekit_agent_inference_proto_goTypes = []any{ (AudioEncoding)(0), // 0: livekit.agent.AudioEncoding - (*SessionSettings)(nil), // 1: livekit.agent.SessionSettings - (*InferenceError)(nil), // 2: livekit.agent.InferenceError - (*EotSettings)(nil), // 3: livekit.agent.EotSettings - (*InterruptionSettings)(nil), // 4: livekit.agent.InterruptionSettings - (*SessionCreate)(nil), // 5: livekit.agent.SessionCreate - (*InputAudio)(nil), // 6: livekit.agent.InputAudio - (*EotInputChatContext)(nil), // 7: livekit.agent.EotInputChatContext - (*SessionFlush)(nil), // 8: livekit.agent.SessionFlush - (*SessionClose)(nil), // 9: livekit.agent.SessionClose - (*InferenceStart)(nil), // 10: livekit.agent.InferenceStart - (*InferenceStop)(nil), // 11: livekit.agent.InferenceStop - (*BufferStart)(nil), // 12: livekit.agent.BufferStart - (*BufferStop)(nil), // 13: livekit.agent.BufferStop - (*ClientMessage)(nil), // 14: livekit.agent.ClientMessage - (*EotInferenceRequest)(nil), // 15: livekit.agent.EotInferenceRequest - (*InterruptionInferenceRequest)(nil), // 16: livekit.agent.InterruptionInferenceRequest - (*InferenceRequest)(nil), // 17: livekit.agent.InferenceRequest - (*InferenceStats)(nil), // 18: livekit.agent.InferenceStats - (*ProcessingStats)(nil), // 19: livekit.agent.ProcessingStats - (*EotInferenceResponse)(nil), // 20: livekit.agent.EotInferenceResponse - (*InterruptionInferenceResponse)(nil), // 21: livekit.agent.InterruptionInferenceResponse - (*InferenceResponse)(nil), // 22: livekit.agent.InferenceResponse - (*SessionCreated)(nil), // 23: livekit.agent.SessionCreated - (*InferenceStarted)(nil), // 24: livekit.agent.InferenceStarted - (*InferenceStopped)(nil), // 25: livekit.agent.InferenceStopped - (*SessionClosed)(nil), // 26: livekit.agent.SessionClosed - (*EotPrediction)(nil), // 27: livekit.agent.EotPrediction - (*InterruptionPrediction)(nil), // 28: livekit.agent.InterruptionPrediction - (*ServerMessage)(nil), // 29: livekit.agent.ServerMessage - (*durationpb.Duration)(nil), // 30: google.protobuf.Duration - (*timestamppb.Timestamp)(nil), // 31: google.protobuf.Timestamp - (*ChatMessage)(nil), // 32: livekit.agent.ChatMessage + (EotPrediction_EotBackend)(0), // 1: livekit.agent.EotPrediction.EotBackend + (*SessionSettings)(nil), // 2: livekit.agent.SessionSettings + (*InferenceError)(nil), // 3: livekit.agent.InferenceError + (*EotSettings)(nil), // 4: livekit.agent.EotSettings + (*InterruptionSettings)(nil), // 5: livekit.agent.InterruptionSettings + (*SessionCreate)(nil), // 6: livekit.agent.SessionCreate + (*InputAudio)(nil), // 7: livekit.agent.InputAudio + (*EotInputChatContext)(nil), // 8: livekit.agent.EotInputChatContext + (*SessionFlush)(nil), // 9: livekit.agent.SessionFlush + (*SessionClose)(nil), // 10: livekit.agent.SessionClose + (*InferenceStart)(nil), // 11: livekit.agent.InferenceStart + (*InferenceStop)(nil), // 12: livekit.agent.InferenceStop + (*BufferStart)(nil), // 13: livekit.agent.BufferStart + (*BufferStop)(nil), // 14: livekit.agent.BufferStop + (*ClientMessage)(nil), // 15: livekit.agent.ClientMessage + (*EotInferenceRequest)(nil), // 16: livekit.agent.EotInferenceRequest + (*InterruptionInferenceRequest)(nil), // 17: livekit.agent.InterruptionInferenceRequest + (*InferenceRequest)(nil), // 18: livekit.agent.InferenceRequest + (*InferenceStats)(nil), // 19: livekit.agent.InferenceStats + (*ProcessingStats)(nil), // 20: livekit.agent.ProcessingStats + (*EotInferenceResponse)(nil), // 21: livekit.agent.EotInferenceResponse + (*InterruptionInferenceResponse)(nil), // 22: livekit.agent.InterruptionInferenceResponse + (*InferenceResponse)(nil), // 23: livekit.agent.InferenceResponse + (*SessionCreated)(nil), // 24: livekit.agent.SessionCreated + (*InferenceStarted)(nil), // 25: livekit.agent.InferenceStarted + (*InferenceStopped)(nil), // 26: livekit.agent.InferenceStopped + (*SessionClosed)(nil), // 27: livekit.agent.SessionClosed + (*EotPrediction)(nil), // 28: livekit.agent.EotPrediction + (*InterruptionPrediction)(nil), // 29: livekit.agent.InterruptionPrediction + (*ServerMessage)(nil), // 30: livekit.agent.ServerMessage + (*durationpb.Duration)(nil), // 31: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 32: google.protobuf.Timestamp + (*ChatMessage)(nil), // 33: livekit.agent.ChatMessage } var file_agent_livekit_agent_inference_proto_depIdxs = []int32{ 0, // 0: livekit.agent.SessionSettings.encoding:type_name -> livekit.agent.AudioEncoding - 3, // 1: livekit.agent.SessionSettings.eot_settings:type_name -> livekit.agent.EotSettings - 4, // 2: livekit.agent.SessionSettings.interruption_settings:type_name -> livekit.agent.InterruptionSettings - 30, // 3: livekit.agent.EotSettings.detection_interval:type_name -> google.protobuf.Duration - 30, // 4: livekit.agent.InterruptionSettings.max_audio_duration:type_name -> google.protobuf.Duration - 30, // 5: livekit.agent.InterruptionSettings.audio_prefix_duration:type_name -> google.protobuf.Duration - 30, // 6: livekit.agent.InterruptionSettings.detection_interval:type_name -> google.protobuf.Duration - 1, // 7: livekit.agent.SessionCreate.settings:type_name -> livekit.agent.SessionSettings - 31, // 8: livekit.agent.InputAudio.created_at:type_name -> google.protobuf.Timestamp - 32, // 9: livekit.agent.EotInputChatContext.messages:type_name -> livekit.agent.ChatMessage - 31, // 10: livekit.agent.ClientMessage.created_at:type_name -> google.protobuf.Timestamp - 5, // 11: livekit.agent.ClientMessage.session_create:type_name -> livekit.agent.SessionCreate - 6, // 12: livekit.agent.ClientMessage.input_audio:type_name -> livekit.agent.InputAudio - 8, // 13: livekit.agent.ClientMessage.session_flush:type_name -> livekit.agent.SessionFlush - 9, // 14: livekit.agent.ClientMessage.session_close:type_name -> livekit.agent.SessionClose - 10, // 15: livekit.agent.ClientMessage.inference_start:type_name -> livekit.agent.InferenceStart - 11, // 16: livekit.agent.ClientMessage.inference_stop:type_name -> livekit.agent.InferenceStop - 12, // 17: livekit.agent.ClientMessage.buffer_start:type_name -> livekit.agent.BufferStart - 13, // 18: livekit.agent.ClientMessage.buffer_stop:type_name -> livekit.agent.BufferStop - 7, // 19: livekit.agent.ClientMessage.eot_input_chat_context:type_name -> livekit.agent.EotInputChatContext + 4, // 1: livekit.agent.SessionSettings.eot_settings:type_name -> livekit.agent.EotSettings + 5, // 2: livekit.agent.SessionSettings.interruption_settings:type_name -> livekit.agent.InterruptionSettings + 31, // 3: livekit.agent.EotSettings.detection_interval:type_name -> google.protobuf.Duration + 31, // 4: livekit.agent.InterruptionSettings.max_audio_duration:type_name -> google.protobuf.Duration + 31, // 5: livekit.agent.InterruptionSettings.audio_prefix_duration:type_name -> google.protobuf.Duration + 31, // 6: livekit.agent.InterruptionSettings.detection_interval:type_name -> google.protobuf.Duration + 2, // 7: livekit.agent.SessionCreate.settings:type_name -> livekit.agent.SessionSettings + 32, // 8: livekit.agent.InputAudio.created_at:type_name -> google.protobuf.Timestamp + 33, // 9: livekit.agent.EotInputChatContext.messages:type_name -> livekit.agent.ChatMessage + 32, // 10: livekit.agent.ClientMessage.created_at:type_name -> google.protobuf.Timestamp + 6, // 11: livekit.agent.ClientMessage.session_create:type_name -> livekit.agent.SessionCreate + 7, // 12: livekit.agent.ClientMessage.input_audio:type_name -> livekit.agent.InputAudio + 9, // 13: livekit.agent.ClientMessage.session_flush:type_name -> livekit.agent.SessionFlush + 10, // 14: livekit.agent.ClientMessage.session_close:type_name -> livekit.agent.SessionClose + 11, // 15: livekit.agent.ClientMessage.inference_start:type_name -> livekit.agent.InferenceStart + 12, // 16: livekit.agent.ClientMessage.inference_stop:type_name -> livekit.agent.InferenceStop + 13, // 17: livekit.agent.ClientMessage.buffer_start:type_name -> livekit.agent.BufferStart + 14, // 18: livekit.agent.ClientMessage.buffer_stop:type_name -> livekit.agent.BufferStop + 8, // 19: livekit.agent.ClientMessage.eot_input_chat_context:type_name -> livekit.agent.EotInputChatContext 0, // 20: livekit.agent.EotInferenceRequest.encoding:type_name -> livekit.agent.AudioEncoding 0, // 21: livekit.agent.InterruptionInferenceRequest.encoding:type_name -> livekit.agent.AudioEncoding - 15, // 22: livekit.agent.InferenceRequest.eot_inference_request:type_name -> livekit.agent.EotInferenceRequest - 16, // 23: livekit.agent.InferenceRequest.interruption_inference_request:type_name -> livekit.agent.InterruptionInferenceRequest - 30, // 24: livekit.agent.InferenceStats.e2e_latency:type_name -> google.protobuf.Duration - 30, // 25: livekit.agent.InferenceStats.preprocessing_duration:type_name -> google.protobuf.Duration - 30, // 26: livekit.agent.InferenceStats.inference_duration:type_name -> google.protobuf.Duration - 31, // 27: livekit.agent.ProcessingStats.earliest_client_created_at:type_name -> google.protobuf.Timestamp - 31, // 28: livekit.agent.ProcessingStats.latest_client_created_at:type_name -> google.protobuf.Timestamp - 30, // 29: livekit.agent.ProcessingStats.e2e_latency:type_name -> google.protobuf.Duration - 18, // 30: livekit.agent.ProcessingStats.inference_stats:type_name -> livekit.agent.InferenceStats - 18, // 31: livekit.agent.EotInferenceResponse.stats:type_name -> livekit.agent.InferenceStats - 18, // 32: livekit.agent.InterruptionInferenceResponse.stats:type_name -> livekit.agent.InferenceStats - 20, // 33: livekit.agent.InferenceResponse.eot_inference_response:type_name -> livekit.agent.EotInferenceResponse - 21, // 34: livekit.agent.InferenceResponse.interruption_inference_response:type_name -> livekit.agent.InterruptionInferenceResponse - 19, // 35: livekit.agent.EotPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats - 19, // 36: livekit.agent.InterruptionPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats - 31, // 37: livekit.agent.InterruptionPrediction.created_at:type_name -> google.protobuf.Timestamp - 30, // 38: livekit.agent.InterruptionPrediction.prediction_duration:type_name -> google.protobuf.Duration - 31, // 39: livekit.agent.ServerMessage.server_created_at:type_name -> google.protobuf.Timestamp - 31, // 40: livekit.agent.ServerMessage.client_created_at:type_name -> google.protobuf.Timestamp - 23, // 41: livekit.agent.ServerMessage.session_created:type_name -> livekit.agent.SessionCreated - 24, // 42: livekit.agent.ServerMessage.inference_started:type_name -> livekit.agent.InferenceStarted - 25, // 43: livekit.agent.ServerMessage.inference_stopped:type_name -> livekit.agent.InferenceStopped - 26, // 44: livekit.agent.ServerMessage.session_closed:type_name -> livekit.agent.SessionClosed - 2, // 45: livekit.agent.ServerMessage.error:type_name -> livekit.agent.InferenceError - 27, // 46: livekit.agent.ServerMessage.eot_prediction:type_name -> livekit.agent.EotPrediction - 28, // 47: livekit.agent.ServerMessage.interruption_prediction:type_name -> livekit.agent.InterruptionPrediction - 48, // [48:48] is the sub-list for method output_type - 48, // [48:48] is the sub-list for method input_type - 48, // [48:48] is the sub-list for extension type_name - 48, // [48:48] is the sub-list for extension extendee - 0, // [0:48] is the sub-list for field type_name + 16, // 22: livekit.agent.InferenceRequest.eot_inference_request:type_name -> livekit.agent.EotInferenceRequest + 17, // 23: livekit.agent.InferenceRequest.interruption_inference_request:type_name -> livekit.agent.InterruptionInferenceRequest + 31, // 24: livekit.agent.InferenceStats.e2e_latency:type_name -> google.protobuf.Duration + 31, // 25: livekit.agent.InferenceStats.preprocessing_duration:type_name -> google.protobuf.Duration + 31, // 26: livekit.agent.InferenceStats.inference_duration:type_name -> google.protobuf.Duration + 32, // 27: livekit.agent.ProcessingStats.earliest_client_created_at:type_name -> google.protobuf.Timestamp + 32, // 28: livekit.agent.ProcessingStats.latest_client_created_at:type_name -> google.protobuf.Timestamp + 31, // 29: livekit.agent.ProcessingStats.e2e_latency:type_name -> google.protobuf.Duration + 19, // 30: livekit.agent.ProcessingStats.inference_stats:type_name -> livekit.agent.InferenceStats + 19, // 31: livekit.agent.EotInferenceResponse.stats:type_name -> livekit.agent.InferenceStats + 19, // 32: livekit.agent.InterruptionInferenceResponse.stats:type_name -> livekit.agent.InferenceStats + 21, // 33: livekit.agent.InferenceResponse.eot_inference_response:type_name -> livekit.agent.EotInferenceResponse + 22, // 34: livekit.agent.InferenceResponse.interruption_inference_response:type_name -> livekit.agent.InterruptionInferenceResponse + 20, // 35: livekit.agent.EotPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats + 1, // 36: livekit.agent.EotPrediction.backend:type_name -> livekit.agent.EotPrediction.EotBackend + 20, // 37: livekit.agent.InterruptionPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats + 32, // 38: livekit.agent.InterruptionPrediction.created_at:type_name -> google.protobuf.Timestamp + 31, // 39: livekit.agent.InterruptionPrediction.prediction_duration:type_name -> google.protobuf.Duration + 32, // 40: livekit.agent.ServerMessage.server_created_at:type_name -> google.protobuf.Timestamp + 32, // 41: livekit.agent.ServerMessage.client_created_at:type_name -> google.protobuf.Timestamp + 24, // 42: livekit.agent.ServerMessage.session_created:type_name -> livekit.agent.SessionCreated + 25, // 43: livekit.agent.ServerMessage.inference_started:type_name -> livekit.agent.InferenceStarted + 26, // 44: livekit.agent.ServerMessage.inference_stopped:type_name -> livekit.agent.InferenceStopped + 27, // 45: livekit.agent.ServerMessage.session_closed:type_name -> livekit.agent.SessionClosed + 3, // 46: livekit.agent.ServerMessage.error:type_name -> livekit.agent.InferenceError + 28, // 47: livekit.agent.ServerMessage.eot_prediction:type_name -> livekit.agent.EotPrediction + 29, // 48: livekit.agent.ServerMessage.interruption_prediction:type_name -> livekit.agent.InterruptionPrediction + 49, // [49:49] is the sub-list for method output_type + 49, // [49:49] is the sub-list for method input_type + 49, // [49:49] is the sub-list for extension type_name + 49, // [49:49] is the sub-list for extension extendee + 0, // [0:49] is the sub-list for field type_name } func init() { file_agent_livekit_agent_inference_proto_init() } @@ -2218,7 +2283,7 @@ func file_agent_livekit_agent_inference_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_agent_livekit_agent_inference_proto_rawDesc), len(file_agent_livekit_agent_inference_proto_rawDesc)), - NumEnums: 1, + NumEnums: 2, NumMessages: 29, NumExtensions: 0, NumServices: 0, From e23016116b8b76bc7ecc2acfbf4fde8ad238f332 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Fri, 24 Apr 2026 10:50:25 +0800 Subject: [PATCH 16/18] simplify stats protos --- protobufs/agent/livekit_agent_inference.proto | 39 ++++++++----------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index 8536e708e..fd6daa34e 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -16,15 +16,15 @@ syntax = "proto3"; package livekit.agent; -option go_package = "github.com/livekit/protocol/livekit/agent"; -option csharp_namespace = "LiveKit.Proto"; -option ruby_package = "LiveKit::Proto"; -option optimize_for = SPEED; - import "agent/livekit_agent_session.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; +option csharp_namespace = "LiveKit.Proto"; +option go_package = "github.com/livekit/protocol/livekit/agent"; +option optimize_for = SPEED; +option ruby_package = "LiveKit::Proto"; + // --- Shared Types --- enum AudioEncoding { @@ -139,21 +139,18 @@ message InferenceRequest { } message InferenceStats { - // server-side e2e latency (server input to server output) - google.protobuf.Duration e2e_latency = 1; - google.protobuf.Duration preprocessing_duration = 2; - google.protobuf.Duration inference_duration = 3; + // proxy -> model: audio window timestamps + optional google.protobuf.Timestamp earliest_client_created_at = 1; + optional google.protobuf.Timestamp latest_client_created_at = 2; + // proxy side e2e latency (proxy send -> proxy receive) + optional google.protobuf.Duration client_e2e_latency = 3; + // model side e2e latency (model endpoint receive -> model endpoint return) + google.protobuf.Duration server_e2e_latency = 4; + // model side inference stats + google.protobuf.Duration preprocessing_duration = 5; + google.protobuf.Duration inference_duration = 6; } -message ProcessingStats { - google.protobuf.Timestamp earliest_client_created_at = 1; - google.protobuf.Timestamp latest_client_created_at = 2; - // client-side e2e latency (client send to client receive) - google.protobuf.Duration e2e_latency = 3; - InferenceStats inference_stats = 4; -} - - message EotInferenceResponse { float probability = 1; InferenceStats stats = 2; @@ -190,16 +187,14 @@ message EotPrediction { EOT_BACKEND_TEXT = 2; } float probability = 1; - ProcessingStats processing_stats = 2; + InferenceStats processing_stats = 2; EotBackend backend = 3; } message InterruptionPrediction { bool is_interruption = 1; repeated float probabilities = 2; - ProcessingStats processing_stats = 3; - google.protobuf.Timestamp created_at = 4; - google.protobuf.Duration prediction_duration = 5; + InferenceStats processing_stats = 3; } message ServerMessage { From e9c9e5d9fd7d5c74de0e258016bf0fb4a6ac0162 Mon Sep 17 00:00:00 2001 From: Chenghao Mou Date: Fri, 24 Apr 2026 10:54:06 +0800 Subject: [PATCH 17/18] clean up --- protobufs/agent/livekit_agent_inference.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobufs/agent/livekit_agent_inference.proto b/protobufs/agent/livekit_agent_inference.proto index fd6daa34e..b2e728405 100644 --- a/protobufs/agent/livekit_agent_inference.proto +++ b/protobufs/agent/livekit_agent_inference.proto @@ -187,14 +187,14 @@ message EotPrediction { EOT_BACKEND_TEXT = 2; } float probability = 1; - InferenceStats processing_stats = 2; + InferenceStats inference_stats = 2; EotBackend backend = 3; } message InterruptionPrediction { bool is_interruption = 1; repeated float probabilities = 2; - InferenceStats processing_stats = 3; + InferenceStats inference_stats = 3; } message ServerMessage { From 3197b215552a6bb03188ea35750d785c9b5340ef Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 24 Apr 2026 02:54:53 +0000 Subject: [PATCH 18/18] generated protobuf --- livekit/agent/livekit_agent_inference.pb.go | 341 ++++++++------------ 1 file changed, 138 insertions(+), 203 deletions(-) diff --git a/livekit/agent/livekit_agent_inference.pb.go b/livekit/agent/livekit_agent_inference.pb.go index 3c350e753..7be8deb3e 100644 --- a/livekit/agent/livekit_agent_inference.pb.go +++ b/livekit/agent/livekit_agent_inference.pb.go @@ -129,7 +129,7 @@ func (x EotPrediction_EotBackend) Number() protoreflect.EnumNumber { // Deprecated: Use EotPrediction_EotBackend.Descriptor instead. func (EotPrediction_EotBackend) EnumDescriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{26, 0} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{25, 0} } type SessionSettings struct { @@ -1203,10 +1203,16 @@ func (*InferenceRequest_InterruptionInferenceRequest) isInferenceRequest_Request type InferenceStats struct { state protoimpl.MessageState `protogen:"open.v1"` - // server-side e2e latency (server input to server output) - E2ELatency *durationpb.Duration `protobuf:"bytes,1,opt,name=e2e_latency,json=e2eLatency,proto3" json:"e2e_latency,omitempty"` - PreprocessingDuration *durationpb.Duration `protobuf:"bytes,2,opt,name=preprocessing_duration,json=preprocessingDuration,proto3" json:"preprocessing_duration,omitempty"` - InferenceDuration *durationpb.Duration `protobuf:"bytes,3,opt,name=inference_duration,json=inferenceDuration,proto3" json:"inference_duration,omitempty"` + // proxy -> model: audio window timestamps + EarliestClientCreatedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=earliest_client_created_at,json=earliestClientCreatedAt,proto3,oneof" json:"earliest_client_created_at,omitempty"` + LatestClientCreatedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=latest_client_created_at,json=latestClientCreatedAt,proto3,oneof" json:"latest_client_created_at,omitempty"` + // proxy side e2e latency (proxy send -> proxy receive) + ClientE2ELatency *durationpb.Duration `protobuf:"bytes,3,opt,name=client_e2e_latency,json=clientE2eLatency,proto3,oneof" json:"client_e2e_latency,omitempty"` + // model side e2e latency (model endpoint receive -> model endpoint return) + ServerE2ELatency *durationpb.Duration `protobuf:"bytes,4,opt,name=server_e2e_latency,json=serverE2eLatency,proto3" json:"server_e2e_latency,omitempty"` + // model side inference stats + PreprocessingDuration *durationpb.Duration `protobuf:"bytes,5,opt,name=preprocessing_duration,json=preprocessingDuration,proto3" json:"preprocessing_duration,omitempty"` + InferenceDuration *durationpb.Duration `protobuf:"bytes,6,opt,name=inference_duration,json=inferenceDuration,proto3" json:"inference_duration,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1241,92 +1247,44 @@ func (*InferenceStats) Descriptor() ([]byte, []int) { return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{17} } -func (x *InferenceStats) GetE2ELatency() *durationpb.Duration { +func (x *InferenceStats) GetEarliestClientCreatedAt() *timestamppb.Timestamp { if x != nil { - return x.E2ELatency + return x.EarliestClientCreatedAt } return nil } -func (x *InferenceStats) GetPreprocessingDuration() *durationpb.Duration { +func (x *InferenceStats) GetLatestClientCreatedAt() *timestamppb.Timestamp { if x != nil { - return x.PreprocessingDuration + return x.LatestClientCreatedAt } return nil } -func (x *InferenceStats) GetInferenceDuration() *durationpb.Duration { +func (x *InferenceStats) GetClientE2ELatency() *durationpb.Duration { if x != nil { - return x.InferenceDuration + return x.ClientE2ELatency } return nil } -type ProcessingStats struct { - state protoimpl.MessageState `protogen:"open.v1"` - EarliestClientCreatedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=earliest_client_created_at,json=earliestClientCreatedAt,proto3" json:"earliest_client_created_at,omitempty"` - LatestClientCreatedAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=latest_client_created_at,json=latestClientCreatedAt,proto3" json:"latest_client_created_at,omitempty"` - // client-side e2e latency (client send to client receive) - E2ELatency *durationpb.Duration `protobuf:"bytes,3,opt,name=e2e_latency,json=e2eLatency,proto3" json:"e2e_latency,omitempty"` - InferenceStats *InferenceStats `protobuf:"bytes,4,opt,name=inference_stats,json=inferenceStats,proto3" json:"inference_stats,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *ProcessingStats) Reset() { - *x = ProcessingStats{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *ProcessingStats) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ProcessingStats) ProtoMessage() {} - -func (x *ProcessingStats) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[18] +func (x *InferenceStats) GetServerE2ELatency() *durationpb.Duration { if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ProcessingStats.ProtoReflect.Descriptor instead. -func (*ProcessingStats) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{18} -} - -func (x *ProcessingStats) GetEarliestClientCreatedAt() *timestamppb.Timestamp { - if x != nil { - return x.EarliestClientCreatedAt + return x.ServerE2ELatency } return nil } -func (x *ProcessingStats) GetLatestClientCreatedAt() *timestamppb.Timestamp { - if x != nil { - return x.LatestClientCreatedAt - } - return nil -} - -func (x *ProcessingStats) GetE2ELatency() *durationpb.Duration { +func (x *InferenceStats) GetPreprocessingDuration() *durationpb.Duration { if x != nil { - return x.E2ELatency + return x.PreprocessingDuration } return nil } -func (x *ProcessingStats) GetInferenceStats() *InferenceStats { +func (x *InferenceStats) GetInferenceDuration() *durationpb.Duration { if x != nil { - return x.InferenceStats + return x.InferenceDuration } return nil } @@ -1341,7 +1299,7 @@ type EotInferenceResponse struct { func (x *EotInferenceResponse) Reset() { *x = EotInferenceResponse{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[19] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1353,7 +1311,7 @@ func (x *EotInferenceResponse) String() string { func (*EotInferenceResponse) ProtoMessage() {} func (x *EotInferenceResponse) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[19] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1366,7 +1324,7 @@ func (x *EotInferenceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EotInferenceResponse.ProtoReflect.Descriptor instead. func (*EotInferenceResponse) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{19} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{18} } func (x *EotInferenceResponse) GetProbability() float32 { @@ -1395,7 +1353,7 @@ type InterruptionInferenceResponse struct { func (x *InterruptionInferenceResponse) Reset() { *x = InterruptionInferenceResponse{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[20] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1407,7 +1365,7 @@ func (x *InterruptionInferenceResponse) String() string { func (*InterruptionInferenceResponse) ProtoMessage() {} func (x *InterruptionInferenceResponse) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[20] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1420,7 +1378,7 @@ func (x *InterruptionInferenceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use InterruptionInferenceResponse.ProtoReflect.Descriptor instead. func (*InterruptionInferenceResponse) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{20} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{19} } func (x *InterruptionInferenceResponse) GetIsInterruption() bool { @@ -1457,7 +1415,7 @@ type InferenceResponse struct { func (x *InferenceResponse) Reset() { *x = InferenceResponse{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[21] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1469,7 +1427,7 @@ func (x *InferenceResponse) String() string { func (*InferenceResponse) ProtoMessage() {} func (x *InferenceResponse) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[21] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1482,7 +1440,7 @@ func (x *InferenceResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use InferenceResponse.ProtoReflect.Descriptor instead. func (*InferenceResponse) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{21} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{20} } func (x *InferenceResponse) GetResponse() isInferenceResponse_Response { @@ -1534,7 +1492,7 @@ type SessionCreated struct { func (x *SessionCreated) Reset() { *x = SessionCreated{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[22] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1546,7 +1504,7 @@ func (x *SessionCreated) String() string { func (*SessionCreated) ProtoMessage() {} func (x *SessionCreated) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[22] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1559,7 +1517,7 @@ func (x *SessionCreated) ProtoReflect() protoreflect.Message { // Deprecated: Use SessionCreated.ProtoReflect.Descriptor instead. func (*SessionCreated) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{22} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{21} } type InferenceStarted struct { @@ -1570,7 +1528,7 @@ type InferenceStarted struct { func (x *InferenceStarted) Reset() { *x = InferenceStarted{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[23] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1582,7 +1540,7 @@ func (x *InferenceStarted) String() string { func (*InferenceStarted) ProtoMessage() {} func (x *InferenceStarted) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[23] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1595,7 +1553,7 @@ func (x *InferenceStarted) ProtoReflect() protoreflect.Message { // Deprecated: Use InferenceStarted.ProtoReflect.Descriptor instead. func (*InferenceStarted) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{23} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{22} } type InferenceStopped struct { @@ -1606,7 +1564,7 @@ type InferenceStopped struct { func (x *InferenceStopped) Reset() { *x = InferenceStopped{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[24] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1618,7 +1576,7 @@ func (x *InferenceStopped) String() string { func (*InferenceStopped) ProtoMessage() {} func (x *InferenceStopped) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[24] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1631,7 +1589,7 @@ func (x *InferenceStopped) ProtoReflect() protoreflect.Message { // Deprecated: Use InferenceStopped.ProtoReflect.Descriptor instead. func (*InferenceStopped) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{24} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{23} } type SessionClosed struct { @@ -1642,7 +1600,7 @@ type SessionClosed struct { func (x *SessionClosed) Reset() { *x = SessionClosed{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[25] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1654,7 +1612,7 @@ func (x *SessionClosed) String() string { func (*SessionClosed) ProtoMessage() {} func (x *SessionClosed) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[25] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1667,21 +1625,21 @@ func (x *SessionClosed) ProtoReflect() protoreflect.Message { // Deprecated: Use SessionClosed.ProtoReflect.Descriptor instead. func (*SessionClosed) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{25} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{24} } type EotPrediction struct { - state protoimpl.MessageState `protogen:"open.v1"` - Probability float32 `protobuf:"fixed32,1,opt,name=probability,proto3" json:"probability,omitempty"` - ProcessingStats *ProcessingStats `protobuf:"bytes,2,opt,name=processing_stats,json=processingStats,proto3" json:"processing_stats,omitempty"` - Backend EotPrediction_EotBackend `protobuf:"varint,3,opt,name=backend,proto3,enum=livekit.agent.EotPrediction_EotBackend" json:"backend,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Probability float32 `protobuf:"fixed32,1,opt,name=probability,proto3" json:"probability,omitempty"` + InferenceStats *InferenceStats `protobuf:"bytes,2,opt,name=inference_stats,json=inferenceStats,proto3" json:"inference_stats,omitempty"` + Backend EotPrediction_EotBackend `protobuf:"varint,3,opt,name=backend,proto3,enum=livekit.agent.EotPrediction_EotBackend" json:"backend,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EotPrediction) Reset() { *x = EotPrediction{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[26] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1693,7 +1651,7 @@ func (x *EotPrediction) String() string { func (*EotPrediction) ProtoMessage() {} func (x *EotPrediction) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[26] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1706,7 +1664,7 @@ func (x *EotPrediction) ProtoReflect() protoreflect.Message { // Deprecated: Use EotPrediction.ProtoReflect.Descriptor instead. func (*EotPrediction) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{26} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{25} } func (x *EotPrediction) GetProbability() float32 { @@ -1716,9 +1674,9 @@ func (x *EotPrediction) GetProbability() float32 { return 0 } -func (x *EotPrediction) GetProcessingStats() *ProcessingStats { +func (x *EotPrediction) GetInferenceStats() *InferenceStats { if x != nil { - return x.ProcessingStats + return x.InferenceStats } return nil } @@ -1731,19 +1689,17 @@ func (x *EotPrediction) GetBackend() EotPrediction_EotBackend { } type InterruptionPrediction struct { - state protoimpl.MessageState `protogen:"open.v1"` - IsInterruption bool `protobuf:"varint,1,opt,name=is_interruption,json=isInterruption,proto3" json:"is_interruption,omitempty"` - Probabilities []float32 `protobuf:"fixed32,2,rep,packed,name=probabilities,proto3" json:"probabilities,omitempty"` - ProcessingStats *ProcessingStats `protobuf:"bytes,3,opt,name=processing_stats,json=processingStats,proto3" json:"processing_stats,omitempty"` - CreatedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` - PredictionDuration *durationpb.Duration `protobuf:"bytes,5,opt,name=prediction_duration,json=predictionDuration,proto3" json:"prediction_duration,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + IsInterruption bool `protobuf:"varint,1,opt,name=is_interruption,json=isInterruption,proto3" json:"is_interruption,omitempty"` + Probabilities []float32 `protobuf:"fixed32,2,rep,packed,name=probabilities,proto3" json:"probabilities,omitempty"` + InferenceStats *InferenceStats `protobuf:"bytes,3,opt,name=inference_stats,json=inferenceStats,proto3" json:"inference_stats,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InterruptionPrediction) Reset() { *x = InterruptionPrediction{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[27] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1755,7 +1711,7 @@ func (x *InterruptionPrediction) String() string { func (*InterruptionPrediction) ProtoMessage() {} func (x *InterruptionPrediction) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[27] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1768,7 +1724,7 @@ func (x *InterruptionPrediction) ProtoReflect() protoreflect.Message { // Deprecated: Use InterruptionPrediction.ProtoReflect.Descriptor instead. func (*InterruptionPrediction) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{27} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{26} } func (x *InterruptionPrediction) GetIsInterruption() bool { @@ -1785,23 +1741,9 @@ func (x *InterruptionPrediction) GetProbabilities() []float32 { return nil } -func (x *InterruptionPrediction) GetProcessingStats() *ProcessingStats { - if x != nil { - return x.ProcessingStats - } - return nil -} - -func (x *InterruptionPrediction) GetCreatedAt() *timestamppb.Timestamp { +func (x *InterruptionPrediction) GetInferenceStats() *InferenceStats { if x != nil { - return x.CreatedAt - } - return nil -} - -func (x *InterruptionPrediction) GetPredictionDuration() *durationpb.Duration { - if x != nil { - return x.PredictionDuration + return x.InferenceStats } return nil } @@ -1828,7 +1770,7 @@ type ServerMessage struct { func (x *ServerMessage) Reset() { *x = ServerMessage{} - mi := &file_agent_livekit_agent_inference_proto_msgTypes[28] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1840,7 +1782,7 @@ func (x *ServerMessage) String() string { func (*ServerMessage) ProtoMessage() {} func (x *ServerMessage) ProtoReflect() protoreflect.Message { - mi := &file_agent_livekit_agent_inference_proto_msgTypes[28] + mi := &file_agent_livekit_agent_inference_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1853,7 +1795,7 @@ func (x *ServerMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use ServerMessage.ProtoReflect.Descriptor instead. func (*ServerMessage) Descriptor() ([]byte, []int) { - return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{28} + return file_agent_livekit_agent_inference_proto_rawDescGZIP(), []int{27} } func (x *ServerMessage) GetServerCreatedAt() *timestamppb.Timestamp { @@ -2069,18 +2011,17 @@ const file_agent_livekit_agent_inference_proto_rawDesc = "" + "\x10InferenceRequest\x12X\n" + "\x15eot_inference_request\x18\x01 \x01(\v2\".livekit.agent.EotInferenceRequestH\x00R\x13eotInferenceRequest\x12s\n" + "\x1einterruption_inference_request\x18\x02 \x01(\v2+.livekit.agent.InterruptionInferenceRequestH\x00R\x1cinterruptionInferenceRequestB\t\n" + - "\arequest\"\xe8\x01\n" + - "\x0eInferenceStats\x12:\n" + - "\ve2e_latency\x18\x01 \x01(\v2\x19.google.protobuf.DurationR\n" + - "e2eLatency\x12P\n" + - "\x16preprocessing_duration\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\x15preprocessingDuration\x12H\n" + - "\x12inference_duration\x18\x03 \x01(\v2\x19.google.protobuf.DurationR\x11inferenceDuration\"\xc3\x02\n" + - "\x0fProcessingStats\x12W\n" + - "\x1aearliest_client_created_at\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x17earliestClientCreatedAt\x12S\n" + - "\x18latest_client_created_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x15latestClientCreatedAt\x12:\n" + - "\ve2e_latency\x18\x03 \x01(\v2\x19.google.protobuf.DurationR\n" + - "e2eLatency\x12F\n" + - "\x0finference_stats\x18\x04 \x01(\v2\x1d.livekit.agent.InferenceStatsR\x0einferenceStats\"m\n" + + "\arequest\"\xce\x04\n" + + "\x0eInferenceStats\x12\\\n" + + "\x1aearliest_client_created_at\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampH\x00R\x17earliestClientCreatedAt\x88\x01\x01\x12X\n" + + "\x18latest_client_created_at\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampH\x01R\x15latestClientCreatedAt\x88\x01\x01\x12L\n" + + "\x12client_e2e_latency\x18\x03 \x01(\v2\x19.google.protobuf.DurationH\x02R\x10clientE2eLatency\x88\x01\x01\x12G\n" + + "\x12server_e2e_latency\x18\x04 \x01(\v2\x19.google.protobuf.DurationR\x10serverE2eLatency\x12P\n" + + "\x16preprocessing_duration\x18\x05 \x01(\v2\x19.google.protobuf.DurationR\x15preprocessingDuration\x12H\n" + + "\x12inference_duration\x18\x06 \x01(\v2\x19.google.protobuf.DurationR\x11inferenceDurationB\x1d\n" + + "\x1b_earliest_client_created_atB\x1b\n" + + "\x19_latest_client_created_atB\x15\n" + + "\x13_client_e2e_latency\"m\n" + "\x14EotInferenceResponse\x12 \n" + "\vprobability\x18\x01 \x01(\x02R\vprobability\x123\n" + "\x05stats\x18\x02 \x01(\v2\x1d.livekit.agent.InferenceStatsR\x05stats\"\xa3\x01\n" + @@ -2096,23 +2037,20 @@ const file_agent_livekit_agent_inference_proto_rawDesc = "" + "\x0eSessionCreated\"\x12\n" + "\x10InferenceStarted\"\x12\n" + "\x10InferenceStopped\"\x0f\n" + - "\rSessionClosed\"\x98\x02\n" + + "\rSessionClosed\"\x95\x02\n" + "\rEotPrediction\x12 \n" + - "\vprobability\x18\x01 \x01(\x02R\vprobability\x12I\n" + - "\x10processing_stats\x18\x02 \x01(\v2\x1e.livekit.agent.ProcessingStatsR\x0fprocessingStats\x12A\n" + + "\vprobability\x18\x01 \x01(\x02R\vprobability\x12F\n" + + "\x0finference_stats\x18\x02 \x01(\v2\x1d.livekit.agent.InferenceStatsR\x0einferenceStats\x12A\n" + "\abackend\x18\x03 \x01(\x0e2'.livekit.agent.EotPrediction.EotBackendR\abackend\"W\n" + "\n" + "EotBackend\x12\x17\n" + "\x13EOT_BACKEND_UNKNOWN\x10\x00\x12\x1a\n" + "\x16EOT_BACKEND_MULTIMODAL\x10\x01\x12\x14\n" + - "\x10EOT_BACKEND_TEXT\x10\x02\"\xb9\x02\n" + + "\x10EOT_BACKEND_TEXT\x10\x02\"\xaf\x01\n" + "\x16InterruptionPrediction\x12'\n" + "\x0fis_interruption\x18\x01 \x01(\bR\x0eisInterruption\x12$\n" + - "\rprobabilities\x18\x02 \x03(\x02R\rprobabilities\x12I\n" + - "\x10processing_stats\x18\x03 \x01(\v2\x1e.livekit.agent.ProcessingStatsR\x0fprocessingStats\x129\n" + - "\n" + - "created_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12J\n" + - "\x13prediction_duration\x18\x05 \x01(\v2\x19.google.protobuf.DurationR\x12predictionDuration\"\x89\x06\n" + + "\rprobabilities\x18\x02 \x03(\x02R\rprobabilities\x12F\n" + + "\x0finference_stats\x18\x03 \x01(\v2\x1d.livekit.agent.InferenceStatsR\x0einferenceStats\"\x89\x06\n" + "\rServerMessage\x12F\n" + "\x11server_created_at\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x0fserverCreatedAt\x12\"\n" + "\n" + @@ -2146,7 +2084,7 @@ func file_agent_livekit_agent_inference_proto_rawDescGZIP() []byte { } var file_agent_livekit_agent_inference_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_agent_livekit_agent_inference_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_agent_livekit_agent_inference_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_agent_livekit_agent_inference_proto_goTypes = []any{ (AudioEncoding)(0), // 0: livekit.agent.AudioEncoding (EotPrediction_EotBackend)(0), // 1: livekit.agent.EotPrediction.EotBackend @@ -2168,33 +2106,32 @@ var file_agent_livekit_agent_inference_proto_goTypes = []any{ (*InterruptionInferenceRequest)(nil), // 17: livekit.agent.InterruptionInferenceRequest (*InferenceRequest)(nil), // 18: livekit.agent.InferenceRequest (*InferenceStats)(nil), // 19: livekit.agent.InferenceStats - (*ProcessingStats)(nil), // 20: livekit.agent.ProcessingStats - (*EotInferenceResponse)(nil), // 21: livekit.agent.EotInferenceResponse - (*InterruptionInferenceResponse)(nil), // 22: livekit.agent.InterruptionInferenceResponse - (*InferenceResponse)(nil), // 23: livekit.agent.InferenceResponse - (*SessionCreated)(nil), // 24: livekit.agent.SessionCreated - (*InferenceStarted)(nil), // 25: livekit.agent.InferenceStarted - (*InferenceStopped)(nil), // 26: livekit.agent.InferenceStopped - (*SessionClosed)(nil), // 27: livekit.agent.SessionClosed - (*EotPrediction)(nil), // 28: livekit.agent.EotPrediction - (*InterruptionPrediction)(nil), // 29: livekit.agent.InterruptionPrediction - (*ServerMessage)(nil), // 30: livekit.agent.ServerMessage - (*durationpb.Duration)(nil), // 31: google.protobuf.Duration - (*timestamppb.Timestamp)(nil), // 32: google.protobuf.Timestamp - (*ChatMessage)(nil), // 33: livekit.agent.ChatMessage + (*EotInferenceResponse)(nil), // 20: livekit.agent.EotInferenceResponse + (*InterruptionInferenceResponse)(nil), // 21: livekit.agent.InterruptionInferenceResponse + (*InferenceResponse)(nil), // 22: livekit.agent.InferenceResponse + (*SessionCreated)(nil), // 23: livekit.agent.SessionCreated + (*InferenceStarted)(nil), // 24: livekit.agent.InferenceStarted + (*InferenceStopped)(nil), // 25: livekit.agent.InferenceStopped + (*SessionClosed)(nil), // 26: livekit.agent.SessionClosed + (*EotPrediction)(nil), // 27: livekit.agent.EotPrediction + (*InterruptionPrediction)(nil), // 28: livekit.agent.InterruptionPrediction + (*ServerMessage)(nil), // 29: livekit.agent.ServerMessage + (*durationpb.Duration)(nil), // 30: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 31: google.protobuf.Timestamp + (*ChatMessage)(nil), // 32: livekit.agent.ChatMessage } var file_agent_livekit_agent_inference_proto_depIdxs = []int32{ 0, // 0: livekit.agent.SessionSettings.encoding:type_name -> livekit.agent.AudioEncoding 4, // 1: livekit.agent.SessionSettings.eot_settings:type_name -> livekit.agent.EotSettings 5, // 2: livekit.agent.SessionSettings.interruption_settings:type_name -> livekit.agent.InterruptionSettings - 31, // 3: livekit.agent.EotSettings.detection_interval:type_name -> google.protobuf.Duration - 31, // 4: livekit.agent.InterruptionSettings.max_audio_duration:type_name -> google.protobuf.Duration - 31, // 5: livekit.agent.InterruptionSettings.audio_prefix_duration:type_name -> google.protobuf.Duration - 31, // 6: livekit.agent.InterruptionSettings.detection_interval:type_name -> google.protobuf.Duration + 30, // 3: livekit.agent.EotSettings.detection_interval:type_name -> google.protobuf.Duration + 30, // 4: livekit.agent.InterruptionSettings.max_audio_duration:type_name -> google.protobuf.Duration + 30, // 5: livekit.agent.InterruptionSettings.audio_prefix_duration:type_name -> google.protobuf.Duration + 30, // 6: livekit.agent.InterruptionSettings.detection_interval:type_name -> google.protobuf.Duration 2, // 7: livekit.agent.SessionCreate.settings:type_name -> livekit.agent.SessionSettings - 32, // 8: livekit.agent.InputAudio.created_at:type_name -> google.protobuf.Timestamp - 33, // 9: livekit.agent.EotInputChatContext.messages:type_name -> livekit.agent.ChatMessage - 32, // 10: livekit.agent.ClientMessage.created_at:type_name -> google.protobuf.Timestamp + 31, // 8: livekit.agent.InputAudio.created_at:type_name -> google.protobuf.Timestamp + 32, // 9: livekit.agent.EotInputChatContext.messages:type_name -> livekit.agent.ChatMessage + 31, // 10: livekit.agent.ClientMessage.created_at:type_name -> google.protobuf.Timestamp 6, // 11: livekit.agent.ClientMessage.session_create:type_name -> livekit.agent.SessionCreate 7, // 12: livekit.agent.ClientMessage.input_audio:type_name -> livekit.agent.InputAudio 9, // 13: livekit.agent.ClientMessage.session_flush:type_name -> livekit.agent.SessionFlush @@ -2208,36 +2145,33 @@ var file_agent_livekit_agent_inference_proto_depIdxs = []int32{ 0, // 21: livekit.agent.InterruptionInferenceRequest.encoding:type_name -> livekit.agent.AudioEncoding 16, // 22: livekit.agent.InferenceRequest.eot_inference_request:type_name -> livekit.agent.EotInferenceRequest 17, // 23: livekit.agent.InferenceRequest.interruption_inference_request:type_name -> livekit.agent.InterruptionInferenceRequest - 31, // 24: livekit.agent.InferenceStats.e2e_latency:type_name -> google.protobuf.Duration - 31, // 25: livekit.agent.InferenceStats.preprocessing_duration:type_name -> google.protobuf.Duration - 31, // 26: livekit.agent.InferenceStats.inference_duration:type_name -> google.protobuf.Duration - 32, // 27: livekit.agent.ProcessingStats.earliest_client_created_at:type_name -> google.protobuf.Timestamp - 32, // 28: livekit.agent.ProcessingStats.latest_client_created_at:type_name -> google.protobuf.Timestamp - 31, // 29: livekit.agent.ProcessingStats.e2e_latency:type_name -> google.protobuf.Duration - 19, // 30: livekit.agent.ProcessingStats.inference_stats:type_name -> livekit.agent.InferenceStats - 19, // 31: livekit.agent.EotInferenceResponse.stats:type_name -> livekit.agent.InferenceStats - 19, // 32: livekit.agent.InterruptionInferenceResponse.stats:type_name -> livekit.agent.InferenceStats - 21, // 33: livekit.agent.InferenceResponse.eot_inference_response:type_name -> livekit.agent.EotInferenceResponse - 22, // 34: livekit.agent.InferenceResponse.interruption_inference_response:type_name -> livekit.agent.InterruptionInferenceResponse - 20, // 35: livekit.agent.EotPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats - 1, // 36: livekit.agent.EotPrediction.backend:type_name -> livekit.agent.EotPrediction.EotBackend - 20, // 37: livekit.agent.InterruptionPrediction.processing_stats:type_name -> livekit.agent.ProcessingStats - 32, // 38: livekit.agent.InterruptionPrediction.created_at:type_name -> google.protobuf.Timestamp - 31, // 39: livekit.agent.InterruptionPrediction.prediction_duration:type_name -> google.protobuf.Duration - 32, // 40: livekit.agent.ServerMessage.server_created_at:type_name -> google.protobuf.Timestamp - 32, // 41: livekit.agent.ServerMessage.client_created_at:type_name -> google.protobuf.Timestamp - 24, // 42: livekit.agent.ServerMessage.session_created:type_name -> livekit.agent.SessionCreated - 25, // 43: livekit.agent.ServerMessage.inference_started:type_name -> livekit.agent.InferenceStarted - 26, // 44: livekit.agent.ServerMessage.inference_stopped:type_name -> livekit.agent.InferenceStopped - 27, // 45: livekit.agent.ServerMessage.session_closed:type_name -> livekit.agent.SessionClosed - 3, // 46: livekit.agent.ServerMessage.error:type_name -> livekit.agent.InferenceError - 28, // 47: livekit.agent.ServerMessage.eot_prediction:type_name -> livekit.agent.EotPrediction - 29, // 48: livekit.agent.ServerMessage.interruption_prediction:type_name -> livekit.agent.InterruptionPrediction - 49, // [49:49] is the sub-list for method output_type - 49, // [49:49] is the sub-list for method input_type - 49, // [49:49] is the sub-list for extension type_name - 49, // [49:49] is the sub-list for extension extendee - 0, // [0:49] is the sub-list for field type_name + 31, // 24: livekit.agent.InferenceStats.earliest_client_created_at:type_name -> google.protobuf.Timestamp + 31, // 25: livekit.agent.InferenceStats.latest_client_created_at:type_name -> google.protobuf.Timestamp + 30, // 26: livekit.agent.InferenceStats.client_e2e_latency:type_name -> google.protobuf.Duration + 30, // 27: livekit.agent.InferenceStats.server_e2e_latency:type_name -> google.protobuf.Duration + 30, // 28: livekit.agent.InferenceStats.preprocessing_duration:type_name -> google.protobuf.Duration + 30, // 29: livekit.agent.InferenceStats.inference_duration:type_name -> google.protobuf.Duration + 19, // 30: livekit.agent.EotInferenceResponse.stats:type_name -> livekit.agent.InferenceStats + 19, // 31: livekit.agent.InterruptionInferenceResponse.stats:type_name -> livekit.agent.InferenceStats + 20, // 32: livekit.agent.InferenceResponse.eot_inference_response:type_name -> livekit.agent.EotInferenceResponse + 21, // 33: livekit.agent.InferenceResponse.interruption_inference_response:type_name -> livekit.agent.InterruptionInferenceResponse + 19, // 34: livekit.agent.EotPrediction.inference_stats:type_name -> livekit.agent.InferenceStats + 1, // 35: livekit.agent.EotPrediction.backend:type_name -> livekit.agent.EotPrediction.EotBackend + 19, // 36: livekit.agent.InterruptionPrediction.inference_stats:type_name -> livekit.agent.InferenceStats + 31, // 37: livekit.agent.ServerMessage.server_created_at:type_name -> google.protobuf.Timestamp + 31, // 38: livekit.agent.ServerMessage.client_created_at:type_name -> google.protobuf.Timestamp + 23, // 39: livekit.agent.ServerMessage.session_created:type_name -> livekit.agent.SessionCreated + 24, // 40: livekit.agent.ServerMessage.inference_started:type_name -> livekit.agent.InferenceStarted + 25, // 41: livekit.agent.ServerMessage.inference_stopped:type_name -> livekit.agent.InferenceStopped + 26, // 42: livekit.agent.ServerMessage.session_closed:type_name -> livekit.agent.SessionClosed + 3, // 43: livekit.agent.ServerMessage.error:type_name -> livekit.agent.InferenceError + 27, // 44: livekit.agent.ServerMessage.eot_prediction:type_name -> livekit.agent.EotPrediction + 28, // 45: livekit.agent.ServerMessage.interruption_prediction:type_name -> livekit.agent.InterruptionPrediction + 46, // [46:46] is the sub-list for method output_type + 46, // [46:46] is the sub-list for method input_type + 46, // [46:46] is the sub-list for extension type_name + 46, // [46:46] is the sub-list for extension extendee + 0, // [0:46] is the sub-list for field type_name } func init() { file_agent_livekit_agent_inference_proto_init() } @@ -2265,11 +2199,12 @@ func file_agent_livekit_agent_inference_proto_init() { (*InferenceRequest_EotInferenceRequest)(nil), (*InferenceRequest_InterruptionInferenceRequest)(nil), } - file_agent_livekit_agent_inference_proto_msgTypes[21].OneofWrappers = []any{ + file_agent_livekit_agent_inference_proto_msgTypes[17].OneofWrappers = []any{} + file_agent_livekit_agent_inference_proto_msgTypes[20].OneofWrappers = []any{ (*InferenceResponse_EotInferenceResponse)(nil), (*InferenceResponse_InterruptionInferenceResponse)(nil), } - file_agent_livekit_agent_inference_proto_msgTypes[28].OneofWrappers = []any{ + file_agent_livekit_agent_inference_proto_msgTypes[27].OneofWrappers = []any{ (*ServerMessage_SessionCreated)(nil), (*ServerMessage_InferenceStarted)(nil), (*ServerMessage_InferenceStopped)(nil), @@ -2284,7 +2219,7 @@ func file_agent_livekit_agent_inference_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_agent_livekit_agent_inference_proto_rawDesc), len(file_agent_livekit_agent_inference_proto_rawDesc)), NumEnums: 2, - NumMessages: 29, + NumMessages: 28, NumExtensions: 0, NumServices: 0, },