diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d5ade7..0c21d2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,3 +93,4 @@ add_subdirectory(simple_joystick_sender) add_subdirectory(simple_joystick_receiver) add_subdirectory(ping_pong_ping) add_subdirectory(ping_pong_pong) +add_subdirectory(user_timestamped_video) diff --git a/user_timestamped_video/CMakeLists.txt b/user_timestamped_video/CMakeLists.txt new file mode 100644 index 0000000..6fdcdd8 --- /dev/null +++ b/user_timestamped_video/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(producer) +add_subdirectory(consumer) diff --git a/user_timestamped_video/README.md b/user_timestamped_video/README.md new file mode 100644 index 0000000..ea9fd27 --- /dev/null +++ b/user_timestamped_video/README.md @@ -0,0 +1,66 @@ +# UserTimestampedVideo + +This example is split into two executables and can demonstrate all four +producer/consumer combinations: + +- `UserTimestampedVideoProducer` publishes a synthetic video track named + `"timestamped-camera"` and stamps each frame with + `VideoCaptureOptions::metadata.user_timestamp_us`. +- `UserTimestampedVideoConsumer` subscribes to the remote + `"timestamped-camera"` track by name with either the rich or legacy callback + path. + +Run them in the same room with different participant identities: + +```sh +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoProducer +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoConsumer +``` + +Requirements: + +- LiveKit C++ SDK `v0.3.4` or newer. This example uses + `VideoFrameMetadata` and `setOnVideoFrameEventCallback`, which are not + available in older SDK releases. +- To pin the SDK version when configuring the examples, pass + `-DLIVEKIT_SDK_VERSION=0.3.4` to CMake. + +Flags: + +- Producer default: sends user timestamps +- Producer `--with-user-timestamp`: explicitly sends user timestamps +- Producer `--without-user-timestamp`: does not send user timestamps +- Consumer default: reads user timestamps through `setOnVideoFrameEventCallback` +- Consumer `--with-user-timestamp`: explicitly reads user timestamps through + `setOnVideoFrameEventCallback` +- Consumer `--without-user-timestamp`: ignores metadata through the legacy + `setOnVideoFrameCallback` + +Matrix: + +```sh +# 1. Producer sends, consumer reads +./UserTimestampedVideoProducer +./UserTimestampedVideoConsumer + +# 2. Producer sends, consumer ignores +./UserTimestampedVideoProducer +./UserTimestampedVideoConsumer --without-user-timestamp + +# 3. Producer does not send, consumer ignores +./UserTimestampedVideoProducer --without-user-timestamp +./UserTimestampedVideoConsumer --without-user-timestamp + +# 4. Producer does not send, consumer reads +./UserTimestampedVideoProducer --without-user-timestamp +./UserTimestampedVideoConsumer +``` + +Timestamp note: + +- `user_ts_us` is application metadata and is the value to compare end to end. +- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. +- `capture_ts_us` on the consumer is the received WebRTC frame timestamp. +- Producer and consumer `capture_ts_us` values are not expected to match exactly, + because WebRTC may translate frame timestamps onto its own internal + capture-time timeline before delivery. diff --git a/user_timestamped_video/common/cli_utils.h b/user_timestamped_video/common/cli_utils.h new file mode 100644 index 0000000..821d882 --- /dev/null +++ b/user_timestamped_video/common/cli_utils.h @@ -0,0 +1,105 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace user_timestamped_video { + +enum class ParseResult { Ok, Help, Error }; + +struct CliOptions { + std::string url; + std::string token; + bool use_user_timestamp = true; +}; + +inline std::atomic g_running{true}; + +inline void handleSignal(int) { g_running.store(false); } + +inline bool isRunning() { return g_running.load(std::memory_order_relaxed); } + +inline void installSignalHandlers() { + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif +} + +inline std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +inline void printUsage(const char *program) { + std::cerr << "Usage:\n" + << " " << program << " " + << "[--with-user-timestamp|--without-user-timestamp]\n" + << "or:\n" + << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program + << " [--with-user-timestamp|--without-user-timestamp]\n"; +} + +inline ParseResult parseArgs(int argc, char *argv[], CliOptions &options) { + std::vector positional; + options = CliOptions{}; + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return ParseResult::Help; + } + if (arg == "--without-user-timestamp") { + options.use_user_timestamp = false; + continue; + } + if (arg == "--with-user-timestamp") { + options.use_user_timestamp = true; + continue; + } + if (!arg.empty() && arg[0] == '-') { + return ParseResult::Error; + } + + positional.push_back(arg); + } + + if (positional.size() > 2) { + return ParseResult::Error; + } + + options.url = getenvOrEmpty("LIVEKIT_URL"); + options.token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (positional.size() == 2) { + options.url = positional[0]; + options.token = positional[1]; + } else if (positional.size() == 1) { + return ParseResult::Error; + } + + return (options.url.empty() || options.token.empty()) ? ParseResult::Error + : ParseResult::Ok; +} + +} // namespace user_timestamped_video diff --git a/user_timestamped_video/consumer/CMakeLists.txt b/user_timestamped_video/consumer/CMakeLists.txt new file mode 100644 index 0000000..d2f80a2 --- /dev/null +++ b/user_timestamped_video/consumer/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(UserTimestampedVideoConsumer + main.cpp +) + +target_include_directories(UserTimestampedVideoConsumer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(UserTimestampedVideoConsumer PRIVATE ${LIVEKIT_CORE_TARGET}) + +livekit_copy_windows_runtime_dlls(UserTimestampedVideoConsumer) diff --git a/user_timestamped_video/consumer/main.cpp b/user_timestamped_video/consumer/main.cpp new file mode 100644 index 0000000..b021a7b --- /dev/null +++ b/user_timestamped_video/consumer/main.cpp @@ -0,0 +1,203 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// UserTimestampedVideoConsumer +/// +/// Receives remote video frames via `Room::setOnVideoFrameEventCallback()` and +/// logs any `VideoFrameMetadata::user_timestamp_us` values that arrive. Pair +/// with `UserTimestampedVideoProducer` running in another process. +/// +/// Usage: +/// UserTimestampedVideoConsumer +/// [--with-user-timestamp|--without-user-timestamp] +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_TOKEN + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../common/cli_utils.h" +#include "livekit/livekit.h" + +using namespace livekit; + +namespace { + +constexpr const char *kTrackName = "timestamped-camera"; + +std::string +formatUserTimestamp(const std::optional &metadata) { + if (!metadata || !metadata->user_timestamp_us.has_value()) { + return "n/a"; + } + + return std::to_string(*metadata->user_timestamp_us); +} + +class UserTimestampedVideoConsumerDelegate : public RoomDelegate { +public: + UserTimestampedVideoConsumerDelegate(Room &room, bool read_user_timestamp) + : room_(room), read_user_timestamp_(read_user_timestamp) {} + + void registerExistingParticipants() { + for (const auto &participant : room_.remoteParticipants()) { + if (participant) { + registerRemoteVideoCallback(participant->identity()); + } + } + } + + void onParticipantConnected(Room &, + const ParticipantConnectedEvent &event) override { + if (!event.participant) { + return; + } + + std::cout << "[consumer] participant connected: " + << event.participant->identity() << "\n"; + registerRemoteVideoCallback(event.participant->identity()); + } + + void onParticipantDisconnected( + Room &, const ParticipantDisconnectedEvent &event) override { + if (!event.participant) { + return; + } + + const std::string identity = event.participant->identity(); + room_.clearOnVideoFrameCallback(identity, std::string(kTrackName)); + + { + std::lock_guard lock(mutex_); + registered_identities_.erase(identity); + } + + std::cout << "[consumer] participant disconnected: " << identity << "\n"; + } + +private: + void registerRemoteVideoCallback(const std::string &identity) { + { + std::lock_guard lock(mutex_); + if (!registered_identities_.insert(identity).second) { + return; + } + } + + VideoStream::Options stream_options; + stream_options.format = VideoBufferType::RGBA; + + if (read_user_timestamp_) { + room_.setOnVideoFrameEventCallback( + identity, std::string(kTrackName), + [identity](const VideoFrameEvent &event) { + std::cout << "[consumer] from=" << identity + << " size=" << event.frame.width() << "x" + << event.frame.height() + << " capture_ts_us=" << event.timestamp_us + << " user_ts_us=" << formatUserTimestamp(event.metadata) + << " rotation=" << static_cast(event.rotation) + << "\n"; + }, + stream_options); + } else { + room_.setOnVideoFrameCallback( + identity, std::string(kTrackName), + [identity](const VideoFrame &frame, const std::int64_t timestamp_us) { + std::cout << "[consumer] from=" << identity + << " size=" << frame.width() << "x" << frame.height() + << " capture_ts_us=" << timestamp_us + << " user_ts_us=ignored\n"; + }, + stream_options); + } + + std::cout << "[consumer] listening for video frames from " << identity + << " track=\"" << kTrackName << "\" with user timestamp " + << (read_user_timestamp_ ? "enabled" : "ignored") << "\n"; + } + + Room &room_; + bool read_user_timestamp_; + std::mutex mutex_; + std::unordered_set registered_identities_; +}; + +} // namespace + +int main(int argc, char *argv[]) { + user_timestamped_video::CliOptions cli_options; + + const user_timestamped_video::ParseResult parse_result = + user_timestamped_video::parseArgs(argc, argv, cli_options); + if (parse_result != user_timestamped_video::ParseResult::Ok) { + user_timestamped_video::printUsage(argv[0]); + return parse_result == user_timestamped_video::ParseResult::Help ? 0 : 1; + } + + user_timestamped_video::installSignalHandlers(); + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + UserTimestampedVideoConsumerDelegate delegate( + room, cli_options.use_user_timestamp); + room.setDelegate(&delegate); + + std::cout << "[consumer] connecting to " << cli_options.url << "\n"; + if (!room.Connect(cli_options.url, cli_options.token, options)) { + std::cerr << "[consumer] failed to connect\n"; + exit_code = 1; + } else { + std::cout << "[consumer] connected as " + << room.localParticipant()->identity() << " to room '" + << room.room_info().name << "' with user timestamp " + << (cli_options.use_user_timestamp ? "enabled" : "ignored") + << "\n"; + + delegate.registerExistingParticipants(); + + while (user_timestamped_video::isRunning()) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + for (const auto &participant : room.remoteParticipants()) { + if (participant) { + room.clearOnVideoFrameCallback(participant->identity(), + std::string(kTrackName)); + } + } + } + + room.setDelegate(nullptr); + } + + livekit::shutdown(); + return exit_code; +} diff --git a/user_timestamped_video/producer/CMakeLists.txt b/user_timestamped_video/producer/CMakeLists.txt new file mode 100644 index 0000000..5c573ea --- /dev/null +++ b/user_timestamped_video/producer/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(UserTimestampedVideoProducer + main.cpp +) + +target_include_directories(UserTimestampedVideoProducer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(UserTimestampedVideoProducer PRIVATE ${LIVEKIT_CORE_TARGET}) + +livekit_copy_windows_runtime_dlls(UserTimestampedVideoProducer) diff --git a/user_timestamped_video/producer/main.cpp b/user_timestamped_video/producer/main.cpp new file mode 100644 index 0000000..95d8a66 --- /dev/null +++ b/user_timestamped_video/producer/main.cpp @@ -0,0 +1,170 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// UserTimestampedVideoProducer +/// +/// Publishes a synthetic camera track and stamps each frame with +/// `VideoCaptureOptions::metadata.user_timestamp_us`. Pair with +/// `UserTimestampedVideoConsumer` in another process to observe the user +/// timestamps flowing end to end. +/// +/// Usage: +/// UserTimestampedVideoProducer +/// [--with-user-timestamp|--without-user-timestamp] +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_TOKEN + +#include +#include +#include +#include +#include +#include + +#include "../common/cli_utils.h" +#include "livekit/livekit.h" + +using namespace livekit; + +namespace { + +constexpr int kFrameWidth = 640; +constexpr int kFrameHeight = 360; +constexpr int kFrameIntervalMs = 200; + +std::uint64_t nowEpochUs() { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); +} + +void fillFrame(VideoFrame &frame, std::uint32_t frame_index) { + const std::uint8_t blue = static_cast((frame_index * 7) % 255); + const std::uint8_t green = + static_cast((frame_index * 13) % 255); + const std::uint8_t red = static_cast((frame_index * 29) % 255); + + std::uint8_t *data = frame.data(); + for (std::size_t i = 0; i < frame.dataSize(); i += 4) { + data[i + 0] = blue; + data[i + 1] = green; + data[i + 2] = red; + data[i + 3] = 255; + } +} + +} // namespace + +int main(int argc, char *argv[]) { + user_timestamped_video::CliOptions cli_options; + + const user_timestamped_video::ParseResult parse_result = + user_timestamped_video::parseArgs(argc, argv, cli_options); + if (parse_result != user_timestamped_video::ParseResult::Ok) { + user_timestamped_video::printUsage(argv[0]); + return parse_result == user_timestamped_video::ParseResult::Help ? 0 : 1; + } + + user_timestamped_video::installSignalHandlers(); + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + std::cout << "[producer] connecting to " << cli_options.url << "\n"; + if (!room.Connect(cli_options.url, cli_options.token, options)) { + std::cerr << "[producer] failed to connect\n"; + exit_code = 1; + } else { + std::cout << "[producer] connected as " + << room.localParticipant()->identity() << " to room '" + << room.room_info().name << "'\n"; + + auto source = std::make_shared(kFrameWidth, kFrameHeight); + auto track = + LocalVideoTrack::createLocalVideoTrack("timestamped-camera", source); + + try { + TrackPublishOptions publish_options; + publish_options.source = TrackSource::SOURCE_CAMERA; + publish_options.packet_trailer_features.user_timestamp = + cli_options.use_user_timestamp; + + room.localParticipant()->publishTrack(track, publish_options); + std::cout << "[producer] published camera track with user timestamp " + << (cli_options.use_user_timestamp ? "enabled" : "disabled") + << "\n"; + + VideoFrame frame = VideoFrame::create(kFrameWidth, kFrameHeight, + VideoBufferType::BGRA); + const auto capture_start = std::chrono::steady_clock::now(); + std::uint32_t frame_index = 0; + auto next_frame_at = std::chrono::steady_clock::now(); + + while (user_timestamped_video::isRunning()) { + fillFrame(frame, frame_index); + + VideoCaptureOptions capture_options; + + // a steady_clock to align with other data/video frames + capture_options.timestamp_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - capture_start) + .count()); + capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; + if (cli_options.use_user_timestamp) { + capture_options.metadata = VideoFrameMetadata{}; + capture_options.metadata->user_timestamp_us = nowEpochUs(); + } + + source->captureFrame(frame, capture_options); + + if (frame_index % 5 == 0) { + std::cout << "[producer] frame=" << frame_index + << " capture_ts_us=" << capture_options.timestamp_us + << " user_ts_us=" + << (cli_options.use_user_timestamp + ? std::to_string(*capture_options.metadata + ->user_timestamp_us) + : std::string("disabled")) + << "\n"; + } + + ++frame_index; + next_frame_at += std::chrono::milliseconds(kFrameIntervalMs); + std::this_thread::sleep_until(next_frame_at); + } + } catch (const std::exception &error) { + std::cerr << "[producer] error: " << error.what() << "\n"; + exit_code = 1; + } + + if (track->publication()) { + room.localParticipant()->unpublishTrack(track->publication()->sid()); + } + } + } + + livekit::shutdown(); + return exit_code; +}