From 08bb32a39bac6079c9ec2e86c7b25a4def4ccbae Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 15:47:36 +0100 Subject: [PATCH 01/15] ci(repo): Add E2E reactions tests --- .../integration_test/allure/allure.dart | 54 +++-- .../integration_test/message_list_test.dart | 17 +- .../mock_server/data_types.dart | 19 +- .../pages/message_list_page.dart | 15 ++ .../integration_test/reactions_test.dart | 211 ++++++++++++++++++ .../robots/participant_robot.dart | 17 ++ .../integration_test/robots/user_robot.dart | 40 ++++ .../user_robot_message_list_asserts.dart | 26 +++ sample_app/integration_test/support/step.dart | 9 +- .../support/stream_test_env.dart | 61 ++++- .../support/widget_test_extensions.dart | 36 +++ sample_app/lib/app.dart | 3 + sample_app/lib/auth/auth_controller.dart | 37 +++ 13 files changed, 504 insertions(+), 41 deletions(-) create mode 100644 sample_app/integration_test/reactions_test.dart diff --git a/sample_app/integration_test/allure/allure.dart b/sample_app/integration_test/allure/allure.dart index b9fb9c851..debdd5ef2 100644 --- a/sample_app/integration_test/allure/allure.dart +++ b/sample_app/integration_test/allure/allure.dart @@ -1,7 +1,6 @@ import 'dart:convert'; import 'dart:math'; -import 'package:flutter_test/flutter_test.dart' show TestFailure; import 'package:uuid/uuid.dart'; const allureResultMarker = 'ALLURE-RESULT::'; @@ -70,7 +69,7 @@ class Allure { static final Allure instance = Allure._(); _Result? _result; - final List<_Step> _stepStack = []; + _Step? _openStep; int get _now => DateTime.now().millisecondsSinceEpoch; @@ -79,7 +78,7 @@ class Allure { required String fullName, Map labels = const {}, }) { - _stepStack.clear(); + _openStep = null; _result = _Result( uuid: const Uuid().v4(), name: name, @@ -91,26 +90,34 @@ class Allure { ); } - Future step(String name, Future Function() body) async { + /// Records a BDD step marker. The statements that run after this call, up to + /// the next [beginStep] (or the end of the test), are the step's body — so a + /// step needs no closure and reads like the native `step("...") { ... }`. + /// + /// A failure in the body is attributed to whichever step is open when the + /// test stops (see [stopTest]). + void beginStep(String name) { + final result = _result; + if (result == null) return; + _closeOpenStep(); final step = _Step(name, _now); - (_stepStack.isNotEmpty ? _stepStack.last.steps : _result?.steps)?.add(step); - _stepStack.add(step); - try { - return await body(); - } on TestFailure catch (e) { - step - ..status = AllureStatus.failed - ..statusDetails = {'message': '$e'}; - rethrow; - } catch (e) { + result.steps.add(step); + _openStep = step; + } + + void _closeOpenStep({AllureStatus? status, String? message, String? trace}) { + final step = _openStep; + if (step == null) return; + step.stop = _now; + if (status != null && status != AllureStatus.passed) { step - ..status = AllureStatus.broken - ..statusDetails = {'message': '$e'}; - rethrow; - } finally { - step.stop = _now; - _stepStack.removeLast(); + ..status = status + ..statusDetails = { + if (message != null) 'message': message, + if (trace != null) 'trace': trace, + }; } + _openStep = null; } void stopTest({ @@ -120,6 +127,13 @@ class Allure { }) { final result = _result; if (result == null) return; + // Close the step that was open when the test ended, propagating a failure + // to it so the report points at the step whose body actually threw. + _closeOpenStep( + status: status, + message: message?.toString(), + trace: trace?.toString(), + ); result ..status = status ..stop = _now; diff --git a/sample_app/integration_test/message_list_test.dart b/sample_app/integration_test/message_list_test.dart index 623f61ef7..b58852acb 100644 --- a/sample_app/integration_test/message_list_test.dart +++ b/sample_app/integration_test/message_list_test.dart @@ -1,3 +1,4 @@ +import 'robots/user_robot.dart'; import 'robots/user_robot_message_list_asserts.dart'; import 'support/step.dart'; import 'support/stream_test_case.dart'; @@ -9,18 +10,14 @@ void main() { allureId: '11188', description: 'message list updates when the user sends a message', body: (env) async { - await step('GIVEN the user opens a channel', () async { - await env.userRobot.login(); - await env.userRobot.openChannel(); - }); + step('GIVEN the user opens a channel'); + await env.userRobot.login().openChannel(); - await step('WHEN the participant sends a message', () async { - await env.participantRobot.sendMessage(sampleText); - }); + step('WHEN the participant sends a message'); + await env.participantRobot.sendMessage(sampleText); - await step('THEN the message is displayed', () async { - await env.userRobot.assertMessage(sampleText); - }); + step('THEN the message is displayed'); + await env.userRobot.assertMessage(sampleText); }, ); } diff --git a/sample_app/integration_test/mock_server/data_types.dart b/sample_app/integration_test/mock_server/data_types.dart index 447662cb0..fdba1af29 100644 --- a/sample_app/integration_test/mock_server/data_types.dart +++ b/sample_app/integration_test/mock_server/data_types.dart @@ -9,15 +9,22 @@ enum AttachmentType { } enum ReactionType { - love('love'), - lol('haha'), - wow('wow'), - sad('sad'), - like('like'); + // Emoji glyphs mirror `streamSupportedEmojis` in stream_core_flutter exactly + // (incl. the U+FE0F variation selector on love) so `find.text` matches. + love('love', '❤️'), + lol('haha', '\u{1F602}'), + wow('wow', '\u{1F62E}'), + sad('sad', '\u{1F44E}'), + like('like', '\u{1F44D}'); - const ReactionType(this.reaction); + const ReactionType(this.reaction, this.emoji); + /// The reaction type string sent to/from the backend (e.g. `like`, `haha`). final String reaction; + + /// The emoji glyph the SDK renders for this reaction, used to locate the + /// reaction on a message in the UI (the display chips expose no keys). + final String emoji; } enum MessageDeliveryStatus { read, pending, sent, failed, nil } diff --git a/sample_app/integration_test/pages/message_list_page.dart b/sample_app/integration_test/pages/message_list_page.dart index ba3ab3f5f..9f7d84701 100644 --- a/sample_app/integration_test/pages/message_list_page.dart +++ b/sample_app/integration_test/pages/message_list_page.dart @@ -1,8 +1,15 @@ import 'package:flutter/widgets.dart'; import 'package:stream_chat_flutter/stream_chat_flutter.dart'; +import '../mock_server/data_types.dart'; + abstract final class MessageListPage { static const composer = _Composer(); + static const reactions = _Reactions(); + + /// The widget that renders a single message row. Long-pressing it opens the + /// message actions modal (which hosts the reaction picker). + static Type get messageItem => StreamMessageItem; } final class _Composer { @@ -11,3 +18,11 @@ final class _Composer { Type get inputField => StreamMessageComposerInputField; Key get sendButton => const ValueKey('send_key'); } + +final class _Reactions { + const _Reactions(); + + /// Key of a reaction in the reaction picker (message actions modal). + /// The SDK keys each picker button by its reaction type string. + Key pickerReaction(ReactionType type) => ValueKey(type.reaction); +} diff --git a/sample_app/integration_test/reactions_test.dart b/sample_app/integration_test/reactions_test.dart new file mode 100644 index 000000000..d228eccc6 --- /dev/null +++ b/sample_app/integration_test/reactions_test.dart @@ -0,0 +1,211 @@ +import 'mock_server/data_types.dart'; +import 'robots/participant_robot.dart'; +import 'robots/user_robot.dart'; +import 'robots/user_robot_message_list_asserts.dart'; +import 'support/step.dart'; +import 'support/stream_test_case.dart'; + +void main() { + const sampleText = 'Test'; + + streamTestWithEnv( + description: 'user adds a reaction to their own message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('WHEN user sends the message'); + await env.userRobot.sendMessage(sampleText); + + step('AND user adds the reaction'); + await env.userRobot.addReaction(ReactionType.like); + + step('THEN the reaction is added'); + await env.userRobot.assertReaction(type: ReactionType.like, isDisplayed: true); + }, + ); + + streamTestWithEnv( + description: 'user deletes a reaction from their own message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('WHEN user sends the message'); + await env.userRobot.sendMessage(sampleText); + + step('AND user adds the reaction'); + await env.userRobot.addReaction(ReactionType.wow).assertReaction(type: ReactionType.wow, isDisplayed: true); + + step('AND user removes the reaction'); + await env.userRobot.deleteReaction(ReactionType.wow); + + step('THEN the reaction is removed'); + await env.userRobot.assertReaction(type: ReactionType.wow, isDisplayed: false); + }, + ); + + streamTestWithEnv( + description: 'user adds a reaction to the participant message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('WHEN participant sends the message'); + await env.participantRobot.sendMessage(sampleText); + + step('AND user adds the reaction'); + await env.userRobot.addReaction(ReactionType.love); + + step('THEN the reaction is added'); + await env.userRobot.assertReaction(type: ReactionType.love, isDisplayed: true); + }, + ); + + streamTestWithEnv( + description: 'user removes a reaction from the participant message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('WHEN participant sends the message'); + await env.participantRobot.sendMessage(sampleText); + + step('AND user adds the reaction'); + await env.userRobot.addReaction(ReactionType.lol).assertReaction(type: ReactionType.lol, isDisplayed: true); + + step('AND user removes the reaction'); + await env.userRobot.deleteReaction(ReactionType.lol); + + step('THEN the reaction is removed'); + await env.userRobot.assertReaction(type: ReactionType.lol, isDisplayed: false); + }, + ); + + streamTestWithEnv( + description: 'participant adds a reaction to the user message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.backendRobot.generateChannels(channelsCount: 1, messagesCount: 1); + await env.userRobot.login().openChannel(); + + step('AND participant adds the reaction to the user message'); + await env.participantRobot.readMessage().addReaction(ReactionType.like); + + step('THEN the reaction is added'); + await env.userRobot.assertReaction(type: ReactionType.like, isDisplayed: true); + }, + ); + + streamTestWithEnv( + description: 'participant removes a reaction from the user message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.backendRobot.generateChannels(channelsCount: 1, messagesCount: 1); + await env.userRobot.login().openChannel(); + + step('AND participant adds the reaction to the user message'); + await env.participantRobot.readMessage().addReaction(ReactionType.lol); + await env.userRobot.assertReaction(type: ReactionType.lol, isDisplayed: true); + + step('AND participant removes the reaction'); + await env.participantRobot.deleteReaction(ReactionType.lol); + + step('THEN the reaction is removed'); + await env.userRobot.assertReaction(type: ReactionType.lol, isDisplayed: false); + }, + ); + + streamTestWithEnv( + description: 'participant adds a reaction to their own message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('WHEN participant sends the message'); + await env.participantRobot.sendMessage(sampleText); + + step('AND participant adds the reaction'); + await env.participantRobot.addReaction(ReactionType.wow); + + step('THEN the reaction is added'); + await env.userRobot.assertReaction(type: ReactionType.wow, isDisplayed: true); + }, + ); + + streamTestWithEnv( + description: 'participant removes a reaction from their own message', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('WHEN participant sends the message'); + await env.participantRobot.sendMessage(sampleText); + + step('AND participant adds the reaction'); + await env.participantRobot.addReaction(ReactionType.sad); + await env.userRobot.assertReaction(type: ReactionType.sad, isDisplayed: true); + + step('AND participant removes the reaction'); + await env.participantRobot.deleteReaction(ReactionType.sad); + + step('THEN the reaction is removed'); + await env.userRobot.assertReaction(type: ReactionType.sad, isDisplayed: false); + }, + ); + + // EXPECTED TO FAIL on Flutter: unlike iOS/Android, the Dart SDK does not + // queue offline reactions — `channel.sendReaction` rolls back its optimistic + // update when the HTTP request fails (channel.dart), and the reaction is + // never replayed on reconnect (the RetryQueue is messages-only). Kept as a + // faithful port of the native case so the gap stays visible; it will pass + // once the SDK gives reactions the same offline-queue treatment as messages. + streamTestWithEnv( + description: 'user adds a reaction while offline', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('AND user sends a message'); + await env.userRobot.sendMessage(sampleText); + + step('AND user becomes offline'); + await env.goOffline(); + + step('WHEN user adds a reaction'); + await env.userRobot.addReaction(ReactionType.like); + + step('THEN the reaction is displayed'); + await env.userRobot.assertReaction(type: ReactionType.like, isDisplayed: true); + + step('WHEN user becomes online'); + await env.goOnline(); + + step('THEN the reaction is still displayed'); + await env.userRobot.assertReaction(type: ReactionType.like, isDisplayed: true); + }, + ); + + streamTestWithEnv( + description: 'participant adds a reaction while the user is offline', + body: (env) async { + step('GIVEN user opens the channel'); + await env.userRobot.login().openChannel(); + + step('AND user sends a message'); + await env.userRobot.sendMessage(sampleText); + + step('AND user becomes offline'); + await env.goOffline(); + + step('WHEN participant adds a reaction'); + await env.participantRobot.addReaction(ReactionType.like); + + step('AND user becomes online'); + await env.goOnline(); + + step('THEN the reaction is displayed'); + await env.userRobot.assertReaction(type: ReactionType.like, isDisplayed: true); + }, + ); +} diff --git a/sample_app/integration_test/robots/participant_robot.dart b/sample_app/integration_test/robots/participant_robot.dart index 4250467cf..4f922b0ed 100644 --- a/sample_app/integration_test/robots/participant_robot.dart +++ b/sample_app/integration_test/robots/participant_robot.dart @@ -174,3 +174,20 @@ class ParticipantRobot { return this; } } + +/// Fluent chaining over async [ParticipantRobot] actions so test steps read like +/// the native robots (`participantRobot.readMessage().addReaction(type)`). +/// +/// Only the methods currently used by the ported suites are mirrored here; add +/// more from [ParticipantRobot] as new suites need to chain them. +extension ParticipantRobotChain on Future { + Future readMessage() async => (await this).readMessage(); + + Future sendMessage(String text, {int delay = 0}) async => + (await this).sendMessage(text, delay: delay); + + Future addReaction(ReactionType type, {int delay = 0}) async => + (await this).addReaction(type, delay: delay); + + Future deleteReaction(ReactionType type) async => (await this).deleteReaction(type); +} diff --git a/sample_app/integration_test/robots/user_robot.dart b/sample_app/integration_test/robots/user_robot.dart index 9cfcb0c17..bef2e6432 100644 --- a/sample_app/integration_test/robots/user_robot.dart +++ b/sample_app/integration_test/robots/user_robot.dart @@ -1,5 +1,6 @@ import 'package:flutter_test/flutter_test.dart'; +import '../mock_server/data_types.dart'; import '../pages/channel_list_page.dart'; import '../pages/message_list_page.dart'; import '../support/predefined_users.dart'; @@ -31,4 +32,43 @@ class UserRobot { await tester.tapByKey(MessageListPage.composer.sendButton); return this; } + + Future addReaction( + ReactionType type, { + int messageIndex = 0, + }) async { + final reaction = MessageListPage.reactions.pickerReaction(type); + final message = find.byType(MessageListPage.messageItem).at(messageIndex); + await tester.longPressUntilVisible(message, find.byKey(reaction)); + await tester.tapByKey(reaction); + return this; + } + + /// Re-selecting an own reaction in the picker toggles it off, so deleting a + /// reaction reuses the same flow as adding it. + Future deleteReaction( + ReactionType type, { + int messageIndex = 0, + }) { + return addReaction(type, messageIndex: messageIndex); + } +} + +/// Fluent chaining over async [UserRobot] actions so test steps read like the +/// native robots (`userRobot.login().openChannel()`) instead of a block of +/// sequential `await`s. Each method mirrors the instance method above. +extension UserRobotChain on Future { + Future login([ + UserCredentials user = PredefinedUsers.currentUser, + ]) async => (await this).login(user); + + Future openChannel({int index = 0}) async => (await this).openChannel(index: index); + + Future sendMessage(String text) async => (await this).sendMessage(text); + + Future addReaction(ReactionType type, {int messageIndex = 0}) async => + (await this).addReaction(type, messageIndex: messageIndex); + + Future deleteReaction(ReactionType type, {int messageIndex = 0}) async => + (await this).deleteReaction(type, messageIndex: messageIndex); } diff --git a/sample_app/integration_test/robots/user_robot_message_list_asserts.dart b/sample_app/integration_test/robots/user_robot_message_list_asserts.dart index fb1fe58cf..421f279fd 100644 --- a/sample_app/integration_test/robots/user_robot_message_list_asserts.dart +++ b/sample_app/integration_test/robots/user_robot_message_list_asserts.dart @@ -1,5 +1,6 @@ import 'package:flutter_test/flutter_test.dart'; +import '../mock_server/data_types.dart'; import '../support/widget_test_extensions.dart'; import 'user_robot.dart'; @@ -8,4 +9,29 @@ extension UserRobotMessageListAsserts on UserRobot { await tester.waitUntilVisible(find.text(text)); return this; } + + Future assertReaction({ + required ReactionType type, + required bool isDisplayed, + }) async { + // Reaction display chips carry no keys, so they're located by emoji glyph. + final reaction = find.text(type.emoji); + if (isDisplayed) { + await tester.waitUntilVisible(reaction); + } else { + await tester.waitUntilNotVisible(reaction); + } + return this; + } +} + +/// Chainable counterparts to [UserRobotMessageListAsserts], so an assertion can +/// follow a fluent action chain (`userRobot.addReaction(x).assertReaction(...)`). +extension UserRobotMessageListAssertsChain on Future { + Future assertMessage(String text) async => (await this).assertMessage(text); + + Future assertReaction({ + required ReactionType type, + required bool isDisplayed, + }) async => (await this).assertReaction(type: type, isDisplayed: isDisplayed); } diff --git a/sample_app/integration_test/support/step.dart b/sample_app/integration_test/support/step.dart index 090299b9f..5e692e667 100644 --- a/sample_app/integration_test/support/step.dart +++ b/sample_app/integration_test/support/step.dart @@ -1,3 +1,10 @@ import '../allure/allure.dart'; -Future step(String description, Future Function() body) => Allure.instance.step(description, body); +/// Records a BDD step marker in the Allure report. Call it on its own line; the +/// statements that follow (up to the next [step]) are the step's body: +/// +/// ```dart +/// step('GIVEN user opens the channel'); +/// await env.userRobot.login().openChannel(); +/// ``` +void step(String description) => Allure.instance.beginStep(description); diff --git a/sample_app/integration_test/support/stream_test_env.dart b/sample_app/integration_test/support/stream_test_env.dart index 765a56c10..70e459bbb 100644 --- a/sample_app/integration_test/support/stream_test_env.dart +++ b/sample_app/integration_test/support/stream_test_env.dart @@ -1,6 +1,9 @@ +import 'dart:async'; + import 'package:flutter_test/flutter_test.dart'; import 'package:sample_app/app.dart'; import 'package:sample_app/auth/auth_controller.dart'; +import 'package:stream_chat_flutter/stream_chat_flutter.dart'; import '../mock_server/mock_server.dart'; import '../robots/backend_robot.dart'; @@ -13,26 +16,76 @@ class StreamTestEnv { late final BackendRobot backendRobot; late final ParticipantRobot participantRobot; late final UserRobot userRobot; + late final WidgetTester _tester; + + // Drives the app's connectivity in place of the real `connectivity_plus` + // monitor, so tests can toggle offline/online deterministically. + final _connectivity = StreamController>.broadcast(); + var _connectivityPrimed = false; Future setUp(WidgetTester tester) async { + _tester = tester; final server = _mockServer = await MockServer.start(); backendRobot = BackendRobot(server); participantRobot = ParticipantRobot(server); userRobot = UserRobot(tester); - authController.debugConnectionOverride = StreamConnectionOverride( - baseURL: server.url, - baseWsUrl: server.wsUrl, - ); + authController + ..debugConnectionOverride = StreamConnectionOverride( + baseURL: server.url, + baseWsUrl: server.wsUrl, + ) + ..debugConnectivityStream = _connectivity.stream; await tester.pumpWidget(const StreamChatSampleApp()); await tester.pumpAndSettle(); } + /// Simulates a full network outage: the SDK's HTTP requests all fail and the + /// WebSocket is closed. Mirrors the native `disableInternetConnection` / + /// `setConnectivity(.off)`. Missed server-side events are recovered on + /// [goOnline]. + Future goOffline() => _setConnectivity(online: false); + + /// Restores the network: HTTP works again and the SDK reconnects + recovers. + Future goOnline() => _setConnectivity(online: true); + + Future _setConnectivity({required bool online}) async { + // Gate HTTP first so it matches the connectivity state before/through the + // transition: block before closing the WS going offline; unblock before + // reconnecting so recovery (sync/queryChannels) can reach the network. + authController.debugForceOffline = !online; + + // StreamChatCore skips the first connectivity event (it races the initial + // connectUser), so prime the stream with a throwaway before the first real + // toggle. The debounce collapses the primer and the real value together. + if (!_connectivityPrimed) { + _connectivity.add([ConnectivityResult.wifi]); + _connectivityPrimed = true; + } + _connectivity.add([if (online) ConnectivityResult.wifi else ConnectivityResult.none]); + await _waitForConnection(online ? ConnectionStatus.connected : ConnectionStatus.disconnected); + } + + Future _waitForConnection( + ConnectionStatus status, { + // Generous: StreamChatCore debounces connectivity changes ~3s before + // acting, then the WebSocket (re)connects. + Duration timeout = const Duration(seconds: 30), + }) async { + final end = DateTime.now().add(timeout); + while (DateTime.now().isBefore(end)) { + await _tester.pump(const Duration(milliseconds: 200)); + if (authController.client?.wsConnectionStatus == status) return; + } + throw TestFailure('Timed out waiting for connection status: $status'); + } + Future tearDown() async { try { await authController.debugReset(); } finally { + await _connectivity.close(); // Null when MockServer.start() itself failed during setUp. await _mockServer?.stop(); } diff --git a/sample_app/integration_test/support/widget_test_extensions.dart b/sample_app/integration_test/support/widget_test_extensions.dart index d7a518ec1..18295e968 100644 --- a/sample_app/integration_test/support/widget_test_extensions.dart +++ b/sample_app/integration_test/support/widget_test_extensions.dart @@ -52,4 +52,40 @@ extension E2EWidgetTester on WidgetTester { } throw TestFailure('Timed out waiting for $finder'); } + + Future waitUntilNotVisible( + Finder finder, { + Duration timeout = const Duration(seconds: 30), + }) async { + final end = DateTime.now().add(timeout); + while (DateTime.now().isBefore(end)) { + await pump(const Duration(milliseconds: 100)); + if (finder.evaluate().isEmpty) { + await pumpAndSettle(); + return; + } + } + throw TestFailure('Timed out waiting for $finder to disappear'); + } + + /// Long-presses [target] until [appears] shows up. + /// + /// A message's long-press handler is disabled while it is still being sent, + /// so a single long-press can be a no-op; retrying until the reaction + /// picker (or any expected widget) appears makes the gesture reliable. + Future longPressUntilVisible( + Finder target, + Finder appears, { + Duration timeout = const Duration(seconds: 30), + }) async { + await waitUntilVisible(target); + final end = DateTime.now().add(timeout); + while (DateTime.now().isBefore(end)) { + await longPress(target); + await pumpAndSettle(); + if (appears.evaluate().isNotEmpty) return; + await pump(const Duration(milliseconds: 250)); + } + throw TestFailure('Timed out long-pressing $target waiting for $appears'); + } } diff --git a/sample_app/lib/app.dart b/sample_app/lib/app.dart index 72ed1cc7a..091b3e64c 100644 --- a/sample_app/lib/app.dart +++ b/sample_app/lib/app.dart @@ -203,6 +203,9 @@ class _StreamChatSampleAppState extends State if (client != null) { return StreamChat( client: client, + // Null in production → the SDK uses the real + // connectivity monitor; set only by e2e tests. + connectivityStream: authController.debugConnectivityStream, componentBuilders: StreamComponentBuilders( extensions: streamChatComponentBuilders( messageItem: customMessageItemBuilder, diff --git a/sample_app/lib/auth/auth_controller.dart b/sample_app/lib/auth/auth_controller.dart index 606fe912a..5cb23f10f 100644 --- a/sample_app/lib/auth/auth_controller.dart +++ b/sample_app/lib/auth/auth_controller.dart @@ -68,9 +68,29 @@ StreamChatClient _buildStreamChatClient( ), baseURL: connectionOverride?.baseURL, baseWsUrl: connectionOverride?.baseWsUrl, + // e2e only: lets the harness simulate a full network outage by failing + // every HTTP request (paired with the WebSocket close from + // debugConnectivityStream). See `AuthController.debugForceOffline`. + chatApiInterceptors: connectionOverride != null ? [_offlineSimulationInterceptor] : null, )..chatPersistenceClient = _chatPersistenceClient; } +/// Rejects every Stream API request with a connection error while +/// [AuthController.debugForceOffline] is set, mimicking a device that has lost +/// its network — the HTTP half of the e2e offline switch. +final _offlineSimulationInterceptor = InterceptorsWrapper( + onRequest: (options, handler) { + if (!authController.debugForceOffline) return handler.next(options); + return handler.reject( + DioException( + requestOptions: options, + type: DioExceptionType.connectionError, + error: 'e2e: simulated offline', + ), + ); + }, +); + /// Authentication state exposed by [AuthController]. sealed class AuthState { const AuthState(); @@ -111,6 +131,21 @@ class AuthController extends ValueNotifier { @visibleForTesting StreamConnectionOverride? debugConnectionOverride; + /// Test-controlled connectivity source fed to the `StreamChat` widget. + /// + /// When set, it replaces the real `connectivity_plus` monitor so e2e tests + /// can deterministically drive the client offline/online (see the e2e + /// harness's `goOffline`/`goOnline`). Null in production (read by `app.dart`, + /// so it can't be `@visibleForTesting` like [debugConnectionOverride]). + Stream>? debugConnectivityStream; + + /// e2e only: when true, every Stream API HTTP request fails with a simulated + /// connection error (see [_offlineSimulationInterceptor]). Paired with the + /// WebSocket close driven by [debugConnectivityStream] to simulate a full + /// network outage. Toggled by the harness's `goOffline`/`goOnline`. + @visibleForTesting + bool debugForceOffline = false; + String? _activeApiKey; PushTokenManager? _pushTokenManager; @@ -226,6 +261,8 @@ class AuthController extends ValueNotifier { _client = null; _activeApiKey = null; debugConnectionOverride = null; + debugConnectivityStream = null; + debugForceOffline = false; if (!CurrentPlatform.isWeb) { const secureStorage = FlutterSecureStorage(); From f0d71fe318bc7239fa9571ea0ed5d077e4949625 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 16:57:04 +0100 Subject: [PATCH 02/15] Fix test --- sample_app/integration_test/reactions_test.dart | 16 ++++++++++------ .../support/widget_test_extensions.dart | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/sample_app/integration_test/reactions_test.dart b/sample_app/integration_test/reactions_test.dart index d228eccc6..6dc4d497b 100644 --- a/sample_app/integration_test/reactions_test.dart +++ b/sample_app/integration_test/reactions_test.dart @@ -9,6 +9,7 @@ void main() { const sampleText = 'Test'; streamTestWithEnv( + allureId: '11293', description: 'user adds a reaction to their own message', body: (env) async { step('GIVEN user opens the channel'); @@ -26,6 +27,7 @@ void main() { ); streamTestWithEnv( + allureId: '11288', description: 'user deletes a reaction from their own message', body: (env) async { step('GIVEN user opens the channel'); @@ -46,6 +48,7 @@ void main() { ); streamTestWithEnv( + allureId: '11292', description: 'user adds a reaction to the participant message', body: (env) async { step('GIVEN user opens the channel'); @@ -63,6 +66,7 @@ void main() { ); streamTestWithEnv( + allureId: '11290', description: 'user removes a reaction from the participant message', body: (env) async { step('GIVEN user opens the channel'); @@ -83,6 +87,7 @@ void main() { ); streamTestWithEnv( + allureId: '11294', description: 'participant adds a reaction to the user message', body: (env) async { step('GIVEN user opens the channel'); @@ -98,6 +103,7 @@ void main() { ); streamTestWithEnv( + allureId: '11296', description: 'participant removes a reaction from the user message', body: (env) async { step('GIVEN user opens the channel'); @@ -117,6 +123,7 @@ void main() { ); streamTestWithEnv( + allureId: '11291', description: 'participant adds a reaction to their own message', body: (env) async { step('GIVEN user opens the channel'); @@ -134,6 +141,7 @@ void main() { ); streamTestWithEnv( + allureId: '11295', description: 'participant removes a reaction from their own message', body: (env) async { step('GIVEN user opens the channel'); @@ -154,13 +162,8 @@ void main() { }, ); - // EXPECTED TO FAIL on Flutter: unlike iOS/Android, the Dart SDK does not - // queue offline reactions — `channel.sendReaction` rolls back its optimistic - // update when the HTTP request fails (channel.dart), and the reaction is - // never replayed on reconnect (the RetryQueue is messages-only). Kept as a - // faithful port of the native case so the gap stays visible; it will pass - // once the SDK gives reactions the same offline-queue treatment as messages. streamTestWithEnv( + allureId: '11287', description: 'user adds a reaction while offline', body: (env) async { step('GIVEN user opens the channel'); @@ -187,6 +190,7 @@ void main() { ); streamTestWithEnv( + allureId: '11289', description: 'participant adds a reaction while the user is offline', body: (env) async { step('GIVEN user opens the channel'); diff --git a/sample_app/integration_test/support/widget_test_extensions.dart b/sample_app/integration_test/support/widget_test_extensions.dart index 18295e968..b7fe0a99d 100644 --- a/sample_app/integration_test/support/widget_test_extensions.dart +++ b/sample_app/integration_test/support/widget_test_extensions.dart @@ -78,13 +78,23 @@ extension E2EWidgetTester on WidgetTester { Finder appears, { Duration timeout = const Duration(seconds: 30), }) async { - await waitUntilVisible(target); final end = DateTime.now().add(timeout); while (DateTime.now().isBefore(end)) { + await pump(const Duration(milliseconds: 100)); + if (appears.evaluate().isNotEmpty) return; + + bool resolves; + try { + resolves = target.evaluate().isNotEmpty; + } catch (_) { + resolves = false; + } + if (!resolves) continue; + await longPress(target); await pumpAndSettle(); if (appears.evaluate().isNotEmpty) return; - await pump(const Duration(milliseconds: 250)); + await pump(const Duration(milliseconds: 150)); } throw TestFailure('Timed out long-pressing $target waiting for $appears'); } From caa944500b30b218909bae2037f81ec0189e2cd4 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 16:57:07 +0100 Subject: [PATCH 03/15] Disable ios --- .github/workflows/e2e_test.yml | 112 ++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 41e119955..139501d46 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -72,59 +72,59 @@ jobs: sample_app/stream-chat-test-mock-server/logs sample_app/build/e2e-test.log - ios: - runs-on: macos-15 - timeout-minutes: 40 - env: - IOS_SIMULATOR_VERSION: "26.2" - IOS_SIMULATOR_DEVICE: "iPhone 17" - steps: - - uses: actions/checkout@v6 - - - uses: ./.github/actions/setup-flutter - - - uses: ./.github/actions/setup-ruby - - - uses: ./.github/actions/cache-cocoapods - with: - key-suffix: ${{ runner.os }} - - - uses: ./.github/actions/cache-xcode-cas - with: - key-suffix: ${{ runner.os }} - xcode-version: ${{ env.IOS_SIMULATOR_VERSION }} - - - name: Bootstrap - run: | - flutter precache --ios - flutter config --no-enable-swift-package-manager - flutter pub global activate melos - melos bootstrap - - - name: Launch Allure TestOps - working-directory: sample_app/ios - run: bundle exec fastlane allure_launch - env: - GITHUB_EVENT: ${{ toJson(github.event) }} - - - name: Run e2e (iOS simulator) - working-directory: sample_app/ios - run: bundle exec fastlane run_e2e_test platform:ios ios:"${{ env.IOS_SIMULATOR_VERSION }}" device:"${{ env.IOS_SIMULATOR_DEVICE }}" - - - name: Upload Allure results - if: env.LAUNCH_ID != '' && (success() || failure()) - working-directory: sample_app/ios - run: bundle exec fastlane allure_upload launch_id:$LAUNCH_ID - - - name: Allure TestOps Launch Removal - if: env.LAUNCH_ID != '' && cancelled() - working-directory: sample_app/ios - run: bundle exec fastlane allure_launch_removal launch_id:$LAUNCH_ID - - - uses: actions/upload-artifact@v7 - if: failure() - with: - name: e2e-ios-logs - path: | - sample_app/stream-chat-test-mock-server/logs - sample_app/build/e2e-test.log + # ios: + # runs-on: macos-15 + # timeout-minutes: 40 + # env: + # IOS_SIMULATOR_VERSION: "26.2" + # IOS_SIMULATOR_DEVICE: "iPhone 17" + # steps: + # - uses: actions/checkout@v6 + + # - uses: ./.github/actions/setup-flutter + + # - uses: ./.github/actions/setup-ruby + + # - uses: ./.github/actions/cache-cocoapods + # with: + # key-suffix: ${{ runner.os }} + + # - uses: ./.github/actions/cache-xcode-cas + # with: + # key-suffix: ${{ runner.os }} + # xcode-version: ${{ env.IOS_SIMULATOR_VERSION }} + + # - name: Bootstrap + # run: | + # flutter precache --ios + # flutter config --no-enable-swift-package-manager + # flutter pub global activate melos + # melos bootstrap + + # - name: Launch Allure TestOps + # working-directory: sample_app/ios + # run: bundle exec fastlane allure_launch + # env: + # GITHUB_EVENT: ${{ toJson(github.event) }} + + # - name: Run e2e (iOS simulator) + # working-directory: sample_app/ios + # run: bundle exec fastlane run_e2e_test platform:ios ios:"${{ env.IOS_SIMULATOR_VERSION }}" device:"${{ env.IOS_SIMULATOR_DEVICE }}" + + # - name: Upload Allure results + # if: env.LAUNCH_ID != '' && (success() || failure()) + # working-directory: sample_app/ios + # run: bundle exec fastlane allure_upload launch_id:$LAUNCH_ID + + # - name: Allure TestOps Launch Removal + # if: env.LAUNCH_ID != '' && cancelled() + # working-directory: sample_app/ios + # run: bundle exec fastlane allure_launch_removal launch_id:$LAUNCH_ID + + # - uses: actions/upload-artifact@v7 + # if: failure() + # with: + # name: e2e-ios-logs + # path: | + # sample_app/stream-chat-test-mock-server/logs + # sample_app/build/e2e-test.log From 8935c5b7c4f8dcae8290e478c25c892046f65ac6 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 17:23:31 +0100 Subject: [PATCH 04/15] test(sample): attach screenshot, hierarchy, log & video to failed e2e tests --- sample_app/fastlane/Allurefile | 16 +++- sample_app/fastlane/Fastfile | 81 ++++++++++++++++++- .../integration_test/allure/allure.dart | 58 +++++++++++-- .../support/failure_artifacts.dart | 60 ++++++++++++++ .../support/stream_test_case.dart | 39 ++++++++- 5 files changed, 243 insertions(+), 11 deletions(-) create mode 100644 sample_app/integration_test/support/failure_artifacts.dart diff --git a/sample_app/fastlane/Allurefile b/sample_app/fastlane/Allurefile index d62e395e5..382697f43 100644 --- a/sample_app/fastlane/Allurefile +++ b/sample_app/fastlane/Allurefile @@ -25,7 +25,21 @@ lane :collect_allure_results do |options| base64 = parts.sort_by(&:first).map(&:last).join File.write("#{results_dir}/#{uuid}-result.json", Base64.decode64(base64)) end - UI.message("Collected #{chunks.size} Allure result(s) into #{results_dir}") + + # Attachments (screenshot / widget hierarchy / log) ride the same log-marker + # transport, keyed by their `-attachment.` source filename; the + # result JSON references them by that name. + attachments = Hash.new { |hash, key| hash[key] = {} } + log.scan(%r{ALLURE-ATTACH::([0-9a-f-]+-attachment\.[a-z0-9]+):(\d+):([A-Za-z0-9+/=]+)}) do |source, seq, chunk| + attachments[source][seq.to_i] = chunk + end + + attachments.each do |source, parts| + base64 = parts.sort_by(&:first).map(&:last).join + File.binwrite("#{results_dir}/#{source}", Base64.decode64(base64)) + end + + UI.message("Collected #{chunks.size} Allure result(s) and #{attachments.size} attachment(s) into #{results_dir}") end lane :install_allurectl do diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index adf4c8ae8..b3f59c32e 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -2,6 +2,9 @@ opt_out_usage require 'net/http' require 'shellwords' +require 'json' +require 'securerandom' +require 'fileutils' import File.expand_path('Allurefile', __dir__) mock_server_repo_name = 'stream-chat-test-mock-server' @@ -17,6 +20,72 @@ def ios_device?(device) device.to_s.match?(/\A[0-9A-F]{8}-[0-9A-F]{4}-/i) end +# Starts a best-effort screen recording of the device/simulator. Returns a +# handle passed back to `stop_screen_recording`, or nil if recording couldn't +# start (recording never fails the test run). +def start_screen_recording(ios:, device:, path:) + FileUtils.mkdir_p(File.dirname(path)) + if ios + # `simctl io recordVideo` records until interrupted; run it detached and + # keep the pid so we can SIGINT it (which finalizes the .mp4). + pid = spawn('xcrun', 'simctl', 'io', device, 'recordVideo', '--codec=h264', '--force', path, + out: File::NULL, err: File::NULL) + { pid: pid, ios: true, path: path } + else + # `screenrecord` writes on-device (max ~180s), pulled after stop. + serial = device ? "-s #{device} " : '' + remote = "/sdcard/#{File.basename(path)}" + pid = spawn("adb #{serial}shell screenrecord --bit-rate 4000000 #{remote}", + out: File::NULL, err: File::NULL) + { pid: pid, ios: false, path: path, remote: remote, serial: serial } + end +rescue StandardError => e + UI.important("Screen recording failed to start: #{e.message}") + nil +end + +def stop_screen_recording(rec) + return unless rec + + Process.kill('INT', rec[:pid]) + Process.wait(rec[:pid]) + unless rec[:ios] + sleep(1) # let screenrecord flush the file before pulling + sh("adb #{rec[:serial]}pull #{rec[:remote]} #{rec[:path]}", log: false) + sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) + end +rescue StandardError => e + UI.important("Screen recording failed to stop: #{e.message}") +end + +# Attaches each recorded video to the failed/broken results of the file it came +# from, matched via the `testFile` label the Dart reporter sets. One video is +# copied once and shared by every failed test in that file. +def attach_videos_to_failed_results(results_dir:, videos_dir:) + return unless Dir.exist?(results_dir) + + sources = {} + Dir["#{results_dir}/*-result.json"].each do |json_path| + result = JSON.parse(File.read(json_path)) + next unless %w[failed broken].include?(result['status']) + + label = (result['labels'] || []).find { |l| l['name'] == 'testFile' } + next unless label + + video = "#{videos_dir}/#{label['value']}.mp4" + next unless File.exist?(video) && File.size(video).positive? + + source = sources[video] ||= begin + name = "#{SecureRandom.uuid}-attachment.mp4" + FileUtils.cp(video, "#{results_dir}/#{name}") + name + end + + (result['attachments'] ||= []) << { 'name' => 'Video', 'source' => source, 'type' => 'video/mp4' } + File.write(json_path, JSON.generate(result)) + end +end + lane :sh_on_root do |options| Dir.chdir(root_path) { sh(options[:command]) } end @@ -88,6 +157,10 @@ lane :run_e2e_test do |options| FileUtils.mkdir_p(File.dirname(log_file)) File.write(log_file, '') + videos_dir = "#{root_path}/build/e2e-videos" + FileUtils.rm_rf(videos_dir) + FileUtils.mkdir_p(videos_dir) + begin Dir.chdir(root_path) do targets = options[:target] ? [options[:target]] : Dir['integration_test/*_test.dart'].sort @@ -95,7 +168,10 @@ lane :run_e2e_test do |options| failed = [] targets.each do |target| - flags = [target] + # Basename ties the recording to the file's failed results via the + # `testFile` label the Dart reporter sets from this dart-define. + test_name = File.basename(target, '.dart') + flags = [target, "--dart-define=E2E_TARGET=#{test_name}"] flags << "-d #{device}" if device if options[:verbose] flags << '--verbose' @@ -104,10 +180,12 @@ lane :run_e2e_test do |options| flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}" end + rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4") ok = sh_with_retries(retries: options[:retries] || 2, label: target) do cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)}" sh("bash -o pipefail -c #{Shellwords.escape(cmd)}") end + stop_screen_recording(rec) failed << target unless ok end @@ -115,6 +193,7 @@ lane :run_e2e_test do |options| end ensure collect_allure_results(log_file: log_file) rescue nil + attach_videos_to_failed_results(results_dir: allure_results_path, videos_dir: videos_dir) rescue nil stop_mock_server end end diff --git a/sample_app/integration_test/allure/allure.dart b/sample_app/integration_test/allure/allure.dart index debdd5ef2..261d9446f 100644 --- a/sample_app/integration_test/allure/allure.dart +++ b/sample_app/integration_test/allure/allure.dart @@ -4,10 +4,21 @@ import 'dart:math'; import 'package:uuid/uuid.dart'; const allureResultMarker = 'ALLURE-RESULT::'; +const allureAttachmentMarker = 'ALLURE-ATTACH::'; const _chunkSize = 900; enum AllureStatus { passed, failed, broken, skipped } +class _Attachment { + _Attachment({required this.name, required this.source, required this.type}); + + final String name; + final String source; + final String type; + + Map toJson() => {'name': name, 'source': source, 'type': type}; +} + class _Step { _Step(this.name, this.start); @@ -47,6 +58,7 @@ class _Result { Map? statusDetails; final List> labels; final List<_Step> steps = []; + final List<_Attachment> attachments = []; Map toJson() => { 'uuid': uuid, @@ -60,6 +72,7 @@ class _Result { 'stop': stop, 'labels': labels, 'steps': [for (final s in steps) s.toJson()], + 'attachments': [for (final a in attachments) a.toJson()], }; } @@ -120,6 +133,45 @@ class Allure { _openStep = null; } + /// Attaches a binary artifact (e.g. a screenshot) to the current test. + /// + /// The bytes are streamed to the device log as chunked + /// `ALLURE-ATTACH::::` markers (the same transport as + /// results); `collect_allure_results` reassembles them into + /// `allure-results/` and this entry references it. Register before + /// [stopTest] so the reference is included in the emitted result JSON. + void attachBytes( + String name, + List bytes, { + required String type, + required String ext, + }) { + final result = _result; + if (result == null || bytes.isEmpty) return; + final source = '${const Uuid().v4()}-attachment.$ext'; + result.attachments.add(_Attachment(name: name, source: source, type: type)); + _emitChunked(allureAttachmentMarker, source, base64.encode(bytes)); + } + + /// Attaches a text artifact (e.g. the widget hierarchy or a log) to the + /// current test. See [attachBytes]. + void attachText( + String name, + String content, { + String type = 'text/plain', + String ext = 'txt', + }) { + if (content.isEmpty) return; + attachBytes(name, utf8.encode(content), type: type, ext: ext); + } + + void _emitChunked(String marker, String key, String encoded) { + for (var i = 0, seq = 0; i < encoded.length; i += _chunkSize, seq++) { + final chunk = encoded.substring(i, min(i + _chunkSize, encoded.length)); + print('$marker$key:$seq:$chunk'); + } + } + void stopTest({ required AllureStatus status, Object? message, @@ -144,11 +196,7 @@ class Allure { }; } - final encoded = base64.encode(utf8.encode(jsonEncode(result.toJson()))); - for (var i = 0, seq = 0; i < encoded.length; i += _chunkSize, seq++) { - final chunk = encoded.substring(i, min(i + _chunkSize, encoded.length)); - print('$allureResultMarker${result.uuid}:$seq:$chunk'); - } + _emitChunked(allureResultMarker, result.uuid, base64.encode(utf8.encode(jsonEncode(result.toJson())))); _result = null; } } diff --git a/sample_app/integration_test/support/failure_artifacts.dart b/sample_app/integration_test/support/failure_artifacts.dart new file mode 100644 index 000000000..374f3f4ab --- /dev/null +++ b/sample_app/integration_test/support/failure_artifacts.dart @@ -0,0 +1,60 @@ +import 'dart:ui' as ui; + +import 'package:flutter/rendering.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import '../allure/allure.dart'; + +/// Captures debug artifacts from the running app and attaches them to the +/// current Allure test. Called on failure, before the result is emitted. +/// +/// Screenshot and widget hierarchy are grabbed in-process from the live render +/// tree; [log] is the output captured during the test body. Video is recorded +/// host-side by the Fastlane lane (it can't be captured in-process). Every +/// capture is best-effort — a failure to grab one artifact never masks the +/// original test failure. +Future captureFailureArtifacts(WidgetTester tester, String log) async { + await _attachScreenshot(tester); + _attachWidgetHierarchy(); + Allure.instance.attachText('Log', log, ext: 'log'); +} + +Future _attachScreenshot(WidgetTester tester) async { + try { + final view = tester.binding.renderViews.first; + final layer = view.debugLayer; + if (layer is! OffsetLayer) return; + + // Capped at 2.0 rather than the full device pixel ratio: the PNG is + // streamed to the device log as base64 chunks, and a huge image risks + // logcat dropping lines (which would corrupt the reassembled file). 2.0 + // stays readable while keeping the volume manageable. + final pixelRatio = view.flutterView.devicePixelRatio.clamp(1.0, 2.0); + final image = await layer.toImage(view.paintBounds, pixelRatio: pixelRatio); + try { + final data = await image.toByteData(format: ui.ImageByteFormat.png); + if (data == null) return; + Allure.instance.attachBytes( + 'Screenshot', + data.buffer.asUint8List(), + type: 'image/png', + ext: 'png', + ); + } finally { + image.dispose(); + } + } catch (_) { + // Best-effort: never let a capture failure hide the real one. + } +} + +void _attachWidgetHierarchy() { + try { + final tree = WidgetsBinding.instance.rootElement?.toStringDeep(); + if (tree == null) return; + Allure.instance.attachText('Widget hierarchy', tree, ext: 'txt'); + } catch (_) { + // Best-effort. + } +} diff --git a/sample_app/integration_test/support/stream_test_case.dart b/sample_app/integration_test/support/stream_test_case.dart index 97a83f6d2..4574bfefd 100644 --- a/sample_app/integration_test/support/stream_test_case.dart +++ b/sample_app/integration_test/support/stream_test_case.dart @@ -1,3 +1,5 @@ +import 'dart:async'; + import 'package:flutter_test/flutter_test.dart'; import 'package:integration_test/integration_test.dart'; @@ -5,8 +7,14 @@ import 'package:integration_test/integration_test.dart'; import 'package:test_api/src/backend/invoker.dart' show Invoker; import '../allure/allure.dart'; +import 'failure_artifacts.dart'; import 'stream_test_env.dart'; +/// The test file currently running, injected by the Fastlane lane +/// (`--dart-define=E2E_TARGET=...`). Lets the lane attach that file's host-side +/// video recording to its failed tests. Empty when run ad-hoc. +const _e2eTarget = String.fromEnvironment('E2E_TARGET'); + void streamTest({ String? allureId, required String description, @@ -18,17 +26,40 @@ void streamTest({ Allure.instance.startTest( name: description, fullName: Invoker.current?.liveTest.test.name ?? description, - labels: {if (allureId != null) 'AS_ID': allureId}, + labels: { + if (allureId != null) 'AS_ID': allureId, + if (_e2eTarget.isNotEmpty) 'testFile': _e2eTarget, + }, ); + + // Capture the test body's output so it can be attached on failure, while + // still forwarding each line to stdout. Result/attachment markers are + // printed outside this zone (in stopTest / captureFailureArtifacts), so + // they never end up in the captured log. + final log = StringBuffer(); + Future runBody() => runZoned( + () async { + await body(tester); + final pendingException = tester.binding.takeException(); + if (pendingException != null) throw pendingException; + }, + zoneSpecification: ZoneSpecification( + print: (self, parent, zone, line) { + log.writeln(line); + parent.print(zone, line); + }, + ), + ); + try { - await body(tester); - final pendingException = tester.binding.takeException(); - if (pendingException != null) throw pendingException; + await runBody(); Allure.instance.stopTest(status: AllureStatus.passed); } on TestFailure catch (e, st) { + await captureFailureArtifacts(tester, log.toString()); Allure.instance.stopTest(status: AllureStatus.failed, message: e, trace: st); rethrow; } catch (e, st) { + await captureFailureArtifacts(tester, log.toString()); Allure.instance.stopTest(status: AllureStatus.broken, message: e, trace: st); rethrow; } From 1c3f13ecfc1d64773b0dcac765e07de5dd454b50 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 17:43:57 +0100 Subject: [PATCH 05/15] test(sample): slice per-test video clips for failed e2e tests --- sample_app/fastlane/Fastfile | 59 ++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index b3f59c32e..da646e8e2 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -25,19 +25,23 @@ end # start (recording never fails the test run). def start_screen_recording(ios:, device:, path:) FileUtils.mkdir_p(File.dirname(path)) + # Wall-clock at recording start, to line the video timeline up with the + # per-test start/stop timestamps the Dart reporter records (same clock on the + # iOS simulator; host-synced on the Android emulator). + started_at_ms = (Time.now.to_f * 1000).round if ios # `simctl io recordVideo` records until interrupted; run it detached and # keep the pid so we can SIGINT it (which finalizes the .mp4). pid = spawn('xcrun', 'simctl', 'io', device, 'recordVideo', '--codec=h264', '--force', path, out: File::NULL, err: File::NULL) - { pid: pid, ios: true, path: path } + { pid: pid, ios: true, path: path, started_at_ms: started_at_ms } else # `screenrecord` writes on-device (max ~180s), pulled after stop. serial = device ? "-s #{device} " : '' remote = "/sdcard/#{File.basename(path)}" pid = spawn("adb #{serial}shell screenrecord --bit-rate 4000000 #{remote}", out: File::NULL, err: File::NULL) - { pid: pid, ios: false, path: path, remote: remote, serial: serial } + { pid: pid, ios: false, path: path, remote: remote, serial: serial, started_at_ms: started_at_ms } end rescue StandardError => e UI.important("Screen recording failed to start: #{e.message}") @@ -58,30 +62,43 @@ rescue StandardError => e UI.important("Screen recording failed to stop: #{e.message}") end -# Attaches each recorded video to the failed/broken results of the file it came -# from, matched via the `testFile` label the Dart reporter sets. One video is -# copied once and shared by every failed test in that file. -def attach_videos_to_failed_results(results_dir:, videos_dir:) +# Attaches a **per-test** video to each failed/broken result. Recording is +# per file (all its tests run in one `flutter test` process, so the host can't +# start/stop between them), so each test's clip is sliced out of its file's +# video using the result's own start/stop timestamps. +# +# `videos` maps a test-file basename => { path:, started_at_ms: }. +def attach_videos_to_failed_results(results_dir:, videos:) return unless Dir.exist?(results_dir) + return if videos.empty? + + unless system('command -v ffmpeg > /dev/null 2>&1') + UI.important('ffmpeg not found; skipping per-test video attachments') + return + end - sources = {} Dir["#{results_dir}/*-result.json"].each do |json_path| result = JSON.parse(File.read(json_path)) next unless %w[failed broken].include?(result['status']) + next unless result['start'] && result['stop'] label = (result['labels'] || []).find { |l| l['name'] == 'testFile' } - next unless label - - video = "#{videos_dir}/#{label['value']}.mp4" - next unless File.exist?(video) && File.size(video).positive? - - source = sources[video] ||= begin - name = "#{SecureRandom.uuid}-attachment.mp4" - FileUtils.cp(video, "#{results_dir}/#{name}") - name - end + meta = label && videos[label['value']] + next unless meta && File.exist?(meta[:path]) && File.size(meta[:path]).positive? + + # Slice [start, stop] out of the file's video, with 1s of lead/tail padding + # to absorb recording-start latency and minor clock skew. + ss = [((result['start'] - meta[:started_at_ms]) / 1000.0) - 1.0, 0].max + duration = ((result['stop'] - result['start']) / 1000.0) + 2.0 + next unless duration.positive? + + clip = "#{results_dir}/#{SecureRandom.uuid}-attachment.mp4" + ok = system('ffmpeg', '-y', '-ss', ss.to_s, '-i', meta[:path], '-t', duration.to_s, + '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-movflags', '+faststart', clip, + out: File::NULL, err: File::NULL) + next unless ok && File.exist?(clip) && File.size(clip).positive? - (result['attachments'] ||= []) << { 'name' => 'Video', 'source' => source, 'type' => 'video/mp4' } + (result['attachments'] ||= []) << { 'name' => 'Video', 'source' => File.basename(clip), 'type' => 'video/mp4' } File.write(json_path, JSON.generate(result)) end end @@ -160,6 +177,7 @@ lane :run_e2e_test do |options| videos_dir = "#{root_path}/build/e2e-videos" FileUtils.rm_rf(videos_dir) FileUtils.mkdir_p(videos_dir) + videos = {} # test-file basename => { path:, started_at_ms: } begin Dir.chdir(root_path) do @@ -168,7 +186,7 @@ lane :run_e2e_test do |options| failed = [] targets.each do |target| - # Basename ties the recording to the file's failed results via the + # Basename ties each test's video clip back to its result via the # `testFile` label the Dart reporter sets from this dart-define. test_name = File.basename(target, '.dart') flags = [target, "--dart-define=E2E_TARGET=#{test_name}"] @@ -186,6 +204,7 @@ lane :run_e2e_test do |options| sh("bash -o pipefail -c #{Shellwords.escape(cmd)}") end stop_screen_recording(rec) + videos[test_name] = { path: rec[:path], started_at_ms: rec[:started_at_ms] } if rec failed << target unless ok end @@ -193,7 +212,7 @@ lane :run_e2e_test do |options| end ensure collect_allure_results(log_file: log_file) rescue nil - attach_videos_to_failed_results(results_dir: allure_results_path, videos_dir: videos_dir) rescue nil + attach_videos_to_failed_results(results_dir: allure_results_path, videos: videos) rescue nil stop_mock_server end end From 7c66922cba84806836e20ad79d3506019a6fa1cb Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 17:48:47 +0100 Subject: [PATCH 06/15] fix(sample): disable offline persistence under e2e --- sample_app/lib/auth/auth_controller.dart | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sample_app/lib/auth/auth_controller.dart b/sample_app/lib/auth/auth_controller.dart index 5cb23f10f..e18e84393 100644 --- a/sample_app/lib/auth/auth_controller.dart +++ b/sample_app/lib/auth/auth_controller.dart @@ -72,7 +72,13 @@ StreamChatClient _buildStreamChatClient( // every HTTP request (paired with the WebSocket close from // debugConnectivityStream). See `AuthController.debugForceOffline`. chatApiInterceptors: connectionOverride != null ? [_offlineSimulationInterceptor] : null, - )..chatPersistenceClient = _chatPersistenceClient; + ) + // No offline persistence under e2e. The persistence client is a shared, + // never-cleared SQLite DB, so state leaks across the tests in a bundle and + // a reaction event referencing an unpersisted message throws an async + // FOREIGN KEY error — which, being async, poisons the shared test binding + // and makes every remaining test in the file fail before it can report. + ..chatPersistenceClient = connectionOverride != null ? null : _chatPersistenceClient; } /// Rejects every Stream API request with a connection error while From c30861262df83f542dfa17eb714146c918c4fa50 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 18:04:59 +0100 Subject: [PATCH 07/15] Remove video recording --- sample_app/fastlane/Fastfile | 100 +----------------- .../support/failure_artifacts.dart | 7 +- .../support/stream_test_case.dart | 10 +- 3 files changed, 5 insertions(+), 112 deletions(-) diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index da646e8e2..adf4c8ae8 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -2,9 +2,6 @@ opt_out_usage require 'net/http' require 'shellwords' -require 'json' -require 'securerandom' -require 'fileutils' import File.expand_path('Allurefile', __dir__) mock_server_repo_name = 'stream-chat-test-mock-server' @@ -20,89 +17,6 @@ def ios_device?(device) device.to_s.match?(/\A[0-9A-F]{8}-[0-9A-F]{4}-/i) end -# Starts a best-effort screen recording of the device/simulator. Returns a -# handle passed back to `stop_screen_recording`, or nil if recording couldn't -# start (recording never fails the test run). -def start_screen_recording(ios:, device:, path:) - FileUtils.mkdir_p(File.dirname(path)) - # Wall-clock at recording start, to line the video timeline up with the - # per-test start/stop timestamps the Dart reporter records (same clock on the - # iOS simulator; host-synced on the Android emulator). - started_at_ms = (Time.now.to_f * 1000).round - if ios - # `simctl io recordVideo` records until interrupted; run it detached and - # keep the pid so we can SIGINT it (which finalizes the .mp4). - pid = spawn('xcrun', 'simctl', 'io', device, 'recordVideo', '--codec=h264', '--force', path, - out: File::NULL, err: File::NULL) - { pid: pid, ios: true, path: path, started_at_ms: started_at_ms } - else - # `screenrecord` writes on-device (max ~180s), pulled after stop. - serial = device ? "-s #{device} " : '' - remote = "/sdcard/#{File.basename(path)}" - pid = spawn("adb #{serial}shell screenrecord --bit-rate 4000000 #{remote}", - out: File::NULL, err: File::NULL) - { pid: pid, ios: false, path: path, remote: remote, serial: serial, started_at_ms: started_at_ms } - end -rescue StandardError => e - UI.important("Screen recording failed to start: #{e.message}") - nil -end - -def stop_screen_recording(rec) - return unless rec - - Process.kill('INT', rec[:pid]) - Process.wait(rec[:pid]) - unless rec[:ios] - sleep(1) # let screenrecord flush the file before pulling - sh("adb #{rec[:serial]}pull #{rec[:remote]} #{rec[:path]}", log: false) - sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) - end -rescue StandardError => e - UI.important("Screen recording failed to stop: #{e.message}") -end - -# Attaches a **per-test** video to each failed/broken result. Recording is -# per file (all its tests run in one `flutter test` process, so the host can't -# start/stop between them), so each test's clip is sliced out of its file's -# video using the result's own start/stop timestamps. -# -# `videos` maps a test-file basename => { path:, started_at_ms: }. -def attach_videos_to_failed_results(results_dir:, videos:) - return unless Dir.exist?(results_dir) - return if videos.empty? - - unless system('command -v ffmpeg > /dev/null 2>&1') - UI.important('ffmpeg not found; skipping per-test video attachments') - return - end - - Dir["#{results_dir}/*-result.json"].each do |json_path| - result = JSON.parse(File.read(json_path)) - next unless %w[failed broken].include?(result['status']) - next unless result['start'] && result['stop'] - - label = (result['labels'] || []).find { |l| l['name'] == 'testFile' } - meta = label && videos[label['value']] - next unless meta && File.exist?(meta[:path]) && File.size(meta[:path]).positive? - - # Slice [start, stop] out of the file's video, with 1s of lead/tail padding - # to absorb recording-start latency and minor clock skew. - ss = [((result['start'] - meta[:started_at_ms]) / 1000.0) - 1.0, 0].max - duration = ((result['stop'] - result['start']) / 1000.0) + 2.0 - next unless duration.positive? - - clip = "#{results_dir}/#{SecureRandom.uuid}-attachment.mp4" - ok = system('ffmpeg', '-y', '-ss', ss.to_s, '-i', meta[:path], '-t', duration.to_s, - '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-movflags', '+faststart', clip, - out: File::NULL, err: File::NULL) - next unless ok && File.exist?(clip) && File.size(clip).positive? - - (result['attachments'] ||= []) << { 'name' => 'Video', 'source' => File.basename(clip), 'type' => 'video/mp4' } - File.write(json_path, JSON.generate(result)) - end -end - lane :sh_on_root do |options| Dir.chdir(root_path) { sh(options[:command]) } end @@ -174,11 +88,6 @@ lane :run_e2e_test do |options| FileUtils.mkdir_p(File.dirname(log_file)) File.write(log_file, '') - videos_dir = "#{root_path}/build/e2e-videos" - FileUtils.rm_rf(videos_dir) - FileUtils.mkdir_p(videos_dir) - videos = {} # test-file basename => { path:, started_at_ms: } - begin Dir.chdir(root_path) do targets = options[:target] ? [options[:target]] : Dir['integration_test/*_test.dart'].sort @@ -186,10 +95,7 @@ lane :run_e2e_test do |options| failed = [] targets.each do |target| - # Basename ties each test's video clip back to its result via the - # `testFile` label the Dart reporter sets from this dart-define. - test_name = File.basename(target, '.dart') - flags = [target, "--dart-define=E2E_TARGET=#{test_name}"] + flags = [target] flags << "-d #{device}" if device if options[:verbose] flags << '--verbose' @@ -198,13 +104,10 @@ lane :run_e2e_test do |options| flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}" end - rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4") ok = sh_with_retries(retries: options[:retries] || 2, label: target) do cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)}" sh("bash -o pipefail -c #{Shellwords.escape(cmd)}") end - stop_screen_recording(rec) - videos[test_name] = { path: rec[:path], started_at_ms: rec[:started_at_ms] } if rec failed << target unless ok end @@ -212,7 +115,6 @@ lane :run_e2e_test do |options| end ensure collect_allure_results(log_file: log_file) rescue nil - attach_videos_to_failed_results(results_dir: allure_results_path, videos: videos) rescue nil stop_mock_server end end diff --git a/sample_app/integration_test/support/failure_artifacts.dart b/sample_app/integration_test/support/failure_artifacts.dart index 374f3f4ab..26c8c0935 100644 --- a/sample_app/integration_test/support/failure_artifacts.dart +++ b/sample_app/integration_test/support/failure_artifacts.dart @@ -10,10 +10,9 @@ import '../allure/allure.dart'; /// current Allure test. Called on failure, before the result is emitted. /// /// Screenshot and widget hierarchy are grabbed in-process from the live render -/// tree; [log] is the output captured during the test body. Video is recorded -/// host-side by the Fastlane lane (it can't be captured in-process). Every -/// capture is best-effort — a failure to grab one artifact never masks the -/// original test failure. +/// tree; [log] is the output captured during the test body. Every capture is +/// best-effort — a failure to grab one artifact never masks the original test +/// failure. Future captureFailureArtifacts(WidgetTester tester, String log) async { await _attachScreenshot(tester); _attachWidgetHierarchy(); diff --git a/sample_app/integration_test/support/stream_test_case.dart b/sample_app/integration_test/support/stream_test_case.dart index 4574bfefd..b98cd9a92 100644 --- a/sample_app/integration_test/support/stream_test_case.dart +++ b/sample_app/integration_test/support/stream_test_case.dart @@ -10,11 +10,6 @@ import '../allure/allure.dart'; import 'failure_artifacts.dart'; import 'stream_test_env.dart'; -/// The test file currently running, injected by the Fastlane lane -/// (`--dart-define=E2E_TARGET=...`). Lets the lane attach that file's host-side -/// video recording to its failed tests. Empty when run ad-hoc. -const _e2eTarget = String.fromEnvironment('E2E_TARGET'); - void streamTest({ String? allureId, required String description, @@ -26,10 +21,7 @@ void streamTest({ Allure.instance.startTest( name: description, fullName: Invoker.current?.liveTest.test.name ?? description, - labels: { - if (allureId != null) 'AS_ID': allureId, - if (_e2eTarget.isNotEmpty) 'testFile': _e2eTarget, - }, + labels: {if (allureId != null) 'AS_ID': allureId}, ); // Capture the test body's output so it can be attached on failure, while From 54d3739fee173c6368bd28e0376e9acacab23f55 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 18:30:30 +0100 Subject: [PATCH 08/15] test(sample): bound e2e failure attachments to avoid log flooding --- .../support/failure_artifacts.dart | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/sample_app/integration_test/support/failure_artifacts.dart b/sample_app/integration_test/support/failure_artifacts.dart index 26c8c0935..893284d0c 100644 --- a/sample_app/integration_test/support/failure_artifacts.dart +++ b/sample_app/integration_test/support/failure_artifacts.dart @@ -6,6 +6,14 @@ import 'package:flutter_test/flutter_test.dart'; import '../allure/allure.dart'; +// Attachments are streamed to the device log as base64 chunks, so they must +// stay small — a full-resolution screenshot or a whole widget tree produces +// thousands of log lines per failure, floods CI, and risks logcat dropping +// chunks (which corrupts the reassembled file). These bounds keep each +// attachment to a few dozen lines while staying useful for triage. +const _maxScreenshotSide = 600.0; // longest edge, logical px * pixelRatio +const _maxTextChars = 20000; + /// Captures debug artifacts from the running app and attaches them to the /// current Allure test. Called on failure, before the result is emitted. /// @@ -16,7 +24,7 @@ import '../allure/allure.dart'; Future captureFailureArtifacts(WidgetTester tester, String log) async { await _attachScreenshot(tester); _attachWidgetHierarchy(); - Allure.instance.attachText('Log', log, ext: 'log'); + Allure.instance.attachText('Log', _cap(log), ext: 'log'); } Future _attachScreenshot(WidgetTester tester) async { @@ -25,11 +33,11 @@ Future _attachScreenshot(WidgetTester tester) async { final layer = view.debugLayer; if (layer is! OffsetLayer) return; - // Capped at 2.0 rather than the full device pixel ratio: the PNG is - // streamed to the device log as base64 chunks, and a huge image risks - // logcat dropping lines (which would corrupt the reassembled file). 2.0 - // stays readable while keeping the volume manageable. - final pixelRatio = view.flutterView.devicePixelRatio.clamp(1.0, 2.0); + // Downscale so the longest edge is ~_maxScreenshotSide px, regardless of + // device resolution — keeps the base64 payload small (see file header). + final longestSide = view.paintBounds.size.longestSide; + final pixelRatio = longestSide <= 0 ? 1.0 : (_maxScreenshotSide / longestSide).clamp(0.1, 1.5); + final image = await layer.toImage(view.paintBounds, pixelRatio: pixelRatio); try { final data = await image.toByteData(format: ui.ImageByteFormat.png); @@ -52,8 +60,14 @@ void _attachWidgetHierarchy() { try { final tree = WidgetsBinding.instance.rootElement?.toStringDeep(); if (tree == null) return; - Allure.instance.attachText('Widget hierarchy', tree, ext: 'txt'); + Allure.instance.attachText('Widget hierarchy', _cap(tree), ext: 'txt'); } catch (_) { // Best-effort. } } + +String _cap(String s) { + if (s.length <= _maxTextChars) return s; + final dropped = s.length - _maxTextChars; + return '${s.substring(0, _maxTextChars)}\n…[truncated $dropped chars]'; +} From 38e2941d76a2468ee7c2f65519ff186a1af9b1ce Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 18:40:10 +0100 Subject: [PATCH 09/15] Restore video tests --- sample_app/fastlane/Fastfile | 100 +++++++++++++++++- .../support/stream_test_case.dart | 10 +- 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index adf4c8ae8..da646e8e2 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -2,6 +2,9 @@ opt_out_usage require 'net/http' require 'shellwords' +require 'json' +require 'securerandom' +require 'fileutils' import File.expand_path('Allurefile', __dir__) mock_server_repo_name = 'stream-chat-test-mock-server' @@ -17,6 +20,89 @@ def ios_device?(device) device.to_s.match?(/\A[0-9A-F]{8}-[0-9A-F]{4}-/i) end +# Starts a best-effort screen recording of the device/simulator. Returns a +# handle passed back to `stop_screen_recording`, or nil if recording couldn't +# start (recording never fails the test run). +def start_screen_recording(ios:, device:, path:) + FileUtils.mkdir_p(File.dirname(path)) + # Wall-clock at recording start, to line the video timeline up with the + # per-test start/stop timestamps the Dart reporter records (same clock on the + # iOS simulator; host-synced on the Android emulator). + started_at_ms = (Time.now.to_f * 1000).round + if ios + # `simctl io recordVideo` records until interrupted; run it detached and + # keep the pid so we can SIGINT it (which finalizes the .mp4). + pid = spawn('xcrun', 'simctl', 'io', device, 'recordVideo', '--codec=h264', '--force', path, + out: File::NULL, err: File::NULL) + { pid: pid, ios: true, path: path, started_at_ms: started_at_ms } + else + # `screenrecord` writes on-device (max ~180s), pulled after stop. + serial = device ? "-s #{device} " : '' + remote = "/sdcard/#{File.basename(path)}" + pid = spawn("adb #{serial}shell screenrecord --bit-rate 4000000 #{remote}", + out: File::NULL, err: File::NULL) + { pid: pid, ios: false, path: path, remote: remote, serial: serial, started_at_ms: started_at_ms } + end +rescue StandardError => e + UI.important("Screen recording failed to start: #{e.message}") + nil +end + +def stop_screen_recording(rec) + return unless rec + + Process.kill('INT', rec[:pid]) + Process.wait(rec[:pid]) + unless rec[:ios] + sleep(1) # let screenrecord flush the file before pulling + sh("adb #{rec[:serial]}pull #{rec[:remote]} #{rec[:path]}", log: false) + sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) + end +rescue StandardError => e + UI.important("Screen recording failed to stop: #{e.message}") +end + +# Attaches a **per-test** video to each failed/broken result. Recording is +# per file (all its tests run in one `flutter test` process, so the host can't +# start/stop between them), so each test's clip is sliced out of its file's +# video using the result's own start/stop timestamps. +# +# `videos` maps a test-file basename => { path:, started_at_ms: }. +def attach_videos_to_failed_results(results_dir:, videos:) + return unless Dir.exist?(results_dir) + return if videos.empty? + + unless system('command -v ffmpeg > /dev/null 2>&1') + UI.important('ffmpeg not found; skipping per-test video attachments') + return + end + + Dir["#{results_dir}/*-result.json"].each do |json_path| + result = JSON.parse(File.read(json_path)) + next unless %w[failed broken].include?(result['status']) + next unless result['start'] && result['stop'] + + label = (result['labels'] || []).find { |l| l['name'] == 'testFile' } + meta = label && videos[label['value']] + next unless meta && File.exist?(meta[:path]) && File.size(meta[:path]).positive? + + # Slice [start, stop] out of the file's video, with 1s of lead/tail padding + # to absorb recording-start latency and minor clock skew. + ss = [((result['start'] - meta[:started_at_ms]) / 1000.0) - 1.0, 0].max + duration = ((result['stop'] - result['start']) / 1000.0) + 2.0 + next unless duration.positive? + + clip = "#{results_dir}/#{SecureRandom.uuid}-attachment.mp4" + ok = system('ffmpeg', '-y', '-ss', ss.to_s, '-i', meta[:path], '-t', duration.to_s, + '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-movflags', '+faststart', clip, + out: File::NULL, err: File::NULL) + next unless ok && File.exist?(clip) && File.size(clip).positive? + + (result['attachments'] ||= []) << { 'name' => 'Video', 'source' => File.basename(clip), 'type' => 'video/mp4' } + File.write(json_path, JSON.generate(result)) + end +end + lane :sh_on_root do |options| Dir.chdir(root_path) { sh(options[:command]) } end @@ -88,6 +174,11 @@ lane :run_e2e_test do |options| FileUtils.mkdir_p(File.dirname(log_file)) File.write(log_file, '') + videos_dir = "#{root_path}/build/e2e-videos" + FileUtils.rm_rf(videos_dir) + FileUtils.mkdir_p(videos_dir) + videos = {} # test-file basename => { path:, started_at_ms: } + begin Dir.chdir(root_path) do targets = options[:target] ? [options[:target]] : Dir['integration_test/*_test.dart'].sort @@ -95,7 +186,10 @@ lane :run_e2e_test do |options| failed = [] targets.each do |target| - flags = [target] + # Basename ties each test's video clip back to its result via the + # `testFile` label the Dart reporter sets from this dart-define. + test_name = File.basename(target, '.dart') + flags = [target, "--dart-define=E2E_TARGET=#{test_name}"] flags << "-d #{device}" if device if options[:verbose] flags << '--verbose' @@ -104,10 +198,13 @@ lane :run_e2e_test do |options| flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}" end + rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4") ok = sh_with_retries(retries: options[:retries] || 2, label: target) do cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)}" sh("bash -o pipefail -c #{Shellwords.escape(cmd)}") end + stop_screen_recording(rec) + videos[test_name] = { path: rec[:path], started_at_ms: rec[:started_at_ms] } if rec failed << target unless ok end @@ -115,6 +212,7 @@ lane :run_e2e_test do |options| end ensure collect_allure_results(log_file: log_file) rescue nil + attach_videos_to_failed_results(results_dir: allure_results_path, videos: videos) rescue nil stop_mock_server end end diff --git a/sample_app/integration_test/support/stream_test_case.dart b/sample_app/integration_test/support/stream_test_case.dart index b98cd9a92..4574bfefd 100644 --- a/sample_app/integration_test/support/stream_test_case.dart +++ b/sample_app/integration_test/support/stream_test_case.dart @@ -10,6 +10,11 @@ import '../allure/allure.dart'; import 'failure_artifacts.dart'; import 'stream_test_env.dart'; +/// The test file currently running, injected by the Fastlane lane +/// (`--dart-define=E2E_TARGET=...`). Lets the lane attach that file's host-side +/// video recording to its failed tests. Empty when run ad-hoc. +const _e2eTarget = String.fromEnvironment('E2E_TARGET'); + void streamTest({ String? allureId, required String description, @@ -21,7 +26,10 @@ void streamTest({ Allure.instance.startTest( name: description, fullName: Invoker.current?.liveTest.test.name ?? description, - labels: {if (allureId != null) 'AS_ID': allureId}, + labels: { + if (allureId != null) 'AS_ID': allureId, + if (_e2eTarget.isNotEmpty) 'testFile': _e2eTarget, + }, ); // Capture the test body's output so it can be attached on failure, while From 956d83fa9f713e8be8c6b988fd3c9d205c53325e Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 20:54:41 +0100 Subject: [PATCH 10/15] Update logs --- sample_app/fastlane/Fastfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index da646e8e2..c0591ee8b 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -200,7 +200,13 @@ lane :run_e2e_test do |options| rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4") ok = sh_with_retries(retries: options[:retries] || 2, label: target) do - cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)}" + # `tee` keeps the FULL output (incl. ALLURE-* markers) in the log file + # that collect_allure_results scans; the trailing grep hides just that + # base64 marker noise from the console. `|| true` keeps grep from + # affecting the pipeline's exit code, so pipefail still surfaces a + # `flutter test` failure. + cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)} | " \ + "{ grep --line-buffered -vE 'ALLURE-(RESULT|ATTACH)::' || true; }" sh("bash -o pipefail -c #{Shellwords.escape(cmd)}") end stop_screen_recording(rec) From 950d1095291f887a062b89e8c34c306c5db79c07 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 21:24:56 +0100 Subject: [PATCH 11/15] Fix video recording --- .github/workflows/e2e_test.yml | 124 +++++++++++++++------------- .github/workflows/e2e_test_cron.yml | 12 +++ sample_app/fastlane/Fastfile | 22 ++++- 3 files changed, 101 insertions(+), 57 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 139501d46..2e1767e72 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -54,6 +54,13 @@ jobs: profile: pixel_6 script: cd sample_app/android && bundle exec fastlane run_e2e_test device:emulator-5554 + - name: Attach videos to failed tests + if: failure() + working-directory: sample_app/android + run: | + sudo apt-get update && sudo apt-get install -y ffmpeg + bundle exec fastlane attach_videos + - name: Upload Allure results if: env.LAUNCH_ID != '' && (success() || failure()) working-directory: sample_app/android @@ -72,59 +79,64 @@ jobs: sample_app/stream-chat-test-mock-server/logs sample_app/build/e2e-test.log - # ios: - # runs-on: macos-15 - # timeout-minutes: 40 - # env: - # IOS_SIMULATOR_VERSION: "26.2" - # IOS_SIMULATOR_DEVICE: "iPhone 17" - # steps: - # - uses: actions/checkout@v6 - - # - uses: ./.github/actions/setup-flutter - - # - uses: ./.github/actions/setup-ruby - - # - uses: ./.github/actions/cache-cocoapods - # with: - # key-suffix: ${{ runner.os }} - - # - uses: ./.github/actions/cache-xcode-cas - # with: - # key-suffix: ${{ runner.os }} - # xcode-version: ${{ env.IOS_SIMULATOR_VERSION }} - - # - name: Bootstrap - # run: | - # flutter precache --ios - # flutter config --no-enable-swift-package-manager - # flutter pub global activate melos - # melos bootstrap - - # - name: Launch Allure TestOps - # working-directory: sample_app/ios - # run: bundle exec fastlane allure_launch - # env: - # GITHUB_EVENT: ${{ toJson(github.event) }} - - # - name: Run e2e (iOS simulator) - # working-directory: sample_app/ios - # run: bundle exec fastlane run_e2e_test platform:ios ios:"${{ env.IOS_SIMULATOR_VERSION }}" device:"${{ env.IOS_SIMULATOR_DEVICE }}" - - # - name: Upload Allure results - # if: env.LAUNCH_ID != '' && (success() || failure()) - # working-directory: sample_app/ios - # run: bundle exec fastlane allure_upload launch_id:$LAUNCH_ID - - # - name: Allure TestOps Launch Removal - # if: env.LAUNCH_ID != '' && cancelled() - # working-directory: sample_app/ios - # run: bundle exec fastlane allure_launch_removal launch_id:$LAUNCH_ID - - # - uses: actions/upload-artifact@v7 - # if: failure() - # with: - # name: e2e-ios-logs - # path: | - # sample_app/stream-chat-test-mock-server/logs - # sample_app/build/e2e-test.log + ios: + runs-on: macos-15 + timeout-minutes: 40 + env: + IOS_SIMULATOR_VERSION: "26.2" + IOS_SIMULATOR_DEVICE: "iPhone 17" + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup-flutter + + - uses: ./.github/actions/setup-ruby + + - uses: ./.github/actions/cache-cocoapods + with: + key-suffix: ${{ runner.os }} + + - uses: ./.github/actions/cache-xcode-cas + with: + key-suffix: ${{ runner.os }} + xcode-version: ${{ env.IOS_SIMULATOR_VERSION }} + + - name: Bootstrap + run: | + flutter precache --ios + flutter config --no-enable-swift-package-manager + flutter pub global activate melos + melos bootstrap + + - name: Launch Allure TestOps + working-directory: sample_app/ios + run: bundle exec fastlane allure_launch + env: + GITHUB_EVENT: ${{ toJson(github.event) }} + + - name: Run e2e (iOS simulator) + working-directory: sample_app/ios + run: bundle exec fastlane run_e2e_test platform:ios ios:"${{ env.IOS_SIMULATOR_VERSION }}" device:"${{ env.IOS_SIMULATOR_DEVICE }}" + + - name: Attach videos to failed tests + if: failure() + working-directory: sample_app/ios + run: bundle exec fastlane attach_videos + + - name: Upload Allure results + if: env.LAUNCH_ID != '' && (success() || failure()) + working-directory: sample_app/ios + run: bundle exec fastlane allure_upload launch_id:$LAUNCH_ID + + - name: Allure TestOps Launch Removal + if: env.LAUNCH_ID != '' && cancelled() + working-directory: sample_app/ios + run: bundle exec fastlane allure_launch_removal launch_id:$LAUNCH_ID + + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: e2e-ios-logs + path: | + sample_app/stream-chat-test-mock-server/logs + sample_app/build/e2e-test.log diff --git a/.github/workflows/e2e_test_cron.yml b/.github/workflows/e2e_test_cron.yml index 4ddd07b66..805882d5d 100644 --- a/.github/workflows/e2e_test_cron.yml +++ b/.github/workflows/e2e_test_cron.yml @@ -62,6 +62,13 @@ jobs: emulator-options: -no-snapshot-save -no-window -no-audio -no-boot-anim -gpu swiftshader_indirect script: cd sample_app/android && bundle exec fastlane run_e2e_test device:emulator-5554 mock_server_branch:${{ env.mock_server_branch }} + - name: Attach videos to failed tests + if: failure() + working-directory: sample_app/android + run: | + sudo apt-get update && sudo apt-get install -y ffmpeg + bundle exec fastlane attach_videos + - name: Upload Allure results if: env.LAUNCH_ID != '' && (success() || failure()) working-directory: sample_app/android @@ -135,6 +142,11 @@ jobs: working-directory: sample_app/ios run: bundle exec fastlane run_e2e_test platform:ios ios:${{ matrix.ios }} device:"${{ matrix.device }} (${{ matrix.ios }})" mock_server_branch:${{ env.mock_server_branch }} + - name: Attach videos to failed tests + if: failure() + working-directory: sample_app/ios + run: bundle exec fastlane attach_videos + - name: Upload Allure results if: env.LAUNCH_ID != '' && (success() || failure()) working-directory: sample_app/ios diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index c0591ee8b..ab459d879 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -218,11 +218,31 @@ lane :run_e2e_test do |options| end ensure collect_allure_results(log_file: log_file) rescue nil - attach_videos_to_failed_results(results_dir: allure_results_path, videos: videos) rescue nil + # Persist the recording metadata so the video clips can be sliced+attached + # in a separate, failure-only CI step (ffmpeg is only needed then, and is + # installed only on the runners/paths that lack it). See `attach_videos`. + File.write("#{videos_dir}/manifest.json", JSON.generate(videos)) unless videos.empty? stop_mock_server end end +# Slices each failed test's window out of its file's recording and attaches it +# to the Allure result. Split out of `run_e2e_test` so CI can run it as an +# independent `if: failure()` step (installing ffmpeg only then). Reads the +# manifest `run_e2e_test` wrote; no-op when there's no manifest. +lane :attach_videos do + manifest = "#{root_path}/build/e2e-videos/manifest.json" + unless File.exist?(manifest) + UI.important("No video manifest at #{manifest}; nothing to attach") + next + end + + videos = JSON.parse(File.read(manifest)).transform_values do |v| + { path: v['path'], started_at_ms: v['started_at_ms'] } + end + attach_videos_to_failed_results(results_dir: allure_results_path, videos: videos) +end + private_lane :sources_matrix do { e2e: ['sample_app', 'packages'], From 2125163cff7dea5c1b4e0cd9d2fff97a44c1051c Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 22:06:29 +0100 Subject: [PATCH 12/15] test(e2e): retry only failed test cases --- sample_app/fastlane/Fastfile | 107 ++++++++++++------ .../integration_test/allure/allure.dart | 11 ++ 2 files changed, 84 insertions(+), 34 deletions(-) diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index ab459d879..9b95b319c 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -5,6 +5,7 @@ require 'shellwords' require 'json' require 'securerandom' require 'fileutils' +require 'base64' import File.expand_path('Allurefile', __dir__) mock_server_repo_name = 'stream-chat-test-mock-server' @@ -133,26 +134,21 @@ lane :stop_mock_server do Net::HTTP.get_response(URI("http://localhost:#{mock_server_driver_port}/stop")) rescue nil end -def sh_with_retries(command: nil, retries: 2, label: nil) - retries = retries.to_i - label ||= command - attempt = 0 - begin - attempt += 1 - if block_given? - yield - else - sh(command) - end - true - rescue StandardError - if attempt <= retries - UI.important("#{label} failed (attempt #{attempt}/#{retries + 1}); retrying…") - retry - end - UI.error("#{label} still failing after #{retries} retries") - false +# Names of the tests that failed/broke in a single `flutter test` run, read +# from the `E2E-TEST::::` markers the Dart reporter emits +# (one per test). Used to retry only the failed cases. Returns [] when nothing +# could be attributed (e.g. a compile error or crash before any test reported), +# which the caller treats as "retry the whole file". +def failed_test_names(log) + return [] unless File.exist?(log) + + failed = [] + File.read(log).scan(%r{E2E-TEST::(passed|failed|broken|skipped)::([A-Za-z0-9+/=]+)}) do |status, name_b64| + next unless %w[failed broken].include?(status) + + failed << Base64.decode64(name_b64).force_encoding('UTF-8') end + failed.uniq end lane :run_e2e_test do |options| @@ -184,31 +180,74 @@ lane :run_e2e_test do |options| targets = options[:target] ? [options[:target]] : Dir['integration_test/*_test.dart'].sort UI.user_error!('No e2e test files found') if targets.empty? + retries = (options[:retries] || 2).to_i + failed = [] targets.each do |target| # Basename ties each test's video clip back to its result via the # `testFile` label the Dart reporter sets from this dart-define. test_name = File.basename(target, '.dart') - flags = [target, "--dart-define=E2E_TARGET=#{test_name}"] - flags << "-d #{device}" if device - if options[:verbose] - flags << '--verbose' - else - flags << '--quiet' - flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}" - end + # Per-attempt log, scanned for the failed-test markers so the *next* + # attempt re-runs only what failed (via --plain-name). Kept distinct + # from the cumulative `log_file` (which accumulates every attempt's + # ALLURE-* markers for Allure) so attribution reflects one attempt only. + attempt_log = "#{videos_dir}/#{test_name}.attempt.log" + + # nil => run the whole file (first attempt); an array => re-run only + # those test descriptions. + only = nil + + # One recording spans all attempts; the video is sliced per failed test + # using each result's start/stop (device-epoch ms) against this offset. rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4") - ok = sh_with_retries(retries: options[:retries] || 2, label: target) do + + ok = false + attempt = 0 + loop do + attempt += 1 + + flags = [target, "--dart-define=E2E_TARGET=#{test_name}"] + flags << "-d #{device}" if device + if options[:verbose] + flags << '--verbose' + else + flags << '--quiet' + flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}" + end + # Retry only the tests that failed last attempt. + only&.each { |name| flags << "--plain-name #{Shellwords.escape(name)}" } + + File.write(attempt_log, '') # `tee` keeps the FULL output (incl. ALLURE-* markers) in the log file - # that collect_allure_results scans; the trailing grep hides just that - # base64 marker noise from the console. `|| true` keeps grep from - # affecting the pipeline's exit code, so pipefail still surfaces a - # `flutter test` failure. + # that collect_allure_results scans, plus a fresh per-attempt copy for + # failure attribution; the trailing grep hides the base64 marker noise + # from the console. `|| true` keeps grep from affecting the pipeline's + # exit code, so pipefail still surfaces a `flutter test` failure. cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)} | " \ - "{ grep --line-buffered -vE 'ALLURE-(RESULT|ATTACH)::' || true; }" - sh("bash -o pipefail -c #{Shellwords.escape(cmd)}") + "tee #{Shellwords.escape(attempt_log)} | " \ + "{ grep --line-buffered -vE 'ALLURE-(RESULT|ATTACH)::|E2E-TEST::' || true; }" + ok = begin + sh("bash -o pipefail -c #{Shellwords.escape(cmd)}") + true + rescue StandardError + false + end + + break if ok || attempt > retries + + only = failed_test_names(attempt_log) + if only.empty? + # No per-test failure attributed (compile error / early crash) — + # retry the whole file. + only = nil + UI.important("#{target} failed (attempt #{attempt}/#{retries + 1}); retrying whole file…") + else + UI.important("#{target} failed (attempt #{attempt}/#{retries + 1}); " \ + "retrying #{only.size} failed test(s): #{only.join(', ')}") + end end + stop_screen_recording(rec) videos[test_name] = { path: rec[:path], started_at_ms: rec[:started_at_ms] } if rec failed << target unless ok diff --git a/sample_app/integration_test/allure/allure.dart b/sample_app/integration_test/allure/allure.dart index 261d9446f..98ab1a922 100644 --- a/sample_app/integration_test/allure/allure.dart +++ b/sample_app/integration_test/allure/allure.dart @@ -5,6 +5,15 @@ import 'package:uuid/uuid.dart'; const allureResultMarker = 'ALLURE-RESULT::'; const allureAttachmentMarker = 'ALLURE-ATTACH::'; + +/// Lightweight per-test outcome marker, emitted once per test as +/// `E2E-TEST::::`. The Fastlane lane parses these to +/// retry **only** the tests that failed in an attempt (rather than re-running +/// the whole file); the name is base64-encoded to survive any characters in +/// the test description. Separate from [allureResultMarker] (which carries the +/// full report payload) so retry attribution stays cheap and delimiter-safe. +const allureTestStatusMarker = 'E2E-TEST::'; + const _chunkSize = 900; enum AllureStatus { passed, failed, broken, skipped } @@ -197,6 +206,8 @@ class Allure { } _emitChunked(allureResultMarker, result.uuid, base64.encode(utf8.encode(jsonEncode(result.toJson())))); + // Compact outcome line for the Fastlane retry logic (see the marker doc). + print('$allureTestStatusMarker${status.name}::${base64.encode(utf8.encode(result.name))}'); _result = null; } } From 26a56e22f7bd776c5f8dd90ec974f14c3c5cf7a6 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 22:38:24 +0100 Subject: [PATCH 13/15] fix(e2e): OR failed tests via single --name alternation on retry --- sample_app/fastlane/Fastfile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index 9b95b319c..71d30ed9d 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -151,6 +151,15 @@ def failed_test_names(log) failed.uniq end +# Builds a single Dart-regex alternation matching any of `names`, for one +# `flutter test --name` (which OR's, unlike passing multiple flags). Each name +# is regex-escaped (ECMAScript metacharacters; spaces left as-is) so the test +# descriptions match literally. +def name_alternation(names) + escaped = names.map { |n| n.gsub(%r{[.*+?^${}()|\[\]\\/]}) { |c| "\\#{c}" } } + "(#{escaped.join('|')})" +end + lane :run_e2e_test do |options| next unless is_check_required(sources: sources_matrix[:e2e], force_check: @force_check) @@ -215,8 +224,10 @@ lane :run_e2e_test do |options| flags << '--quiet' flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}" end - # Retry only the tests that failed last attempt. - only&.each { |name| flags << "--plain-name #{Shellwords.escape(name)}" } + # Retry only the tests that failed last attempt. `flutter test` AND's + # multiple --name/--plain-name flags (a test must match them all), so + # a single --name with a regex alternation is the only way to OR them. + flags << "--name #{Shellwords.escape(name_alternation(only))}" if only File.write(attempt_log, '') # `tee` keeps the FULL output (incl. ALLURE-* markers) in the log file From 2e08044783cdcc8147afd13746bb76683d626b43 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Fri, 17 Jul 2026 23:41:11 +0100 Subject: [PATCH 14/15] fix(e2e): attach per-test failure videos reliably --- .github/workflows/e2e_test.yml | 4 ++- .github/workflows/e2e_test_cron.yml | 4 ++- sample_app/fastlane/Fastfile | 48 +++++++++++++++++++++-------- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 2e1767e72..ee32d6175 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -121,7 +121,9 @@ jobs: - name: Attach videos to failed tests if: failure() working-directory: sample_app/ios - run: bundle exec fastlane attach_videos + run: | + command -v ffmpeg >/dev/null 2>&1 || brew install ffmpeg + bundle exec fastlane attach_videos - name: Upload Allure results if: env.LAUNCH_ID != '' && (success() || failure()) diff --git a/.github/workflows/e2e_test_cron.yml b/.github/workflows/e2e_test_cron.yml index 805882d5d..ad0bf6c7e 100644 --- a/.github/workflows/e2e_test_cron.yml +++ b/.github/workflows/e2e_test_cron.yml @@ -145,7 +145,9 @@ jobs: - name: Attach videos to failed tests if: failure() working-directory: sample_app/ios - run: bundle exec fastlane attach_videos + run: | + command -v ffmpeg >/dev/null 2>&1 || brew install ffmpeg + bundle exec fastlane attach_videos - name: Upload Allure results if: env.LAUNCH_ID != '' && (success() || failure()) diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index 71d30ed9d..84fd0cb5f 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -52,12 +52,21 @@ end def stop_screen_recording(rec) return unless rec - Process.kill('INT', rec[:pid]) - Process.wait(rec[:pid]) - unless rec[:ios] - sleep(1) # let screenrecord flush the file before pulling + if rec[:ios] + # simctl finalizes the .mp4 when its own process gets SIGINT. + Process.kill('INT', rec[:pid]) + Process.wait(rec[:pid]) + else + # `screenrecord` writes the moov atom only when the ON-DEVICE process gets + # SIGINT. Killing the local `adb shell` client instead lets adbd send the + # child SIGHUP, which leaves a truncated/empty (~4 byte) file — so signal + # the device-side process directly, let it flush, then pull. + sh("adb #{rec[:serial]}shell pkill -INT screenrecord", log: false) rescue nil + sleep(3) # let screenrecord finalize + flush the file on-device + Process.kill('INT', rec[:pid]) rescue nil # reap the local adb client + Process.wait(rec[:pid]) rescue nil sh("adb #{rec[:serial]}pull #{rec[:remote]} #{rec[:path]}", log: false) - sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) + sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) rescue nil end rescue StandardError => e UI.important("Screen recording failed to stop: #{e.message}") @@ -97,7 +106,13 @@ def attach_videos_to_failed_results(results_dir:, videos:) ok = system('ffmpeg', '-y', '-ss', ss.to_s, '-i', meta[:path], '-t', duration.to_s, '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-movflags', '+faststart', clip, out: File::NULL, err: File::NULL) - next unless ok && File.exist?(clip) && File.size(clip).positive? + # Guard against a degenerate clip (e.g. the slice window fell past the end + # of the recording, or ffmpeg failed): a real clip is many KB — anything + # tiny is an empty/broken mp4 we must not attach. + unless ok && File.exist?(clip) && File.size(clip) > 1024 + File.delete(clip) if File.exist?(clip) + next + end (result['attachments'] ||= []) << { 'name' => 'Video', 'source' => File.basename(clip), 'type' => 'video/mp4' } File.write(json_path, JSON.generate(result)) @@ -207,10 +222,6 @@ lane :run_e2e_test do |options| # those test descriptions. only = nil - # One recording spans all attempts; the video is sliced per failed test - # using each result's start/stop (device-epoch ms) against this offset. - rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4") - ok = false attempt = 0 loop do @@ -229,6 +240,16 @@ lane :run_e2e_test do |options| # a single --name with a regex alternation is the only way to OR them. flags << "--name #{Shellwords.escape(name_alternation(only))}" if only + # Record PER ATTEMPT (not once across all attempts): a still-failing + # test's final Allure result comes from the LAST attempt it ran in, so + # that attempt's recording is the one we slice. Because retries re-run + # only the failing subset, that last attempt is short — its clip fits + # within Android's ~180s screenrecord cap, whereas a recording + # spanning every attempt would put the failure minutes past the cap + # (→ an empty, past-the-end slice). Same path each attempt: the last + # one wins and is what the manifest points at. + rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4") + File.write(attempt_log, '') # `tee` keeps the FULL output (incl. ALLURE-* markers) in the log file # that collect_allure_results scans, plus a fresh per-attempt copy for @@ -245,6 +266,11 @@ lane :run_e2e_test do |options| false end + stop_screen_recording(rec) + # Keep the latest attempt's recording metadata (see the per-attempt + # rationale above). + videos[test_name] = { path: rec[:path], started_at_ms: rec[:started_at_ms] } if rec + break if ok || attempt > retries only = failed_test_names(attempt_log) @@ -259,8 +285,6 @@ lane :run_e2e_test do |options| end end - stop_screen_recording(rec) - videos[test_name] = { path: rec[:path], started_at_ms: rec[:started_at_ms] } if rec failed << target unless ok end From 7d654121253911b6ab68d705ebbcc025317865a8 Mon Sep 17 00:00:00 2001 From: Alexey Alter-Pesotskiy Date: Sat, 18 Jul 2026 01:15:23 +0100 Subject: [PATCH 15/15] Do not record video on iOS --- .github/workflows/e2e_test.yml | 7 ---- .github/workflows/e2e_test_cron.yml | 7 ---- sample_app/fastlane/Fastfile | 56 ++++++++++++----------------- 3 files changed, 23 insertions(+), 47 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index ee32d6175..a60a02e72 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -118,13 +118,6 @@ jobs: working-directory: sample_app/ios run: bundle exec fastlane run_e2e_test platform:ios ios:"${{ env.IOS_SIMULATOR_VERSION }}" device:"${{ env.IOS_SIMULATOR_DEVICE }}" - - name: Attach videos to failed tests - if: failure() - working-directory: sample_app/ios - run: | - command -v ffmpeg >/dev/null 2>&1 || brew install ffmpeg - bundle exec fastlane attach_videos - - name: Upload Allure results if: env.LAUNCH_ID != '' && (success() || failure()) working-directory: sample_app/ios diff --git a/.github/workflows/e2e_test_cron.yml b/.github/workflows/e2e_test_cron.yml index ad0bf6c7e..6550e3593 100644 --- a/.github/workflows/e2e_test_cron.yml +++ b/.github/workflows/e2e_test_cron.yml @@ -142,13 +142,6 @@ jobs: working-directory: sample_app/ios run: bundle exec fastlane run_e2e_test platform:ios ios:${{ matrix.ios }} device:"${{ matrix.device }} (${{ matrix.ios }})" mock_server_branch:${{ env.mock_server_branch }} - - name: Attach videos to failed tests - if: failure() - working-directory: sample_app/ios - run: | - command -v ffmpeg >/dev/null 2>&1 || brew install ffmpeg - bundle exec fastlane attach_videos - - name: Upload Allure results if: env.LAUNCH_ID != '' && (success() || failure()) working-directory: sample_app/ios diff --git a/sample_app/fastlane/Fastfile b/sample_app/fastlane/Fastfile index 84fd0cb5f..a41d5e8bf 100644 --- a/sample_app/fastlane/Fastfile +++ b/sample_app/fastlane/Fastfile @@ -21,29 +21,25 @@ def ios_device?(device) device.to_s.match?(/\A[0-9A-F]{8}-[0-9A-F]{4}-/i) end -# Starts a best-effort screen recording of the device/simulator. Returns a +# Starts a best-effort screen recording of the device/emulator. Returns a # handle passed back to `stop_screen_recording`, or nil if recording couldn't # start (recording never fails the test run). +# +# Video is recorded on ANDROID ONLY — iOS recording is intentionally disabled. def start_screen_recording(ios:, device:, path:) + return nil if ios + FileUtils.mkdir_p(File.dirname(path)) # Wall-clock at recording start, to line the video timeline up with the - # per-test start/stop timestamps the Dart reporter records (same clock on the - # iOS simulator; host-synced on the Android emulator). + # per-test start/stop timestamps the Dart reporter records (host-synced on + # the Android emulator). started_at_ms = (Time.now.to_f * 1000).round - if ios - # `simctl io recordVideo` records until interrupted; run it detached and - # keep the pid so we can SIGINT it (which finalizes the .mp4). - pid = spawn('xcrun', 'simctl', 'io', device, 'recordVideo', '--codec=h264', '--force', path, - out: File::NULL, err: File::NULL) - { pid: pid, ios: true, path: path, started_at_ms: started_at_ms } - else - # `screenrecord` writes on-device (max ~180s), pulled after stop. - serial = device ? "-s #{device} " : '' - remote = "/sdcard/#{File.basename(path)}" - pid = spawn("adb #{serial}shell screenrecord --bit-rate 4000000 #{remote}", - out: File::NULL, err: File::NULL) - { pid: pid, ios: false, path: path, remote: remote, serial: serial, started_at_ms: started_at_ms } - end + # `screenrecord` writes on-device (max ~180s), pulled after stop. + serial = device ? "-s #{device} " : '' + remote = "/sdcard/#{File.basename(path)}" + pid = spawn("adb #{serial}shell screenrecord --bit-rate 4000000 #{remote}", + out: File::NULL, err: File::NULL) + { pid: pid, path: path, remote: remote, serial: serial, started_at_ms: started_at_ms } rescue StandardError => e UI.important("Screen recording failed to start: #{e.message}") nil @@ -52,22 +48,16 @@ end def stop_screen_recording(rec) return unless rec - if rec[:ios] - # simctl finalizes the .mp4 when its own process gets SIGINT. - Process.kill('INT', rec[:pid]) - Process.wait(rec[:pid]) - else - # `screenrecord` writes the moov atom only when the ON-DEVICE process gets - # SIGINT. Killing the local `adb shell` client instead lets adbd send the - # child SIGHUP, which leaves a truncated/empty (~4 byte) file — so signal - # the device-side process directly, let it flush, then pull. - sh("adb #{rec[:serial]}shell pkill -INT screenrecord", log: false) rescue nil - sleep(3) # let screenrecord finalize + flush the file on-device - Process.kill('INT', rec[:pid]) rescue nil # reap the local adb client - Process.wait(rec[:pid]) rescue nil - sh("adb #{rec[:serial]}pull #{rec[:remote]} #{rec[:path]}", log: false) - sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) rescue nil - end + # `screenrecord` writes the moov atom only when the ON-DEVICE process gets + # SIGINT. Killing the local `adb shell` client instead lets adbd send the + # child SIGHUP, which leaves a truncated/empty (~4 byte) file — so signal + # the device-side process directly, let it flush, then pull. + sh("adb #{rec[:serial]}shell pkill -INT screenrecord", log: false) rescue nil + sleep(3) # let screenrecord finalize + flush the file on-device + Process.kill('INT', rec[:pid]) rescue nil # reap the local adb client + Process.wait(rec[:pid]) rescue nil + sh("adb #{rec[:serial]}pull #{rec[:remote]} #{rec[:path]}", log: false) + sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) rescue nil rescue StandardError => e UI.important("Screen recording failed to stop: #{e.message}") end