ci(repo): Add E2E reactions tests

.github/workflows/e2e_test.yml

@@ -54,6 +54,13 @@ jobs:
           profile: pixel_6
           script: cd sample_app/android && bundle exec fastlane run_e2e_test device:emulator-5554
 
+      - name: Attach videos to failed tests
+        if: failure()
+        working-directory: sample_app/android
+        run: |
+          sudo apt-get update && sudo apt-get install -y ffmpeg
+          bundle exec fastlane attach_videos
+
       - name: Upload Allure results
         if: env.LAUNCH_ID != '' && (success() || failure())
         working-directory: sample_app/android

.github/workflows/e2e_test_cron.yml

@@ -62,6 +62,13 @@ jobs:
           emulator-options: -no-snapshot-save -no-window -no-audio -no-boot-anim -gpu swiftshader_indirect
           script: cd sample_app/android && bundle exec fastlane run_e2e_test device:emulator-5554 mock_server_branch:${{ env.mock_server_branch }}
 
+      - name: Attach videos to failed tests
+        if: failure()
+        working-directory: sample_app/android
+        run: |
+          sudo apt-get update && sudo apt-get install -y ffmpeg
+          bundle exec fastlane attach_videos
+
       - name: Upload Allure results
         if: env.LAUNCH_ID != '' && (success() || failure())
         working-directory: sample_app/android

sample_app/fastlane/Allurefile

@@ -25,7 +25,21 @@ lane :collect_allure_results do |options|
     base64 = parts.sort_by(&:first).map(&:last).join
     File.write("#{results_dir}/#{uuid}-result.json", Base64.decode64(base64))
   end
-  UI.message("Collected #{chunks.size} Allure result(s) into #{results_dir}")
+
+  # Attachments (screenshot / widget hierarchy / log) ride the same log-marker
+  # transport, keyed by their `<uuid>-attachment.<ext>` source filename; the
+  # result JSON references them by that name.
+  attachments = Hash.new { |hash, key| hash[key] = {} }
+  log.scan(%r{ALLURE-ATTACH::([0-9a-f-]+-attachment\.[a-z0-9]+):(\d+):([A-Za-z0-9+/=]+)}) do |source, seq, chunk|
+    attachments[source][seq.to_i] = chunk
+  end
+
+  attachments.each do |source, parts|
+    base64 = parts.sort_by(&:first).map(&:last).join
+    File.binwrite("#{results_dir}/#{source}", Base64.decode64(base64))
+  end
+
+  UI.message("Collected #{chunks.size} Allure result(s) and #{attachments.size} attachment(s) into #{results_dir}")
 end
 
 lane :install_allurectl do

sample_app/fastlane/Fastfile

@@ -2,6 +2,10 @@ opt_out_usage
 
 require 'net/http'
 require 'shellwords'
+require 'json'
+require 'securerandom'
+require 'fileutils'
+require 'base64'
 import File.expand_path('Allurefile', __dir__)
 
 mock_server_repo_name = 'stream-chat-test-mock-server'
@@ -17,6 +21,94 @@ def ios_device?(device)
   device.to_s.match?(/\A[0-9A-F]{8}-[0-9A-F]{4}-/i)
 end
 
+# Starts a best-effort screen recording of the device/emulator. Returns a
+# handle passed back to `stop_screen_recording`, or nil if recording couldn't
+# start (recording never fails the test run).
+#
+# Video is recorded on ANDROID ONLY — iOS recording is intentionally disabled.
+def start_screen_recording(ios:, device:, path:)
+  return nil if ios
+
+  FileUtils.mkdir_p(File.dirname(path))
+  # Wall-clock at recording start, to line the video timeline up with the
+  # per-test start/stop timestamps the Dart reporter records (host-synced on
+  # the Android emulator).
+  started_at_ms = (Time.now.to_f * 1000).round
+  # `screenrecord` writes on-device (max ~180s), pulled after stop.
+  serial = device ? "-s #{device} " : ''
+  remote = "/sdcard/#{File.basename(path)}"
+  pid = spawn("adb #{serial}shell screenrecord --bit-rate 4000000 #{remote}",
+              out: File::NULL, err: File::NULL)
+  { pid: pid, path: path, remote: remote, serial: serial, started_at_ms: started_at_ms }
+rescue StandardError => e
+  UI.important("Screen recording failed to start: #{e.message}")
+  nil
+end
+
+def stop_screen_recording(rec)
+  return unless rec
+
+  # `screenrecord` writes the moov atom only when the ON-DEVICE process gets
+  # SIGINT. Killing the local `adb shell` client instead lets adbd send the
+  # child SIGHUP, which leaves a truncated/empty (~4 byte) file — so signal
+  # the device-side process directly, let it flush, then pull.
+  sh("adb #{rec[:serial]}shell pkill -INT screenrecord", log: false) rescue nil
+  sleep(3) # let screenrecord finalize + flush the file on-device
+  Process.kill('INT', rec[:pid]) rescue nil # reap the local adb client
+  Process.wait(rec[:pid]) rescue nil
+  sh("adb #{rec[:serial]}pull #{rec[:remote]} #{rec[:path]}", log: false)
+  sh("adb #{rec[:serial]}shell rm #{rec[:remote]}", log: false) rescue nil
+rescue StandardError => e
+  UI.important("Screen recording failed to stop: #{e.message}")
+end
+
+# Attaches a **per-test** video to each failed/broken result. Recording is
+# per file (all its tests run in one `flutter test` process, so the host can't
+# start/stop between them), so each test's clip is sliced out of its file's
+# video using the result's own start/stop timestamps.
+#
+# `videos` maps a test-file basename => { path:, started_at_ms: }.
+def attach_videos_to_failed_results(results_dir:, videos:)
+  return unless Dir.exist?(results_dir)
+  return if videos.empty?
+
+  unless system('command -v ffmpeg > /dev/null 2>&1')
+    UI.important('ffmpeg not found; skipping per-test video attachments')
+    return
+  end
+
+  Dir["#{results_dir}/*-result.json"].each do |json_path|
+    result = JSON.parse(File.read(json_path))
+    next unless %w[failed broken].include?(result['status'])
+    next unless result['start'] && result['stop']
+
+    label = (result['labels'] || []).find { |l| l['name'] == 'testFile' }
+    meta = label && videos[label['value']]
+    next unless meta && File.exist?(meta[:path]) && File.size(meta[:path]).positive?
+
+    # Slice [start, stop] out of the file's video, with 1s of lead/tail padding
+    # to absorb recording-start latency and minor clock skew.
+    ss = [((result['start'] - meta[:started_at_ms]) / 1000.0) - 1.0, 0].max
+    duration = ((result['stop'] - result['start']) / 1000.0) + 2.0
+    next unless duration.positive?
+
+    clip = "#{results_dir}/#{SecureRandom.uuid}-attachment.mp4"
+    ok = system('ffmpeg', '-y', '-ss', ss.to_s, '-i', meta[:path], '-t', duration.to_s,
+                '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-movflags', '+faststart', clip,
+                out: File::NULL, err: File::NULL)
+    # Guard against a degenerate clip (e.g. the slice window fell past the end
+    # of the recording, or ffmpeg failed): a real clip is many KB — anything
+    # tiny is an empty/broken mp4 we must not attach.
+    unless ok && File.exist?(clip) && File.size(clip) > 1024
+      File.delete(clip) if File.exist?(clip)
+      next
+    end
+
+    (result['attachments'] ||= []) << { 'name' => 'Video', 'source' => File.basename(clip), 'type' => 'video/mp4' }
+    File.write(json_path, JSON.generate(result))
+  end
+end
+
 lane :sh_on_root do |options|
   Dir.chdir(root_path) { sh(options[:command]) }
 end
@@ -47,26 +139,30 @@ lane :stop_mock_server do
   Net::HTTP.get_response(URI("http://localhost:#{mock_server_driver_port}/stop")) rescue nil
 end
 
-def sh_with_retries(command: nil, retries: 2, label: nil)
-  retries = retries.to_i
-  label ||= command
-  attempt = 0
-  begin
-    attempt += 1
-    if block_given?
-      yield
-    else
-      sh(command)
-    end
-    true
-  rescue StandardError
-    if attempt <= retries
-      UI.important("#{label} failed (attempt #{attempt}/#{retries + 1}); retrying…")
-      retry
-    end
-    UI.error("#{label} still failing after #{retries} retries")
-    false
+# Names of the tests that failed/broke in a single `flutter test` run, read
+# from the `E2E-TEST::<status>::<base64(name)>` markers the Dart reporter emits
+# (one per test). Used to retry only the failed cases. Returns [] when nothing
+# could be attributed (e.g. a compile error or crash before any test reported),
+# which the caller treats as "retry the whole file".
+def failed_test_names(log)
+  return [] unless File.exist?(log)
+
+  failed = []
+  File.read(log).scan(%r{E2E-TEST::(passed|failed|broken|skipped)::([A-Za-z0-9+/=]+)}) do |status, name_b64|
+    next unless %w[failed broken].include?(status)
+
+    failed << Base64.decode64(name_b64).force_encoding('UTF-8')
   end
+  failed.uniq
+end
+
+# Builds a single Dart-regex alternation matching any of `names`, for one
+# `flutter test --name` (which OR's, unlike passing multiple flags). Each name
+# is regex-escaped (ECMAScript metacharacters; spaces left as-is) so the test
+# descriptions match literally.
+def name_alternation(names)
+  escaped = names.map { |n| n.gsub(%r{[.*+?^${}()|\[\]\\/]}) { |c| "\\#{c}" } }
+  "(#{escaped.join('|')})"
 end
 
 lane :run_e2e_test do |options|
@@ -88,37 +184,129 @@ lane :run_e2e_test do |options|
   FileUtils.mkdir_p(File.dirname(log_file))
   File.write(log_file, '')
 
+  videos_dir = "#{root_path}/build/e2e-videos"
+  FileUtils.rm_rf(videos_dir)
+  FileUtils.mkdir_p(videos_dir)
+  videos = {} # test-file basename => { path:, started_at_ms: }
+
   begin
     Dir.chdir(root_path) do
       targets = options[:target] ? [options[:target]] : Dir['integration_test/*_test.dart'].sort
       UI.user_error!('No e2e test files found') if targets.empty?
 
+      retries = (options[:retries] || 2).to_i
+
       failed = []
       targets.each do |target|
-        flags = [target]
-        flags << "-d #{device}" if device
-        if options[:verbose]
-          flags << '--verbose'
-        else
-          flags << '--quiet'
-          flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}"
-        end
+        # Basename ties each test's video clip back to its result via the
+        # `testFile` label the Dart reporter sets from this dart-define.
+        test_name = File.basename(target, '.dart')
+
+        # Per-attempt log, scanned for the failed-test markers so the *next*
+        # attempt re-runs only what failed (via --plain-name). Kept distinct
+        # from the cumulative `log_file` (which accumulates every attempt's
+        # ALLURE-* markers for Allure) so attribution reflects one attempt only.
+        attempt_log = "#{videos_dir}/#{test_name}.attempt.log"
+
+        # nil => run the whole file (first attempt); an array => re-run only
+        # those test descriptions.
+        only = nil
+
+        ok = false
+        attempt = 0
+        loop do
+          attempt += 1
+
+          flags = [target, "--dart-define=E2E_TARGET=#{test_name}"]
+          flags << "-d #{device}" if device
+          if options[:verbose]
+            flags << '--verbose'
+          else
+            flags << '--quiet'
+            flags << "--reporter #{ENV['GITHUB_ACTIONS'] ? 'github' : 'expanded'}"
+          end
+          # Retry only the tests that failed last attempt. `flutter test` AND's
+          # multiple --name/--plain-name flags (a test must match them all), so
+          # a single --name with a regex alternation is the only way to OR them.
+          flags << "--name #{Shellwords.escape(name_alternation(only))}" if only
 
-        ok = sh_with_retries(retries: options[:retries] || 2, label: target) do
-          cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)}"
-          sh("bash -o pipefail -c #{Shellwords.escape(cmd)}")
+          # Record PER ATTEMPT (not once across all attempts): a still-failing
+          # test's final Allure result comes from the LAST attempt it ran in, so
+          # that attempt's recording is the one we slice. Because retries re-run
+          # only the failing subset, that last attempt is short — its clip fits
+          # within Android's ~180s screenrecord cap, whereas a recording
+          # spanning every attempt would put the failure minutes past the cap
+          # (→ an empty, past-the-end slice). Same path each attempt: the last
+          # one wins and is what the manifest points at.
+          rec = start_screen_recording(ios: ios, device: device, path: "#{videos_dir}/#{test_name}.mp4")
+
+          File.write(attempt_log, '')
+          # `tee` keeps the FULL output (incl. ALLURE-* markers) in the log file
+          # that collect_allure_results scans, plus a fresh per-attempt copy for
+          # failure attribution; the trailing grep hides the base64 marker noise
+          # from the console. `|| true` keeps grep from affecting the pipeline's
+          # exit code, so pipefail still surfaces a `flutter test` failure.
+          cmd = "flutter test #{flags.join(' ')} 2>&1 | tee -a #{Shellwords.escape(log_file)} | " \
+                "tee #{Shellwords.escape(attempt_log)} | " \
+                "{ grep --line-buffered -vE 'ALLURE-(RESULT|ATTACH)::|E2E-TEST::' || true; }"
+          ok = begin
+            sh("bash -o pipefail -c #{Shellwords.escape(cmd)}")
+            true
+          rescue StandardError
+            false
+          end
+
+          stop_screen_recording(rec)
+          # Keep the latest attempt's recording metadata (see the per-attempt
+          # rationale above).
+          videos[test_name] = { path: rec[:path], started_at_ms: rec[:started_at_ms] } if rec
+
+          break if ok || attempt > retries
+
+          only = failed_test_names(attempt_log)
+          if only.empty?
+            # No per-test failure attributed (compile error / early crash) —
+            # retry the whole file.
+            only = nil
+            UI.important("#{target} failed (attempt #{attempt}/#{retries + 1}); retrying whole file…")
+          else
+            UI.important("#{target} failed (attempt #{attempt}/#{retries + 1}); " \
+                         "retrying #{only.size} failed test(s): #{only.join(', ')}")
+          end
         end
+
         failed << target unless ok
       end
 
       UI.user_error!("E2E failed after retries: #{failed.join(', ')}") unless failed.empty?
     end
   ensure
     collect_allure_results(log_file: log_file) rescue nil
+    # Persist the recording metadata so the video clips can be sliced+attached
+    # in a separate, failure-only CI step (ffmpeg is only needed then, and is
+    # installed only on the runners/paths that lack it). See `attach_videos`.
+    File.write("#{videos_dir}/manifest.json", JSON.generate(videos)) unless videos.empty?
     stop_mock_server
   end
 end
 
+# Slices each failed test's window out of its file's recording and attaches it
+# to the Allure result. Split out of `run_e2e_test` so CI can run it as an
+# independent `if: failure()` step (installing ffmpeg only then). Reads the
+# manifest `run_e2e_test` wrote; no-op when there's no manifest.
+lane :attach_videos do
+  manifest = "#{root_path}/build/e2e-videos/manifest.json"
+  unless File.exist?(manifest)
+    UI.important("No video manifest at #{manifest}; nothing to attach")
+    next
+  end
+
+  videos = JSON.parse(File.read(manifest)).transform_values do |v|
+    { path: v['path'], started_at_ms: v['started_at_ms'] }
+  end
+  attach_videos_to_failed_results(results_dir: allure_results_path, videos: videos)
+end
+
 private_lane :sources_matrix do
   {
     e2e: ['sample_app', 'packages'],