From b13fb0279cfee39c4fa6f7449951ee5aac3f9ab0 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Sat, 27 Jun 2026 14:37:43 +0200 Subject: [PATCH 1/3] chore: add SDK compliance harness 0.8.0 --- .github/workflows/sdk-compliance.yml | 21 ++ lib/posthog/backoff_policy.rb | 2 +- lib/posthog/client.rb | 6 +- lib/posthog/feature_flags.rb | 2 +- lib/posthog/send_worker.rb | 2 + lib/posthog/transport.rb | 47 ++- sdk_compliance_adapter/Dockerfile | 14 + sdk_compliance_adapter/adapter.rb | 353 ++++++++++++++++++++++ sdk_compliance_adapter/docker-compose.yml | 19 ++ spec/posthog/send_worker_spec.rb | 14 + 10 files changed, 471 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/sdk-compliance.yml create mode 100644 sdk_compliance_adapter/Dockerfile create mode 100644 sdk_compliance_adapter/adapter.rb create mode 100644 sdk_compliance_adapter/docker-compose.yml diff --git a/.github/workflows/sdk-compliance.yml b/.github/workflows/sdk-compliance.yml new file mode 100644 index 00000000..22853555 --- /dev/null +++ b/.github/workflows/sdk-compliance.yml @@ -0,0 +1,21 @@ +name: SDK Compliance Tests + +permissions: + contents: read + packages: read + pull-requests: write + +on: + pull_request: + push: + branches: + - main + +jobs: + compliance: + name: PostHog SDK compliance tests + uses: PostHog/posthog-sdk-test-harness/.github/workflows/test-sdk-action.yml@be8b8d5a3f94a249659844e94832e874f049c1e4 + with: + adapter-dockerfile: "sdk_compliance_adapter/Dockerfile" + adapter-context: "." + test-harness-version: "0.8.0" diff --git a/lib/posthog/backoff_policy.rb b/lib/posthog/backoff_policy.rb index 816f1401..31e8ff0f 100644 --- a/lib/posthog/backoff_policy.rb +++ b/lib/posthog/backoff_policy.rb @@ -33,7 +33,7 @@ def next_interval @attempts += 1 - [interval, @max_timeout_ms].min + interval.clamp(@min_timeout_ms, @max_timeout_ms) end private diff --git a/lib/posthog/client.rb b/lib/posthog/client.rb index c87f23fb..7cb8f5a8 100644 --- a/lib/posthog/client.rb +++ b/lib/posthog/client.rb @@ -67,6 +67,9 @@ def _decrement_instance_count(api_key) # in seconds. Defaults to 30. # @option opts [Integer] :feature_flag_request_timeout_seconds How long to wait for feature flag evaluation, # in seconds. Defaults to 3. + # @option opts [Integer] :max_retries How many times to retry batch uploads after the first send attempt. + # Defaults to the transport default. Set to 0 to disable retrying. + # @option opts [Boolean] :enable_compression +true+ to gzip batch upload request bodies. # @option opts [Integer] :feature_flag_request_max_retries How many times to retry a flag request after a # transient network error. Each retry sleeps on the calling thread before retrying, so this adds to # worst-case latency. Defaults to 1. Set to 0 to disable retrying. @@ -109,7 +112,8 @@ def initialize(opts = {}) @transport = Transport.new( api_host: opts[:host], skip_ssl_verification: opts[:skip_ssl_verification], - retries: 3 + retries: opts.key?(:max_retries) ? opts[:max_retries].to_i + 1 : 3, + gzip: opts[:enable_compression] == true ) @sync_lock = Mutex.new end diff --git a/lib/posthog/feature_flags.rb b/lib/posthog/feature_flags.rb index 44906529..9e82229d 100644 --- a/lib/posthog/feature_flags.rb +++ b/lib/posthog/feature_flags.rb @@ -150,7 +150,7 @@ def get_flags(distinct_id, groups = {}, person_properties = {}, group_properties group_properties: group_properties } request_data[:flag_keys_to_evaluate] = flag_keys if flag_keys && !flag_keys.empty? - request_data[:geoip_disable] = true if disable_geoip + request_data[:geoip_disable] = disable_geoip unless disable_geoip.nil? flags_response = _request_feature_flag_evaluation(request_data) diff --git a/lib/posthog/send_worker.rb b/lib/posthog/send_worker.rb index 3218d375..752e7ae4 100644 --- a/lib/posthog/send_worker.rb +++ b/lib/posthog/send_worker.rb @@ -43,6 +43,8 @@ def initialize(queue, api_key, options = {}) @shutdown = false @pid = Process.pid @transport_options = { api_host: options[:host], skip_ssl_verification: options[:skip_ssl_verification] } + @transport_options[:retries] = options[:max_retries].to_i + 1 if options.key?(:max_retries) + @transport_options[:gzip] = true if options[:enable_compression] == true @transport = Transport.new(@transport_options) end diff --git a/lib/posthog/transport.rb b/lib/posthog/transport.rb index d7d26ac0..8547737e 100644 --- a/lib/posthog/transport.rb +++ b/lib/posthog/transport.rb @@ -8,6 +8,9 @@ require 'net/http' require 'net/https' require 'json' +require 'stringio' +require 'time' +require 'zlib' module PostHog # HTTP transport used by the SDK workers. @@ -40,10 +43,12 @@ def initialize(options = {}) options[:port] = options[:port].nil? ? PORT : options[:port] options[:ssl] = options[:ssl].nil? ? SSL : options[:ssl] - @headers = options[:headers] || HEADERS + @headers = (options[:headers] || HEADERS).dup @path = options[:path] || PATH @retries = options[:retries] || RETRIES @backoff_policy = options[:backoff_policy] || PostHog::BackoffPolicy.new + @gzip = options[:gzip] == true + @last_retry_after = nil http = Net::HTTP.new(options[:host], options[:port]) http.use_ssl = options[:ssl] @@ -100,10 +105,8 @@ def shutdown private def should_retry_request?(status_code, body) - if status_code >= 500 - true # Server error - elsif status_code == 429 # rubocop:disable Lint/DuplicateBranch - true # Rate limited + if status_code >= 500 || [408, 429].include?(status_code) + true # Server error, request timeout, or rate limited elsif status_code >= 400 logger.error(body) false # Client error. Do not retry, but log @@ -133,18 +136,49 @@ def retry_with_backoff(retries_remaining, &block) if should_retry && (retries_remaining > 1) logger.debug("Retrying request, #{retries_remaining} retries left") - sleep(@backoff_policy.next_interval.to_f / 1000) + sleep(retry_delay_seconds) retry_with_backoff(retries_remaining - 1, &block) else [result, caught_exception] end end + def retry_delay_seconds + retry_after = parse_retry_after(@last_retry_after) + @last_retry_after = nil + return retry_after if retry_after + + @backoff_policy.next_interval.to_f / 1000 + end + + def parse_retry_after(value) + return nil if value.nil? || value.empty? + + seconds = Float(value, exception: false) + return seconds if seconds&.positive? + + parsed_time = Time.httpdate(value) + delay = parsed_time - Time.now + delay.positive? ? delay : nil + rescue ArgumentError + nil + end + + def gzip(payload) + io = StringIO.new + Zlib::GzipWriter.wrap(io) { |gzip| gzip.write(payload) } + io.string + end + # Sends a request for the batch, returns [status_code, body] def send_request(api_key, batch) payload = JSON.generate(api_key: api_key, batch: batch) request = Net::HTTP::Post.new(@path, @headers) + if @gzip + payload = gzip(payload) + request['Content-Encoding'] = 'gzip' + end if self.class.stub logger.debug "stubbed request to #{@path}: " \ @@ -155,6 +189,7 @@ def send_request(api_key, batch) @http_mutex.synchronize do @http.start unless @http.started? # Maintain a persistent connection response = @http.request(request, payload) + @last_retry_after = response['Retry-After'] [response.code.to_i, response.body] end end diff --git a/sdk_compliance_adapter/Dockerfile b/sdk_compliance_adapter/Dockerfile new file mode 100644 index 00000000..5bb59454 --- /dev/null +++ b/sdk_compliance_adapter/Dockerfile @@ -0,0 +1,14 @@ +FROM ruby:3.3-slim + +WORKDIR /app + +RUN gem install concurrent-ruby --no-document + +COPY lib/ /app/lib/ +COPY sdk_compliance_adapter/adapter.rb /app/adapter.rb + +ENV RUBYLIB=/app/lib + +EXPOSE 8080 + +CMD ["ruby", "/app/adapter.rb"] diff --git a/sdk_compliance_adapter/adapter.rb b/sdk_compliance_adapter/adapter.rb new file mode 100644 index 00000000..93461aad --- /dev/null +++ b/sdk_compliance_adapter/adapter.rb @@ -0,0 +1,353 @@ +# frozen_string_literal: true + +require 'json' +require 'securerandom' +require 'socket' +require 'time' + +require 'posthog' + +module SDKComplianceAdapter + class State + def initialize + @mutex = Mutex.new + @client = nil + reset_counters + end + + def reset + client = @mutex.synchronize do + current = @client + @client = nil + current + end + + begin + client&.shutdown + rescue StandardError => e + record_error(e.message) + end + + @mutex.synchronize { reset_counters } + end + + def client + @mutex.synchronize { @client } + end + + def client=(new_client) + @mutex.synchronize { @client = new_client } + end + + def increment_captured + @mutex.synchronize do + @total_events_captured += 1 + @pending_events += 1 + end + end + + def record_request(status_code, batch) + events = batch_events(batch) + batch_id = batch_id_for(events) + uuids = events.filter_map { |event| event['uuid'] || event[:uuid] } + + @mutex.synchronize do + retry_attempt = @retry_attempts[batch_id] || 0 + @requests_made << { + timestamp_ms: (Time.now.to_f * 1000).to_i, + status_code: status_code, + retry_attempt: retry_attempt, + event_count: events.length, + uuid_list: uuids + } + + @total_retries += 1 if retry_attempt.positive? + + if status_code == 200 + @total_events_sent += events.length + @pending_events = [@pending_events - events.length, 0].max + @retry_attempts.delete(batch_id) + else + @retry_attempts[batch_id] = retry_attempt + 1 + end + end + end + + def record_error(error) + @mutex.synchronize { @last_error = error } + end + + def snapshot + @mutex.synchronize do + { + pending_events: @pending_events, + total_events_captured: @total_events_captured, + total_events_sent: @total_events_sent, + total_retries: @total_retries, + last_error: @last_error, + requests_made: @requests_made.map(&:dup) + } + end + end + + private + + def reset_counters + @pending_events = 0 + @total_events_captured = 0 + @total_events_sent = 0 + @total_retries = 0 + @last_error = nil + @requests_made = [] + @retry_attempts = {} + end + + def batch_events(batch) + JSON.parse(JSON.generate(batch)) + rescue StandardError + [] + end + + def batch_id_for(events) + uuids = events.filter_map { |event| event['uuid'] || event[:uuid] }.sort + return SecureRandom.uuid if uuids.empty? + + uuids.first(3).join('-') + end + end + + STATE = State.new + + def self.state + STATE + end +end + +module PostHog + class Transport + unless method_defined?( + :sdk_compliance_original_send_request + ) + alias sdk_compliance_original_send_request send_request + end + + def send_request(api_key, batch) + status_code, body = sdk_compliance_original_send_request(api_key, batch) + SDKComplianceAdapter.state.record_request(status_code, batch) + [status_code, body] + rescue StandardError => e + SDKComplianceAdapter.state.record_request(0, batch) + SDKComplianceAdapter.state.record_error(e.message) + raise + end + end +end + +class ComplianceServer + def initialize(host: '0.0.0.0', port: 8080) + @server = TCPServer.new(host, port) + end + + def run + loop do + socket = @server.accept + Thread.new(socket) { |client| handle_connection(client) } + end + end + + private + + def handle_connection(socket) + request_line = socket.gets + return if request_line.nil? || request_line.strip.empty? + + method, path, = request_line.split + headers = read_headers(socket) + body = read_body(socket, headers) + + status, payload = route(method, path, body) + write_response(socket, status, payload) + rescue StandardError => e + write_response(socket, 500, { error: e.message }) + ensure + begin + socket.close + rescue StandardError + nil + end + end + + def read_headers(socket) + headers = {} + while (line = socket.gets) + line = line.chomp + break if line.empty? + + key, value = line.split(':', 2) + headers[key.downcase] = value.strip if key && value + end + headers + end + + def read_body(socket, headers) + length = headers.fetch('content-length', '0').to_i + return '' if length.zero? + + socket.read(length) + end + + def route(method, path, body) + case [method, path] + when ['GET', '/health'] + health + when ['POST', '/init'] + init(parse_json(body)) + when ['POST', '/capture'] + capture(parse_json(body)) + when ['POST', '/flush'] + flush + when ['POST', '/get_feature_flag'] + get_feature_flag(parse_json(body)) + when ['GET', '/state'] + [200, SDKComplianceAdapter.state.snapshot] + when ['POST', '/reset'] + SDKComplianceAdapter.state.reset + [200, { success: true }] + else + [404, { error: 'not found' }] + end + rescue JSON::ParserError => e + [400, { error: "invalid JSON: #{e.message}" }] + rescue ArgumentError => e + SDKComplianceAdapter.state.record_error(e.message) + [400, { error: e.message }] + rescue StandardError => e + SDKComplianceAdapter.state.record_error(e.message) + [500, { error: e.message }] + end + + def parse_json(body) + return {} if body.nil? || body.empty? + + JSON.parse(body) + end + + def health + [ + 200, + { + sdk_name: 'posthog-ruby', + sdk_version: PostHog::VERSION, + adapter_version: '1.0.0', + capabilities: %w[capture_v0 encoding_gzip] + } + ] + end + + def init(data) + SDKComplianceAdapter.state.reset + + api_key = data['api_key'] + host = data['host'] + return [400, { error: 'api_key is required' }] if api_key.nil? || api_key.empty? + return [400, { error: 'host is required' }] if host.nil? || host.empty? + + options = { + api_key: api_key, + host: host, + batch_size: data.fetch('flush_at', 100), + flush_interval_seconds: data.fetch('flush_interval_ms', 500).to_f / 1000.0, + on_error: proc { |_status, error| SDKComplianceAdapter.state.record_error(error) }, + disable_singleton_warning: true + } + options[:max_retries] = data['max_retries'] if data.key?('max_retries') + options[:enable_compression] = true if data['enable_compression'] == true + + SDKComplianceAdapter.state.client = PostHog::Client.new(options) + [200, { success: true }] + end + + def capture(data) + client = SDKComplianceAdapter.state.client + return [400, { error: 'SDK not initialized' }] unless client + + distinct_id = data['distinct_id'] + event = data['event'] + return [400, { error: 'distinct_id is required' }] if distinct_id.nil? || distinct_id.empty? + return [400, { error: 'event is required' }] if event.nil? || event.empty? + + uuid = SecureRandom.uuid + attrs = { + distinct_id: distinct_id, + event: event, + properties: data['properties'] || {}, + uuid: uuid + } + attrs[:timestamp] = Time.iso8601(data['timestamp']) if data['timestamp'] + + if client.capture(attrs) + SDKComplianceAdapter.state.increment_captured + [200, { success: true, uuid: uuid }] + else + [500, { error: 'capture was not queued' }] + end + end + + def flush + client = SDKComplianceAdapter.state.client + return [400, { error: 'SDK not initialized' }] unless client + + client.flush + [200, { success: true, events_flushed: SDKComplianceAdapter.state.snapshot[:total_events_sent] }] + rescue StandardError => e + SDKComplianceAdapter.state.record_error(e.message) + [500, { error: e.message, errors: [e.message] }] + end + + def get_feature_flag(data) + client = SDKComplianceAdapter.state.client + return [400, { error: 'SDK not initialized' }] unless client + + key = data['key'] + distinct_id = data['distinct_id'] + return [400, { error: 'key is required' }] if key.nil? || key.empty? + return [400, { error: 'distinct_id is required' }] if distinct_id.nil? || distinct_id.empty? + + disable_geoip = data.key?('disable_geoip') ? data['disable_geoip'] : false + flags = client.evaluate_flags( + distinct_id, + groups: data['groups'] || {}, + person_properties: data['person_properties'] || {}, + group_properties: data['group_properties'] || {}, + only_evaluate_locally: data.fetch('force_remote', true) == false, + disable_geoip: disable_geoip, + flag_keys: [key] + ) + value = flags.get_flag(key) + client.flush + + [200, { success: true, value: value }] + rescue StandardError => e + SDKComplianceAdapter.state.record_error(e.message) + [500, { error: e.message }] + end + + def write_response(socket, status, payload) + body = JSON.generate(payload) + reason = { + 200 => 'OK', 400 => 'Bad Request', 404 => 'Not Found', 500 => 'Internal Server Error' + }.fetch(status, 'OK') + + socket.write "HTTP/1.1 #{status} #{reason}\r\n" + socket.write "Content-Type: application/json\r\n" + socket.write "Content-Length: #{body.bytesize}\r\n" + socket.write "Connection: close\r\n" + socket.write "\r\n" + socket.write body + end +end + +trap('TERM') { exit } +trap('INT') { exit } + +ComplianceServer.new.run diff --git a/sdk_compliance_adapter/docker-compose.yml b/sdk_compliance_adapter/docker-compose.yml new file mode 100644 index 00000000..d057a86e --- /dev/null +++ b/sdk_compliance_adapter/docker-compose.yml @@ -0,0 +1,19 @@ +services: + sdk-adapter: + build: + context: .. + dockerfile: sdk_compliance_adapter/Dockerfile + networks: + - test-network + + test-harness: + image: ghcr.io/posthog/sdk-test-harness:0.8.0 + command: ["run", "--adapter-url", "http://sdk-adapter:8080", "--mock-url", "http://test-harness:8081"] + networks: + - test-network + depends_on: + - sdk-adapter + +networks: + test-network: + driver: bridge diff --git a/spec/posthog/send_worker_spec.rb b/spec/posthog/send_worker_spec.rb index bfd9f2a1..c5d26fe6 100644 --- a/spec/posthog/send_worker_spec.rb +++ b/spec/posthog/send_worker_spec.rb @@ -33,6 +33,20 @@ def run_worker_until_idle(worker, queue) expect(worker.instance_variable_get(:@flush_interval_seconds)).to eq(5.0) end + + it 'passes max_retries to the transport as total attempts' do + queue = Queue.new + worker = described_class.new(queue, 'secret', max_retries: 2) + + expect(worker.instance_variable_get(:@transport_options)[:retries]).to eq(3) + end + + it 'passes compression to the transport when enabled' do + queue = Queue.new + worker = described_class.new(queue, 'secret', enable_compression: true) + + expect(worker.instance_variable_get(:@transport_options)[:gzip]).to eq(true) + end end describe '#run' do From e9799987ac5e1631dad12eee691920a3b7ffe617 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto <5731772+marandaneto@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:30:44 +0200 Subject: [PATCH 2/3] fix retry-after handling in transport --- lib/posthog/transport.rb | 3 +- spec/posthog/send_worker_spec.rb | 48 +++++++++++++++++++++++++------- spec/posthog/transport_spec.rb | 46 ++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/lib/posthog/transport.rb b/lib/posthog/transport.rb index 8547737e..0c6b18df 100644 --- a/lib/posthog/transport.rb +++ b/lib/posthog/transport.rb @@ -155,7 +155,7 @@ def parse_retry_after(value) return nil if value.nil? || value.empty? seconds = Float(value, exception: false) - return seconds if seconds&.positive? + return seconds if seconds && seconds >= 0 parsed_time = Time.httpdate(value) delay = parsed_time - Time.now @@ -172,6 +172,7 @@ def gzip(payload) # Sends a request for the batch, returns [status_code, body] def send_request(api_key, batch) + @last_retry_after = nil payload = JSON.generate(api_key: api_key, batch: batch) request = Net::HTTP::Post.new(@path, @headers) diff --git a/spec/posthog/send_worker_spec.rb b/spec/posthog/send_worker_spec.rb index c5d26fe6..041e5ca1 100644 --- a/spec/posthog/send_worker_spec.rb +++ b/spec/posthog/send_worker_spec.rb @@ -34,18 +34,46 @@ def run_worker_until_idle(worker, queue) expect(worker.instance_variable_get(:@flush_interval_seconds)).to eq(5.0) end - it 'passes max_retries to the transport as total attempts' do - queue = Queue.new - worker = described_class.new(queue, 'secret', max_retries: 2) - - expect(worker.instance_variable_get(:@transport_options)[:retries]).to eq(3) - end + [ + { + description: 'passes max_retries: 0 to the transport as one total attempt', + options: { max_retries: 0 }, + expected_options: { retries: 1 } + }, + { + description: 'passes max_retries to the transport as total attempts', + options: { max_retries: 2 }, + expected_options: { retries: 3 } + }, + { + description: 'passes compression to the transport when enabled', + options: { enable_compression: true }, + expected_options: { gzip: true } + }, + { + description: 'leaves compression unset when disabled', + options: { enable_compression: false }, + absent_options: [:gzip] + }, + { + description: 'leaves compression unset when nil', + options: { enable_compression: nil }, + absent_options: [:gzip] + } + ].each do |configuration| + it configuration[:description] do + queue = Queue.new + worker = described_class.new(queue, 'secret', configuration[:options]) + transport_options = worker.instance_variable_get(:@transport_options) - it 'passes compression to the transport when enabled' do - queue = Queue.new - worker = described_class.new(queue, 'secret', enable_compression: true) + configuration.fetch(:expected_options, {}).each do |key, value| + expect(transport_options[key]).to eq(value) + end - expect(worker.instance_variable_get(:@transport_options)[:gzip]).to eq(true) + configuration.fetch(:absent_options, []).each do |key| + expect(transport_options).not_to have_key(key) + end + end end end diff --git a/spec/posthog/transport_spec.rb b/spec/posthog/transport_spec.rb index 93fe80d7..9bac1dac 100644 --- a/spec/posthog/transport_spec.rb +++ b/spec/posthog/transport_spec.rb @@ -235,6 +235,52 @@ module PostHog it_behaves_like('non-retried request', 400, '{}') end + context 'a retryable response includes Retry-After' do + let(:status_code) { 429 } + let(:retries) { 2 } + let(:backoff_policy) { FakeBackoffPolicy.new([1000]) } + + subject do + described_class.new( + retries: retries, + backoff_policy: backoff_policy + ) + end + + it 'honors Retry-After: 0 as an immediate retry' do + allow(response).to receive(:[]).with('Retry-After').and_return('0') + expect(subject).to receive(:sleep).once.with(0.0).and_return(nil) + + subject.send(api_key, batch) + end + + it 'does not reuse a stale Retry-After header after retries are exhausted' do + http = subject.instance_variable_get(:@http) + rate_limited_response = Net::HTTPResponse.new(http_version, 429, 'Too Many Requests') + allow(rate_limited_response).to receive(:body).and_return(response_body) + allow(rate_limited_response).to receive(:[]).with('Retry-After').and_return('123') + + success_response = Net::HTTPResponse.new(http_version, 200, 'OK') + allow(success_response).to receive(:body).and_return(response_body) + allow(success_response).to receive(:[]).with('Retry-After').and_return(nil) + + requests = [rate_limited_response, IOError.new('connection reset'), success_response] + allow(http).to receive(:request) do + next_request = requests.shift + raise next_request if next_request.is_a?(StandardError) + + next_request + end + + subject.instance_variable_set(:@retries, 1) + subject.send(api_key, batch) + + subject.instance_variable_set(:@retries, 2) + expect(subject).to receive(:sleep).once.with(1).and_return(nil) + subject.send(api_key, batch) + end + end + context 'response body is malformed JSON' do let(:response_body) { 'Malformed JSON ---' } From ba47218c487aa89dacd1d980198209c7e4782ae4 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Mon, 29 Jun 2026 16:14:32 +0200 Subject: [PATCH 3/3] chore: add retry policy changeset --- .changeset/curly-snakes-retry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/curly-snakes-retry.md diff --git a/.changeset/curly-snakes-retry.md b/.changeset/curly-snakes-retry.md new file mode 100644 index 00000000..997699f6 --- /dev/null +++ b/.changeset/curly-snakes-retry.md @@ -0,0 +1,5 @@ +--- +"posthog-ruby": patch +--- + +Retry capture delivery on transient HTTP errors such as 408, 429, and 5xx while continuing to avoid retries for non-retryable 4xx responses.