From ca94c2941fd7a4978f716b0133ce36af259c495f Mon Sep 17 00:00:00 2001 From: Cesar Okuti Date: Fri, 7 Aug 2026 21:33:48 +0100 Subject: [PATCH 1/3] fix: scrub invalid UTF-8 in CloudFront log lines CloudFront logs can contain raw invalid UTF-8 that breaks CGI.unescape/split and JSON emit. Scrub with String#scrub while preserving valid Unicode. Co-authored-by: Cursor --- CHANGELOG.md | 3 ++ ...nt-plugin-cloudfront-log-optimized.gemspec | 2 +- lib/fluent/plugin/in_cloudfront_log.rb | 14 ++++++ test/plugin/test_in_cloudfrontlog.rb | 46 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e249947..abb3a99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## Release 0.2.3 +- [fix] scrub invalid UTF-8 in CloudFront log lines before CGI.unescape/split so parse and JSON emit stay valid without dropping Unicode + ## Release 0.2.2 - [end] replace slow gsub with more efficient string replace diff --git a/fluent-plugin-cloudfront-log-optimized.gemspec b/fluent-plugin-cloudfront-log-optimized.gemspec index aa946f2..282c589 100644 --- a/fluent-plugin-cloudfront-log-optimized.gemspec +++ b/fluent-plugin-cloudfront-log-optimized.gemspec @@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| spec.name = "fluent-plugin-cloudfront-log-optimized" - spec.version = "0.2.2" + spec.version = "0.2.3" spec.authors = ["kubihee", "lenfree", "kjwierenga"] spec.email = ["kubihie@gmail.com", "lenfree.yeung@gmail.com", "k.j.wierenga@gmail.com"] diff --git a/lib/fluent/plugin/in_cloudfront_log.rb b/lib/fluent/plugin/in_cloudfront_log.rb index bc21a87..d22e529 100644 --- a/lib/fluent/plugin/in_cloudfront_log.rb +++ b/lib/fluent/plugin/in_cloudfront_log.rb @@ -29,6 +29,7 @@ def initialize require 'aws-sdk-s3' require 'time' require 'uri' + require 'cgi' end def configure(conf) @@ -126,12 +127,21 @@ def purge(filename) end end + # Replace invalid UTF-8 sequences while preserving valid multi-byte Unicode. + # CloudFront access logs can contain raw high bytes that break CGI.unescape/split + # and later JSON serialization (e.g. Loki). + def scrub_utf8(value) + value.to_s.dup.force_encoding('UTF-8').scrub('') + end + def process_line(line) if line[0.1] == '#' parse_header(line) return end + line = scrub_utf8(line) + # replace %09 (tab) with space to avoid incorrect introduction of tab character by CGI.unescape line["%09"] = " " if line.include?("%09") @@ -140,6 +150,10 @@ def process_line(line) CGI.unescape(line).split("\t") ].transpose.to_h + record.each do |key, value| + record[key] = scrub_utf8(value) if value.is_a?(String) + end + timestamp = if @parse_date_time Time.iso8601("#{record['date']}T#{record['time']}+00:00").to_i else diff --git a/test/plugin/test_in_cloudfrontlog.rb b/test/plugin/test_in_cloudfrontlog.rb index 8735e76..40eea46 100644 --- a/test/plugin/test_in_cloudfrontlog.rb +++ b/test/plugin/test_in_cloudfrontlog.rb @@ -139,4 +139,50 @@ def create_driver(conf = MINIMAL_CONFIG) end end + sub_test_case "invalid UTF-8 in log lines" do + FIELDS_LINE = "#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end" + + def build_line(uri_query:, user_agent: 'Mozilla/5.0') + [ + '2026-08-07', '19:49:19', 'IAD55-C1', '1234', '1.2.3.4', 'GET', 'd111.cloudfront.net', + '/path', '200', 'https://example.com/', user_agent, uri_query, '-', 'Hit', 'AbCdEf', + 'www.example.com', 'https', '200', '0.050', '-', 'TLSv1.2', 'ECDHE-RSA-AES128-GCM-SHA256', + 'Hit', 'HTTP/2.0', '-', '-', '54321', '0.010', 'Hit', 'text/html', '1234', '-', '-' + ].join("\t") + end + + def prime_and_process(instance, line) + instance.process_line("#Version: 1.0") + instance.process_line(FIELDS_LINE) + instance.process_line(line) + end + + test "scrubs invalid UTF-8 bytes without raising" do + driver = create_driver(MINIMAL_CONFIG) + line = build_line(uri_query: "q=bad\xFFbyte&x=1", user_agent: "Mozilla/\x80Bot") + + emitted_event = nil + assert_nothing_raised { + emitted_event = prime_and_process(driver.instance, line) + } + + assert_equal(true, emitted_event['cs-uri-query'].encoding == Encoding::UTF_8) + assert_equal(true, emitted_event['cs-uri-query'].valid_encoding?) + assert_equal(true, emitted_event['cs(User-Agent)'].valid_encoding?) + assert_equal(false, emitted_event['cs-uri-query'].bytes.include?(0xFF)) + assert_equal('q=badbyte&x=1', emitted_event['cs-uri-query']) + assert_equal('Mozilla/Bot', emitted_event['cs(User-Agent)']) + end + + test "preserves valid Unicode percent-encoding" do + driver = create_driver(MINIMAL_CONFIG) + line = build_line(uri_query: 'q=%C3%A7af%C3%A9&name=Jo%C3%A3o') + + emitted_event = prime_and_process(driver.instance, line) + + assert_equal(true, emitted_event['cs-uri-query'].include?('çafé')) + assert_equal(true, emitted_event['cs-uri-query'].include?('João')) + end + end + end From f99e262c84960547e95dedbf8b849967549052e7 Mon Sep 17 00:00:00 2001 From: Cesar Okuti Date: Mon, 10 Aug 2026 10:52:31 +0100 Subject: [PATCH 2/3] fix: handle Latin-1 percent-encoding in CloudFront log fields CGI.unescape of values like %E9 yields high bytes that break UTF-8 String#split. Parse as ASCII-8BIT and fall back to ISO-8859-1 so records stay valid UTF-8 for Loki. Co-authored-by: Cursor --- lib/fluent/plugin/in_cloudfront_log.rb | 20 +++++++++++++------- test/plugin/test_in_cloudfrontlog.rb | 19 ++++++++++++++++--- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/lib/fluent/plugin/in_cloudfront_log.rb b/lib/fluent/plugin/in_cloudfront_log.rb index d22e529..c2fd064 100644 --- a/lib/fluent/plugin/in_cloudfront_log.rb +++ b/lib/fluent/plugin/in_cloudfront_log.rb @@ -127,21 +127,27 @@ def purge(filename) end end - # Replace invalid UTF-8 sequences while preserving valid multi-byte Unicode. - # CloudFront access logs can contain raw high bytes that break CGI.unescape/split - # and later JSON serialization (e.g. Loki). + # Normalize field values to valid UTF-8 for Loki JSON. + # Prefer keeping already-valid UTF-8; otherwise treat bytes as ISO-8859-1 + # (common for percent-encoded Latin-1 in query strings, e.g. %E9 → é). def scrub_utf8(value) - value.to_s.dup.force_encoding('UTF-8').scrub('') + s = value.to_s.dup + s.force_encoding('UTF-8') + return s if s.valid_encoding? + + s.force_encoding('ISO-8859-1').encode('UTF-8') end def process_line(line) + # Parse as binary: CGI.unescape of %E9/%FF etc. yields high bytes that are + # invalid when the string is tagged UTF-8, and String#split then raises. + line = line.to_s.dup.force_encoding('ASCII-8BIT') + if line[0.1] == '#' - parse_header(line) + parse_header(scrub_utf8(line)) return end - line = scrub_utf8(line) - # replace %09 (tab) with space to avoid incorrect introduction of tab character by CGI.unescape line["%09"] = " " if line.include?("%09") diff --git a/test/plugin/test_in_cloudfrontlog.rb b/test/plugin/test_in_cloudfrontlog.rb index 40eea46..02726bc 100644 --- a/test/plugin/test_in_cloudfrontlog.rb +++ b/test/plugin/test_in_cloudfrontlog.rb @@ -169,9 +169,8 @@ def prime_and_process(instance, line) assert_equal(true, emitted_event['cs-uri-query'].encoding == Encoding::UTF_8) assert_equal(true, emitted_event['cs-uri-query'].valid_encoding?) assert_equal(true, emitted_event['cs(User-Agent)'].valid_encoding?) - assert_equal(false, emitted_event['cs-uri-query'].bytes.include?(0xFF)) - assert_equal('q=badbyte&x=1', emitted_event['cs-uri-query']) - assert_equal('Mozilla/Bot', emitted_event['cs(User-Agent)']) + assert_equal(true, emitted_event['cs-uri-query'].include?('q=bad')) + assert_equal(true, emitted_event['cs(User-Agent)'].include?('Mozilla/')) end test "preserves valid Unicode percent-encoding" do @@ -183,6 +182,20 @@ def prime_and_process(instance, line) assert_equal(true, emitted_event['cs-uri-query'].include?('çafé')) assert_equal(true, emitted_event['cs-uri-query'].include?('João')) end + + test "decodes Latin-1 percent-encoding without raising" do + driver = create_driver(MINIMAL_CONFIG) + # Real CloudFront case: "métricas" encoded as Latin-1 %E9 instead of UTF-8 %C3%A9 + line = build_line(uri_query: 'q=5%20m%E9tricas%20que%20te%20pueden%20servir') + + emitted_event = nil + assert_nothing_raised { + emitted_event = prime_and_process(driver.instance, line) + } + + assert_equal(true, emitted_event['cs-uri-query'].valid_encoding?) + assert_equal(true, emitted_event['cs-uri-query'].include?('métricas')) + end end end From ff76aae0f0cef0e104cfbf46809a55b29735683e Mon Sep 17 00:00:00 2001 From: Cesar Okuti Date: Mon, 10 Aug 2026 11:06:45 +0100 Subject: [PATCH 3/3] docs: clarify 0.2.3 changelog for Latin-1 percent-encoding fix Co-authored-by: Cursor --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abb3a99..93c533e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## Release 0.2.3 -- [fix] scrub invalid UTF-8 in CloudFront log lines before CGI.unescape/split so parse and JSON emit stay valid without dropping Unicode +- [fix] parse CloudFront lines as binary and normalize fields to UTF-8 (ISO-8859-1 fallback) so Latin-1 percent-encoding like `%E9` and invalid UTF-8 no longer break `CGI.unescape`/`.split` or Loki JSON ## Release 0.2.2 - [end] replace slow gsub with more efficient string replace