From 3df980492732d294792c19a8c34f1c1dcb9ce8e4 Mon Sep 17 00:00:00 2001 From: Theo Gravity Date: Mon, 11 May 2026 23:42:45 -0700 Subject: [PATCH 1/2] feat(transports/newrelic): align encoder with TypeScript transport format Update encoder to emit timestamp/level/log/attributes instead of logtype/loglevel/message with flat fields. Adds attribute validation (max 255 attributes, 255-char names, 4094-char values) at encode time. Documentation and tests updated to match. --- docs/src/transports/newrelic.md | 49 ++++++++-------- docs/src/whats-new.md | 6 ++ transports/betterstack/go.sum | 4 +- transports/newrelic/go.sum | 6 +- transports/newrelic/newrelic.go | 83 ++++++++++++++++++++++++---- transports/newrelic/newrelic_test.go | 50 +++++++++-------- 6 files changed, 138 insertions(+), 60 deletions(-) diff --git a/docs/src/transports/newrelic.md b/docs/src/transports/newrelic.md index ff8a056..f7ff492 100644 --- a/docs/src/transports/newrelic.md +++ b/docs/src/transports/newrelic.md @@ -7,7 +7,7 @@ description: Ship logs to the New Relic Log Ingest API. -Sends log entries to the [New Relic Log Ingest API](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/). Built on the [HTTP transport](/transports/http) with a New Relic-specific encoder, site-aware URL, and `api-key` header. +Sends log entries to the [New Relic Log Ingest API](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/). Built on the [HTTP transport](/transports/http) with a New Relic-specific encoder (timestamp, level, log, attributes), site-aware URL, and `Api-Key` header. Includes attribute validation enforced at encode time (max 255 attributes, 255-char names, 4,094-char values). Log format matches the [TypeScript transport](https://loglayer.dev/transports/new-relic). ```sh go get go.loglayer.dev/transports/newrelic @@ -134,47 +134,52 @@ See the [HTTP transport docs](/transports/http) for the full HTTP config surface ## Encoded Body Shape -Each log entry becomes one object in a JSON array: +Each log entry becomes one object in a JSON array. The format matches the [TypeScript New Relic transport](https://loglayer.dev/transports/new-relic): ```json [ { - "logtype": "LogEvent", "timestamp": 1745616000123, - "loglevel": "info", - "message": "served request", - "requestId": "abc", - "durationMs": 42 + "level": "info", + "log": "served request", + "attributes": { + "requestId": "abc", + "durationMs": 42 + } } ] ``` -Every object includes `logtype: "LogEvent"` and `timestamp` (epoch milliseconds) as required by the New Relic API. Persistent fields (`WithFields`) and metadata (`WithMetadata`) follow the [core placement rules](/configuration#fieldskey): when `FieldsKey` is empty, fields merge at the root of each log object; when `MetadataFieldName` is empty, map metadata merges at the root and non-map metadata nests under `metadata`. Fields and metadata are merged via `transport.MergeIntoMap` so they coexist cleanly with the system fields. Set either knob on `loglayer.Config` to nest under a configured key instead. +Every object includes `timestamp` (epoch milliseconds), `level` (the log layer severity), and `log` (the message text). Persistent fields and metadata are merged under the `attributes` key with New Relic's API constraints enforced: maximum 255 attributes, 255-character attribute names, and string values truncated at 4,094 characters. Reserved fields (`timestamp`, `level`, `log`) are excluded from attributes to prevent collisions. -## Level → loglevel Mapping +## Level → level Mapping -New Relic uses a `loglevel` string per entry. The transport maps loglayer levels: +New Relic uses a `level` string per entry. The transport maps loglayer levels: -| LogLayer Level | New Relic loglevel | -|------------------|--------------------| -| `LogLevelTrace` | `trace` | -| `LogLevelDebug` | `debug` | -| `LogLevelInfo` | `info` | -| `LogLevelWarn` | `warn` | -| `LogLevelError` | `error` | -| `LogLevelFatal` | `critical` | -| `LogLevelPanic` | `critical` | +| LogLayer Level | New Relic level | +|------------------|------------------| +| `LogLevelTrace` | `trace` | +| `LogLevelDebug` | `debug` | +| `LogLevelInfo` | `info` | +| `LogLevelWarn` | `warn` | +| `LogLevelError` | `error` | +| `LogLevelFatal` | `critical` | +| `LogLevelPanic` | `critical` | New Relic has no distinction between Fatal and Panic, so both map to `critical` (the highest-severity level in the Log Ingest API). ## API Limits -New Relic enforces these restrictions on Log Ingest API requests ([reference](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#limits)): +The transport enforces these limits at encode time ([reference](https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/#limits)): + +- 255 attributes maximum per event (excess attributes are dropped, not merged) +- 255 characters maximum per attribute name (longer names are dropped silently) +- String values truncated at 4,094 characters + +New Relic also enforces these on the server side: - 1MB maximum payload per POST (compression recommended) - Payload must be UTF-8 encoded -- 255 attributes maximum per event -- 255 characters maximum per attribute name - 4,094 characters stored in NRDB; longer values stored as a blob The default `BatchSize` of 100 stays well under the 1MB payload limit for typical entries. If you bump `BatchSize` for higher throughput or ship large attributes, watch for the boundary. diff --git a/docs/src/whats-new.md b/docs/src/whats-new.md index 3350fb7..8a5f68b 100644 --- a/docs/src/whats-new.md +++ b/docs/src/whats-new.md @@ -7,6 +7,12 @@ description: Latest features and improvements in LogLayer for Go. - See the [main `CHANGELOG.md`](https://github.com/loglayer/loglayer-go/blob/main/CHANGELOG.md) for the auto-generated per-release log. +## May 11, 2026 + +`transports/newrelic`: + +Encoder updated to match the [TypeScript transport format](https://loglayer.dev/transports/new-relic). Log entries now use `timestamp`, `level`, `log`, and `attributes` (nested) instead of the previous flat shape (`logtype`, `loglevel`, `message`, fields at root). Attribute validation added at encode time: maximum 255 attributes, 255-char names, 4,094-char string values. + ## May 10, 2026 `transports/newrelic`: diff --git a/transports/betterstack/go.sum b/transports/betterstack/go.sum index 04aba65..512eb95 100644 --- a/transports/betterstack/go.sum +++ b/transports/betterstack/go.sum @@ -2,7 +2,7 @@ github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= go.loglayer.dev/transports/http/v2 v2.1.0 h1:+9kaawgxkbFYKvIliTWwSjj9WKUnG9Uzjm3dRmuCkRY= go.loglayer.dev/transports/http/v2 v2.1.0/go.mod h1:OeIaQoUHcT3Qeb8HI8CaY4OFhQwwpC/Pb0yXGTrHxIM= -go.loglayer.dev/v2 v2.0.1 h1:B7oYpkfMky0UG/N8IdKeIiiTi6h7eyj5NvRyEth9DHI= -go.loglayer.dev/v2 v2.0.1/go.mod h1:+BWhs5AyICvCLBz07qHnCE12W34tArUWfXOba0Ct/QI= +go.loglayer.dev/v2 v2.1.0 h1:8/fq8Z1NNLtjfKgaPn6S0Hy7zWPnkjn9Gj3YgEDFk4w= +go.loglayer.dev/v2 v2.1.0/go.mod h1:+BWhs5AyICvCLBz07qHnCE12W34tArUWfXOba0Ct/QI= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= diff --git a/transports/newrelic/go.sum b/transports/newrelic/go.sum index 9feb45c..43b75e5 100644 --- a/transports/newrelic/go.sum +++ b/transports/newrelic/go.sum @@ -6,8 +6,10 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -go.loglayer.dev/v2 v2.0.1 h1:B7oYpkfMky0UG/N8IdKeIiiTi6h7eyj5NvRyEth9DHI= -go.loglayer.dev/v2 v2.0.1/go.mod h1:+BWhs5AyICvCLBz07qHnCE12W34tArUWfXOba0Ct/QI= +go.loglayer.dev/transports/http/v2 v2.1.0 h1:+9kaawgxkbFYKvIliTWwSjj9WKUnG9Uzjm3dRmuCkRY= +go.loglayer.dev/transports/http/v2 v2.1.0/go.mod h1:OeIaQoUHcT3Qeb8HI8CaY4OFhQwwpC/Pb0yXGTrHxIM= +go.loglayer.dev/v2 v2.1.0 h1:8/fq8Z1NNLtjfKgaPn6S0Hy7zWPnkjn9Gj3YgEDFk4w= +go.loglayer.dev/v2 v2.1.0/go.mod h1:+BWhs5AyICvCLBz07qHnCE12W34tArUWfXOba0Ct/QI= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/transports/newrelic/newrelic.go b/transports/newrelic/newrelic.go index 820d4c3..b741f35 100644 --- a/transports/newrelic/newrelic.go +++ b/transports/newrelic/newrelic.go @@ -3,9 +3,8 @@ // Wraps transports/http with New Relic-specific defaults: // - Site-aware intake URL (US, EU) // - Api-Key header from Config.LicenseKey -// - Encoder that emits New Relic's expected log shape (logtype, -// timestamp, loglevel, message) merged with the user's fields -// and metadata +// - Encoder that emits the New Relic log shape (timestamp, level, log, +// attributes) with attribute validation enforced at encode time // // API reference: // https://docs.newrelic.com/docs/logs/log-api/introduction-log-api/ @@ -155,18 +154,21 @@ func Build(cfg Config) (*Transport, error) { } // newEncoder produces the JSON-array encoder for New Relic's intake format. -// Each entry is a JSON object with logtype, timestamp, loglevel, message, -// plus any user fields and metadata. +// Each entry is a JSON object with timestamp, level, log, and attributes +// (merged data + metadata), matching the TypeScript transport format. func newEncoder() httptr.Encoder { return httptr.EncoderFunc(func(entries []httptr.Entry) ([]byte, string, error) { objs := make([]map[string]any, len(entries)) for i, e := range entries { - obj := make(map[string]any, 4+len(e.Data)) - obj["logtype"] = "LogEvent" - obj["timestamp"] = e.Time.UnixMilli() - obj["loglevel"] = loglevelFor(e.Level) - obj["message"] = transport.JoinMessages(e.Messages) - transport.MergeIntoMap(obj, e.Data, e.Metadata, e.Schema.MetadataFieldName) + obj := map[string]any{ + "timestamp": e.Time.UnixMilli(), + "level": loglevelFor(e.Level), + "log": transport.JoinMessages(e.Messages), + } + attrs := mergeAttributes(e) + if len(attrs) > 0 { + obj["attributes"] = attrs + } objs[i] = obj } body, err := json.Marshal(objs) @@ -174,6 +176,65 @@ func newEncoder() httptr.Encoder { }) } +const ( + maxAttributes = 255 + maxAttributeNameLength = 255 + maxAttributeValueLength = 4094 +) + +// mergeAttributes merges entry data and metadata into a single attributes +// map, enforcing New Relic's API constraints: max 255 attributes, max 255- +// char attribute names, and values truncated at 4094 chars. Reserved fields +// (timestamp, level, log) are excluded to prevent collisions. +func mergeAttributes(e httptr.Entry) map[string]any { + attrs := make(map[string]any) + for k, v := range e.Data { + if reserved(k) { + continue + } + if len(attrs)+1 > maxAttributes { + break + } + setAttr(attrs, k, v) + } + + if m, ok := transport.MetadataAsRootMap(e.Metadata); ok { + for k, v := range m { + if reserved(k) { + continue + } + if len(attrs)+1 > maxAttributes { + break + } + setAttr(attrs, k, v) + } + } + + return attrs +} + +// reserved returns true if k is a top-level New Relic log field that should +// not appear inside the attributes map. +func reserved(k string) bool { + switch k { + case "timestamp", "level", "log": + return true + } + return false +} + +// setAttr validates and sets a single attribute, truncating string values +// that exceed the New Relic limit. +func setAttr(attrs map[string]any, key string, val any) { + if len(key) > maxAttributeNameLength { + return + } + if s, ok := val.(string); ok && len(s) > maxAttributeValueLength { + val = s[:maxAttributeValueLength] + } + attrs[key] = val +} + // loglevelFor maps a loglayer LogLevel to New Relic's loglevel string. // Fatal and Panic both map to "critical" (New Relic's highest-severity // log level). diff --git a/transports/newrelic/newrelic_test.go b/transports/newrelic/newrelic_test.go index fb3a300..3472f36 100644 --- a/transports/newrelic/newrelic_test.go +++ b/transports/newrelic/newrelic_test.go @@ -79,8 +79,8 @@ func TestNewRelic_BasicBatchOnClose(t *testing.T) { } // First entry should be "hello" - if arr[0]["message"] != "hello" { - t.Errorf("first message: got %v, want hello", arr[0]["message"]) + if arr[0]["log"] != "hello" { + t.Errorf("first log: got %v, want hello", arr[0]["log"]) } } @@ -292,9 +292,9 @@ func TestNewRelic_LevelFiltering(t *testing.T) { // Check that no debug or info entries made it through. for i, obj := range arr { - if loglevel, ok := obj["loglevel"].(string); ok { - if loglevel == "debug" || loglevel == "info" { - t.Errorf("entry %d should have been filtered, got loglevel %q", i, loglevel) + if level, ok := obj["level"].(string); ok { + if level == "debug" || level == "info" { + t.Errorf("entry %d should have been filtered, got level %q", i, level) } } } @@ -339,8 +339,8 @@ func TestNewRelic_EncodedBodyShape(t *testing.T) { obj := arr[0] - if obj["logtype"] != "LogEvent" { - t.Errorf("logtype: got %v, want LogEvent", obj["logtype"]) + if level, ok := obj["level"].(string); !ok || level == "" { + t.Errorf("level: missing or empty, got %v", obj["level"]) } // Timestamp should be a number (epoch milliseconds). @@ -350,12 +350,12 @@ func TestNewRelic_EncodedBodyShape(t *testing.T) { t.Errorf("timestamp too small for a valid epoch-ms: %v", ts) } - if obj["loglevel"] != "info" { - t.Errorf("loglevel: got %v, want info", obj["loglevel"]) + if obj["level"] != "info" { + t.Errorf("level: got %v, want info", obj["level"]) } - if obj["message"] != "test body shape" { - t.Errorf("message: got %v, want 'test body shape'", obj["message"]) + if obj["log"] != "test body shape" { + t.Errorf("log: got %v, want 'test body shape'", obj["log"]) } } @@ -393,15 +393,19 @@ func TestNewRelic_FieldAndMetadataInBody(t *testing.T) { _ = json.Unmarshal(cap.bodies[0], &arr) obj := arr[0] - if obj["requestId"] != "req-42" { - t.Errorf("requestId: got %v", obj["requestId"]) + attrs, ok := obj["attributes"].(map[string]any) + if !ok { + t.Fatal("expected attributes map") + } + if attrs["requestId"] != "req-42" { + t.Errorf("requestId: got %v", attrs["requestId"]) } // JSON unmarshals numbers as float64. - if obj["durationMs"] != float64(123) { - t.Errorf("durationMs: got %v", obj["durationMs"]) + if attrs["durationMs"] != float64(123) { + t.Errorf("durationMs: got %v", attrs["durationMs"]) } - if obj["endpoint"] != "/api/v1" { - t.Errorf("endpoint: got %v", obj["endpoint"]) + if attrs["endpoint"] != "/api/v1" { + t.Errorf("endpoint: got %v", attrs["endpoint"]) } } @@ -585,11 +589,11 @@ func TestNewRelic_GroupsWork(t *testing.T) { t.Fatalf("expected 1 entry, got %d", len(arr)) } obj := arr[0] - if obj["logtype"] != "LogEvent" { - t.Errorf("logtype: got %v", obj["logtype"]) + if obj["level"] == nil { + t.Errorf("level: missing") } - if obj["message"] != "grouped log" { - t.Errorf("message: got %v", obj["message"]) + if obj["log"] != "grouped log" { + t.Errorf("log: got %v", obj["log"]) } } @@ -650,9 +654,9 @@ func TestNewRelic_LevelMapping(t *testing.T) { "critical": {}, } for i, obj := range allEntries { - if lv, ok := obj["loglevel"].(string); ok { + if lv, ok := obj["level"].(string); ok { if _, ok := wantMap[lv]; !ok { - t.Errorf("entry %d has unexpected loglevel %q", i, lv) + t.Errorf("entry %d has unexpected level %q", i, lv) } } } From 66b17d5d09a907d4c07a35c456c4d54271e6e045 Mon Sep 17 00:00:00 2001 From: Theo Gravity Date: Mon, 11 May 2026 23:43:05 -0700 Subject: [PATCH 2/2] chore: add changeset for newrelic encoder alignment --- .changeset/new-relic-encoder-align.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/new-relic-encoder-align.md diff --git a/.changeset/new-relic-encoder-align.md b/.changeset/new-relic-encoder-align.md new file mode 100644 index 0000000..c72e1b7 --- /dev/null +++ b/.changeset/new-relic-encoder-align.md @@ -0,0 +1,5 @@ +--- +"transports/newrelic": minor +--- + +Align encoder with TypeScript transport format: emit `timestamp`, `level`, `log`, and `attributes` instead of the previous flat `logtype`/`loglevel`/`message` shape. Adds attribute validation at encode time (max 255 attributes, 255-char names, 4,094-char string values).