[APMSVLS-469] feat(traces): rescue errored traces via agent-side error sampler

bottlecap/Cargo.toml

@@ -82,9 +82,10 @@ libdd-trace-normalization = { git = "https://github.com/DataDog/libdatadog", rev
 libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "85ce322a1dcb1eda7df9bcc021223b2d1a236783", default-features = false }
 libdd-trace-stats = { git = "https://github.com/DataDog/libdatadog", rev = "85ce322a1dcb1eda7df9bcc021223b2d1a236783", default-features = false }
 datadog-opentelemetry = { git = "https://github.com/DataDog/dd-trace-rs", rev = "50bfea8755b75e448a80ac04d53fa7edd414eefe", default-features = false, features = ["_unstable_propagation"] }
-dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "d0c7f44191445e20d309734675d5e8b91d2a5d51", default-features = false }
-datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "d0c7f44191445e20d309734675d5e8b91d2a5d51", default-features = false }
-datadog-agent-config = { git = "https://github.com/DataDog/serverless-components", rev = "d0c7f44191445e20d309734675d5e8b91d2a5d51", default-features = false }
+dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "54e570ae60d169acfd48800b1805a9bbdbbeafb2", default-features = false }
+datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "54e570ae60d169acfd48800b1805a9bbdbbeafb2", default-features = false }
+datadog-agent-config = { git = "https://github.com/DataDog/serverless-components", rev = "54e570ae60d169acfd48800b1805a9bbdbbeafb2", default-features = false }
+datadog-agent-trace-sampler = { git = "https://github.com/DataDog/serverless-components", rev = "54e570ae60d169acfd48800b1805a9bbdbbeafb2", default-features = false }
 libddwaf = { version = "1.28.1", git = "https://github.com/DataDog/libddwaf-rust", rev = "d1534a158d976bd4f747bf9fcc58e0712d2d17fc", default-features = false, features = ["serde"] }
 
 [dev-dependencies]

bottlecap/LICENSE-3rdparty.csv

@@ -44,6 +44,7 @@ crossbeam-utils,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The
 crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers
 ctor,https://github.com/mmastrac/rust-ctor,Apache-2.0 OR MIT,Matt Mastracci <matthew@mastracci.com>
 datadog-agent-config,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-agent-config Authors
+datadog-agent-trace-sampler,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-agent-trace-sampler Authors
 datadog-fips,https://github.com/DataDog/serverless-components,Apache-2.0,The datadog-fips Authors
 datadog-opentelemetry,https://github.com/DataDog/dd-trace-rs/tree/main/datadog-opentelemetry,Apache-2.0,Datadog Inc. <info@datadoghq.com>
 datadog-protos,https://github.com/DataDog/saluki,Apache-2.0,The datadog-protos Authors

bottlecap/src/bin/bottlecap/main.rs

@@ -1162,6 +1162,7 @@ fn start_trace_agent(
 
     let trace_processor = Arc::new(trace_processor::ServerlessTraceProcessor {
         obfuscation_config: Arc::new(obfuscation_config),
+        error_sampler: trace_processor::new_error_sampler(config.ext.apm_error_sampler_enabled),
     });
 
     let (span_dedup_service, span_dedup_handle) = span_dedup_service::DedupService::new();

bottlecap/src/config/mod.rs

@@ -66,6 +66,13 @@ pub struct LambdaConfig {
     pub capture_lambda_payload: bool,
     pub capture_lambda_payload_max_depth: u32,
     pub lambda_extension_compute_stats: bool,
+
+    /// `DD_APM_ERROR_SAMPLER_ENABLED`: rescue errored trace chunks that would
+    /// otherwise be dropped, on the `lambda_extension_compute_stats` path. The
+    /// sampler runs in `AlwaysKeep` mode, so this is a plain on/off switch:
+    /// enabled rescues every errored chunk. See APMSVLS-469.
+    pub apm_error_sampler_enabled: bool,
+
     pub span_dedup_timeout: Option<Duration>,
     pub api_key_secret_reload_interval: Option<Duration>,
     pub serverless_appsec_enabled: bool,
@@ -95,6 +102,7 @@ impl Default for LambdaConfig {
             capture_lambda_payload: false,
             capture_lambda_payload_max_depth: 10,
             lambda_extension_compute_stats: false,
+            apm_error_sampler_enabled: false,
             span_dedup_timeout: None,
             api_key_secret_reload_interval: None,
             serverless_appsec_enabled: false,
@@ -153,6 +161,9 @@ pub struct LambdaConfigSource {
     #[serde(deserialize_with = "deser_opt_bool")]
     pub lambda_extension_compute_stats: Option<bool>,
 
+    #[serde(deserialize_with = "deser_opt_bool")]
+    pub apm_error_sampler_enabled: Option<bool>,
+
     #[serde(deserialize_with = "deser_dur_secs_ignore_zero")]
     pub span_dedup_timeout: Option<Duration>,
     #[serde(deserialize_with = "deser_dur_secs_ignore_zero")]
@@ -199,6 +210,7 @@ impl DatadogConfigExtension for LambdaConfig {
                 capture_lambda_payload,
                 capture_lambda_payload_max_depth,
                 lambda_extension_compute_stats,
+                apm_error_sampler_enabled,
                 serverless_appsec_enabled,
                 appsec_waf_timeout,
                 api_security_enabled,
@@ -536,6 +548,32 @@ mod lambda_config_tests {
         assert!(!config.ext.lambda_extension_compute_stats);
     }
 
+    // ---- error sampler (apm_error_sampler_enabled) ----
+
+    #[test]
+    fn apm_error_sampler_enabled_defaults_to_false() {
+        let config = load(|_| Ok(()));
+        assert!(!config.ext.apm_error_sampler_enabled);
+    }
+
+    #[test]
+    fn apm_error_sampler_enabled_from_env() {
+        let config = load(|jail| {
+            jail.set_env("DD_APM_ERROR_SAMPLER_ENABLED", "true");
+            Ok(())
+        });
+        assert!(config.ext.apm_error_sampler_enabled);
+    }
+
+    #[test]
+    fn apm_error_sampler_enabled_from_yaml() {
+        let config = load(|jail| {
+            jail.create_file("datadog.yaml", "apm_error_sampler_enabled: true\n")?;
+            Ok(())
+        });
+        assert!(config.ext.apm_error_sampler_enabled);
+    }
+
     // ---- Duration fields ----
 
     #[test]

bottlecap/src/lifecycle/invocation/processor.rs

@@ -1918,6 +1918,7 @@ mod tests {
                         appsec: None,
                         processor: Arc::new(trace_processor::ServerlessTraceProcessor {
                             obfuscation_config: Arc::new(ObfuscationConfig::new().expect("Failed to create ObfuscationConfig")),
+                            error_sampler: trace_processor::default_error_sampler(),
                         }),
                         trace_tx: tokio::sync::mpsc::channel(1).0,
                         stats_generator: Arc::new(StatsGenerator::new(stats_concentrator_handle)),
@@ -2028,6 +2029,7 @@ mod tests {
                 obfuscation_config: Arc::new(
                     ObfuscationConfig::new().expect("Failed to create ObfuscationConfig"),
                 ),
+                error_sampler: trace_processor::default_error_sampler(),
             }),
             trace_tx: tokio::sync::mpsc::channel(1).0,
             stats_generator: Arc::new(StatsGenerator::new(stats_concentrator_handle)),
@@ -2667,6 +2669,7 @@ mod tests {
                 obfuscation_config: Arc::new(
                     ObfuscationConfig::new().expect("Failed to create ObfuscationConfig"),
                 ),
+                error_sampler: trace_processor::default_error_sampler(),
             }),
             trace_tx: tokio::sync::mpsc::channel(1).0,
             stats_generator: Arc::new(StatsGenerator::new(stats_concentrator_handle)),
@@ -3176,6 +3179,7 @@ mod tests {
                 obfuscation_config: Arc::new(
                     ObfuscationConfig::new().expect("Failed to create ObfuscationConfig"),
                 ),
+                error_sampler: trace_processor::default_error_sampler(),
             }),
             trace_tx,
             stats_generator: Arc::new(StatsGenerator::new(stats_concentrator_handle)),