From 1089f32eb3224183d2eca6c652a436cc50b3f94c Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Tue, 21 Apr 2026 14:54:10 -0400 Subject: [PATCH 01/26] feat(datadog-aws-lambda): add root invocation span with OTel tracing Implements the core Lambda handler wrapper with Datadog tracing: - WrappedHandler: tower::Service that wraps user handlers with OTel spans - LambdaSpan: aws.lambda root span with cold_start, request_id, function metadata - Invocation lifecycle: start/handler_context/finish with error recording - Config: service/env/version or full DatadogTracingBuilder control - Lambda-appropriate OTel defaults (sync writes, no client-side stats) Trigger extraction and inferred spans will follow in a subsequent PR. --- instrumentation/Cargo.lock | 856 +++++++++++++++++- instrumentation/datadog-aws-lambda/Cargo.toml | 7 +- .../datadog-aws-lambda/src/attribute_keys.rs | 18 + .../datadog-aws-lambda/src/invocation.rs | 265 ++++++ instrumentation/datadog-aws-lambda/src/lib.rs | 221 +++++ 5 files changed, 1341 insertions(+), 26 deletions(-) create mode 100644 instrumentation/datadog-aws-lambda/src/attribute_keys.rs create mode 100644 instrumentation/datadog-aws-lambda/src/invocation.rs diff --git a/instrumentation/Cargo.lock b/instrumentation/Cargo.lock index c21728ff..6920f7ef 100644 --- a/instrumentation/Cargo.lock +++ b/instrumentation/Cargo.lock @@ -17,6 +17,18 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "anyhow" version = "1.0.102" @@ -54,6 +66,17 @@ dependencies = [ "syn", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -114,6 +137,12 @@ dependencies = [ "crossbeam-channel", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.60" @@ -136,13 +165,66 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "const_format" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" dependencies = [ "const_format_proc_macros", + "konst", ] [[package]] @@ -165,6 +247,42 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -174,12 +292,37 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -194,11 +337,9 @@ dependencies = [ name = "datadog-aws-lambda" version = "0.1.0" dependencies = [ - "base64 0.22.1", "datadog-opentelemetry", "lambda_runtime", "opentelemetry", - "opentelemetry-semantic-conventions", "opentelemetry_sdk", "serde", "serde_json", @@ -213,6 +354,7 @@ dependencies = [ "anyhow", "arc-swap", "base64 0.21.7", + "criterion", "foldhash 0.1.5", "hashbrown 0.15.5", "http-body-util", @@ -227,9 +369,10 @@ dependencies = [ "libdd-trace-utils", "lru", "opentelemetry", + "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", - "rand 0.8.5", + "rand 0.8.6", "rustc_version_runtime", "serde", "serde_json", @@ -250,6 +393,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -278,6 +432,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -290,6 +450,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.32" @@ -424,6 +593,36 @@ dependencies = [ "wasip3", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -458,6 +657,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -523,6 +728,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -533,19 +739,35 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -553,12 +775,115 @@ dependencies = [ "tracing", ] +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -571,6 +896,42 @@ dependencies = [ "serde_core", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -592,10 +953,27 @@ version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "lambda_runtime" version = "0.13.0" @@ -619,7 +997,7 @@ dependencies = [ "serde_path_to_error", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tracing", ] @@ -639,7 +1017,7 @@ dependencies = [ "hyper", "hyper-util", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", "tracing-subscriber", @@ -757,7 +1135,7 @@ dependencies = [ "libdd-trace-protobuf 2.0.0", "memfd", "prost", - "rand 0.8.5", + "rand 0.8.6", "rmp", "rmp-serde", "rustix", @@ -861,7 +1239,7 @@ dependencies = [ "libdd-trace-normalization", "libdd-trace-protobuf 3.0.1", "prost", - "rand 0.8.5", + "rand 0.8.6", "rmp", "rmp-serde", "rmpv", @@ -877,6 +1255,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "log" version = "0.4.29" @@ -954,6 +1338,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opentelemetry" version = "0.31.0" @@ -967,6 +1357,50 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror 2.0.18", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.31.0" @@ -1022,6 +1456,43 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1067,7 +1538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -1096,9 +1567,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -1153,6 +1624,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.12.3" @@ -1182,6 +1673,40 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "rmp" version = "0.8.15" @@ -1254,6 +1779,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.28" @@ -1324,6 +1858,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -1385,6 +1931,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1402,6 +1954,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sys-info" version = "0.9.1" @@ -1461,11 +2033,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" -version = "1.51.1" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", @@ -1511,6 +2103,43 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.4.13" @@ -1526,6 +2155,43 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -1607,9 +2273,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicode-ident" @@ -1629,11 +2295,29 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -1652,6 +2336,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -1669,11 +2363,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -1682,7 +2376,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] @@ -1698,6 +2392,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.118" @@ -1764,6 +2468,25 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows" version = "0.48.0" @@ -1936,6 +2659,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -2015,6 +2744,35 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.48" @@ -2035,6 +2793,60 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/instrumentation/datadog-aws-lambda/Cargo.toml b/instrumentation/datadog-aws-lambda/Cargo.toml index 5e75bb24..16865171 100644 --- a/instrumentation/datadog-aws-lambda/Cargo.toml +++ b/instrumentation/datadog-aws-lambda/Cargo.toml @@ -13,16 +13,15 @@ authors.workspace = true publish.workspace = true [dependencies] -base64 = "0.22" -datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry" } +datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry", features = ["test-utils"] } lambda_runtime = "0.13" serde = { workspace = true } -serde_json = { workspace = true } +serde_json = { workspace = true, features = ["raw_value"] } opentelemetry = { workspace = true } -opentelemetry-semantic-conventions = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["trace", "metrics", "logs"] } tracing = { workspace = true } [dev-dependencies] +datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry", features = ["test-utils"] } opentelemetry_sdk = { workspace = true, features = ["trace", "metrics", "logs", "testing"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs new file mode 100644 index 00000000..aa1834bd --- /dev/null +++ b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs @@ -0,0 +1,18 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) const OPERATION_NAME: &str = "operation.name"; +pub(crate) const RESOURCE_NAME: &str = "resource.name"; +pub(crate) const SPAN_TYPE: &str = "span.type"; +pub(crate) const ERROR: &str = "error"; +pub(crate) const ERROR_MESSAGE: &str = "error.message"; + +// Root span tags +pub(crate) const LANGUAGE: &str = "language"; +pub(crate) const REQUEST_ID: &str = "request_id"; +pub(crate) const COLD_START: &str = "cold_start"; +pub(crate) const FUNCTION_ARN: &str = "function_arn"; +pub(crate) const FUNCTION_VERSION: &str = "function_version"; +pub(crate) const FUNCTION_NAME: &str = "functionname"; +pub(crate) const RESOURCE_NAMES: &str = "resource_names"; +pub(crate) const DD_ORIGIN: &str = "_dd.origin"; diff --git a/instrumentation/datadog-aws-lambda/src/invocation.rs b/instrumentation/datadog-aws-lambda/src/invocation.rs new file mode 100644 index 00000000..2086d417 --- /dev/null +++ b/instrumentation/datadog-aws-lambda/src/invocation.rs @@ -0,0 +1,265 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Lifecycle management for a single Lambda invocation. +//! +//! Each invocation produces one *root span* (`aws.lambda`) that wraps the handler call. +//! +//! Typical usage: +//! 1. [`Invocation::start`] — create the root span before the handler runs. +//! 2. [`Invocation::handler_context`] — pass the returned context to the handler so its OTel spans +//! are correctly parented. +//! 3. [`Invocation::finish`] — record errors and end the span after the handler returns. + +use crate::attribute_keys as attr; + +use opentelemetry::trace::{SpanKind, TraceContextExt, Tracer}; +use opentelemetry::{Context, KeyValue}; +use opentelemetry_sdk::trace::SdkTracer; + +pub(crate) static TRACER_NAME: &str = "datadog-lambda-rs"; + +/// The Lambda invocation (`aws.lambda`) root span. +/// +/// Wraps the OTel context so the span can be ended via [`finish`](Self::finish) +/// and errors can be recorded via [`set_error`](Self::set_error). +pub(crate) struct LambdaSpan { + /// OTel context whose active span is the `aws.lambda` root span. + cx: Context, + /// Lambda request ID, copied here for structured log correlation. + request_id: String, +} + +impl LambdaSpan { + /// Creates and starts the `aws.lambda` root span. + /// + /// The span is parented to the ambient OTel context, which is typically a new root trace. + pub(crate) fn start( + tracer: &SdkTracer, + lambda_cx: &lambda_runtime::Context, + cold_start: bool, + ) -> Self { + let function_name = &lambda_cx.env_config.function_name; + let request_id = lambda_cx.request_id.clone(); + + tracing::debug!(request_id, "creating invocation root span"); + + let parent_cx = Context::current(); + + let mut builder = tracer.span_builder(TRACER_NAME); + builder.span_kind = Some(SpanKind::Server); + let attrs = vec![ + KeyValue::new(attr::OPERATION_NAME, "aws.lambda"), + KeyValue::new(attr::LANGUAGE, "rust"), + KeyValue::new(attr::RESOURCE_NAME, function_name.clone()), + KeyValue::new(attr::SPAN_TYPE, "serverless"), + KeyValue::new(attr::REQUEST_ID, request_id.clone()), + KeyValue::new(attr::COLD_START, cold_start), + KeyValue::new(attr::FUNCTION_ARN, lambda_cx.invoked_function_arn.clone()), + KeyValue::new(attr::FUNCTION_VERSION, lambda_cx.env_config.version.clone()), + KeyValue::new(attr::FUNCTION_NAME, function_name.to_lowercase()), + KeyValue::new(attr::RESOURCE_NAMES, function_name.clone()), + KeyValue::new(attr::DD_ORIGIN, "lambda"), + ]; + builder.attributes = Some(attrs); + + let span = tracer.build_with_context(builder, &parent_cx); + let cx = parent_cx.with_span(span); + + Self { cx, request_id } + } + + pub(crate) fn set_error(&self, err: &impl std::fmt::Display) { + let err_msg = err.to_string(); + tracing::warn!(request_id = self.request_id, "handler returned error"); + let span = self.cx.span(); + span.set_status(opentelemetry::trace::Status::Error { + description: err_msg.clone().into(), + }); + span.set_attribute(KeyValue::new(attr::ERROR, true)); + span.set_attribute(KeyValue::new(attr::ERROR_MESSAGE, err_msg)); + // error.type is omitted: lambda_runtime boxes the error, erasing its type. + } + + /// Ends the root span. + pub(crate) fn finish(self) { + self.cx.span().end(); + } +} + +/// Owns the full lifecycle of a single Lambda invocation's tracing state. +/// +/// Holds the root span created for the handler call. +/// Call [`start`](Self::start) before the handler, then [`finish`](Self::finish) after. +pub(crate) struct Invocation { + /// The `aws.lambda` root span for this invocation. + lambda_span: LambdaSpan, +} + +impl Invocation { + pub(crate) fn start( + tracer: &SdkTracer, + lambda_cx: &lambda_runtime::Context, + cold_start: bool, + ) -> Self { + let lambda_span = LambdaSpan::start(tracer, lambda_cx, cold_start); + Self { lambda_span } + } + + /// Returns the OTel context to use as the active context during handler execution. + /// + /// User handler spans should be children of the root span, so this context must + /// be passed to [`FutureExt::with_context`](opentelemetry::trace::FutureExt::with_context) + /// (or equivalent) when running the handler future. + #[must_use] + pub(crate) fn handler_context(&self) -> Context { + self.lambda_span.cx.clone() + } + + /// Records any handler error and ends the root span. + /// + /// The result is returned unchanged; this method only has side effects + /// (error attributes, span end times). + pub(crate) fn finish(self, result: Result) -> Result + where + Err: std::fmt::Display, + { + if let Err(ref err) = result { + self.lambda_span.set_error(err); + } + self.lambda_span.finish(); + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use opentelemetry::trace::{ + SpanContext, SpanId, TraceFlags, TraceId, TraceState, TracerProvider as _, + }; + use opentelemetry::Value; + use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData}; + + fn test_provider() -> (SdkTracerProvider, InMemorySpanExporter) { + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + (provider, exporter) + } + + fn test_lambda_cx() -> lambda_runtime::Context { + let mut cx = lambda_runtime::Context::default(); + cx.request_id = "req-123".to_string(); + cx.invoked_function_arn = "arn:aws:lambda:us-east-1:123:function:my-function".to_string(); + std::sync::Arc::make_mut(&mut cx.env_config).function_name = "My-Function".to_string(); + std::sync::Arc::make_mut(&mut cx.env_config).version = "$LATEST".to_string(); + cx + } + + fn find_attr<'a>(attrs: &'a [KeyValue], key: &str) -> Option<&'a Value> { + attrs + .iter() + .find(|kv| kv.key.as_str() == key) + .map(|kv| &kv.value) + } + + fn finished_spans(exporter: &InMemorySpanExporter) -> Vec { + exporter.get_finished_spans().unwrap() + } + + #[test] + fn sets_expected_attributes_on_root_span() { + let (provider, exporter) = test_provider(); + let tracer = provider.tracer("test"); + + let lambda_cx = test_lambda_cx(); + + let span = LambdaSpan::start(&tracer, &lambda_cx, true); + span.finish(); + provider.force_flush().unwrap(); + + let spans = finished_spans(&exporter); + assert_eq!(spans.len(), 1); + let attrs = &spans[0].attributes; + + assert_eq!( + find_attr(attrs, "_dd.origin"), + Some(&Value::String("lambda".into())) + ); + assert_eq!( + find_attr(attrs, "request_id"), + Some(&Value::String("req-123".into())) + ); + assert_eq!( + find_attr(attrs, "resource.name"), + Some(&Value::String("My-Function".into())) + ); + assert_eq!( + find_attr(attrs, "operation.name"), + Some(&Value::String("aws.lambda".into())) + ); + assert_eq!( + find_attr(attrs, "functionname"), + Some(&Value::String("my-function".into())) + ); + assert_eq!(find_attr(attrs, "cold_start"), Some(&Value::Bool(true))); + } + + #[test] + fn inherits_parent_trace_id_for_root_span() { + let (provider, _) = test_provider(); + let tracer = provider.tracer("test"); + + let trace_id = TraceId::from_hex("4bf92f3577b34da6a3ce929d0e0e4736").unwrap(); + let parent_sc = SpanContext::new( + trace_id, + SpanId::from_hex("00f067aa0ba902b7").unwrap(), + TraceFlags::SAMPLED, + true, + TraceState::default(), + ); + let _guard = Context::current() + .with_remote_span_context(parent_sc) + .attach(); + + let span = LambdaSpan::start(&tracer, &test_lambda_cx(), false); + assert_eq!(span.cx.span().span_context().trace_id(), trace_id); + } + + #[tokio::test] + async fn error_handler_sets_error_attributes() { + let (provider, exporter) = test_provider(); + let invocation = Invocation { + lambda_span: LambdaSpan::start(&provider.tracer("test"), &test_lambda_cx(), false), + }; + + let _: Result<(), String> = invocation.finish(Err::<(), String>("boom".to_string())); + provider.force_flush().unwrap(); + + let spans = finished_spans(&exporter); + let attrs = &spans[0].attributes; + assert_eq!(find_attr(attrs, "error"), Some(&Value::Bool(true))); + assert_eq!( + find_attr(attrs, "error.message"), + Some(&Value::String("boom".into())) + ); + } + + #[tokio::test] + async fn successful_handler_sets_no_error_attributes() { + let (provider, exporter) = test_provider(); + let invocation = Invocation { + lambda_span: LambdaSpan::start(&provider.tracer("test"), &test_lambda_cx(), false), + }; + + let _: Result<(), String> = invocation.finish(Ok(())); + provider.force_flush().unwrap(); + + let spans = finished_spans(&exporter); + let attrs = &spans[0].attributes; + assert!(find_attr(attrs, "error").is_none()); + assert!(find_attr(attrs, "error.message").is_none()); + } +} diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 5a97e724..1f773155 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -1,2 +1,223 @@ // Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 + +// Panic and unwrap are banned in production code to prevent silent Lambda crashes. +// Tests are exempt so they can use `.unwrap()` freely. +#![cfg_attr(not(test), deny(clippy::panic))] +#![cfg_attr(not(test), deny(clippy::unwrap_used))] +#![cfg_attr(not(test), deny(clippy::expect_used))] + +mod attribute_keys; +mod invocation; + +use invocation::{Invocation, TRACER_NAME}; +use lambda_runtime::tower::Service; +use lambda_runtime::LambdaEvent; +use opentelemetry::trace::FutureExt; +use serde::de::DeserializeOwned; +use serde_json::value::RawValue; +use std::future::Future; +use std::marker::PhantomData; +use std::sync::Arc; +use std::task::{self, Poll}; + +type BoxFuture = std::pin::Pin + Send>>; + +#[derive(Default)] +pub struct Config { + /// Service name. Overrides `DD_SERVICE`. Ignored when [`tracing`](Self::tracing) is `Some`. + pub service: Option, + /// Deployment environment. Overrides `DD_ENV`. Ignored when [`tracing`](Self::tracing) is + /// `Some`. + pub env: Option, + /// Service version. Overrides `DD_VERSION`. Ignored when [`tracing`](Self::tracing) is `Some`. + pub version: Option, + /// Full control over the OTel SDK and Datadog tracer config. + /// + /// When `None` (default), Lambda-appropriate defaults are applied and + /// `service`/`env`/`version` above are forwarded. When `Some`, the builder is used as-is; + /// `service`/`env`/`version` are ignored and you are responsible for setting: + /// - `trace_stats_computation_enabled = false` (the Datadog agent handles stats for serverless + /// environments) + /// - `trace_writer_synchronous_write = true` (so `force_flush()` blocks until spans reach + /// agent) + pub tracing: Option, +} + +fn build_tracing( + service: Option, + env: Option, + version: Option, +) -> datadog_opentelemetry::DatadogTracingBuilder { + let mut builder = datadog_opentelemetry::configuration::Config::builder(); + // Stats are computed server-side by the extension; client-side computation is redundant. + builder.set_trace_stats_computation_enabled(false); + // Synchronous writes make force_flush() block until data reaches the agent, + // this helps reduce span loss when the Lambda process freezes after the handler returns. + builder.set_trace_writer_synchronous_write(true); + if let Some(s) = service { + builder.set_service(s); + } + if let Some(e) = env { + builder.set_env(e); + } + if let Some(v) = version { + builder.set_version(v); + } + datadog_opentelemetry::tracing().with_config(builder.build()) +} + +/// A Lambda handler wrapped with Datadog tracing. +/// +/// Owns the [`SdkTracerProvider`](opentelemetry_sdk::trace::SdkTracerProvider) lifecycle, +/// applies Lambda-appropriate defaults, and implements [`tower::Service`] so it composes +/// naturally with tower middleware. +/// +/// # Examples +/// +/// ```ignore +/// // Zero-config +/// lambda_runtime::run(WrappedHandler::new(my_handler, Config::default())).await +/// +/// // Set service/env/version +/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { +/// service: Some("my-service".into()), +/// env: Some("prod".into()), +/// ..Default::default() +/// })).await +/// +/// // Full tracer control +/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { +/// tracing: Some( +/// datadog_opentelemetry::tracing() +/// .with_config(builder_config_here) +/// .with_span_processor(MyProcessor), +/// ), +/// ..Default::default() +/// })).await +/// +/// // With tower middleware +/// lambda_runtime::run( +/// tower::ServiceBuilder::new() +/// .layer(some_middleware) +/// .service(WrappedHandler::new(my_handler, Config::default())) +/// ).await +/// ``` +pub struct WrappedHandler { + inner: Arc, + provider: opentelemetry_sdk::trace::SdkTracerProvider, + tracer: opentelemetry_sdk::trace::SdkTracer, + cold_start: bool, + _phantom: PhantomData R>, +} + +impl WrappedHandler { + pub fn new(handler: F, config: Config) -> Self { + let Config { + tracing, + service, + env, + version, + } = config; + let provider = tracing + .unwrap_or_else(|| build_tracing(service, env, version)) + .init(); + let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); + Self { + inner: Arc::new(handler), + provider, + tracer, + cold_start: true, + _phantom: PhantomData, + } + } + + fn take_cold_start(&mut self) -> bool { + std::mem::replace(&mut self.cold_start, false) + } +} + +impl Service>> for WrappedHandler +where + F: Fn(LambdaEvent) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + E: DeserializeOwned + Send + Sync + 'static, + R: Send + 'static, +{ + type Response = R; + type Error = lambda_runtime::Error; + type Future = BoxFuture>; + + fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, event: LambdaEvent>) -> Self::Future { + let cold_start = self.take_cold_start(); + let inner_handler = Arc::clone(&self.inner); + let provider = self.provider.clone(); + let invocation = Invocation::start(&self.tracer, &event.context, cold_start); + let typed_payload = match serde_json::from_str::(event.payload.get()) { + Ok(payload) => payload, + Err(err) => { + return Box::pin(async move { + let result: Result = Err(err.into()); + let result = invocation.finish(result); + if let Err(err) = provider.force_flush() { + tracing::error!("flush failed: {err}"); + } + result + }); + } + }; + let typed_event = LambdaEvent::new(typed_payload, event.context); + let fut = inner_handler(typed_event); + Box::pin(async move { + let result = fut.with_context(invocation.handler_context()).await; + let result = invocation.finish(result); + if let Err(err) = provider.force_flush() { + tracing::error!("flush failed: {err}"); + } + result + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn noop_handler( + _: LambdaEvent, + ) -> std::future::Ready> { + std::future::ready(Ok(())) + } + + #[allow(clippy::type_complexity)] + fn test_handler() -> WrappedHandler< + fn(LambdaEvent) -> std::future::Ready>, + serde_json::Value, + (), + > { + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); + let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); + WrappedHandler { + inner: Arc::new(noop_handler), + provider, + tracer, + cold_start: true, + _phantom: PhantomData, + } + } + + #[test] + fn cold_start_is_tracked_per_handler() { + let mut first = test_handler(); + assert!(first.take_cold_start()); + assert!(!first.take_cold_start()); + + let mut second = test_handler(); + assert!(second.take_cold_start()); + assert!(!second.take_cold_start()); + } +} From 75b9f9cd012ed7225b3109a4e15b80561c6f4556 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Wed, 22 Apr 2026 11:29:36 -0400 Subject: [PATCH 02/26] refactor(datadog-aws-lambda): address pr feedback and accept tower::Service - Change OTel span name from tracer scope name to "aws.lambda" - Remove redundant "language" tag - Remove logging from LambdaSpan (error info captured in span attributes) - Accept tower::Service instead of Fn for inner handler, enabling middleware composition inside the traced span - Replace custom Config struct with Option, applying Lambda defaults (stats disabled, sync writes) when None --- .../datadog-aws-lambda/src/attribute_keys.rs | 1 - .../datadog-aws-lambda/src/invocation.rs | 20 +-- instrumentation/datadog-aws-lambda/src/lib.rs | 146 +++++++----------- 3 files changed, 65 insertions(+), 102 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs index aa1834bd..b0ebad4d 100644 --- a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs +++ b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs @@ -8,7 +8,6 @@ pub(crate) const ERROR: &str = "error"; pub(crate) const ERROR_MESSAGE: &str = "error.message"; // Root span tags -pub(crate) const LANGUAGE: &str = "language"; pub(crate) const REQUEST_ID: &str = "request_id"; pub(crate) const COLD_START: &str = "cold_start"; pub(crate) const FUNCTION_ARN: &str = "function_arn"; diff --git a/instrumentation/datadog-aws-lambda/src/invocation.rs b/instrumentation/datadog-aws-lambda/src/invocation.rs index 2086d417..8c395ab3 100644 --- a/instrumentation/datadog-aws-lambda/src/invocation.rs +++ b/instrumentation/datadog-aws-lambda/src/invocation.rs @@ -6,10 +6,10 @@ //! Each invocation produces one *root span* (`aws.lambda`) that wraps the handler call. //! //! Typical usage: -//! 1. [`Invocation::start`] — create the root span before the handler runs. -//! 2. [`Invocation::handler_context`] — pass the returned context to the handler so its OTel spans +//! 1. [`Invocation::start`] - create the root span before the handler runs. +//! 2. [`Invocation::handler_context`] - pass the returned context to the handler so its OTel spans //! are correctly parented. -//! 3. [`Invocation::finish`] — record errors and end the span after the handler returns. +//! 3. [`Invocation::finish`] - record errors and end the span after the handler returns. use crate::attribute_keys as attr; @@ -26,8 +26,6 @@ pub(crate) static TRACER_NAME: &str = "datadog-lambda-rs"; pub(crate) struct LambdaSpan { /// OTel context whose active span is the `aws.lambda` root span. cx: Context, - /// Lambda request ID, copied here for structured log correlation. - request_id: String, } impl LambdaSpan { @@ -41,19 +39,15 @@ impl LambdaSpan { ) -> Self { let function_name = &lambda_cx.env_config.function_name; let request_id = lambda_cx.request_id.clone(); - - tracing::debug!(request_id, "creating invocation root span"); - let parent_cx = Context::current(); - let mut builder = tracer.span_builder(TRACER_NAME); + let mut builder = tracer.span_builder("aws.lambda"); builder.span_kind = Some(SpanKind::Server); let attrs = vec![ KeyValue::new(attr::OPERATION_NAME, "aws.lambda"), - KeyValue::new(attr::LANGUAGE, "rust"), KeyValue::new(attr::RESOURCE_NAME, function_name.clone()), KeyValue::new(attr::SPAN_TYPE, "serverless"), - KeyValue::new(attr::REQUEST_ID, request_id.clone()), + KeyValue::new(attr::REQUEST_ID, request_id), KeyValue::new(attr::COLD_START, cold_start), KeyValue::new(attr::FUNCTION_ARN, lambda_cx.invoked_function_arn.clone()), KeyValue::new(attr::FUNCTION_VERSION, lambda_cx.env_config.version.clone()), @@ -66,12 +60,11 @@ impl LambdaSpan { let span = tracer.build_with_context(builder, &parent_cx); let cx = parent_cx.with_span(span); - Self { cx, request_id } + Self { cx } } pub(crate) fn set_error(&self, err: &impl std::fmt::Display) { let err_msg = err.to_string(); - tracing::warn!(request_id = self.request_id, "handler returned error"); let span = self.cx.span(); span.set_status(opentelemetry::trace::Status::Error { description: err_msg.clone().into(), @@ -182,6 +175,7 @@ mod tests { let spans = finished_spans(&exporter); assert_eq!(spans.len(), 1); + assert_eq!(spans[0].name, "aws.lambda"); let attrs = &spans[0].attributes; assert_eq!( diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 1f773155..a10ce8dc 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -16,54 +16,19 @@ use lambda_runtime::LambdaEvent; use opentelemetry::trace::FutureExt; use serde::de::DeserializeOwned; use serde_json::value::RawValue; -use std::future::Future; use std::marker::PhantomData; -use std::sync::Arc; use std::task::{self, Poll}; type BoxFuture = std::pin::Pin + Send>>; -#[derive(Default)] -pub struct Config { - /// Service name. Overrides `DD_SERVICE`. Ignored when [`tracing`](Self::tracing) is `Some`. - pub service: Option, - /// Deployment environment. Overrides `DD_ENV`. Ignored when [`tracing`](Self::tracing) is - /// `Some`. - pub env: Option, - /// Service version. Overrides `DD_VERSION`. Ignored when [`tracing`](Self::tracing) is `Some`. - pub version: Option, - /// Full control over the OTel SDK and Datadog tracer config. - /// - /// When `None` (default), Lambda-appropriate defaults are applied and - /// `service`/`env`/`version` above are forwarded. When `Some`, the builder is used as-is; - /// `service`/`env`/`version` are ignored and you are responsible for setting: - /// - `trace_stats_computation_enabled = false` (the Datadog agent handles stats for serverless - /// environments) - /// - `trace_writer_synchronous_write = true` (so `force_flush()` blocks until spans reach - /// agent) - pub tracing: Option, -} - -fn build_tracing( - service: Option, - env: Option, - version: Option, -) -> datadog_opentelemetry::DatadogTracingBuilder { +/// Builds a [`DatadogTracingBuilder`](datadog_opentelemetry::DatadogTracingBuilder) with +/// Lambda-appropriate defaults: +/// - `trace_stats_computation_enabled = false` (the extension handles stats) +/// - `trace_writer_synchronous_write = true` (flush blocks until spans reach agent) +fn default_lambda_tracing() -> datadog_opentelemetry::DatadogTracingBuilder { let mut builder = datadog_opentelemetry::configuration::Config::builder(); - // Stats are computed server-side by the extension; client-side computation is redundant. builder.set_trace_stats_computation_enabled(false); - // Synchronous writes make force_flush() block until data reaches the agent, - // this helps reduce span loss when the Lambda process freezes after the handler returns. builder.set_trace_writer_synchronous_write(true); - if let Some(s) = service { - builder.set_service(s); - } - if let Some(e) = env { - builder.set_env(e); - } - if let Some(v) = version { - builder.set_version(v); - } datadog_opentelemetry::tracing().with_config(builder.build()) } @@ -73,58 +38,56 @@ fn build_tracing( /// applies Lambda-appropriate defaults, and implements [`tower::Service`] so it composes /// naturally with tower middleware. /// +/// The inner handler is any [`tower::Service`] that accepts `LambdaEvent`. Plain async +/// functions can be converted with [`service_fn`](lambda_runtime::service_fn). +/// +/// When `config` is `None`, Lambda-appropriate defaults are applied: +/// - `trace_stats_computation_enabled = false` (extension handles stats) +/// - `trace_writer_synchronous_write = true` (flush blocks until spans reach agent) +/// +/// When providing a custom config, set these values yourself to avoid redundant stats +/// computation or span loss during Lambda freezes. +/// /// # Examples /// /// ```ignore -/// // Zero-config -/// lambda_runtime::run(WrappedHandler::new(my_handler, Config::default())).await +/// use lambda_runtime::service_fn; /// -/// // Set service/env/version -/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { -/// service: Some("my-service".into()), -/// env: Some("prod".into()), -/// ..Default::default() -/// })).await +/// // Zero-config (Lambda defaults applied automatically) +/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), None)).await /// -/// // Full tracer control -/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { -/// tracing: Some( -/// datadog_opentelemetry::tracing() -/// .with_config(builder_config_here) -/// .with_span_processor(MyProcessor), -/// ), -/// ..Default::default() -/// })).await +/// // Custom config +/// let mut builder = datadog_opentelemetry::configuration::Config::builder(); +/// builder.set_service("my-svc".into()); +/// builder.set_env("prod".into()); +/// builder.set_trace_stats_computation_enabled(false); +/// builder.set_trace_writer_synchronous_write(true); +/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), Some(builder.build()))).await /// -/// // With tower middleware -/// lambda_runtime::run( -/// tower::ServiceBuilder::new() -/// .layer(some_middleware) -/// .service(WrappedHandler::new(my_handler, Config::default())) -/// ).await +/// // With tower middleware between tracing and the handler +/// let svc = tower::ServiceBuilder::new() +/// .layer(some_middleware) +/// .service(service_fn(my_handler)); +/// lambda_runtime::run(WrappedHandler::new(svc, None)).await /// ``` -pub struct WrappedHandler { - inner: Arc, +pub struct WrappedHandler { + inner: S, provider: opentelemetry_sdk::trace::SdkTracerProvider, tracer: opentelemetry_sdk::trace::SdkTracer, cold_start: bool, _phantom: PhantomData R>, } -impl WrappedHandler { - pub fn new(handler: F, config: Config) -> Self { - let Config { - tracing, - service, - env, - version, - } = config; - let provider = tracing - .unwrap_or_else(|| build_tracing(service, env, version)) - .init(); +impl WrappedHandler { + pub fn new(handler: S, config: Option) -> Self { + let tracing_builder = match config { + Some(cfg) => datadog_opentelemetry::tracing().with_config(cfg), + None => default_lambda_tracing(), + }; + let provider = tracing_builder.init(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); Self { - inner: Arc::new(handler), + inner: handler, provider, tracer, cold_start: true, @@ -137,24 +100,27 @@ impl WrappedHandler { } } -impl Service>> for WrappedHandler +impl Service>> for WrappedHandler where - F: Fn(LambdaEvent) -> Fut + Send + Sync + 'static, - Fut: Future> + Send + 'static, - E: DeserializeOwned + Send + Sync + 'static, + S: Service, Response = R, Error = lambda_runtime::Error> + + Clone + + Send + + 'static, + S::Future: Send + 'static, + E: DeserializeOwned + Send + 'static, R: Send + 'static, { type Response = R; type Error = lambda_runtime::Error; type Future = BoxFuture>; - fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { - Poll::Ready(Ok(())) + fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx) } fn call(&mut self, event: LambdaEvent>) -> Self::Future { let cold_start = self.take_cold_start(); - let inner_handler = Arc::clone(&self.inner); + let mut inner = self.inner.clone(); let provider = self.provider.clone(); let invocation = Invocation::start(&self.tracer, &event.context, cold_start); let typed_payload = match serde_json::from_str::(event.payload.get()) { @@ -171,7 +137,7 @@ where } }; let typed_event = LambdaEvent::new(typed_payload, event.context); - let fut = inner_handler(typed_event); + let fut = inner.call(typed_event); Box::pin(async move { let result = fut.with_context(invocation.handler_context()).await; let result = invocation.finish(result); @@ -186,6 +152,7 @@ where #[cfg(test)] mod tests { use super::*; + use lambda_runtime::service_fn; fn noop_handler( _: LambdaEvent, @@ -193,16 +160,19 @@ mod tests { std::future::ready(Ok(())) } - #[allow(clippy::type_complexity)] fn test_handler() -> WrappedHandler< - fn(LambdaEvent) -> std::future::Ready>, + lambda_runtime::tower::util::ServiceFn< + fn( + LambdaEvent, + ) -> std::future::Ready>, + >, serde_json::Value, (), > { let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); WrappedHandler { - inner: Arc::new(noop_handler), + inner: service_fn(noop_handler as fn(_) -> _), provider, tracer, cold_start: true, From 5c920bf345721a17d1e90a4d217ff0c087d4a8ca Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Wed, 22 Apr 2026 12:46:02 -0400 Subject: [PATCH 03/26] test(datadog-aws-lambda): add WrappedHandler call and middleware tests - Verify payload flows through WrappedHandler to inner service and back - Verify tower middleware composed between tracing and handler executes --- instrumentation/datadog-aws-lambda/src/lib.rs | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index a10ce8dc..4d58be7f 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -153,6 +153,8 @@ where mod tests { use super::*; use lambda_runtime::service_fn; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; fn noop_handler( _: LambdaEvent, @@ -190,4 +192,71 @@ mod tests { assert!(second.take_cold_start()); assert!(!second.take_cold_start()); } + + /// Sends a JSON payload through WrappedHandler and verifies the inner handler receives + /// the deserialized payload and its response is returned unchanged. + #[tokio::test] + async fn handler_receives_payload_and_returns_response() { + // Handler that echoes the payload back as the response. + async fn echo( + event: LambdaEvent, + ) -> Result { + Ok(event.payload) + } + + let input = r#"{"key":"value"}"#; + let raw = RawValue::from_string(input.to_string()).unwrap(); + let event = LambdaEvent::new(raw, lambda_runtime::Context::default()); + + let mut handler = WrappedHandler::new(service_fn(echo), None); + let response = handler.call(event).await.unwrap(); + + assert_eq!(response, serde_json::json!({"key": "value"})); + } + + /// Composes a middleware layer between tracing and the handler, then verifies + /// both the middleware and handler execute. + /// + /// ```text + /// WrappedHandler (tracing) -> counter middleware -> echo handler + /// ``` + #[tokio::test] + async fn middleware_between_tracing_and_handler_executes() { + // Handler that echoes the payload back as the response. + async fn echo( + event: LambdaEvent, + ) -> Result { + Ok(event.payload) + } + + // Counter that the middleware increments to prove it ran. + let middleware_call_count = Arc::new(AtomicUsize::new(0)); + let counter = middleware_call_count.clone(); + + // Compose: counter middleware -> echo handler. + let service_with_middleware = lambda_runtime::tower::ServiceBuilder::new() + .map_request(move |req: LambdaEvent| { + counter.fetch_add(1, Ordering::SeqCst); + req + }) + .service(service_fn(echo)); + + let input = r#"{"hello":"world"}"#; + let raw = RawValue::from_string(input.to_string()).unwrap(); + let event = LambdaEvent::new(raw, lambda_runtime::Context::default()); + + let mut handler = WrappedHandler::new(service_with_middleware, None); + let response = handler.call(event).await.unwrap(); + + assert_eq!( + middleware_call_count.load(Ordering::SeqCst), + 1, + "middleware should have run" + ); + assert_eq!( + response, + serde_json::json!({"hello": "world"}), + "handler should have echoed payload" + ); + } } From 530847bcffabd8b5174b1c246c6e1c09283c381c Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Wed, 22 Apr 2026 12:51:06 -0400 Subject: [PATCH 04/26] fix(datadog-aws-lambda): resolve clippy type_complexity with type alias and apply nightly fmt --- instrumentation/datadog-aws-lambda/src/lib.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 4d58be7f..dab69327 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -156,21 +156,17 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; + type NoopService = lambda_runtime::tower::util::ServiceFn< + fn(LambdaEvent) -> std::future::Ready>, + >; + fn noop_handler( _: LambdaEvent, ) -> std::future::Ready> { std::future::ready(Ok(())) } - fn test_handler() -> WrappedHandler< - lambda_runtime::tower::util::ServiceFn< - fn( - LambdaEvent, - ) -> std::future::Ready>, - >, - serde_json::Value, - (), - > { + fn test_handler() -> WrappedHandler { let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); WrappedHandler { From 868056803f40e24a11d5cf9b17abd7136b0627d3 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Wed, 22 Apr 2026 14:29:23 -0400 Subject: [PATCH 05/26] chore(datadog-aws-lambda): clarify why payload is deserialized manually --- instrumentation/datadog-aws-lambda/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index dab69327..12f4de5e 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -123,6 +123,10 @@ where let mut inner = self.inner.clone(); let provider = self.provider.clone(); let invocation = Invocation::start(&self.tracer, &event.context, cold_start); + // Deserialize here rather than letting the runtime do it so that + // deserialization errors are captured on the span. If we took + // LambdaEvent directly, the runtime would handle the error + // before our code runs and the invocation would not be traced. let typed_payload = match serde_json::from_str::(event.payload.get()) { Ok(payload) => payload, Err(err) => { From 5b4d53a3be8866766f0762ad53d7d124c4ce1801 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Thu, 23 Apr 2026 09:42:08 -0400 Subject: [PATCH 06/26] feat(datadog-aws-lambda): tag root span with crate version --- instrumentation/datadog-aws-lambda/src/attribute_keys.rs | 1 + instrumentation/datadog-aws-lambda/src/invocation.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs index b0ebad4d..2af40a48 100644 --- a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs +++ b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs @@ -15,3 +15,4 @@ pub(crate) const FUNCTION_VERSION: &str = "function_version"; pub(crate) const FUNCTION_NAME: &str = "functionname"; pub(crate) const RESOURCE_NAMES: &str = "resource_names"; pub(crate) const DD_ORIGIN: &str = "_dd.origin"; +pub(crate) const DATADOG_LAMBDA: &str = "datadog_lambda"; diff --git a/instrumentation/datadog-aws-lambda/src/invocation.rs b/instrumentation/datadog-aws-lambda/src/invocation.rs index 8c395ab3..2870458f 100644 --- a/instrumentation/datadog-aws-lambda/src/invocation.rs +++ b/instrumentation/datadog-aws-lambda/src/invocation.rs @@ -54,6 +54,7 @@ impl LambdaSpan { KeyValue::new(attr::FUNCTION_NAME, function_name.to_lowercase()), KeyValue::new(attr::RESOURCE_NAMES, function_name.clone()), KeyValue::new(attr::DD_ORIGIN, "lambda"), + KeyValue::new(attr::DATADOG_LAMBDA, env!("CARGO_PKG_VERSION")), ]; builder.attributes = Some(attrs); @@ -199,6 +200,10 @@ mod tests { Some(&Value::String("my-function".into())) ); assert_eq!(find_attr(attrs, "cold_start"), Some(&Value::Bool(true))); + assert_eq!( + find_attr(attrs, "datadog_lambda"), + Some(&Value::String(env!("CARGO_PKG_VERSION").into())) + ); } #[test] From 2cd84c31ac2a915891030c716897a45849d0ecdd Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Thu, 23 Apr 2026 09:46:53 -0400 Subject: [PATCH 07/26] refactor(datadog-aws-lambda): take config builder --- instrumentation/datadog-aws-lambda/src/lib.rs | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 12f4de5e..a3a3e41e 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -21,15 +21,17 @@ use std::task::{self, Poll}; type BoxFuture = std::pin::Pin + Send>>; -/// Builds a [`DatadogTracingBuilder`](datadog_opentelemetry::DatadogTracingBuilder) with -/// Lambda-appropriate defaults: +/// Applies Lambda-appropriate tracing defaults to a [`ConfigBuilder`]. +/// +/// These settings are enforced for correctness in Lambda: /// - `trace_stats_computation_enabled = false` (the extension handles stats) /// - `trace_writer_synchronous_write = true` (flush blocks until spans reach agent) -fn default_lambda_tracing() -> datadog_opentelemetry::DatadogTracingBuilder { - let mut builder = datadog_opentelemetry::configuration::Config::builder(); +fn apply_lambda_tracing_defaults( + mut builder: datadog_opentelemetry::configuration::ConfigBuilder, +) -> datadog_opentelemetry::configuration::ConfigBuilder { builder.set_trace_stats_computation_enabled(false); builder.set_trace_writer_synchronous_write(true); - datadog_opentelemetry::tracing().with_config(builder.build()) + builder } /// A Lambda handler wrapped with Datadog tracing. @@ -41,34 +43,26 @@ fn default_lambda_tracing() -> datadog_opentelemetry::DatadogTracingBuilder { /// The inner handler is any [`tower::Service`] that accepts `LambdaEvent`. Plain async /// functions can be converted with [`service_fn`](lambda_runtime::service_fn). /// -/// When `config` is `None`, Lambda-appropriate defaults are applied: -/// - `trace_stats_computation_enabled = false` (extension handles stats) -/// - `trace_writer_synchronous_write = true` (flush blocks until spans reach agent) -/// -/// When providing a custom config, set these values yourself to avoid redundant stats -/// computation or span loss during Lambda freezes. -/// /// # Examples /// /// ```ignore /// use lambda_runtime::service_fn; /// /// // Zero-config (Lambda defaults applied automatically) -/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), None)).await +/// let config = datadog_opentelemetry::configuration::Config::builder(); +/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), config)).await /// /// // Custom config /// let mut builder = datadog_opentelemetry::configuration::Config::builder(); /// builder.set_service("my-svc".into()); /// builder.set_env("prod".into()); -/// builder.set_trace_stats_computation_enabled(false); -/// builder.set_trace_writer_synchronous_write(true); -/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), Some(builder.build()))).await +/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), builder)).await /// /// // With tower middleware between tracing and the handler /// let svc = tower::ServiceBuilder::new() /// .layer(some_middleware) /// .service(service_fn(my_handler)); -/// lambda_runtime::run(WrappedHandler::new(svc, None)).await +/// lambda_runtime::run(WrappedHandler::new(svc, datadog_opentelemetry::configuration::Config::builder())).await /// ``` pub struct WrappedHandler { inner: S, @@ -79,12 +73,12 @@ pub struct WrappedHandler { } impl WrappedHandler { - pub fn new(handler: S, config: Option) -> Self { - let tracing_builder = match config { - Some(cfg) => datadog_opentelemetry::tracing().with_config(cfg), - None => default_lambda_tracing(), - }; - let provider = tracing_builder.init(); + pub fn new( + handler: S, + config: datadog_opentelemetry::configuration::ConfigBuilder, + ) -> Self { + let config = apply_lambda_tracing_defaults(config); + let provider = datadog_opentelemetry::tracing().with_config(config.build()).init(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); Self { inner: handler, @@ -208,7 +202,10 @@ mod tests { let raw = RawValue::from_string(input.to_string()).unwrap(); let event = LambdaEvent::new(raw, lambda_runtime::Context::default()); - let mut handler = WrappedHandler::new(service_fn(echo), None); + let mut handler = WrappedHandler::new( + service_fn(echo), + datadog_opentelemetry::configuration::Config::builder(), + ); let response = handler.call(event).await.unwrap(); assert_eq!(response, serde_json::json!({"key": "value"})); @@ -245,7 +242,10 @@ mod tests { let raw = RawValue::from_string(input.to_string()).unwrap(); let event = LambdaEvent::new(raw, lambda_runtime::Context::default()); - let mut handler = WrappedHandler::new(service_with_middleware, None); + let mut handler = WrappedHandler::new( + service_with_middleware, + datadog_opentelemetry::configuration::Config::builder(), + ); let response = handler.call(event).await.unwrap(); assert_eq!( From ff2cb0fede68927754e2c1fc5e1c1ed1a0556aa6 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Thu, 23 Apr 2026 11:31:26 -0400 Subject: [PATCH 08/26] fix(datadog-aws-lambda): remove test-utils feature from production dependency --- instrumentation/datadog-aws-lambda/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/datadog-aws-lambda/Cargo.toml b/instrumentation/datadog-aws-lambda/Cargo.toml index 16865171..e720e2d6 100644 --- a/instrumentation/datadog-aws-lambda/Cargo.toml +++ b/instrumentation/datadog-aws-lambda/Cargo.toml @@ -13,7 +13,7 @@ authors.workspace = true publish.workspace = true [dependencies] -datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry", features = ["test-utils"] } +datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry" } lambda_runtime = "0.13" serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } From 72fa4848cbd8e086e13c21aaf7fa73779a67565b Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Thu, 23 Apr 2026 11:32:37 -0400 Subject: [PATCH 09/26] Revert "fix(datadog-aws-lambda): remove test-utils feature from production dependency" This reverts commit ff2cb0fede68927754e2c1fc5e1c1ed1a0556aa6. --- instrumentation/datadog-aws-lambda/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/datadog-aws-lambda/Cargo.toml b/instrumentation/datadog-aws-lambda/Cargo.toml index e720e2d6..16865171 100644 --- a/instrumentation/datadog-aws-lambda/Cargo.toml +++ b/instrumentation/datadog-aws-lambda/Cargo.toml @@ -13,7 +13,7 @@ authors.workspace = true publish.workspace = true [dependencies] -datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry" } +datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry", features = ["test-utils"] } lambda_runtime = "0.13" serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } From e5de6a312be6b6a3137e0a0d4829f774bc397e0b Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Mon, 27 Apr 2026 12:46:29 -0700 Subject: [PATCH 10/26] Fix format --- instrumentation/datadog-aws-lambda/src/lib.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index a3a3e41e..e5e8a316 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -73,12 +73,11 @@ pub struct WrappedHandler { } impl WrappedHandler { - pub fn new( - handler: S, - config: datadog_opentelemetry::configuration::ConfigBuilder, - ) -> Self { + pub fn new(handler: S, config: datadog_opentelemetry::configuration::ConfigBuilder) -> Self { let config = apply_lambda_tracing_defaults(config); - let provider = datadog_opentelemetry::tracing().with_config(config.build()).init(); + let provider = datadog_opentelemetry::tracing() + .with_config(config.build()) + .init(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); Self { inner: handler, From 8cab1d92c4548c1bb40a92951308c4c447fdfc69 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Tue, 21 Apr 2026 14:54:10 -0400 Subject: [PATCH 11/26] feat(datadog-aws-lambda): add root invocation span with OTel tracing Implements the core Lambda handler wrapper with Datadog tracing: - WrappedHandler: tower::Service that wraps user handlers with OTel spans - LambdaSpan: aws.lambda root span with cold_start, request_id, function metadata - Invocation lifecycle: start/handler_context/finish with error recording - Config: service/env/version or full DatadogTracingBuilder control - Lambda-appropriate OTel defaults (sync writes, no client-side stats) Trigger extraction and inferred spans will follow in a subsequent PR. --- instrumentation/Cargo.lock | 350 +++++++++++++++++- .../datadog-aws-lambda/src/attribute_keys.rs | 2 +- .../datadog-aws-lambda/src/invocation.rs | 25 +- instrumentation/datadog-aws-lambda/src/lib.rs | 224 +++++------ 4 files changed, 449 insertions(+), 152 deletions(-) diff --git a/instrumentation/Cargo.lock b/instrumentation/Cargo.lock index 6920f7ef..3374dbf9 100644 --- a/instrumentation/Cargo.lock +++ b/instrumentation/Cargo.lock @@ -217,6 +217,58 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "const_format" version = "0.2.36" @@ -323,6 +375,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -404,6 +462,17 @@ dependencies = [ "syn", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.15.0" @@ -426,6 +495,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -752,6 +831,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.9.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -1135,7 +1227,7 @@ dependencies = [ "libdd-trace-protobuf 2.0.0", "memfd", "prost", - "rand 0.8.6", + "rand 0.8.5", "rmp", "rmp-serde", "rustix", @@ -1344,6 +1436,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "opentelemetry" version = "0.31.0" @@ -1365,7 +1463,7 @@ checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", - "http", + "http 1.4.0", "opentelemetry", "reqwest", ] @@ -1376,7 +1474,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ - "http", + "http 1.4.0", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -1469,6 +1567,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "plotters-backend" version = "0.3.7" @@ -1493,6 +1604,30 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1678,6 +1813,40 @@ name = "reqwest" version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.9.0", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower 0.5.3", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", @@ -1767,6 +1936,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1788,6 +1970,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.28" @@ -1937,6 +2128,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "static_assertions" version = "1.1.0" @@ -2053,6 +2250,26 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.52.1" @@ -2112,10 +2329,10 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.9.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -2183,8 +2400,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tower 0.5.3", @@ -2295,6 +2512,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "url" version = "2.5.8" @@ -2307,6 +2530,24 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -2346,6 +2587,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2750,6 +3001,35 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "yoke" version = "0.8.2" @@ -2814,6 +3094,60 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerotrie" version = "0.2.4" diff --git a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs index 2af40a48..aa1834bd 100644 --- a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs +++ b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs @@ -8,6 +8,7 @@ pub(crate) const ERROR: &str = "error"; pub(crate) const ERROR_MESSAGE: &str = "error.message"; // Root span tags +pub(crate) const LANGUAGE: &str = "language"; pub(crate) const REQUEST_ID: &str = "request_id"; pub(crate) const COLD_START: &str = "cold_start"; pub(crate) const FUNCTION_ARN: &str = "function_arn"; @@ -15,4 +16,3 @@ pub(crate) const FUNCTION_VERSION: &str = "function_version"; pub(crate) const FUNCTION_NAME: &str = "functionname"; pub(crate) const RESOURCE_NAMES: &str = "resource_names"; pub(crate) const DD_ORIGIN: &str = "_dd.origin"; -pub(crate) const DATADOG_LAMBDA: &str = "datadog_lambda"; diff --git a/instrumentation/datadog-aws-lambda/src/invocation.rs b/instrumentation/datadog-aws-lambda/src/invocation.rs index 2870458f..2086d417 100644 --- a/instrumentation/datadog-aws-lambda/src/invocation.rs +++ b/instrumentation/datadog-aws-lambda/src/invocation.rs @@ -6,10 +6,10 @@ //! Each invocation produces one *root span* (`aws.lambda`) that wraps the handler call. //! //! Typical usage: -//! 1. [`Invocation::start`] - create the root span before the handler runs. -//! 2. [`Invocation::handler_context`] - pass the returned context to the handler so its OTel spans +//! 1. [`Invocation::start`] — create the root span before the handler runs. +//! 2. [`Invocation::handler_context`] — pass the returned context to the handler so its OTel spans //! are correctly parented. -//! 3. [`Invocation::finish`] - record errors and end the span after the handler returns. +//! 3. [`Invocation::finish`] — record errors and end the span after the handler returns. use crate::attribute_keys as attr; @@ -26,6 +26,8 @@ pub(crate) static TRACER_NAME: &str = "datadog-lambda-rs"; pub(crate) struct LambdaSpan { /// OTel context whose active span is the `aws.lambda` root span. cx: Context, + /// Lambda request ID, copied here for structured log correlation. + request_id: String, } impl LambdaSpan { @@ -39,33 +41,37 @@ impl LambdaSpan { ) -> Self { let function_name = &lambda_cx.env_config.function_name; let request_id = lambda_cx.request_id.clone(); + + tracing::debug!(request_id, "creating invocation root span"); + let parent_cx = Context::current(); - let mut builder = tracer.span_builder("aws.lambda"); + let mut builder = tracer.span_builder(TRACER_NAME); builder.span_kind = Some(SpanKind::Server); let attrs = vec![ KeyValue::new(attr::OPERATION_NAME, "aws.lambda"), + KeyValue::new(attr::LANGUAGE, "rust"), KeyValue::new(attr::RESOURCE_NAME, function_name.clone()), KeyValue::new(attr::SPAN_TYPE, "serverless"), - KeyValue::new(attr::REQUEST_ID, request_id), + KeyValue::new(attr::REQUEST_ID, request_id.clone()), KeyValue::new(attr::COLD_START, cold_start), KeyValue::new(attr::FUNCTION_ARN, lambda_cx.invoked_function_arn.clone()), KeyValue::new(attr::FUNCTION_VERSION, lambda_cx.env_config.version.clone()), KeyValue::new(attr::FUNCTION_NAME, function_name.to_lowercase()), KeyValue::new(attr::RESOURCE_NAMES, function_name.clone()), KeyValue::new(attr::DD_ORIGIN, "lambda"), - KeyValue::new(attr::DATADOG_LAMBDA, env!("CARGO_PKG_VERSION")), ]; builder.attributes = Some(attrs); let span = tracer.build_with_context(builder, &parent_cx); let cx = parent_cx.with_span(span); - Self { cx } + Self { cx, request_id } } pub(crate) fn set_error(&self, err: &impl std::fmt::Display) { let err_msg = err.to_string(); + tracing::warn!(request_id = self.request_id, "handler returned error"); let span = self.cx.span(); span.set_status(opentelemetry::trace::Status::Error { description: err_msg.clone().into(), @@ -176,7 +182,6 @@ mod tests { let spans = finished_spans(&exporter); assert_eq!(spans.len(), 1); - assert_eq!(spans[0].name, "aws.lambda"); let attrs = &spans[0].attributes; assert_eq!( @@ -200,10 +205,6 @@ mod tests { Some(&Value::String("my-function".into())) ); assert_eq!(find_attr(attrs, "cold_start"), Some(&Value::Bool(true))); - assert_eq!( - find_attr(attrs, "datadog_lambda"), - Some(&Value::String(env!("CARGO_PKG_VERSION").into())) - ); } #[test] diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index e5e8a316..1f773155 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -16,22 +16,55 @@ use lambda_runtime::LambdaEvent; use opentelemetry::trace::FutureExt; use serde::de::DeserializeOwned; use serde_json::value::RawValue; +use std::future::Future; use std::marker::PhantomData; +use std::sync::Arc; use std::task::{self, Poll}; type BoxFuture = std::pin::Pin + Send>>; -/// Applies Lambda-appropriate tracing defaults to a [`ConfigBuilder`]. -/// -/// These settings are enforced for correctness in Lambda: -/// - `trace_stats_computation_enabled = false` (the extension handles stats) -/// - `trace_writer_synchronous_write = true` (flush blocks until spans reach agent) -fn apply_lambda_tracing_defaults( - mut builder: datadog_opentelemetry::configuration::ConfigBuilder, -) -> datadog_opentelemetry::configuration::ConfigBuilder { +#[derive(Default)] +pub struct Config { + /// Service name. Overrides `DD_SERVICE`. Ignored when [`tracing`](Self::tracing) is `Some`. + pub service: Option, + /// Deployment environment. Overrides `DD_ENV`. Ignored when [`tracing`](Self::tracing) is + /// `Some`. + pub env: Option, + /// Service version. Overrides `DD_VERSION`. Ignored when [`tracing`](Self::tracing) is `Some`. + pub version: Option, + /// Full control over the OTel SDK and Datadog tracer config. + /// + /// When `None` (default), Lambda-appropriate defaults are applied and + /// `service`/`env`/`version` above are forwarded. When `Some`, the builder is used as-is; + /// `service`/`env`/`version` are ignored and you are responsible for setting: + /// - `trace_stats_computation_enabled = false` (the Datadog agent handles stats for serverless + /// environments) + /// - `trace_writer_synchronous_write = true` (so `force_flush()` blocks until spans reach + /// agent) + pub tracing: Option, +} + +fn build_tracing( + service: Option, + env: Option, + version: Option, +) -> datadog_opentelemetry::DatadogTracingBuilder { + let mut builder = datadog_opentelemetry::configuration::Config::builder(); + // Stats are computed server-side by the extension; client-side computation is redundant. builder.set_trace_stats_computation_enabled(false); + // Synchronous writes make force_flush() block until data reaches the agent, + // this helps reduce span loss when the Lambda process freezes after the handler returns. builder.set_trace_writer_synchronous_write(true); - builder + if let Some(s) = service { + builder.set_service(s); + } + if let Some(e) = env { + builder.set_env(e); + } + if let Some(v) = version { + builder.set_version(v); + } + datadog_opentelemetry::tracing().with_config(builder.build()) } /// A Lambda handler wrapped with Datadog tracing. @@ -40,47 +73,58 @@ fn apply_lambda_tracing_defaults( /// applies Lambda-appropriate defaults, and implements [`tower::Service`] so it composes /// naturally with tower middleware. /// -/// The inner handler is any [`tower::Service`] that accepts `LambdaEvent`. Plain async -/// functions can be converted with [`service_fn`](lambda_runtime::service_fn). -/// /// # Examples /// /// ```ignore -/// use lambda_runtime::service_fn; +/// // Zero-config +/// lambda_runtime::run(WrappedHandler::new(my_handler, Config::default())).await /// -/// // Zero-config (Lambda defaults applied automatically) -/// let config = datadog_opentelemetry::configuration::Config::builder(); -/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), config)).await +/// // Set service/env/version +/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { +/// service: Some("my-service".into()), +/// env: Some("prod".into()), +/// ..Default::default() +/// })).await /// -/// // Custom config -/// let mut builder = datadog_opentelemetry::configuration::Config::builder(); -/// builder.set_service("my-svc".into()); -/// builder.set_env("prod".into()); -/// lambda_runtime::run(WrappedHandler::new(service_fn(my_handler), builder)).await +/// // Full tracer control +/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { +/// tracing: Some( +/// datadog_opentelemetry::tracing() +/// .with_config(builder_config_here) +/// .with_span_processor(MyProcessor), +/// ), +/// ..Default::default() +/// })).await /// -/// // With tower middleware between tracing and the handler -/// let svc = tower::ServiceBuilder::new() -/// .layer(some_middleware) -/// .service(service_fn(my_handler)); -/// lambda_runtime::run(WrappedHandler::new(svc, datadog_opentelemetry::configuration::Config::builder())).await +/// // With tower middleware +/// lambda_runtime::run( +/// tower::ServiceBuilder::new() +/// .layer(some_middleware) +/// .service(WrappedHandler::new(my_handler, Config::default())) +/// ).await /// ``` -pub struct WrappedHandler { - inner: S, +pub struct WrappedHandler { + inner: Arc, provider: opentelemetry_sdk::trace::SdkTracerProvider, tracer: opentelemetry_sdk::trace::SdkTracer, cold_start: bool, _phantom: PhantomData R>, } -impl WrappedHandler { - pub fn new(handler: S, config: datadog_opentelemetry::configuration::ConfigBuilder) -> Self { - let config = apply_lambda_tracing_defaults(config); - let provider = datadog_opentelemetry::tracing() - .with_config(config.build()) +impl WrappedHandler { + pub fn new(handler: F, config: Config) -> Self { + let Config { + tracing, + service, + env, + version, + } = config; + let provider = tracing + .unwrap_or_else(|| build_tracing(service, env, version)) .init(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); Self { - inner: handler, + inner: Arc::new(handler), provider, tracer, cold_start: true, @@ -93,33 +137,26 @@ impl WrappedHandler { } } -impl Service>> for WrappedHandler +impl Service>> for WrappedHandler where - S: Service, Response = R, Error = lambda_runtime::Error> - + Clone - + Send - + 'static, - S::Future: Send + 'static, - E: DeserializeOwned + Send + 'static, + F: Fn(LambdaEvent) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + E: DeserializeOwned + Send + Sync + 'static, R: Send + 'static, { type Response = R; type Error = lambda_runtime::Error; type Future = BoxFuture>; - fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { - self.inner.poll_ready(cx) + fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { + Poll::Ready(Ok(())) } fn call(&mut self, event: LambdaEvent>) -> Self::Future { let cold_start = self.take_cold_start(); - let mut inner = self.inner.clone(); + let inner_handler = Arc::clone(&self.inner); let provider = self.provider.clone(); let invocation = Invocation::start(&self.tracer, &event.context, cold_start); - // Deserialize here rather than letting the runtime do it so that - // deserialization errors are captured on the span. If we took - // LambdaEvent directly, the runtime would handle the error - // before our code runs and the invocation would not be traced. let typed_payload = match serde_json::from_str::(event.payload.get()) { Ok(payload) => payload, Err(err) => { @@ -134,7 +171,7 @@ where } }; let typed_event = LambdaEvent::new(typed_payload, event.context); - let fut = inner.call(typed_event); + let fut = inner_handler(typed_event); Box::pin(async move { let result = fut.with_context(invocation.handler_context()).await; let result = invocation.finish(result); @@ -149,13 +186,6 @@ where #[cfg(test)] mod tests { use super::*; - use lambda_runtime::service_fn; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - - type NoopService = lambda_runtime::tower::util::ServiceFn< - fn(LambdaEvent) -> std::future::Ready>, - >; fn noop_handler( _: LambdaEvent, @@ -163,11 +193,16 @@ mod tests { std::future::ready(Ok(())) } - fn test_handler() -> WrappedHandler { + #[allow(clippy::type_complexity)] + fn test_handler() -> WrappedHandler< + fn(LambdaEvent) -> std::future::Ready>, + serde_json::Value, + (), + > { let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); WrappedHandler { - inner: service_fn(noop_handler as fn(_) -> _), + inner: Arc::new(noop_handler), provider, tracer, cold_start: true, @@ -185,77 +220,4 @@ mod tests { assert!(second.take_cold_start()); assert!(!second.take_cold_start()); } - - /// Sends a JSON payload through WrappedHandler and verifies the inner handler receives - /// the deserialized payload and its response is returned unchanged. - #[tokio::test] - async fn handler_receives_payload_and_returns_response() { - // Handler that echoes the payload back as the response. - async fn echo( - event: LambdaEvent, - ) -> Result { - Ok(event.payload) - } - - let input = r#"{"key":"value"}"#; - let raw = RawValue::from_string(input.to_string()).unwrap(); - let event = LambdaEvent::new(raw, lambda_runtime::Context::default()); - - let mut handler = WrappedHandler::new( - service_fn(echo), - datadog_opentelemetry::configuration::Config::builder(), - ); - let response = handler.call(event).await.unwrap(); - - assert_eq!(response, serde_json::json!({"key": "value"})); - } - - /// Composes a middleware layer between tracing and the handler, then verifies - /// both the middleware and handler execute. - /// - /// ```text - /// WrappedHandler (tracing) -> counter middleware -> echo handler - /// ``` - #[tokio::test] - async fn middleware_between_tracing_and_handler_executes() { - // Handler that echoes the payload back as the response. - async fn echo( - event: LambdaEvent, - ) -> Result { - Ok(event.payload) - } - - // Counter that the middleware increments to prove it ran. - let middleware_call_count = Arc::new(AtomicUsize::new(0)); - let counter = middleware_call_count.clone(); - - // Compose: counter middleware -> echo handler. - let service_with_middleware = lambda_runtime::tower::ServiceBuilder::new() - .map_request(move |req: LambdaEvent| { - counter.fetch_add(1, Ordering::SeqCst); - req - }) - .service(service_fn(echo)); - - let input = r#"{"hello":"world"}"#; - let raw = RawValue::from_string(input.to_string()).unwrap(); - let event = LambdaEvent::new(raw, lambda_runtime::Context::default()); - - let mut handler = WrappedHandler::new( - service_with_middleware, - datadog_opentelemetry::configuration::Config::builder(), - ); - let response = handler.call(event).await.unwrap(); - - assert_eq!( - middleware_call_count.load(Ordering::SeqCst), - 1, - "middleware should have run" - ); - assert_eq!( - response, - serde_json::json!({"hello": "world"}), - "handler should have echoed payload" - ); - } } From 90a9c840cc5722a74cfe4850d76180bb7529c694 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 28 Apr 2026 11:10:31 -0700 Subject: [PATCH 12/26] Fix Cargo.lock --- instrumentation/Cargo.lock | 390 +++---------------------------------- 1 file changed, 28 insertions(+), 362 deletions(-) diff --git a/instrumentation/Cargo.lock b/instrumentation/Cargo.lock index 3374dbf9..ad5dd822 100644 --- a/instrumentation/Cargo.lock +++ b/instrumentation/Cargo.lock @@ -145,9 +145,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.60" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", "shlex", @@ -217,58 +217,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstyle", - "clap_lex", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - [[package]] name = "const_format" version = "0.2.36" @@ -375,12 +323,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -462,17 +404,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "either" version = "1.15.0" @@ -495,16 +426,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "fastrand" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -831,19 +752,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-timeout" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" -dependencies = [ - "hyper 1.9.0", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" @@ -968,9 +876,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1041,9 +949,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" dependencies = [ "cfg-if", "futures-util", @@ -1129,9 +1037,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libdd-common" @@ -1227,7 +1135,7 @@ dependencies = [ "libdd-trace-protobuf 2.0.0", "memfd", "prost", - "rand 0.8.5", + "rand 0.8.6", "rmp", "rmp-serde", "rustix", @@ -1436,12 +1344,6 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "opentelemetry" version = "0.31.0" @@ -1463,7 +1365,7 @@ checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", - "http 1.4.0", + "http", "opentelemetry", "reqwest", ] @@ -1474,7 +1376,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ - "http 1.4.0", + "http", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -1567,19 +1469,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "plotters-backend" version = "0.3.7" @@ -1604,30 +1493,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1813,40 +1678,6 @@ name = "reqwest" version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "hyper 1.9.0", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower 0.5.3", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "rfc6979" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", @@ -1936,19 +1767,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - [[package]] name = "rustversion" version = "1.0.22" @@ -1970,15 +1788,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "semver" version = "1.0.28" @@ -2128,12 +1937,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - [[package]] name = "static_assertions" version = "1.1.0" @@ -2250,26 +2053,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tokio" version = "1.52.1" @@ -2329,10 +2112,10 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", - "hyper 1.9.0", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", @@ -2400,8 +2183,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.0", - "http-body 1.0.1", + "http", + "http-body", "iri-string", "pin-project-lite", "tower 0.5.3", @@ -2512,12 +2295,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "url" version = "2.5.8" @@ -2530,24 +2307,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -2587,16 +2346,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "want" version = "0.3.1" @@ -2632,9 +2381,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" dependencies = [ "cfg-if", "once_cell", @@ -2645,9 +2394,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" dependencies = [ "js-sys", "wasm-bindgen", @@ -2655,9 +2404,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2665,9 +2414,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" dependencies = [ "bumpalo", "proc-macro2", @@ -2678,9 +2427,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" dependencies = [ "unicode-ident", ] @@ -2721,9 +2470,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" dependencies = [ "js-sys", "wasm-bindgen", @@ -3001,35 +2750,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - [[package]] name = "yoke" version = "0.8.2" @@ -3094,60 +2814,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zerotrie" version = "0.2.4" From 70f982c0d40e85056aa090b9a3478c6ba728470a Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 5 May 2026 18:21:50 -0700 Subject: [PATCH 13/26] refactor(datadog-aws-lambda): wrap a Service instead of a plain handler fn --- instrumentation/datadog-aws-lambda/src/lib.rs | 163 +++++++++++++----- 1 file changed, 117 insertions(+), 46 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 1f773155..d9f6adb9 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -18,7 +18,6 @@ use serde::de::DeserializeOwned; use serde_json::value::RawValue; use std::future::Future; use std::marker::PhantomData; -use std::sync::Arc; use std::task::{self, Poll}; type BoxFuture = std::pin::Pin + Send>>; @@ -77,54 +76,73 @@ fn build_tracing( /// /// ```ignore /// // Zero-config -/// lambda_runtime::run(WrappedHandler::new(my_handler, Config::default())).await +/// lambda_runtime::run(WrappedHandler::new( +/// lambda_runtime::service_fn(my_handler), +/// Config::default(), +/// )).await /// /// // Set service/env/version -/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { -/// service: Some("my-service".into()), -/// env: Some("prod".into()), -/// ..Default::default() -/// })).await +/// lambda_runtime::run(WrappedHandler::new( +/// lambda_runtime::service_fn(my_handler), +/// Config { +/// service: Some("my-service".into()), +/// env: Some("prod".into()), +/// ..Default::default() +/// }, +/// )).await /// /// // Full tracer control -/// lambda_runtime::run(WrappedHandler::new(my_handler, Config { -/// tracing: Some( -/// datadog_opentelemetry::tracing() -/// .with_config(builder_config_here) -/// .with_span_processor(MyProcessor), -/// ), -/// ..Default::default() -/// })).await +/// lambda_runtime::run(WrappedHandler::new( +/// lambda_runtime::service_fn(my_handler), +/// Config { +/// tracing: Some( +/// datadog_opentelemetry::tracing() +/// .with_config(builder_config_here) +/// .with_span_processor(MyProcessor), +/// ), +/// ..Default::default() +/// }, +/// )).await /// /// // With tower middleware /// lambda_runtime::run( -/// tower::ServiceBuilder::new() -/// .layer(some_middleware) -/// .service(WrappedHandler::new(my_handler, Config::default())) +/// WrappedHandler::new( +/// tower::ServiceBuilder::new() +/// .layer(some_middleware) +/// .service(lambda_runtime::service_fn(my_handler)), +/// Config::default(), +/// ) /// ).await /// ``` -pub struct WrappedHandler { - inner: Arc, +pub struct WrappedHandler { + inner: S, provider: opentelemetry_sdk::trace::SdkTracerProvider, tracer: opentelemetry_sdk::trace::SdkTracer, cold_start: bool, _phantom: PhantomData R>, } -impl WrappedHandler { - pub fn new(handler: F, config: Config) -> Self { +impl WrappedHandler { + /// Wraps a Tower service with Datadog tracing. + /// + /// Use this constructor when you want to apply Tower middleware after tracing + /// has started but before your Lambda handler executes. + pub fn new(inner: S, config: Config) -> Self + where + S: Service, Response = R>, + { let Config { tracing, - service, + service: service_name, env, version, } = config; let provider = tracing - .unwrap_or_else(|| build_tracing(service, env, version)) + .unwrap_or_else(|| build_tracing(service_name, env, version)) .init(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); Self { - inner: Arc::new(handler), + inner, provider, tracer, cold_start: true, @@ -137,24 +155,24 @@ impl WrappedHandler { } } -impl Service>> for WrappedHandler +impl Service>> for WrappedHandler where - F: Fn(LambdaEvent) -> Fut + Send + Sync + 'static, - Fut: Future> + Send + 'static, - E: DeserializeOwned + Send + Sync + 'static, + S: Service, Response = R>, + S::Future: Future> + Send + 'static, + S::Error: Into, + E: DeserializeOwned + Send + 'static, R: Send + 'static, { type Response = R; type Error = lambda_runtime::Error; type Future = BoxFuture>; - fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { - Poll::Ready(Ok(())) + fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) } fn call(&mut self, event: LambdaEvent>) -> Self::Future { let cold_start = self.take_cold_start(); - let inner_handler = Arc::clone(&self.inner); let provider = self.provider.clone(); let invocation = Invocation::start(&self.tracer, &event.context, cold_start); let typed_payload = match serde_json::from_str::(event.payload.get()) { @@ -163,29 +181,35 @@ where return Box::pin(async move { let result: Result = Err(err.into()); let result = invocation.finish(result); - if let Err(err) = provider.force_flush() { - tracing::error!("flush failed: {err}"); - } + flush_provider(&provider); result }); } }; let typed_event = LambdaEvent::new(typed_payload, event.context); - let fut = inner_handler(typed_event); + let fut = self.inner.call(typed_event); Box::pin(async move { let result = fut.with_context(invocation.handler_context()).await; - let result = invocation.finish(result); - if let Err(err) = provider.force_flush() { - tracing::error!("flush failed: {err}"); - } + let result = invocation.finish(result.map_err(Into::into)); + flush_provider(&provider); result }) } } +fn flush_provider(provider: &opentelemetry_sdk::trace::SdkTracerProvider) { + if let Err(err) = provider.force_flush() { + tracing::error!("flush failed: {err}"); + } +} + #[cfg(test)] mod tests { use super::*; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; fn noop_handler( _: LambdaEvent, @@ -194,15 +218,11 @@ mod tests { } #[allow(clippy::type_complexity)] - fn test_handler() -> WrappedHandler< - fn(LambdaEvent) -> std::future::Ready>, - serde_json::Value, - (), - > { + fn test_handler() -> WrappedHandler { let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); WrappedHandler { - inner: Arc::new(noop_handler), + inner: ReadyService, provider, tracer, cold_start: true, @@ -210,6 +230,41 @@ mod tests { } } + struct ReadyService; + + impl Service> for ReadyService { + type Response = (); + type Error = lambda_runtime::Error; + type Future = std::future::Ready>; + + fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, event: LambdaEvent) -> Self::Future { + noop_handler(event) + } + } + + struct ReadyCountingService { + ready_calls: Arc, + } + + impl Service> for ReadyCountingService { + type Response = (); + type Error = lambda_runtime::Error; + type Future = std::future::Ready>; + + fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { + self.ready_calls.fetch_add(1, Ordering::Relaxed); + Poll::Ready(Ok(())) + } + + fn call(&mut self, _: LambdaEvent) -> Self::Future { + std::future::ready(Ok(())) + } + } + #[test] fn cold_start_is_tracked_per_handler() { let mut first = test_handler(); @@ -220,4 +275,20 @@ mod tests { assert!(second.take_cold_start()); assert!(!second.take_cold_start()); } + + #[test] + fn poll_ready_delegates_to_inner_service() { + let ready_calls = Arc::new(AtomicUsize::new(0)); + let mut wrapped = WrappedHandler::new( + ReadyCountingService { + ready_calls: Arc::clone(&ready_calls), + }, + Config::default(), + ); + let waker = std::task::Waker::noop(); + let mut cx = task::Context::from_waker(waker); + + assert!(wrapped.poll_ready(&mut cx).is_ready()); + assert_eq!(ready_calls.load(Ordering::Relaxed), 1); + } } From 06dc26076c4d45af20dc1465502bad20f5b8651c Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 11:26:36 -0700 Subject: [PATCH 14/26] refactor(datadog-aws-lambda): import opentelemetry_sdk trace types --- instrumentation/datadog-aws-lambda/src/lib.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index d9f6adb9..d70fc1f5 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -14,6 +14,7 @@ use invocation::{Invocation, TRACER_NAME}; use lambda_runtime::tower::Service; use lambda_runtime::LambdaEvent; use opentelemetry::trace::FutureExt; +use opentelemetry_sdk::trace::{SdkTracer, SdkTracerProvider}; use serde::de::DeserializeOwned; use serde_json::value::RawValue; use std::future::Future; @@ -68,8 +69,8 @@ fn build_tracing( /// A Lambda handler wrapped with Datadog tracing. /// -/// Owns the [`SdkTracerProvider`](opentelemetry_sdk::trace::SdkTracerProvider) lifecycle, -/// applies Lambda-appropriate defaults, and implements [`tower::Service`] so it composes +/// Owns the [`SdkTracerProvider`] lifecycle, +/// applies Lambda-appropriate defaults, and implements [`Service`] so it composes /// naturally with tower middleware. /// /// # Examples @@ -116,8 +117,8 @@ fn build_tracing( /// ``` pub struct WrappedHandler { inner: S, - provider: opentelemetry_sdk::trace::SdkTracerProvider, - tracer: opentelemetry_sdk::trace::SdkTracer, + provider: SdkTracerProvider, + tracer: SdkTracer, cold_start: bool, _phantom: PhantomData R>, } @@ -197,7 +198,7 @@ where } } -fn flush_provider(provider: &opentelemetry_sdk::trace::SdkTracerProvider) { +fn flush_provider(provider: &SdkTracerProvider) { if let Err(err) = provider.force_flush() { tracing::error!("flush failed: {err}"); } @@ -219,7 +220,7 @@ mod tests { #[allow(clippy::type_complexity)] fn test_handler() -> WrappedHandler { - let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder().build(); + let provider = SdkTracerProvider::builder().build(); let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); WrappedHandler { inner: ReadyService, From 54b30c5291bc5d5a27bbb3bc33573ab83abb136f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 12:13:20 -0700 Subject: [PATCH 15/26] fix(datadog-aws-lambda): use aws.lambda as root span name --- instrumentation/datadog-aws-lambda/src/invocation.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/invocation.rs b/instrumentation/datadog-aws-lambda/src/invocation.rs index 2086d417..58317603 100644 --- a/instrumentation/datadog-aws-lambda/src/invocation.rs +++ b/instrumentation/datadog-aws-lambda/src/invocation.rs @@ -18,6 +18,7 @@ use opentelemetry::{Context, KeyValue}; use opentelemetry_sdk::trace::SdkTracer; pub(crate) static TRACER_NAME: &str = "datadog-lambda-rs"; +pub(crate) const ROOT_SPAN_NAME: &str = "aws.lambda"; /// The Lambda invocation (`aws.lambda`) root span. /// @@ -46,10 +47,10 @@ impl LambdaSpan { let parent_cx = Context::current(); - let mut builder = tracer.span_builder(TRACER_NAME); + let mut builder = tracer.span_builder(ROOT_SPAN_NAME); builder.span_kind = Some(SpanKind::Server); let attrs = vec![ - KeyValue::new(attr::OPERATION_NAME, "aws.lambda"), + KeyValue::new(attr::OPERATION_NAME, ROOT_SPAN_NAME), KeyValue::new(attr::LANGUAGE, "rust"), KeyValue::new(attr::RESOURCE_NAME, function_name.clone()), KeyValue::new(attr::SPAN_TYPE, "serverless"), @@ -182,6 +183,7 @@ mod tests { let spans = finished_spans(&exporter); assert_eq!(spans.len(), 1); + assert_eq!(spans[0].name.as_ref(), ROOT_SPAN_NAME); let attrs = &spans[0].attributes; assert_eq!( From ddfe75cfb8a23c51d6248ca8c0702442c5c3b600 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 12:29:18 -0700 Subject: [PATCH 16/26] refactor(datadog-aws-lambda): simplify tracing imports --- instrumentation/datadog-aws-lambda/src/invocation.rs | 4 ++-- instrumentation/datadog-aws-lambda/src/lib.rs | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/invocation.rs b/instrumentation/datadog-aws-lambda/src/invocation.rs index 58317603..e159bf0f 100644 --- a/instrumentation/datadog-aws-lambda/src/invocation.rs +++ b/instrumentation/datadog-aws-lambda/src/invocation.rs @@ -13,7 +13,7 @@ use crate::attribute_keys as attr; -use opentelemetry::trace::{SpanKind, TraceContextExt, Tracer}; +use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer}; use opentelemetry::{Context, KeyValue}; use opentelemetry_sdk::trace::SdkTracer; @@ -74,7 +74,7 @@ impl LambdaSpan { let err_msg = err.to_string(); tracing::warn!(request_id = self.request_id, "handler returned error"); let span = self.cx.span(); - span.set_status(opentelemetry::trace::Status::Error { + span.set_status(Status::Error { description: err_msg.clone().into(), }); span.set_attribute(KeyValue::new(attr::ERROR, true)); diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index d70fc1f5..7e0ea75f 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -10,10 +10,11 @@ mod attribute_keys; mod invocation; +use datadog_opentelemetry::DatadogTracingBuilder; use invocation::{Invocation, TRACER_NAME}; use lambda_runtime::tower::Service; use lambda_runtime::LambdaEvent; -use opentelemetry::trace::FutureExt; +use opentelemetry::trace::{FutureExt, TracerProvider}; use opentelemetry_sdk::trace::{SdkTracer, SdkTracerProvider}; use serde::de::DeserializeOwned; use serde_json::value::RawValue; @@ -41,14 +42,14 @@ pub struct Config { /// environments) /// - `trace_writer_synchronous_write = true` (so `force_flush()` blocks until spans reach /// agent) - pub tracing: Option, + pub tracing: Option, } fn build_tracing( service: Option, env: Option, version: Option, -) -> datadog_opentelemetry::DatadogTracingBuilder { +) -> DatadogTracingBuilder { let mut builder = datadog_opentelemetry::configuration::Config::builder(); // Stats are computed server-side by the extension; client-side computation is redundant. builder.set_trace_stats_computation_enabled(false); @@ -141,7 +142,7 @@ impl WrappedHandler { let provider = tracing .unwrap_or_else(|| build_tracing(service_name, env, version)) .init(); - let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); + let tracer = TracerProvider::tracer(&provider, TRACER_NAME); Self { inner, provider, @@ -221,7 +222,7 @@ mod tests { #[allow(clippy::type_complexity)] fn test_handler() -> WrappedHandler { let provider = SdkTracerProvider::builder().build(); - let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); + let tracer = TracerProvider::tracer(&provider, TRACER_NAME); WrappedHandler { inner: ReadyService, provider, From 2dbf5520aaeb15deb0dae35e7dff2314915d2f0f Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 13:24:03 -0700 Subject: [PATCH 17/26] refactor(datadog-aws-lambda): simplify tracing config for WrappedHandler Remove the local config wrapper and accept Datadog's ConfigBuilder directly for customized tracing setup. Add a zero-config WrappedHandler::new constructor and move the explicit builder path to WrappedHandler::with_config. Force the Lambda-required tracing defaults internally, clean up Datadog/OpenTelemetry imports, and fix the WrappedHandler rustdoc examples to be rendered as ignored examples instead of failing doctests. --- instrumentation/datadog-aws-lambda/src/lib.rs | 121 ++++++------------ 1 file changed, 39 insertions(+), 82 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 7e0ea75f..3dd401ff 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -10,7 +10,7 @@ mod attribute_keys; mod invocation; -use datadog_opentelemetry::DatadogTracingBuilder; +use datadog_opentelemetry::configuration::{Config, ConfigBuilder}; use invocation::{Invocation, TRACER_NAME}; use lambda_runtime::tower::Service; use lambda_runtime::LambdaEvent; @@ -24,50 +24,6 @@ use std::task::{self, Poll}; type BoxFuture = std::pin::Pin + Send>>; -#[derive(Default)] -pub struct Config { - /// Service name. Overrides `DD_SERVICE`. Ignored when [`tracing`](Self::tracing) is `Some`. - pub service: Option, - /// Deployment environment. Overrides `DD_ENV`. Ignored when [`tracing`](Self::tracing) is - /// `Some`. - pub env: Option, - /// Service version. Overrides `DD_VERSION`. Ignored when [`tracing`](Self::tracing) is `Some`. - pub version: Option, - /// Full control over the OTel SDK and Datadog tracer config. - /// - /// When `None` (default), Lambda-appropriate defaults are applied and - /// `service`/`env`/`version` above are forwarded. When `Some`, the builder is used as-is; - /// `service`/`env`/`version` are ignored and you are responsible for setting: - /// - `trace_stats_computation_enabled = false` (the Datadog agent handles stats for serverless - /// environments) - /// - `trace_writer_synchronous_write = true` (so `force_flush()` blocks until spans reach - /// agent) - pub tracing: Option, -} - -fn build_tracing( - service: Option, - env: Option, - version: Option, -) -> DatadogTracingBuilder { - let mut builder = datadog_opentelemetry::configuration::Config::builder(); - // Stats are computed server-side by the extension; client-side computation is redundant. - builder.set_trace_stats_computation_enabled(false); - // Synchronous writes make force_flush() block until data reaches the agent, - // this helps reduce span loss when the Lambda process freezes after the handler returns. - builder.set_trace_writer_synchronous_write(true); - if let Some(s) = service { - builder.set_service(s); - } - if let Some(e) = env { - builder.set_env(e); - } - if let Some(v) = version { - builder.set_version(v); - } - datadog_opentelemetry::tracing().with_config(builder.build()) -} - /// A Lambda handler wrapped with Datadog tracing. /// /// Owns the [`SdkTracerProvider`] lifecycle, @@ -80,30 +36,16 @@ fn build_tracing( /// // Zero-config /// lambda_runtime::run(WrappedHandler::new( /// lambda_runtime::service_fn(my_handler), -/// Config::default(), /// )).await /// /// // Set service/env/version -/// lambda_runtime::run(WrappedHandler::new( -/// lambda_runtime::service_fn(my_handler), -/// Config { -/// service: Some("my-service".into()), -/// env: Some("prod".into()), -/// ..Default::default() -/// }, -/// )).await +/// let mut config = Config::builder(); +/// config.set_service("my-service".into()); +/// config.set_env("prod".into()); /// -/// // Full tracer control -/// lambda_runtime::run(WrappedHandler::new( +/// lambda_runtime::run(WrappedHandler::with_config( /// lambda_runtime::service_fn(my_handler), -/// Config { -/// tracing: Some( -/// datadog_opentelemetry::tracing() -/// .with_config(builder_config_here) -/// .with_span_processor(MyProcessor), -/// ), -/// ..Default::default() -/// }, +/// config, /// )).await /// /// // With tower middleware @@ -112,7 +54,6 @@ fn build_tracing( /// tower::ServiceBuilder::new() /// .layer(some_middleware) /// .service(lambda_runtime::service_fn(my_handler)), -/// Config::default(), /// ) /// ).await /// ``` @@ -125,23 +66,42 @@ pub struct WrappedHandler { } impl WrappedHandler { - /// Wraps a Tower service with Datadog tracing. + /// Wraps a Tower service with Datadog tracing using the default Datadog config sources. + /// + /// This is equivalent to calling [`with_config`](Self::with_config) with + /// [`Config::builder()`], then forcing the Lambda-safe tracing defaults. + pub fn new(inner: S) -> Self + where + S: Service, Response = R>, + { + Self::with_config(inner, Config::builder()) + } + + /// Wraps a Tower service with Datadog tracing using a caller-provided config builder. /// /// Use this constructor when you want to apply Tower middleware after tracing /// has started but before your Lambda handler executes. - pub fn new(inner: S, config: Config) -> Self + /// + /// The provided Datadog config builder is always forced to Lambda-safe defaults: + /// - `trace_stats_computation_enabled = false` + /// - `trace_writer_synchronous_write = true` + pub fn with_config(inner: S, config: ConfigBuilder) -> Self where S: Service, Response = R>, { - let Config { - tracing, - service: service_name, - env, - version, - } = config; - let provider = tracing - .unwrap_or_else(|| build_tracing(service_name, env, version)) - .init(); + let provider = { + let mut config = config; + // Stats are computed server-side by the extension; client-side computation is + // redundant. + config.set_trace_stats_computation_enabled(false); + // Synchronous writes make force_flush() block until data reaches the agent, + // this helps reduce span loss when the Lambda process freezes after the handler + // returns. + config.set_trace_writer_synchronous_write(true); + datadog_opentelemetry::tracing() + .with_config(config.build()) + .init() + }; let tracer = TracerProvider::tracer(&provider, TRACER_NAME); Self { inner, @@ -281,12 +241,9 @@ mod tests { #[test] fn poll_ready_delegates_to_inner_service() { let ready_calls = Arc::new(AtomicUsize::new(0)); - let mut wrapped = WrappedHandler::new( - ReadyCountingService { - ready_calls: Arc::clone(&ready_calls), - }, - Config::default(), - ); + let mut wrapped = WrappedHandler::new(ReadyCountingService { + ready_calls: Arc::clone(&ready_calls), + }); let waker = std::task::Waker::noop(); let mut cx = task::Context::from_waker(waker); From 19feab9d3695d1f2f40dc84a56068b7c42811860 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 13:43:49 -0700 Subject: [PATCH 18/26] refactor(datadog-aws-lambda): rename WrappedHandler to TracedService WrappedHandler was too generic, and the type's actual contract is a tower::Service over LambdaEvent rather than a handler function. Rename it to TracedService to better reflect both its tracing behavior and its service-based API, and update the docs/examples accordingly, including setting version in the config example. --- instrumentation/datadog-aws-lambda/src/lib.rs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 3dd401ff..287754dd 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -24,7 +24,7 @@ use std::task::{self, Poll}; type BoxFuture = std::pin::Pin + Send>>; -/// A Lambda handler wrapped with Datadog tracing. +/// A Lambda service wrapped with Datadog tracing. /// /// Owns the [`SdkTracerProvider`] lifecycle, /// applies Lambda-appropriate defaults, and implements [`Service`] so it composes @@ -34,7 +34,7 @@ type BoxFuture = std::pin::Pin + Send /// /// ```ignore /// // Zero-config -/// lambda_runtime::run(WrappedHandler::new( +/// lambda_runtime::run(TracedService::new( /// lambda_runtime::service_fn(my_handler), /// )).await /// @@ -42,22 +42,23 @@ type BoxFuture = std::pin::Pin + Send /// let mut config = Config::builder(); /// config.set_service("my-service".into()); /// config.set_env("prod".into()); +/// config.set_version("1.2.3".into()); /// -/// lambda_runtime::run(WrappedHandler::with_config( +/// lambda_runtime::run(TracedService::with_config( /// lambda_runtime::service_fn(my_handler), /// config, /// )).await /// /// // With tower middleware /// lambda_runtime::run( -/// WrappedHandler::new( +/// TracedService::new( /// tower::ServiceBuilder::new() /// .layer(some_middleware) /// .service(lambda_runtime::service_fn(my_handler)), /// ) /// ).await /// ``` -pub struct WrappedHandler { +pub struct TracedService { inner: S, provider: SdkTracerProvider, tracer: SdkTracer, @@ -65,7 +66,7 @@ pub struct WrappedHandler { _phantom: PhantomData R>, } -impl WrappedHandler { +impl TracedService { /// Wraps a Tower service with Datadog tracing using the default Datadog config sources. /// /// This is equivalent to calling [`with_config`](Self::with_config) with @@ -80,7 +81,7 @@ impl WrappedHandler { /// Wraps a Tower service with Datadog tracing using a caller-provided config builder. /// /// Use this constructor when you want to apply Tower middleware after tracing - /// has started but before your Lambda handler executes. + /// has started but before your Lambda service executes. /// /// The provided Datadog config builder is always forced to Lambda-safe defaults: /// - `trace_stats_computation_enabled = false` @@ -117,7 +118,7 @@ impl WrappedHandler { } } -impl Service>> for WrappedHandler +impl Service>> for TracedService where S: Service, Response = R>, S::Future: Future> + Send + 'static, @@ -180,10 +181,10 @@ mod tests { } #[allow(clippy::type_complexity)] - fn test_handler() -> WrappedHandler { + fn test_handler() -> TracedService { let provider = SdkTracerProvider::builder().build(); let tracer = TracerProvider::tracer(&provider, TRACER_NAME); - WrappedHandler { + TracedService { inner: ReadyService, provider, tracer, @@ -241,7 +242,7 @@ mod tests { #[test] fn poll_ready_delegates_to_inner_service() { let ready_calls = Arc::new(AtomicUsize::new(0)); - let mut wrapped = WrappedHandler::new(ReadyCountingService { + let mut wrapped = TracedService::new(ReadyCountingService { ready_calls: Arc::clone(&ready_calls), }); let waker = std::task::Waker::noop(); From b9e8297bdb568d83410b570388285134a5a6fab5 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 15:07:35 -0700 Subject: [PATCH 19/26] refactor(datadog-aws-lambda): align TracedService error bounds with lambda_runtime TracedService previously required inner service errors to convert into lambda_runtime::Error, which was narrower than lambda_runtime::run. Relax the bound to Into + Debug and introduce TracedServiceError to normalize wrapped service errors and local deserialization failures into a single outer error type that is compatible with both Lambda diagnostics and invocation span reporting. --- instrumentation/datadog-aws-lambda/src/lib.rs | 75 +++++++++++++++++-- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 287754dd..69af1938 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -18,12 +18,51 @@ use opentelemetry::trace::{FutureExt, TracerProvider}; use opentelemetry_sdk::trace::{SdkTracer, SdkTracerProvider}; use serde::de::DeserializeOwned; use serde_json::value::RawValue; +use std::fmt; use std::future::Future; use std::marker::PhantomData; use std::task::{self, Poll}; type BoxFuture = std::pin::Pin + Send>>; +/// Error returned by [`TracedService`]. +/// +/// This remains compatible with [`lambda_runtime::run`] by converting into +/// [`lambda_runtime::Diagnostic`], while also providing a stable display string +/// for invocation span error reporting. +#[derive(Debug)] +pub struct TracedServiceError(lambda_runtime::Diagnostic); + +impl fmt::Display for TracedServiceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0.error_message) + } +} + +impl From for TracedServiceError { + fn from(value: lambda_runtime::Diagnostic) -> Self { + Self(value) + } +} + +impl From for lambda_runtime::Diagnostic { + fn from(value: TracedServiceError) -> Self { + value.0 + } +} + +impl From for TracedServiceError { + fn from(value: lambda_runtime::Error) -> Self { + Self(value.into()) + } +} + +impl From for TracedServiceError { + fn from(value: serde_json::Error) -> Self { + lambda_runtime::Error::from(value).into() + } +} + /// A Lambda service wrapped with Datadog tracing. /// /// Owns the [`SdkTracerProvider`] lifecycle, @@ -122,16 +161,18 @@ impl Service>> for TracedService where S: Service, Response = R>, S::Future: Future> + Send + 'static, - S::Error: Into, + S::Error: Into + fmt::Debug, E: DeserializeOwned + Send + 'static, R: Send + 'static, { type Response = R; - type Error = lambda_runtime::Error; - type Future = BoxFuture>; + type Error = TracedServiceError; + type Future = BoxFuture>; fn poll_ready(&mut self, cx: &mut task::Context<'_>) -> Poll> { - self.inner.poll_ready(cx).map_err(Into::into) + self.inner + .poll_ready(cx) + .map_err(|err| TracedServiceError::from(err.into())) } fn call(&mut self, event: LambdaEvent>) -> Self::Future { @@ -142,7 +183,7 @@ where Ok(payload) => payload, Err(err) => { return Box::pin(async move { - let result: Result = Err(err.into()); + let result: Result = Err(err.into()); let result = invocation.finish(result); flush_provider(&provider); result @@ -153,7 +194,8 @@ where let fut = self.inner.call(typed_event); Box::pin(async move { let result = fut.with_context(invocation.handler_context()).await; - let result = invocation.finish(result.map_err(Into::into)); + let result = + invocation.finish(result.map_err(|err| TracedServiceError::from(err.into()))); flush_provider(&provider); result }) @@ -213,6 +255,8 @@ mod tests { ready_calls: Arc, } + struct StringErrorReadyService; + impl Service> for ReadyCountingService { type Response = (); type Error = lambda_runtime::Error; @@ -228,6 +272,20 @@ mod tests { } } + impl Service> for StringErrorReadyService { + type Response = (); + type Error = String; + type Future = std::future::Ready>; + + fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _: LambdaEvent) -> Self::Future { + std::future::ready(Ok(())) + } + } + #[test] fn cold_start_is_tracked_per_handler() { let mut first = test_handler(); @@ -251,4 +309,9 @@ mod tests { assert!(wrapped.poll_ready(&mut cx).is_ready()); assert_eq!(ready_calls.load(Ordering::Relaxed), 1); } + + #[test] + fn supports_non_lambda_runtime_error_types() { + let _wrapped = TracedService::new(StringErrorReadyService); + } } From aa5fc55afc24f7edb9dcad71e3929c49f68747d8 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 15:23:40 -0700 Subject: [PATCH 20/26] refactor(datadog-aws-lambda): remove redundant provider flush With synchronous trace writes enabled, the Datadog exporter already waits for the completed trace chunk to flush when the root span ends. Remove the extra provider.force_flush() calls, drop the now-redundant stored SdkTracerProvider from TracedService, and update the comment to describe the actual flush behavior. Also add a TODO in Cargo.toml to drop the test-utils feature from the production datadog-opentelemetry dependency once ConfigBuilder::set_trace_writer_synchronous_write is ungated upstream. --- instrumentation/datadog-aws-lambda/Cargo.toml | 2 ++ instrumentation/datadog-aws-lambda/src/lib.rs | 28 +++++-------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/Cargo.toml b/instrumentation/datadog-aws-lambda/Cargo.toml index 16865171..db00b910 100644 --- a/instrumentation/datadog-aws-lambda/Cargo.toml +++ b/instrumentation/datadog-aws-lambda/Cargo.toml @@ -13,6 +13,8 @@ authors.workspace = true publish.workspace = true [dependencies] +# TODO: Drop `test-utils` from this production dependency once +# `ConfigBuilder::set_trace_writer_synchronous_write` is ungated upstream. datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry", features = ["test-utils"] } lambda_runtime = "0.13" serde = { workspace = true } diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 69af1938..2ea0382f 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -15,7 +15,7 @@ use invocation::{Invocation, TRACER_NAME}; use lambda_runtime::tower::Service; use lambda_runtime::LambdaEvent; use opentelemetry::trace::{FutureExt, TracerProvider}; -use opentelemetry_sdk::trace::{SdkTracer, SdkTracerProvider}; +use opentelemetry_sdk::trace::SdkTracer; use serde::de::DeserializeOwned; use serde_json::value::RawValue; use std::fmt; @@ -99,7 +99,6 @@ impl From for TracedServiceError { /// ``` pub struct TracedService { inner: S, - provider: SdkTracerProvider, tracer: SdkTracer, cold_start: bool, _phantom: PhantomData R>, @@ -134,9 +133,9 @@ impl TracedService { // Stats are computed server-side by the extension; client-side computation is // redundant. config.set_trace_stats_computation_enabled(false); - // Synchronous writes make force_flush() block until data reaches the agent, - // this helps reduce span loss when the Lambda process freezes after the handler - // returns. + // Synchronous writes make the Datadog exporter wait for the completed trace chunk + // to flush when the root span ends, which helps reduce span loss when the Lambda + // process freezes after the handler returns. config.set_trace_writer_synchronous_write(true); datadog_opentelemetry::tracing() .with_config(config.build()) @@ -145,7 +144,6 @@ impl TracedService { let tracer = TracerProvider::tracer(&provider, TRACER_NAME); Self { inner, - provider, tracer, cold_start: true, _phantom: PhantomData, @@ -177,16 +175,13 @@ where fn call(&mut self, event: LambdaEvent>) -> Self::Future { let cold_start = self.take_cold_start(); - let provider = self.provider.clone(); let invocation = Invocation::start(&self.tracer, &event.context, cold_start); let typed_payload = match serde_json::from_str::(event.payload.get()) { Ok(payload) => payload, Err(err) => { return Box::pin(async move { let result: Result = Err(err.into()); - let result = invocation.finish(result); - flush_provider(&provider); - result + invocation.finish(result) }); } }; @@ -194,23 +189,15 @@ where let fut = self.inner.call(typed_event); Box::pin(async move { let result = fut.with_context(invocation.handler_context()).await; - let result = - invocation.finish(result.map_err(|err| TracedServiceError::from(err.into()))); - flush_provider(&provider); - result + invocation.finish(result.map_err(|err| TracedServiceError::from(err.into()))) }) } } -fn flush_provider(provider: &SdkTracerProvider) { - if let Err(err) = provider.force_flush() { - tracing::error!("flush failed: {err}"); - } -} - #[cfg(test)] mod tests { use super::*; + use opentelemetry_sdk::trace::SdkTracerProvider; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -228,7 +215,6 @@ mod tests { let tracer = TracerProvider::tracer(&provider, TRACER_NAME); TracedService { inner: ReadyService, - provider, tracer, cold_start: true, _phantom: PhantomData, From 817e9eaf75a7bf48d713a6cc9ef544df5e9fee69 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 6 May 2026 15:44:00 -0700 Subject: [PATCH 21/26] fix(datadog-aws-lambda): propagate invocation context through sync service call Attach the invocation context around inner.call(...) and use with_current_context() so both the synchronous call phase and the returned future run under the same active Lambda invocation context. Add a regression test covering services that inspect the active span in call(). --- instrumentation/datadog-aws-lambda/src/lib.rs | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index 2ea0382f..eef6b976 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -186,9 +186,12 @@ where } }; let typed_event = LambdaEvent::new(typed_payload, event.context); - let fut = self.inner.call(typed_event); + let fut = { + let _guard = invocation.handler_context().attach(); + self.inner.call(typed_event).with_current_context() + }; Box::pin(async move { - let result = fut.with_context(invocation.handler_context()).await; + let result = fut.await; invocation.finish(result.map_err(|err| TracedServiceError::from(err.into()))) }) } @@ -197,9 +200,11 @@ where #[cfg(test)] mod tests { use super::*; + use opentelemetry::trace::TraceContextExt; + use opentelemetry::Context; use opentelemetry_sdk::trace::SdkTracerProvider; use std::sync::{ - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, }; @@ -241,6 +246,10 @@ mod tests { ready_calls: Arc, } + struct ContextRecordingService { + saw_active_span_in_call: Arc, + } + struct StringErrorReadyService; impl Service> for ReadyCountingService { @@ -272,6 +281,22 @@ mod tests { } } + impl Service> for ContextRecordingService { + type Response = (); + type Error = lambda_runtime::Error; + type Future = std::future::Ready>; + + fn poll_ready(&mut self, _cx: &mut task::Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _: LambdaEvent) -> Self::Future { + self.saw_active_span_in_call + .store(Context::current().has_active_span(), Ordering::Relaxed); + std::future::ready(Ok(())) + } + } + #[test] fn cold_start_is_tracked_per_handler() { let mut first = test_handler(); @@ -300,4 +325,18 @@ mod tests { fn supports_non_lambda_runtime_error_types() { let _wrapped = TracedService::new(StringErrorReadyService); } + + #[tokio::test] + async fn call_runs_sync_phase_under_invocation_context() { + let saw_active_span_in_call = Arc::new(AtomicBool::new(false)); + let mut wrapped = TracedService::new(ContextRecordingService { + saw_active_span_in_call: Arc::clone(&saw_active_span_in_call), + }); + + let payload = RawValue::from_string("null".to_string()).unwrap(); + let event = LambdaEvent::new(payload, lambda_runtime::Context::default()); + wrapped.call(event).await.unwrap(); + + assert!(saw_active_span_in_call.load(Ordering::Relaxed)); + } } From aaf242a492b48d8800f7f9788ab528cb5c67e6c6 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Tue, 21 Apr 2026 15:02:04 -0400 Subject: [PATCH 22/26] feat(datadog-aws-lambda): add trigger extraction and inferred spans Adds libdd-trace-inferrer integration to parse Lambda event payloads and create inferred spans for upstream triggers (SQS, SNS, EventBridge, API Gateway, Lambda Function URLs). - span_inferrer module: bridges libdd-trace-inferrer with OTel SDK - TriggerExtraction: parses event payload, extracts carrier headers - InferredSpanScope: manages 0-2 inferred spans per invocation - Root span gains trigger metadata (event_source, async_invocation) - Correct span timing: async spans end at start, sync at end --- instrumentation/Cargo.lock | 123 ++++++ instrumentation/Cargo.toml | 2 + instrumentation/datadog-aws-lambda/Cargo.toml | 4 +- .../datadog-aws-lambda/src/attribute_keys.rs | 8 + .../datadog-aws-lambda/src/invocation.rs | 178 +++++++- instrumentation/datadog-aws-lambda/src/lib.rs | 13 +- .../src/span_inferrer/mod.rs | 408 ++++++++++++++++++ 7 files changed, 714 insertions(+), 22 deletions(-) create mode 100644 instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs diff --git a/instrumentation/Cargo.lock b/instrumentation/Cargo.lock index ad5dd822..76e1be0a 100644 --- a/instrumentation/Cargo.lock +++ b/instrumentation/Cargo.lock @@ -17,6 +17,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -165,6 +174,20 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -238,6 +261,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -339,7 +368,9 @@ version = "0.1.0" dependencies = [ "datadog-opentelemetry", "lambda_runtime", + "libdd-trace-inferrer", "opentelemetry", + "opentelemetry-semantic-conventions", "opentelemetry_sdk", "serde", "serde_json", @@ -775,6 +806,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -1177,6 +1232,21 @@ dependencies = [ "serde", ] +[[package]] +name = "libdd-trace-inferrer" +version = "1.0.0" +source = "git+https://github.com/DataDog/libdatadog?branch=david.ogbureke%2Flibdd-trace-inferrer#3d602313b31724e8d287e0c6a22d29919374998e" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "regex", + "serde", + "serde_json", + "sha2", + "tracing", +] + [[package]] name = "libdd-trace-normalization" version = "2.0.0" @@ -2496,12 +2566,65 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/instrumentation/Cargo.toml b/instrumentation/Cargo.toml index 55c9c8d7..3e8cdab5 100644 --- a/instrumentation/Cargo.toml +++ b/instrumentation/Cargo.toml @@ -15,6 +15,8 @@ authors = ["Datadog Inc. "] publish = false [workspace.dependencies] +datadog-aws-core = { path = "datadog-aws-core" } +libdd-trace-inferrer = { git = "https://github.com/DataDog/libdatadog", branch = "david.ogbureke/libdd-trace-inferrer" } serde = "1.0.194" serde_json = "1.0.140" opentelemetry = { version = "0.31.0", features = ["trace", "metrics", "logs"], default-features = false } diff --git a/instrumentation/datadog-aws-lambda/Cargo.toml b/instrumentation/datadog-aws-lambda/Cargo.toml index db00b910..4293b877 100644 --- a/instrumentation/datadog-aws-lambda/Cargo.toml +++ b/instrumentation/datadog-aws-lambda/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "datadog-aws-lambda" -description = "Datadog distributed tracing for AWS Lambda." +description = "Datadog distributed tracing for AWS Lambda. Provides inferred spans and trace context extraction for SQS, SNS, EventBridge, API Gateway (REST and HTTP APIs), and Lambda Function URLs." version.workspace = true edition.workspace = true rust-version = "1.85.0" @@ -16,10 +16,12 @@ publish.workspace = true # TODO: Drop `test-utils` from this production dependency once # `ConfigBuilder::set_trace_writer_synchronous_write` is ungated upstream. datadog-opentelemetry = { version = "0.3", path = "../../datadog-opentelemetry", features = ["test-utils"] } +libdd-trace-inferrer = { workspace = true } lambda_runtime = "0.13" serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } opentelemetry = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["trace", "metrics", "logs"] } tracing = { workspace = true } diff --git a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs index aa1834bd..b9406803 100644 --- a/instrumentation/datadog-aws-lambda/src/attribute_keys.rs +++ b/instrumentation/datadog-aws-lambda/src/attribute_keys.rs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 pub(crate) const OPERATION_NAME: &str = "operation.name"; +// `operation_name` duplicates `operation.name` as a custom attribute +pub(crate) const OPERATION_NAME_CUSTOM: &str = "operation_name"; pub(crate) const RESOURCE_NAME: &str = "resource.name"; pub(crate) const SPAN_TYPE: &str = "span.type"; pub(crate) const ERROR: &str = "error"; @@ -11,8 +13,14 @@ pub(crate) const ERROR_MESSAGE: &str = "error.message"; pub(crate) const LANGUAGE: &str = "language"; pub(crate) const REQUEST_ID: &str = "request_id"; pub(crate) const COLD_START: &str = "cold_start"; +pub(crate) const ASYNC_INVOCATION: &str = "async_invocation"; pub(crate) const FUNCTION_ARN: &str = "function_arn"; pub(crate) const FUNCTION_VERSION: &str = "function_version"; pub(crate) const FUNCTION_NAME: &str = "functionname"; pub(crate) const RESOURCE_NAMES: &str = "resource_names"; pub(crate) const DD_ORIGIN: &str = "_dd.origin"; +pub(crate) const FUNCTION_TRIGGER_EVENT_SOURCE: &str = "function_trigger.event_source"; +pub(crate) const FUNCTION_TRIGGER_EVENT_SOURCE_ARN: &str = "function_trigger.event_source_arn"; + +// OpenTelemetry semantic convention keys +pub(crate) use opentelemetry_semantic_conventions::attribute::{PEER_SERVICE, SERVICE_NAME}; diff --git a/instrumentation/datadog-aws-lambda/src/invocation.rs b/instrumentation/datadog-aws-lambda/src/invocation.rs index e159bf0f..ee03031c 100644 --- a/instrumentation/datadog-aws-lambda/src/invocation.rs +++ b/instrumentation/datadog-aws-lambda/src/invocation.rs @@ -3,19 +3,24 @@ //! Lifecycle management for a single Lambda invocation. //! -//! Each invocation produces one *root span* (`aws.lambda`) that wraps the handler call. +//! Each invocation produces: +//! - Zero or more *inferred spans* derived from the trigger payload (e.g., an SQS span) +//! - One *root span* (`aws.lambda`) that wraps the handler call //! //! Typical usage: -//! 1. [`Invocation::start`] — create the root span before the handler runs. +//! 1. [`Invocation::start`] — create all spans before the handler runs. //! 2. [`Invocation::handler_context`] — pass the returned context to the handler so its OTel spans //! are correctly parented. -//! 3. [`Invocation::finish`] — record errors and end the span after the handler returns. +//! 3. [`Invocation::finish`] — record errors and end all spans after the handler returns. use crate::attribute_keys as attr; +use crate::span_inferrer::{extract_trigger, InferredSpanScope, TriggerContext}; +use libdd_trace_inferrer::SpanInferrer; use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer}; use opentelemetry::{Context, KeyValue}; use opentelemetry_sdk::trace::SdkTracer; +use std::time::SystemTime; pub(crate) static TRACER_NAME: &str = "datadog-lambda-rs"; pub(crate) const ROOT_SPAN_NAME: &str = "aws.lambda"; @@ -34,9 +39,12 @@ pub(crate) struct LambdaSpan { impl LambdaSpan { /// Creates and starts the `aws.lambda` root span. /// - /// The span is parented to the ambient OTel context, which is typically a new root trace. + /// The span is parented to `trigger.parent_cx` when that context contains a valid + /// span (i.e., a propagated or inferred upstream span exists). Otherwise it falls + /// back to the ambient OTel context, which is typically a new root trace. pub(crate) fn start( tracer: &SdkTracer, + trigger: &TriggerContext, lambda_cx: &lambda_runtime::Context, cold_start: bool, ) -> Self { @@ -45,27 +53,44 @@ impl LambdaSpan { tracing::debug!(request_id, "creating invocation root span"); - let parent_cx = Context::current(); + let effective_cx = if trigger.parent_cx.span().span_context().is_valid() { + trigger.parent_cx.clone() + } else { + Context::current() + }; let mut builder = tracer.span_builder(ROOT_SPAN_NAME); builder.span_kind = Some(SpanKind::Server); - let attrs = vec![ + let mut attrs = vec![ KeyValue::new(attr::OPERATION_NAME, ROOT_SPAN_NAME), KeyValue::new(attr::LANGUAGE, "rust"), KeyValue::new(attr::RESOURCE_NAME, function_name.clone()), KeyValue::new(attr::SPAN_TYPE, "serverless"), KeyValue::new(attr::REQUEST_ID, request_id.clone()), KeyValue::new(attr::COLD_START, cold_start), + KeyValue::new(attr::ASYNC_INVOCATION, trigger.is_async), KeyValue::new(attr::FUNCTION_ARN, lambda_cx.invoked_function_arn.clone()), KeyValue::new(attr::FUNCTION_VERSION, lambda_cx.env_config.version.clone()), KeyValue::new(attr::FUNCTION_NAME, function_name.to_lowercase()), KeyValue::new(attr::RESOURCE_NAMES, function_name.clone()), KeyValue::new(attr::DD_ORIGIN, "lambda"), ]; + if let Some(ref source) = trigger.event_source { + attrs.push(KeyValue::new( + attr::FUNCTION_TRIGGER_EVENT_SOURCE, + source.clone(), + )); + } + if let Some(ref arn) = trigger.event_source_arn { + attrs.push(KeyValue::new( + attr::FUNCTION_TRIGGER_EVENT_SOURCE_ARN, + arn.clone(), + )); + } builder.attributes = Some(attrs); - let span = tracer.build_with_context(builder, &parent_cx); - let cx = parent_cx.with_span(span); + let span = tracer.build_with_context(builder, &effective_cx); + let cx = effective_cx.with_span(span); Self { cx, request_id } } @@ -90,21 +115,45 @@ impl LambdaSpan { /// Owns the full lifecycle of a single Lambda invocation's tracing state. /// -/// Holds the root span created for the handler call. +/// Holds the root span and all inferred spans created from the trigger payload. /// Call [`start`](Self::start) before the handler, then [`finish`](Self::finish) after. pub(crate) struct Invocation { /// The `aws.lambda` root span for this invocation. lambda_span: LambdaSpan, + /// Inferred spans derived from the trigger payload (e.g., SQS, SNS). + /// May be empty when the payload has no recognisable trigger. + inferred_spans: InferredSpanScope, + /// Wall-clock time at invocation start, used to compute inferred span durations. + started_at: SystemTime, } impl Invocation { pub(crate) fn start( tracer: &SdkTracer, + inferrer: &SpanInferrer, + payload: &str, lambda_cx: &lambda_runtime::Context, cold_start: bool, ) -> Self { - let lambda_span = LambdaSpan::start(tracer, lambda_cx, cold_start); - Self { lambda_span } + let extraction = extract_trigger(inferrer, payload); + let inferred_spans = InferredSpanScope::start( + tracer, + &extraction.upstream_cx, + &extraction.inference_result, + ); + let trigger = TriggerContext { + parent_cx: inferred_spans.innermost_context(&extraction.upstream_cx), + is_async: extraction.is_async, + event_source: extraction.event_source, + event_source_arn: extraction.event_source_arn, + }; + let lambda_span = LambdaSpan::start(tracer, &trigger, lambda_cx, cold_start); + + Self { + lambda_span, + inferred_spans, + started_at: SystemTime::now(), + } } /// Returns the OTel context to use as the active context during handler execution. @@ -117,7 +166,7 @@ impl Invocation { self.lambda_span.cx.clone() } - /// Records any handler error and ends the root span. + /// Records any handler error and ends all spans. /// /// The result is returned unchanged; this method only has side effects /// (error attributes, span end times). @@ -125,17 +174,29 @@ impl Invocation { where Err: std::fmt::Display, { + let invocation_end = SystemTime::now(); + if let Err(ref err) = result { self.lambda_span.set_error(err); } - self.lambda_span.finish(); + self.finish_spans(invocation_end); result } + + /// Ends all spans in the correct order: inferred spans first, then the root span. + /// + /// Inferred spans must be ended before the root span so that timing relationships + /// are correct in the Datadog backend. + fn finish_spans(self, invocation_end: SystemTime) { + self.inferred_spans.end(self.started_at, invocation_end); + self.lambda_span.finish(); + } } #[cfg(test)] mod tests { use super::*; + use crate::span_inferrer::TriggerContext; use opentelemetry::trace::{ SpanContext, SpanId, TraceFlags, TraceId, TraceState, TracerProvider as _, }; @@ -159,6 +220,15 @@ mod tests { cx } + fn test_trigger() -> TriggerContext { + TriggerContext { + parent_cx: Context::current(), + is_async: false, + event_source: None, + event_source_arn: None, + } + } + fn find_attr<'a>(attrs: &'a [KeyValue], key: &str) -> Option<&'a Value> { attrs .iter() @@ -176,8 +246,12 @@ mod tests { let tracer = provider.tracer("test"); let lambda_cx = test_lambda_cx(); + let trigger = TriggerContext { + is_async: true, + ..test_trigger() + }; - let span = LambdaSpan::start(&tracer, &lambda_cx, true); + let span = LambdaSpan::start(&tracer, &trigger, &lambda_cx, true); span.finish(); provider.force_flush().unwrap(); @@ -207,6 +281,10 @@ mod tests { Some(&Value::String("my-function".into())) ); assert_eq!(find_attr(attrs, "cold_start"), Some(&Value::Bool(true))); + assert_eq!( + find_attr(attrs, "async_invocation"), + Some(&Value::Bool(true)) + ); } #[test] @@ -222,11 +300,13 @@ mod tests { true, TraceState::default(), ); - let _guard = Context::current() - .with_remote_span_context(parent_sc) - .attach(); + let parent_cx = Context::current().with_remote_span_context(parent_sc); + let trigger = TriggerContext { + parent_cx, + ..test_trigger() + }; - let span = LambdaSpan::start(&tracer, &test_lambda_cx(), false); + let span = LambdaSpan::start(&tracer, &trigger, &test_lambda_cx(), false); assert_eq!(span.cx.span().span_context().trace_id(), trace_id); } @@ -234,7 +314,14 @@ mod tests { async fn error_handler_sets_error_attributes() { let (provider, exporter) = test_provider(); let invocation = Invocation { - lambda_span: LambdaSpan::start(&provider.tracer("test"), &test_lambda_cx(), false), + lambda_span: LambdaSpan::start( + &provider.tracer("test"), + &test_trigger(), + &test_lambda_cx(), + false, + ), + inferred_spans: InferredSpanScope::empty(), + started_at: SystemTime::now(), }; let _: Result<(), String> = invocation.finish(Err::<(), String>("boom".to_string())); @@ -253,7 +340,14 @@ mod tests { async fn successful_handler_sets_no_error_attributes() { let (provider, exporter) = test_provider(); let invocation = Invocation { - lambda_span: LambdaSpan::start(&provider.tracer("test"), &test_lambda_cx(), false), + lambda_span: LambdaSpan::start( + &provider.tracer("test"), + &test_trigger(), + &test_lambda_cx(), + false, + ), + inferred_spans: InferredSpanScope::empty(), + started_at: SystemTime::now(), }; let _: Result<(), String> = invocation.finish(Ok(())); @@ -264,4 +358,48 @@ mod tests { assert!(find_attr(attrs, "error").is_none()); assert!(find_attr(attrs, "error.message").is_none()); } + + #[test] + fn start_invocation_materializes_inferred_spans_for_sqs_events() { + let (provider, _) = test_provider(); + let payload = serde_json::json!({ + "Records": [{ + "messageId": "msg-001", + "receiptHandle": "receipt-001", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789:test-queue", + "awsRegion": "us-east-1", + "body": "hello", + "md5OfBody": "d8e8fca2dc0f896fd7cb4cb0031ba249", + "attributes": { + "SentTimestamp": "1718444400000", + "ApproximateFirstReceiveTimestamp": "1718444400100", + "ApproximateReceiveCount": "1", + "SenderId": "AIDAIENQZJOLO23YVJ4VO" + }, + "messageAttributes": { + "_datadog": { + "stringValue": serde_json::to_string(&serde_json::json!({ + "x-datadog-trace-id": "12345", + "x-datadog-parent-id": "67890", + "x-datadog-sampling-priority": "1" + })) + .unwrap(), + "dataType": "String" + } + } + }] + }) + .to_string(); + + let inferrer = crate::span_inferrer::build_inferrer(); + let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, TRACER_NAME); + let invocation = Invocation::start(&tracer, &inferrer, &payload, &test_lambda_cx(), false); + assert!(invocation + .handler_context() + .span() + .span_context() + .is_valid()); + assert!(!invocation.inferred_spans.is_empty()); + } } diff --git a/instrumentation/datadog-aws-lambda/src/lib.rs b/instrumentation/datadog-aws-lambda/src/lib.rs index eef6b976..5ced2088 100644 --- a/instrumentation/datadog-aws-lambda/src/lib.rs +++ b/instrumentation/datadog-aws-lambda/src/lib.rs @@ -9,11 +9,13 @@ mod attribute_keys; mod invocation; +mod span_inferrer; use datadog_opentelemetry::configuration::{Config, ConfigBuilder}; use invocation::{Invocation, TRACER_NAME}; use lambda_runtime::tower::Service; use lambda_runtime::LambdaEvent; +use libdd_trace_inferrer::SpanInferrer; use opentelemetry::trace::{FutureExt, TracerProvider}; use opentelemetry_sdk::trace::SdkTracer; use serde::de::DeserializeOwned; @@ -100,6 +102,7 @@ impl From for TracedServiceError { pub struct TracedService { inner: S, tracer: SdkTracer, + inferrer: SpanInferrer, cold_start: bool, _phantom: PhantomData R>, } @@ -145,6 +148,7 @@ impl TracedService { Self { inner, tracer, + inferrer: span_inferrer::build_inferrer(), cold_start: true, _phantom: PhantomData, } @@ -175,7 +179,13 @@ where fn call(&mut self, event: LambdaEvent>) -> Self::Future { let cold_start = self.take_cold_start(); - let invocation = Invocation::start(&self.tracer, &event.context, cold_start); + let invocation = Invocation::start( + &self.tracer, + &self.inferrer, + event.payload.get(), + &event.context, + cold_start, + ); let typed_payload = match serde_json::from_str::(event.payload.get()) { Ok(payload) => payload, Err(err) => { @@ -221,6 +231,7 @@ mod tests { TracedService { inner: ReadyService, tracer, + inferrer: span_inferrer::build_inferrer(), cold_start: true, _phantom: PhantomData, } diff --git a/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs new file mode 100644 index 00000000..c8ea7d42 --- /dev/null +++ b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs @@ -0,0 +1,408 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Bridge between [`libdd_trace_inferrer`] and the OpenTelemetry SDK. +//! +//! Responsibilities: +//! - Parse the raw Lambda event payload with `libdd_trace_inferrer` to identify the trigger type +//! and extract span metadata and carrier headers. +//! - Convert the inferred [`SpanData`](libdd_trace_inferrer::SpanData) into live OTel spans using +//! the Datadog tracer. +//! - Expose [`TriggerContext`] so [`crate::invocation`] can parent the root span correctly. + +use crate::attribute_keys as attr; +use libdd_trace_inferrer::{InferConfig, InferenceResult, SpanInferrer}; + +pub(crate) fn build_inferrer() -> SpanInferrer { + #[allow(clippy::disallowed_methods)] + let region = std::env::var("AWS_REGION").unwrap_or_default(); + SpanInferrer::new(InferConfig { + region, + ..InferConfig::default() + }) +} + +use opentelemetry::trace::{SpanKind, TraceContextExt, Tracer}; +use opentelemetry::{global, Context, KeyValue}; +use opentelemetry_sdk::trace::SdkTracer; +use std::time::{Duration, SystemTime}; + +/// Metadata extracted from the trigger event, used to parent the root span. +pub(crate) struct TriggerContext { + /// OTel context whose active span is the innermost inferred span (or the extracted + /// propagation context when there are no inferred spans). + pub parent_cx: Context, + /// Whether this trigger uses asynchronous invocation semantics. + /// + /// When `true`, inferred span duration = `invocation_start - event_time`. + /// When `false`, inferred span duration = `invocation_end - event_time`. + pub is_async: bool, + /// Short name of the outermost trigger, e.g. `"sqs"`. Added as a tag on the root span. + pub event_source: Option, + /// ARN of the outermost trigger resource. Added as a tag on the root span when present. + pub event_source_arn: Option, +} + +/// An inferred span that is currently open, held as an OTel context. +struct ActiveInferredSpan { + /// OTel context whose active span is this inferred span. + cx: Context, + /// Mirrors [`TriggerContext::is_async`] — determines the end-time passed to + /// [`InferredSpanScope::end`]. + is_async: bool, +} + +/// Handle for the set of inferred spans created for a trigger. +pub(crate) struct InferredSpanScope { + outer: Option, + inner: Option, +} + +impl InferredSpanScope { + pub(crate) fn empty() -> Self { + Self { + outer: None, + inner: None, + } + } + + #[cfg(test)] + pub(crate) fn is_empty(&self) -> bool { + self.inner.is_none() + } + + /// Creates inferred spans from an [`InferenceResult`]. + /// + /// Call [`innermost_context`](Self::innermost_context) after construction to get the + /// OTel context whose active span is the innermost inferred span (e.g., SQS inside + /// SNS). Callers should use that context as the parent for the `aws.lambda` root span. + pub(crate) fn start(tracer: &SdkTracer, parent_cx: &Context, result: &InferenceResult) -> Self { + if !result.should_create_inferred_span() { + return Self::empty(); + } + + let mut current_cx = parent_cx.clone(); + + let outer = result + .wrapped_span + .as_ref() + .filter(|w| w.should_create_inferred_span()) + .map(|w| { + current_cx = build_inferred_span(tracer, &w.span_data, ¤t_cx); + ActiveInferredSpan { + cx: current_cx.clone(), + is_async: w.is_async, + } + }); + + current_cx = build_inferred_span(tracer, &result.span_data, ¤t_cx); + let inner = Some(ActiveInferredSpan { + cx: current_cx, + is_async: result.is_async, + }); + + Self { outer, inner } + } + + /// Returns the OTel context of the innermost inferred span. + /// + /// Falls back to `fallback` (typically the upstream propagation context) when no + /// inferred spans were created. Callers should use this as the parent for the root span. + pub(crate) fn innermost_context(&self, fallback: &Context) -> Context { + self.inner + .as_ref() + .map(|s| s.cx.clone()) + .unwrap_or_else(|| fallback.clone()) + } + + /// End all inferred spans with correct timing. + /// + /// Async spans end at invocation start (propagation delay). + /// Sync spans end at invocation end (full request duration). + pub(crate) fn end(&self, invocation_start: SystemTime, invocation_end: SystemTime) { + for span in [self.outer.as_ref(), self.inner.as_ref()] + .into_iter() + .flatten() + { + let end_time = if span.is_async { + invocation_start + } else { + invocation_end + }; + span.cx.span().end_with_timestamp(end_time); + } + } +} + +/// Converts a [`libdd_trace_inferrer::SpanData`] into a live OTel span. +/// +/// Returns a new OTel context with the new span as the active span. All metadata +/// from `span_data.meta` is added as span attributes. +/// +/// If `span_data.start` is zero or negative (unknown event time), the OTel SDK +/// assigns the current wall-clock time as the start time. +fn build_inferred_span( + tracer: &SdkTracer, + span_data: &libdd_trace_inferrer::SpanData, + parent_cx: &Context, +) -> Context { + let mut builder = tracer.span_builder(span_data.name.clone()); + builder.span_kind = Some(SpanKind::Server); + + let start_ns = u64::try_from(span_data.start).unwrap_or(0); + if start_ns > 0 { + builder.start_time = Some(SystemTime::UNIX_EPOCH + Duration::from_nanos(start_ns)); + } + + let mut attrs = vec![ + KeyValue::new(attr::SERVICE_NAME, span_data.service.clone()), + KeyValue::new(attr::RESOURCE_NAME, span_data.resource.clone()), + KeyValue::new(attr::SPAN_TYPE, span_data.r#type.clone()), + KeyValue::new(attr::OPERATION_NAME, span_data.name.clone()), + KeyValue::new(attr::OPERATION_NAME_CUSTOM, span_data.name.clone()), + KeyValue::new(attr::PEER_SERVICE, span_data.service.clone()), + ]; + for (k, v) in &span_data.meta { + attrs.push(KeyValue::new(k.clone(), v.clone())); + } + builder.attributes = Some(attrs); + + let span = tracer.build_with_context(builder, parent_cx); + parent_cx.with_span(span) +} + +/// Output of [`extract_trigger`]. +pub(crate) struct TriggerExtraction { + /// OTel context extracted from the trigger's carrier headers. + /// + /// Contains the upstream trace/span IDs when the trigger carries Datadog propagation + /// headers. Falls back to the ambient context when no valid carrier is found. + pub upstream_cx: Context, + /// Full inference result from `libdd_trace_inferrer`, including span data, trigger + /// tags, and async/sync classification. + pub inference_result: InferenceResult, + /// Whether this trigger uses asynchronous invocation semantics. + pub is_async: bool, + /// Short name of the outermost trigger, e.g. `"sqs"`. Added as a tag on the root span. + pub event_source: Option, + /// ARN of the outermost trigger resource. Added as a tag on the root span when present. + pub event_source_arn: Option, +} + +/// Infers trigger metadata from `payload` and extracts the upstream OTel context. +/// +/// Carrier extraction uses `x-datadog-trace-id` as a sentinel: a missing or zero +/// trace ID means there are no upstream headers to propagate, so we fall back to +/// the ambient context rather than accidentally creating a span parented to trace ID 0. +pub(crate) fn extract_trigger(inferrer: &SpanInferrer, payload: &str) -> TriggerExtraction { + let result = inferrer.infer_span(payload).unwrap_or_default(); + + let upstream_cx = global::get_text_map_propagator(|p| { + if result + .carrier + .get("x-datadog-trace-id") + .and_then(|id| id.parse::().ok()) + .is_some_and(|id| id != 0) + { + tracing::debug!( + trace_id = result + .carrier + .get("x-datadog-trace-id") + .map(String::as_str) + .unwrap_or("?"), + "extracted trace context from trigger" + ); + p.extract(&result.carrier) + } else { + tracing::debug!("no trace context found in event"); + Context::current() + } + }); + + TriggerExtraction { + upstream_cx, + is_async: result.is_async, + event_source: result + .trigger_tags + .get("function_trigger.event_source") + .cloned(), + event_source_arn: result + .trigger_tags + .get("function_trigger.event_source_arn") + .cloned(), + inference_result: result, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use libdd_trace_inferrer::{InferenceResult, SpanData}; + use opentelemetry::trace::{TraceContextExt, TracerProvider as _}; + use opentelemetry::Value as OtelValue; + use opentelemetry_sdk::trace::{ + InMemorySpanExporter, SdkTracerProvider, SpanData as OtelSpanData, + }; + use serde_json::json; + use std::collections::HashMap; + use std::time::SystemTime; + + fn test_provider() -> (SdkTracerProvider, InMemorySpanExporter) { + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + (provider, exporter) + } + + fn find_attr<'a>(attrs: &'a [KeyValue], key: &str) -> Option<&'a OtelValue> { + attrs + .iter() + .find(|kv| kv.key.as_str() == key) + .map(|kv| &kv.value) + } + + fn finished_spans(exporter: &InMemorySpanExporter) -> Vec { + exporter.get_finished_spans().unwrap() + } + + fn make_result(name: &str, service: &str) -> InferenceResult { + InferenceResult { + span_data: SpanData { + name: name.to_string(), + service: service.to_string(), + resource: service.to_string(), + r#type: "web".to_string(), + start: 0, + meta: HashMap::new(), + }, + is_async: false, + ..InferenceResult::default() + } + } + + #[test] + fn sets_expected_attributes_on_inferred_spans() { + let (provider, exporter) = test_provider(); + let tracer = provider.tracer("test"); + + let result = make_result("aws.sqs", "my-queue"); + let scope = InferredSpanScope::start(&tracer, &Context::current(), &result); + let now = SystemTime::now(); + scope.end(now, now); + provider.force_flush().ok(); + + let spans = finished_spans(&exporter); + assert_eq!(spans.len(), 1); + let attrs = &spans[0].attributes; + + assert_eq!( + find_attr(attrs, "service.name"), + Some(&OtelValue::String("my-queue".into())) + ); + assert_eq!( + find_attr(attrs, "resource.name"), + Some(&OtelValue::String("my-queue".into())) + ); + assert_eq!( + find_attr(attrs, "operation.name"), + Some(&OtelValue::String("aws.sqs".into())) + ); + assert_eq!( + find_attr(attrs, "operation_name"), + Some(&OtelValue::String("aws.sqs".into())) + ); + } + + #[test] + fn chains_inferred_spans() { + let (provider, exporter) = test_provider(); + let tracer = provider.tracer("test"); + + let mut result = make_result("aws.sqs", "my-queue"); + result.wrapped_span = Some(Box::new(make_result("aws.sns", "my-topic"))); + + let parent_cx = Context::current(); + let scope = InferredSpanScope::start(&tracer, &parent_cx, &result); + let cx = scope.innermost_context(&parent_cx); + let now = SystemTime::now(); + scope.end(now, now); + provider.force_flush().ok(); + + let spans = finished_spans(&exporter); + assert_eq!(spans.len(), 2); + // Innermost span (sqs) should be the active span in the returned context + assert_eq!( + cx.span().span_context().span_id(), + spans + .iter() + .find(|s| s.name == "aws.sqs") + .unwrap() + .span_context + .span_id() + ); + } + + #[test] + fn preserves_parent_context_when_no_inferred_spans_are_created() { + let (provider, _) = test_provider(); + let tracer = provider.tracer("test"); + let parent_cx = Context::current(); + let empty_result = InferenceResult::default(); + let scope = InferredSpanScope::start(&tracer, &parent_cx, &empty_result); + let result_cx = scope.innermost_context(&parent_cx); + + assert_eq!( + result_cx.span().span_context().trace_id(), + parent_cx.span().span_context().trace_id() + ); + assert!(scope.is_empty()); + } + + #[test] + fn extracts_trigger_context_from_sqs_event() { + let carrier_json = json!({ + "x-datadog-trace-id": "12345", + "x-datadog-parent-id": "67890", + "x-datadog-sampling-priority": "1" + }); + let event = json!({ + "Records": [{ + "messageId": "msg-001", + "receiptHandle": "receipt-001", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789:test-queue", + "awsRegion": "us-east-1", + "body": "hello", + "md5OfBody": "d8e8fca2dc0f896fd7cb4cb0031ba249", + "attributes": { + "SentTimestamp": "1718444400000", + "ApproximateFirstReceiveTimestamp": "1718444400100", + "ApproximateReceiveCount": "1", + "SenderId": "AIDAIENQZJOLO23YVJ4VO" + }, + "messageAttributes": { + "_datadog": { + "stringValue": serde_json::to_string(&carrier_json).unwrap(), + "dataType": "String" + } + } + }] + }); + + let inferrer = build_inferrer(); + let extraction = extract_trigger(&inferrer, &event.to_string()); + + assert!(extraction.inference_result.is_async); + assert_eq!( + extraction + .inference_result + .trigger_tags + .get("function_trigger.event_source") + .map(String::as_str), + Some("sqs") + ); + assert!(extraction.inference_result.should_create_inferred_span()); + assert_eq!(extraction.inference_result.span_data.name, "aws.sqs"); + } +} From 6f6a9e0549aef21e0fe4694d20e2556a3317db76 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Tue, 28 Apr 2026 16:03:15 -0700 Subject: [PATCH 23/26] Fix Cargo.toml --- instrumentation/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/instrumentation/Cargo.toml b/instrumentation/Cargo.toml index 3e8cdab5..b251868b 100644 --- a/instrumentation/Cargo.toml +++ b/instrumentation/Cargo.toml @@ -15,7 +15,6 @@ authors = ["Datadog Inc. "] publish = false [workspace.dependencies] -datadog-aws-core = { path = "datadog-aws-core" } libdd-trace-inferrer = { git = "https://github.com/DataDog/libdatadog", branch = "david.ogbureke/libdd-trace-inferrer" } serde = "1.0.194" serde_json = "1.0.140" From c5fd42f20aa747b9d5927e9815593d227da9f3aa Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Wed, 29 Apr 2026 14:53:50 -0700 Subject: [PATCH 24/26] refactor(aws-lambda): remove duplicate x-datadog-trace-id literal in trigger extraction --- .../datadog-aws-lambda/src/span_inferrer/mod.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs index c8ea7d42..8d29e65e 100644 --- a/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs +++ b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs @@ -196,20 +196,15 @@ pub(crate) struct TriggerExtraction { /// the ambient context rather than accidentally creating a span parented to trace ID 0. pub(crate) fn extract_trigger(inferrer: &SpanInferrer, payload: &str) -> TriggerExtraction { let result = inferrer.infer_span(payload).unwrap_or_default(); + let trace_id = result.carrier.get("x-datadog-trace-id").map(String::as_str); + let has_upstream_trace = trace_id + .and_then(|id| id.parse::().ok()) + .is_some_and(|id| id != 0); let upstream_cx = global::get_text_map_propagator(|p| { - if result - .carrier - .get("x-datadog-trace-id") - .and_then(|id| id.parse::().ok()) - .is_some_and(|id| id != 0) - { + if has_upstream_trace { tracing::debug!( - trace_id = result - .carrier - .get("x-datadog-trace-id") - .map(String::as_str) - .unwrap_or("?"), + trace_id = trace_id.unwrap_or("?"), "extracted trace context from trigger" ); p.extract(&result.carrier) From db83efe1caa828db7e1a442b9abe30bed3a3a873 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 22 May 2026 17:31:07 -0700 Subject: [PATCH 25/26] test(datadog-aws-lambda): add regression for wrapped inferred span timing --- .../src/span_inferrer/mod.rs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs index 8d29e65e..ae115a76 100644 --- a/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs +++ b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs @@ -276,6 +276,10 @@ mod tests { } } + fn nanos_to_system_time(ns: u64) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_nanos(ns) + } + #[test] fn sets_expected_attributes_on_inferred_spans() { let (provider, exporter) = test_provider(); @@ -338,6 +342,35 @@ mod tests { ); } + #[test] + fn wrapped_span_ends_when_inner_span_starts() { + let (provider, exporter) = test_provider(); + let tracer = provider.tracer("test"); + + let outer_start_ns = 1_000_000_000; + let inner_start_ns = 2_000_000_000; + let invocation_start = nanos_to_system_time(10_000_000_000); + let invocation_end = nanos_to_system_time(20_000_000_000); + + let mut result = make_result("aws.sqs", "my-queue"); + result.span_data.start = inner_start_ns as i64; + result.is_async = true; + + let mut wrapped = make_result("aws.sns", "my-topic"); + wrapped.span_data.start = outer_start_ns as i64; + result.wrapped_span = Some(Box::new(wrapped)); + + let scope = InferredSpanScope::start(&tracer, &Context::current(), &result); + scope.end(invocation_start, invocation_end); + provider.force_flush().ok(); + + let spans = finished_spans(&exporter); + let outer = spans.iter().find(|s| s.name == "aws.sns").unwrap(); + let inner = spans.iter().find(|s| s.name == "aws.sqs").unwrap(); + + assert_eq!(outer.end_time, inner.start_time); + } + #[test] fn preserves_parent_context_when_no_inferred_spans_are_created() { let (provider, _) = test_provider(); From bc898f1564d645d2f9096aaa1d404d87d87c5093 Mon Sep 17 00:00:00 2001 From: Yury Gribkov Date: Fri, 22 May 2026 18:09:15 -0700 Subject: [PATCH 26/26] fix(datadog-aws-lambda): end wrapped inferred spans at inner event time In wrapped cases such as EventBridge -> SQS, inference creates both an outer span for the upstream trigger and an inner span for the delivery event. The outer wrapped span should end when the inner event begins, not when the Lambda invocation ends, otherwise it incorrectly covers handler execution and inflates the upstream trigger duration. --- .../src/span_inferrer/mod.rs | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs index ae115a76..9a96050c 100644 --- a/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs +++ b/instrumentation/datadog-aws-lambda/src/span_inferrer/mod.rs @@ -50,6 +50,8 @@ struct ActiveInferredSpan { /// Mirrors [`TriggerContext::is_async`] — determines the end-time passed to /// [`InferredSpanScope::end`]. is_async: bool, + /// Source event timestamp for this inferred span, when known. + start_time: Option, } /// Handle for the set of inferred spans created for a trigger. @@ -92,6 +94,7 @@ impl InferredSpanScope { ActiveInferredSpan { cx: current_cx.clone(), is_async: w.is_async, + start_time: inferred_span_start_time(&w.span_data), } }); @@ -99,6 +102,7 @@ impl InferredSpanScope { let inner = Some(ActiveInferredSpan { cx: current_cx, is_async: result.is_async, + start_time: inferred_span_start_time(&result.span_data), }); Self { outer, inner } @@ -117,23 +121,35 @@ impl InferredSpanScope { /// End all inferred spans with correct timing. /// - /// Async spans end at invocation start (propagation delay). - /// Sync spans end at invocation end (full request duration). + /// Wrapped outer spans end when the inner event begins. + /// Inner async spans end at invocation start (propagation delay). + /// Inner sync spans end at invocation end (full request duration). pub(crate) fn end(&self, invocation_start: SystemTime, invocation_end: SystemTime) { - for span in [self.outer.as_ref(), self.inner.as_ref()] - .into_iter() - .flatten() - { - let end_time = if span.is_async { + if let Some(outer) = self.outer.as_ref() { + let outer_end_time = self + .inner + .as_ref() + .and_then(|inner| inner.start_time) + .unwrap_or(invocation_start); + outer.cx.span().end_with_timestamp(outer_end_time); + } + + if let Some(inner) = self.inner.as_ref() { + let end_time = if inner.is_async { invocation_start } else { invocation_end }; - span.cx.span().end_with_timestamp(end_time); + inner.cx.span().end_with_timestamp(end_time); } } } +fn inferred_span_start_time(span_data: &libdd_trace_inferrer::SpanData) -> Option { + let start_ns = u64::try_from(span_data.start).ok()?; + (start_ns > 0).then(|| SystemTime::UNIX_EPOCH + Duration::from_nanos(start_ns)) +} + /// Converts a [`libdd_trace_inferrer::SpanData`] into a live OTel span. /// /// Returns a new OTel context with the new span as the active span. All metadata @@ -149,9 +165,8 @@ fn build_inferred_span( let mut builder = tracer.span_builder(span_data.name.clone()); builder.span_kind = Some(SpanKind::Server); - let start_ns = u64::try_from(span_data.start).unwrap_or(0); - if start_ns > 0 { - builder.start_time = Some(SystemTime::UNIX_EPOCH + Duration::from_nanos(start_ns)); + if let Some(start_time) = inferred_span_start_time(span_data) { + builder.start_time = Some(start_time); } let mut attrs = vec![