From dc928d10e9e417360f151df2c4aed34aaede96d6 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Sun, 12 Jul 2026 13:42:31 -0500 Subject: [PATCH 1/7] opentelemetry-configuration: substitute env vars after parsing (#5406) Environment variable substitution ran on the raw configuration file text before parsing, so a ${VAR} reference inside a YAML comment was treated as a real substitution -- an undefined variable in a comment crashed loading, and documenting the substitution mechanism was impossible. Parse the file first, then substitute only within scalar values (never in comments or mapping keys), matching the configuration spec and the Java and Node.js implementations. For an unquoted standalone ${VAR} reference the YAML node's type tag is re-resolved from the substituted value so type coercion still applies (e.g. ${LIMIT} -> int); quoted or embedded references resolve to strings. JSON string values are substituted in place. --- .changelog/5406.fixed | 1 + .../configuration/file/_env_substitution.py | 33 ++--- .../configuration/file/_loader.py | 132 ++++++++++++++---- .../tests/file/test_env_substitution.py | 37 ++--- .../tests/file/test_loader.py | 128 +++++++++++++++++ 5 files changed, 258 insertions(+), 73 deletions(-) create mode 100644 .changelog/5406.fixed diff --git a/.changelog/5406.fixed b/.changelog/5406.fixed new file mode 100644 index 00000000000..e6c5f221467 --- /dev/null +++ b/.changelog/5406.fixed @@ -0,0 +1 @@ +`opentelemetry-configuration`: perform environment variable substitution on scalar values after parsing the configuration file, so `${VAR}` references inside comments and mapping keys are no longer substituted and undefined references in comments no longer abort loading diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py index 9d3a1b815aa..882437c70f3 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py @@ -19,18 +19,21 @@ class EnvSubstitutionError(Exception): def substitute_env_vars(text: str) -> str: - """Substitute environment variables in configuration text. + """Substitute environment variables within a configuration scalar value. + + Substitution is applied per scalar value after the configuration file has + been parsed, so comments and mapping keys are never touched. Supports the + following syntax: - Supports the following syntax: - ${VAR}: Substitute with environment variable VAR. Raises error if not found. - ${VAR:-default}: Substitute with VAR if set, otherwise use default value. - $$: Escape sequence for literal $. Args: - text: Configuration text with potential ${VAR} placeholders. + text: A scalar value with potential ${VAR} placeholders. Returns: - Text with environment variables substituted. + The value with environment variables substituted. Raises: EnvSubstitutionError: If a required environment variable is not found. @@ -70,23 +73,11 @@ def replace_var(match) -> str: f"Environment variable '{var_name}' not found and no default provided" ) - # Per spec: "It MUST NOT be possible to inject YAML structures by - # environment variables." Newlines are the primary injection vector — - # a value like "legit\nmalicious_key: val" would create extra YAML - # keys if substituted verbatim. Wrap such values in a YAML - # double-quoted scalar so the newline is treated as literal text. - # Simple values (no newlines) are returned as-is so that YAML type - # coercion still applies per spec ("Node types MUST be interpreted - # after environment variable substitution takes place"). - if "\n" in value or "\r" in value: - escaped = ( - value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - ) - return f'"{escaped}"' + # Substitution runs on an already-parsed scalar value, so the result + # cannot inject new YAML structure regardless of its contents (a + # newline in the value stays a literal character within this one + # scalar). Type interpretation for standalone references is handled by + # the loader, which re-resolves the node tag after substitution. return value return re.sub(pattern, replace_var, text) diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py index 6131e3c1b08..6bb13f2dcd2 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py @@ -7,6 +7,7 @@ import json import logging import os +import re from pathlib import Path from typing import Any @@ -16,6 +17,7 @@ MissingDependencyError, ) from opentelemetry.configuration.file._env_substitution import ( + EnvSubstitutionError, substitute_env_vars, ) from opentelemetry.configuration.models import OpenTelemetryConfiguration @@ -65,14 +67,107 @@ def _get_schema() -> dict: _logger = logging.getLogger(__name__) +_YAML_STR_TAG = "tag:yaml.org,2002:str" + +# A scalar whose entire value is a single ``${VAR}`` / ``${VAR:-default}`` +# reference. Only such standalone references (when unquoted) have their type +# re-interpreted after substitution; embedded or multiple references resolve to +# a string per the configuration spec. A leading ``$$`` escape does not match, +# so ``$${VAR}`` is treated as an embedded (string) value. +_STANDALONE_ENV_REF = re.compile( + r"\A\$\{[A-Za-z_][A-Za-z0-9_]*(?::-[^}]*)?\}\Z" +) + + +def _substitute_env_in_yaml_node(node: yaml.Node, loader: yaml.SafeLoader): + """Apply env-var substitution to string scalar values in a YAML node tree. + + Substitution runs after parsing on scalar *values* only, so comments and + mapping keys are never candidates (per the configuration spec). For an + unquoted standalone ``${VAR}`` reference the node's type tag is re-resolved + from the substituted value so YAML type coercion still applies (e.g. + ``${LIMIT}`` -> int); quoted or embedded references stay strings. + """ + if isinstance(node, yaml.ScalarNode): + if node.tag == _YAML_STR_TAG: + raw = node.value + node.value = substitute_env_vars(raw) + if node.style is None and _STANDALONE_ENV_REF.match(raw): + node.tag = loader.resolve( + yaml.ScalarNode, node.value, (True, False) + ) + elif isinstance(node, yaml.SequenceNode): + for item in node.value: + _substitute_env_in_yaml_node(item, loader) + elif isinstance(node, yaml.MappingNode): + # Recurse into values only; keys are not substitution candidates. + for _key_node, value_node in node.value: + _substitute_env_in_yaml_node(value_node, loader) + + +def _substitute_env_in_json_value(value: Any) -> Any: + """Recursively apply env-var substitution to string values in JSON data. + + JSON has explicit types and no comments, so substitution applies only to + string values (the result stays a string) and mapping keys are left as-is. + """ + if isinstance(value, str): + return substitute_env_vars(value) + if isinstance(value, list): + return [_substitute_env_in_json_value(item) for item in value] + if isinstance(value, dict): + return { + key: _substitute_env_in_json_value(val) + for key, val in value.items() + } + return value + + +def _parse_config_content( + content: str, suffix: str, file_path: str | os.PathLike[str] +) -> Any: + """Parse configuration text and substitute environment variables. + + Parsing happens first, so ``${VAR}`` references in comments and mapping + keys are never substitution candidates; substitution then runs on scalar + values, with YAML node types re-resolved for standalone references. + + Raises: + ConfigurationError: If the content cannot be parsed or substitution of + a required environment variable fails. + """ + try: + if suffix == ".json": + return _substitute_env_in_json_value(json.loads(content)) + yaml_loader = yaml.SafeLoader(content) + try: + root_node = yaml_loader.get_single_node() + if root_node is None: + return None + _substitute_env_in_yaml_node(root_node, yaml_loader) + return yaml_loader.construct_document(root_node) + finally: + yaml_loader.dispose() + except EnvSubstitutionError as exc: + raise ConfigurationError( + f"Environment variable substitution failed: {exc}" + ) from exc + except yaml.YAMLError as exc: + _logger.exception("Failed to parse YAML from %s", file_path) + raise ConfigurationError(f"Failed to parse YAML: {exc}") from exc + except json.JSONDecodeError as exc: + _logger.exception("Failed to parse JSON from %s", file_path) + raise ConfigurationError(f"Failed to parse JSON: {exc}") from exc + def load_config_file( file_path: str | os.PathLike[str], ) -> OpenTelemetryConfiguration: """Load and parse an OpenTelemetry configuration file. - Supports YAML and JSON formats. Performs environment variable substitution - before parsing. + Supports YAML and JSON formats. Environment variable substitution is + performed after parsing, on scalar values only, so ``${VAR}`` references in + comments or mapping keys are left untouched. Args: file_path: Path to the configuration file (.yaml, .yml, or .json). @@ -109,32 +204,17 @@ def load_config_file( f"Failed to read configuration file: {file_path}" ) from exc - # Perform environment variable substitution - try: - content = substitute_env_vars(content) - except Exception as exc: + # Parse the file, then substitute environment variables in scalar values. + # Parsing first means comments and mapping keys are never substitution + # candidates, and node types are still resolved after substitution. + suffix = path.suffix.lower() + if suffix not in (".yaml", ".yml", ".json"): + _logger.error("Unsupported file format: %s", suffix) raise ConfigurationError( - f"Environment variable substitution failed: {exc}" - ) from exc + f"Unsupported file format: {suffix}. Use .yaml, .yml, or .json" + ) - # Parse based on file extension - suffix = path.suffix.lower() - try: - if suffix in (".yaml", ".yml"): - data = yaml.safe_load(content) - elif suffix == ".json": - data = json.loads(content) - else: - _logger.error("Unsupported file format: %s", suffix) - raise ConfigurationError( - f"Unsupported file format: {suffix}. Use .yaml, .yml, or .json" - ) - except yaml.YAMLError as exc: - _logger.exception("Failed to parse YAML from %s", file_path) - raise ConfigurationError(f"Failed to parse YAML: {exc}") from exc - except json.JSONDecodeError as exc: - _logger.exception("Failed to parse JSON from %s", file_path) - raise ConfigurationError(f"Failed to parse JSON: {exc}") from exc + data = _parse_config_content(content, suffix, file_path) if data is None: _logger.error("Configuration file is empty: %s", file_path) diff --git a/opentelemetry-configuration/tests/file/test_env_substitution.py b/opentelemetry-configuration/tests/file/test_env_substitution.py index 14c79f20fd5..9bb19d5eca5 100644 --- a/opentelemetry-configuration/tests/file/test_env_substitution.py +++ b/opentelemetry-configuration/tests/file/test_env_substitution.py @@ -107,37 +107,22 @@ def test_only_dollar_signs(self): result = substitute_env_vars("$$$$") self.assertEqual(result, "$$") - def test_newline_in_value_prevents_yaml_injection(self): - """Values containing newlines must not inject YAML structure. + def test_newline_in_value_returned_verbatim(self): + """A newline in a value is returned as-is. - Per spec: "It MUST NOT be possible to inject YAML structures by - environment variables." A value like "legit\\nmalicious_key: val" - must be emitted as a quoted scalar, not raw YAML. + Substitution runs per scalar value after parsing, so it does not + escape or quote newlines; YAML injection is prevented structurally + (see the loader tests), not by rewriting the value here. """ - with patch.dict( - os.environ, - {"SERVICE_NAME": "legit-service\nmalicious_key: injected_value"}, - ): - result = substitute_env_vars( - "file_format: '1.0'\nservice_name: ${SERVICE_NAME}" - ) - parsed = yaml.safe_load(result) - self.assertNotIn("malicious_key", parsed) - self.assertIn("legit-service", parsed["service_name"]) - - def test_newline_in_value_preserved_as_literal(self): - """Newline within a value is preserved as a literal newline character.""" with patch.dict(os.environ, {"MULTI": "line1\nline2"}): - result = substitute_env_vars("key: ${MULTI}") - parsed = yaml.safe_load(result) - self.assertEqual(parsed["key"], "line1\nline2") + result = substitute_env_vars("${MULTI}") + self.assertEqual(result, "line1\nline2") - def test_carriage_return_in_value_is_escaped(self): - """Carriage return in value is escaped, not injected.""" + def test_carriage_return_in_value_returned_verbatim(self): + """A carriage return in a value is returned as-is, not escaped.""" with patch.dict(os.environ, {"VAL": "text\r\nmore"}): - result = substitute_env_vars("key: ${VAL}") - parsed = yaml.safe_load(result) - self.assertIsInstance(parsed["key"], str) + result = substitute_env_vars("${VAL}") + self.assertEqual(result, "text\r\nmore") def test_type_coercion_preserved_for_simple_values(self): """Simple values without newlines still undergo YAML type coercion per spec.""" diff --git a/opentelemetry-configuration/tests/file/test_loader.py b/opentelemetry-configuration/tests/file/test_loader.py index c9dcbee8e67..dd99bdd27e0 100644 --- a/opentelemetry-configuration/tests/file/test_loader.py +++ b/opentelemetry-configuration/tests/file/test_loader.py @@ -7,6 +7,8 @@ from pathlib import Path from unittest.mock import patch +import yaml + from opentelemetry.configuration._tracer_provider import ( create_tracer_provider, ) @@ -14,6 +16,10 @@ ConfigurationError, load_config_file, ) +from opentelemetry.configuration.file._loader import ( + _substitute_env_in_json_value, + _substitute_env_in_yaml_node, +) from opentelemetry.configuration.models import ( BatchSpanProcessor as BatchSpanProcessorConfig, ) @@ -367,3 +373,125 @@ def test_malformed_version_is_rejected(self): with self.assertRaises(ConfigurationError) as ctx: self._load("not-a-version") self.assertIn("file_format", str(ctx.exception)) + + +class TestEnvVarSubstitutionScope(unittest.TestCase): + """Substitution applies only to scalar values, per the config spec. + + These exercise the YAML node walker directly so the spec's type and + scope rules can be asserted without the JSON schema constraining shape. + """ + + @staticmethod + def _substitute_yaml(text: str): + loader = yaml.SafeLoader(text) + try: + node = loader.get_single_node() + _substitute_env_in_yaml_node(node, loader) + return loader.construct_document(node) + finally: + loader.dispose() + + def test_unquoted_standalone_reference_is_type_coerced(self): + with patch.dict(os.environ, {"N": "42", "FLAG": "true"}): + result = self._substitute_yaml("count: ${N}\nflag: ${FLAG}") + self.assertEqual(result["count"], 42) + self.assertIsInstance(result["count"], int) + self.assertIs(result["flag"], True) + + def test_quoted_reference_stays_string(self): + with patch.dict(os.environ, {"N": "42"}): + result = self._substitute_yaml('count: "${N}"') + self.assertEqual(result["count"], "42") + + def test_embedded_reference_resolves_to_string(self): + with patch.dict(os.environ, {"N": "42"}): + result = self._substitute_yaml("name: svc-${N}") + self.assertEqual(result["name"], "svc-42") + + def test_mapping_key_is_not_substituted(self): + # A ${VAR} in a key position is left verbatim and triggers no lookup, + # so an undefined variable there does not raise. + with patch.dict(os.environ, {}, clear=True): + result = self._substitute_yaml("${UNDEFINED_KEY}: value") + self.assertEqual(result, {"${UNDEFINED_KEY}": "value"}) + + def test_escape_sequence_is_not_a_reference(self): + with patch.dict(os.environ, {}, clear=True): + result = self._substitute_yaml("literal: $${NOT_A_VAR}") + self.assertEqual(result["literal"], "${NOT_A_VAR}") + + def test_value_newline_cannot_inject_mapping_keys(self): + with patch.dict(os.environ, {"VAL": "legit\nmalicious_key: injected"}): + result = self._substitute_yaml("service_name: ${VAL}") + self.assertEqual(list(result), ["service_name"]) + self.assertEqual( + result["service_name"], "legit\nmalicious_key: injected" + ) + + +class TestJsonEnvVarSubstitution(unittest.TestCase): + """JSON substitution touches only string values, not keys or non-strings.""" + + def test_string_values_substituted_keys_untouched(self): + with patch.dict(os.environ, {"V": "resolved"}): + result = _substitute_env_in_json_value( + {"${KEY}": "${V}", "nested": ["${V}", 1, True, None]} + ) + self.assertEqual( + result, + {"${KEY}": "resolved", "nested": ["resolved", 1, True, None]}, + ) + + +class TestEnvVarSubstitutionEndToEnd(unittest.TestCase): + """End-to-end loader behavior for issue #5406.""" + + @staticmethod + def _load_yaml(text: str) -> OpenTelemetryConfiguration: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False + ) as fh: + fh.write(text) + path = fh.name + try: + return load_config_file(path) + finally: + os.unlink(path) + + def test_undefined_variable_in_comment_does_not_crash(self): + # The reported bug: a ${VAR} inside a comment must be ignored, so an + # undefined variable there no longer aborts loading. + text = ( + "file_format: '1.0'\n" + "# documented default uses ${UNDEFINED_VAR} - not substituted\n" + "disabled: false\n" + ) + with patch.dict(os.environ, {}, clear=True): + config = self._load_yaml(text) + self.assertEqual(config.file_format, "1.0") + self.assertIs(config.disabled, False) + + def test_standalone_reference_coerces_type_for_schema(self): + # An integer field populated from ${VAR} must be an int so it passes + # JSON-schema validation. + text = ( + "file_format: '1.0'\n" + "attribute_limits:\n" + " attribute_count_limit: ${LIMIT}\n" + ) + with patch.dict(os.environ, {"LIMIT": "100"}): + config = self._load_yaml(text) + self.assertEqual(config.attribute_limits.attribute_count_limit, 100) + + def test_quoted_reference_for_int_field_fails_schema(self): + # Quoting forces a string, which is invalid for an integer field. + text = ( + "file_format: '1.0'\n" + "attribute_limits:\n" + ' attribute_count_limit: "${LIMIT}"\n' + ) + with patch.dict(os.environ, {"LIMIT": "100"}): + with self.assertRaises(ConfigurationError) as ctx: + self._load_yaml(text) + self.assertIn("schema", str(ctx.exception).lower()) From 340aa7de192038d9305c1e057eae7987ba7113bd Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Sun, 12 Jul 2026 15:11:38 -0500 Subject: [PATCH 2/7] Rename changelog fragment to PR number --- .changelog/{5406.fixed => 5407.fixed} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changelog/{5406.fixed => 5407.fixed} (100%) diff --git a/.changelog/5406.fixed b/.changelog/5407.fixed similarity index 100% rename from .changelog/5406.fixed rename to .changelog/5407.fixed From 5edcc93fdadbed9ded7458f0cc31d692ac9275a2 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Sun, 12 Jul 2026 15:20:47 -0500 Subject: [PATCH 3/7] Rename substitute_env_vars parameter to configuration_value and clarify docstring --- .../configuration/file/_env_substitution.py | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py index 882437c70f3..52008970b84 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py @@ -18,34 +18,47 @@ class EnvSubstitutionError(Exception): """ -def substitute_env_vars(text: str) -> str: - """Substitute environment variables within a configuration scalar value. +def substitute_env_vars(configuration_value: str) -> str: + """Substitute environment variables within a configuration value. - Substitution is applied per scalar value after the configuration file has - been parsed, so comments and mapping keys are never touched. Supports the - following syntax: + A configuration value is a single value from the parsed configuration file + (the value of one key or one list item), never a key, a comment, or a whole + mapping/list. Substitution is applied per configuration value after the + file has been parsed, so comments and mapping keys are never touched. + + For example, given the YAML:: + + service_name: ${SERVICE_NAME} + endpoint: http://${HOST}:${PORT} + + the configuration values are ``${SERVICE_NAME}`` and + ``http://${HOST}:${PORT}``; this function is called once with each, and + never with the keys ``service_name`` or ``endpoint``. + + Supports the following syntax: - ${VAR}: Substitute with environment variable VAR. Raises error if not found. - ${VAR:-default}: Substitute with VAR if set, otherwise use default value. - $$: Escape sequence for literal $. Args: - text: A scalar value with potential ${VAR} placeholders. + configuration_value: A configuration value with potential ${VAR} + placeholders. Returns: - The value with environment variables substituted. + The configuration value with environment variables substituted. Raises: EnvSubstitutionError: If a required environment variable is not found. Examples: >>> os.environ['SERVICE_NAME'] = 'my-service' - >>> substitute_env_vars('name: ${SERVICE_NAME}') - 'name: my-service' - >>> substitute_env_vars('name: ${MISSING:-default}') - 'name: default' - >>> substitute_env_vars('price: $$100') - 'price: $100' + >>> substitute_env_vars('${SERVICE_NAME}') + 'my-service' + >>> substitute_env_vars('${MISSING:-default}') + 'default' + >>> substitute_env_vars('$$100') + '$100' """ # Pattern matches $$ (escape sequence) or ${VAR_NAME} / ${VAR_NAME:-default_value} # Handling both in a single pass ensures $$ followed by ${VAR} works correctly @@ -80,4 +93,4 @@ def replace_var(match) -> str: # the loader, which re-resolves the node tag after substitution. return value - return re.sub(pattern, replace_var, text) + return re.sub(pattern, replace_var, configuration_value) From 7b40fa86208663c9721dcc31051d708d39779433 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Sun, 12 Jul 2026 15:24:10 -0500 Subject: [PATCH 4/7] Use 'configuration value' instead of 'scalar value' in docs and comments --- .../configuration/file/_env_substitution.py | 8 +++--- .../configuration/file/_loader.py | 28 ++++++++++--------- .../tests/file/test_env_substitution.py | 4 +-- .../tests/file/test_loader.py | 2 +- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py index 52008970b84..8addd46fa8d 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py @@ -86,10 +86,10 @@ def replace_var(match) -> str: f"Environment variable '{var_name}' not found and no default provided" ) - # Substitution runs on an already-parsed scalar value, so the result - # cannot inject new YAML structure regardless of its contents (a - # newline in the value stays a literal character within this one - # scalar). Type interpretation for standalone references is handled by + # Substitution runs on an already-parsed configuration value, so the + # result cannot inject new YAML structure regardless of its contents + # (a newline in the value stays a literal character within this one + # value). Type interpretation for standalone references is handled by # the loader, which re-resolves the node tag after substitution. return value diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py index 6bb13f2dcd2..312ffd301c0 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py @@ -69,21 +69,22 @@ def _get_schema() -> dict: _YAML_STR_TAG = "tag:yaml.org,2002:str" -# A scalar whose entire value is a single ``${VAR}`` / ``${VAR:-default}`` -# reference. Only such standalone references (when unquoted) have their type -# re-interpreted after substitution; embedded or multiple references resolve to -# a string per the configuration spec. A leading ``$$`` escape does not match, -# so ``$${VAR}`` is treated as an embedded (string) value. +# A configuration value whose entire content is a single ``${VAR}`` / +# ``${VAR:-default}`` reference. Only such standalone references (when +# unquoted) have their type re-interpreted after substitution; embedded or +# multiple references resolve to a string per the configuration spec. A +# leading ``$$`` escape does not match, so ``$${VAR}`` is treated as an +# embedded (string) value. _STANDALONE_ENV_REF = re.compile( r"\A\$\{[A-Za-z_][A-Za-z0-9_]*(?::-[^}]*)?\}\Z" ) def _substitute_env_in_yaml_node(node: yaml.Node, loader: yaml.SafeLoader): - """Apply env-var substitution to string scalar values in a YAML node tree. + """Apply env-var substitution to string configuration values in a node tree. - Substitution runs after parsing on scalar *values* only, so comments and - mapping keys are never candidates (per the configuration spec). For an + Substitution runs after parsing on configuration values only, so comments + and mapping keys are never candidates (per the configuration spec). For an unquoted standalone ``${VAR}`` reference the node's type tag is re-resolved from the substituted value so YAML type coercion still applies (e.g. ``${LIMIT}`` -> int); quoted or embedded references stay strings. @@ -129,8 +130,9 @@ def _parse_config_content( """Parse configuration text and substitute environment variables. Parsing happens first, so ``${VAR}`` references in comments and mapping - keys are never substitution candidates; substitution then runs on scalar - values, with YAML node types re-resolved for standalone references. + keys are never substitution candidates; substitution then runs on + configuration values, with YAML node types re-resolved for standalone + references. Raises: ConfigurationError: If the content cannot be parsed or substitution of @@ -166,8 +168,8 @@ def load_config_file( """Load and parse an OpenTelemetry configuration file. Supports YAML and JSON formats. Environment variable substitution is - performed after parsing, on scalar values only, so ``${VAR}`` references in - comments or mapping keys are left untouched. + performed after parsing, on configuration values only, so ``${VAR}`` + references in comments or mapping keys are left untouched. Args: file_path: Path to the configuration file (.yaml, .yml, or .json). @@ -204,7 +206,7 @@ def load_config_file( f"Failed to read configuration file: {file_path}" ) from exc - # Parse the file, then substitute environment variables in scalar values. + # Parse the file, then substitute env vars in configuration values only. # Parsing first means comments and mapping keys are never substitution # candidates, and node types are still resolved after substitution. suffix = path.suffix.lower() diff --git a/opentelemetry-configuration/tests/file/test_env_substitution.py b/opentelemetry-configuration/tests/file/test_env_substitution.py index 9bb19d5eca5..035feb31c66 100644 --- a/opentelemetry-configuration/tests/file/test_env_substitution.py +++ b/opentelemetry-configuration/tests/file/test_env_substitution.py @@ -110,8 +110,8 @@ def test_only_dollar_signs(self): def test_newline_in_value_returned_verbatim(self): """A newline in a value is returned as-is. - Substitution runs per scalar value after parsing, so it does not - escape or quote newlines; YAML injection is prevented structurally + Substitution runs per configuration value after parsing, so it does + not escape or quote newlines; YAML injection is prevented structurally (see the loader tests), not by rewriting the value here. """ with patch.dict(os.environ, {"MULTI": "line1\nline2"}): diff --git a/opentelemetry-configuration/tests/file/test_loader.py b/opentelemetry-configuration/tests/file/test_loader.py index dd99bdd27e0..014914b7565 100644 --- a/opentelemetry-configuration/tests/file/test_loader.py +++ b/opentelemetry-configuration/tests/file/test_loader.py @@ -376,7 +376,7 @@ def test_malformed_version_is_rejected(self): class TestEnvVarSubstitutionScope(unittest.TestCase): - """Substitution applies only to scalar values, per the config spec. + """Substitution applies only to configuration values, per the config spec. These exercise the YAML node walker directly so the spec's type and scope rules can be asserted without the JSON schema constraining shape. From 217c4fee95d96c9677e57aa008a5023d75a01f70 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Sun, 12 Jul 2026 15:39:13 -0500 Subject: [PATCH 5/7] Explain the YAML node substitution and type re-resolution logic --- .../configuration/file/_loader.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py index 312ffd301c0..60f01cd21af 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py @@ -90,18 +90,37 @@ def _substitute_env_in_yaml_node(node: yaml.Node, loader: yaml.SafeLoader): ``${LIMIT}`` -> int); quoted or embedded references stay strings. """ if isinstance(node, yaml.ScalarNode): + # A ``ScalarNode`` is a leaf -- one configuration value. Only a + # ``str``-tagged one can hold a ``${VAR}`` reference (nodes the parser + # already typed as int, float, bool or null cannot), so those are the + # only substitution candidates. if node.tag == _YAML_STR_TAG: + # ``node.value`` is the raw text of this one value (never a key or + # comment). Replace any ${VAR} / ${VAR:-default} / $$ in it. raw = node.value node.value = substitute_env_vars(raw) + # The spec says node types are interpreted *after* substitution, + # but only for a value that is exactly one reference and written + # unquoted (``style is None``). Example: ``count: ${N}`` with + # ``N=42`` must yield the int 42, not the string "42". In that case + # re-run YAML's implicit type resolution on the substituted text + # and retag the node. A quoted (``"${N}"``) or embedded + # (``foo-${N}``) value keeps its ``str`` tag and stays a string. if node.style is None and _STANDALONE_ENV_REF.match(raw): + # ``resolve`` returns the implicit tag (int/float/bool/null/ + # str) YAML would assign to ``node.value``; the ``(True, False)`` + # implicit-flag pair marks it as a plain scalar so that + # type-guessing applies, exactly as during normal parsing. node.tag = loader.resolve( yaml.ScalarNode, node.value, (True, False) ) elif isinstance(node, yaml.SequenceNode): + # A sequence (list): each item is itself a value node -- recurse. for item in node.value: _substitute_env_in_yaml_node(item, loader) elif isinstance(node, yaml.MappingNode): - # Recurse into values only; keys are not substitution candidates. + # A mapping (dict): recurse into values only; keys are not + # substitution candidates per the spec. for _key_node, value_node in node.value: _substitute_env_in_yaml_node(value_node, loader) From 8f8b4b2fd0ac2fb0152364ed8d90efda213de1b5 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Sun, 12 Jul 2026 15:44:15 -0500 Subject: [PATCH 6/7] Add worked example tracing an unquoted standalone reference through the node walker --- .../configuration/file/_loader.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py index 60f01cd21af..cc51cdd4f0e 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py @@ -89,6 +89,26 @@ def _substitute_env_in_yaml_node(node: yaml.Node, loader: yaml.SafeLoader): from the substituted value so YAML type coercion still applies (e.g. ``${LIMIT}`` -> int); quoted or embedded references stay strings. """ + # Worked example: a node value that makes all three checks below true. + # + # YAML (with environment LIMIT=100): + # attribute_limits: + # attribute_count_limit: ${LIMIT} + # + # The parser hands this function the value node of attribute_count_limit, + # which has value="${LIMIT}", tag=str, style=None (plain/unquoted). Then: + # + # 1. isinstance(node, yaml.ScalarNode) -> True: it is a leaf value. + # 2. node.tag == _YAML_STR_TAG -> True: "${LIMIT}" is not a number or + # bool, so the parser tagged it str. We substitute in place, giving + # node.value = "100". + # 3. node.style is None and _STANDALONE_ENV_REF.match(raw) -> True: + # it was unquoted and the whole value is a single reference. We + # re-resolve the tag: resolve("100") -> int, and construct_document() + # later builds the integer 100 (not the string "100"). + # + # Counter-cases that reach check 3 but stop there: "${LIMIT}" quoted fails + # the style test; x${LIMIT} fails the regex. Both keep the str tag. if isinstance(node, yaml.ScalarNode): # A ``ScalarNode`` is a leaf -- one configuration value. Only a # ``str``-tagged one can hold a ``${VAR}`` reference (nodes the parser From 8de1d268105e082f06605c9d86257c5fd575ff14 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Wed, 15 Jul 2026 18:43:19 -0500 Subject: [PATCH 7/7] Keep public parameter name of substitute_env_vars substitute_env_vars is re-exported in opentelemetry.configuration.file.__all__, so renaming its parameter is a breaking public API change. Restore the parameter name to text to satisfy the public-symbols-check while keeping the docstring that explains the configuration-value semantics. --- .../opentelemetry/configuration/file/_env_substitution.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py index 8addd46fa8d..bbf41a4f112 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py @@ -18,7 +18,7 @@ class EnvSubstitutionError(Exception): """ -def substitute_env_vars(configuration_value: str) -> str: +def substitute_env_vars(text: str) -> str: """Substitute environment variables within a configuration value. A configuration value is a single value from the parsed configuration file @@ -42,8 +42,7 @@ def substitute_env_vars(configuration_value: str) -> str: - $$: Escape sequence for literal $. Args: - configuration_value: A configuration value with potential ${VAR} - placeholders. + text: A configuration value with potential ${VAR} placeholders. Returns: The configuration value with environment variables substituted. @@ -93,4 +92,4 @@ def replace_var(match) -> str: # the loader, which re-resolves the node tag after substitution. return value - return re.sub(pattern, replace_var, configuration_value) + return re.sub(pattern, replace_var, text)