diff --git a/.changelog/5407.fixed b/.changelog/5407.fixed new file mode 100644 index 00000000000..e6c5f221467 --- /dev/null +++ b/.changelog/5407.fixed @@ -0,0 +1 @@ +`opentelemetry-configuration`: perform environment variable substitution on scalar values after parsing the configuration file, so `${VAR}` references inside comments and mapping keys are no longer substituted and undefined references in comments no longer abort loading diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py index 9d3a1b815aa..bbf41a4f112 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_env_substitution.py @@ -19,30 +19,45 @@ class EnvSubstitutionError(Exception): def substitute_env_vars(text: str) -> str: - """Substitute environment variables in configuration text. + """Substitute environment variables within a configuration value. + + A configuration value is a single value from the parsed configuration file + (the value of one key or one list item), never a key, a comment, or a whole + mapping/list. Substitution is applied per configuration value after the + file has been parsed, so comments and mapping keys are never touched. + + For example, given the YAML:: + + service_name: ${SERVICE_NAME} + endpoint: http://${HOST}:${PORT} + + the configuration values are ``${SERVICE_NAME}`` and + ``http://${HOST}:${PORT}``; this function is called once with each, and + never with the keys ``service_name`` or ``endpoint``. Supports the following syntax: + - ${VAR}: Substitute with environment variable VAR. Raises error if not found. - ${VAR:-default}: Substitute with VAR if set, otherwise use default value. - $$: Escape sequence for literal $. Args: - text: Configuration text with potential ${VAR} placeholders. + text: A configuration value with potential ${VAR} placeholders. Returns: - Text with environment variables substituted. + The configuration value with environment variables substituted. Raises: EnvSubstitutionError: If a required environment variable is not found. Examples: >>> os.environ['SERVICE_NAME'] = 'my-service' - >>> substitute_env_vars('name: ${SERVICE_NAME}') - 'name: my-service' - >>> substitute_env_vars('name: ${MISSING:-default}') - 'name: default' - >>> substitute_env_vars('price: $$100') - 'price: $100' + >>> substitute_env_vars('${SERVICE_NAME}') + 'my-service' + >>> substitute_env_vars('${MISSING:-default}') + 'default' + >>> substitute_env_vars('$$100') + '$100' """ # Pattern matches $$ (escape sequence) or ${VAR_NAME} / ${VAR_NAME:-default_value} # Handling both in a single pass ensures $$ followed by ${VAR} works correctly @@ -70,23 +85,11 @@ def replace_var(match) -> str: f"Environment variable '{var_name}' not found and no default provided" ) - # Per spec: "It MUST NOT be possible to inject YAML structures by - # environment variables." Newlines are the primary injection vector — - # a value like "legit\nmalicious_key: val" would create extra YAML - # keys if substituted verbatim. Wrap such values in a YAML - # double-quoted scalar so the newline is treated as literal text. - # Simple values (no newlines) are returned as-is so that YAML type - # coercion still applies per spec ("Node types MUST be interpreted - # after environment variable substitution takes place"). - if "\n" in value or "\r" in value: - escaped = ( - value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - ) - return f'"{escaped}"' + # Substitution runs on an already-parsed configuration value, so the + # result cannot inject new YAML structure regardless of its contents + # (a newline in the value stays a literal character within this one + # value). Type interpretation for standalone references is handled by + # the loader, which re-resolves the node tag after substitution. return value return re.sub(pattern, replace_var, text) diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py index 6131e3c1b08..cc51cdd4f0e 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/file/_loader.py @@ -7,6 +7,7 @@ import json import logging import os +import re from pathlib import Path from typing import Any @@ -16,6 +17,7 @@ MissingDependencyError, ) from opentelemetry.configuration.file._env_substitution import ( + EnvSubstitutionError, substitute_env_vars, ) from opentelemetry.configuration.models import OpenTelemetryConfiguration @@ -65,14 +67,148 @@ def _get_schema() -> dict: _logger = logging.getLogger(__name__) +_YAML_STR_TAG = "tag:yaml.org,2002:str" + +# A configuration value whose entire content is a single ``${VAR}`` / +# ``${VAR:-default}`` reference. Only such standalone references (when +# unquoted) have their type re-interpreted after substitution; embedded or +# multiple references resolve to a string per the configuration spec. A +# leading ``$$`` escape does not match, so ``$${VAR}`` is treated as an +# embedded (string) value. +_STANDALONE_ENV_REF = re.compile( + r"\A\$\{[A-Za-z_][A-Za-z0-9_]*(?::-[^}]*)?\}\Z" +) + + +def _substitute_env_in_yaml_node(node: yaml.Node, loader: yaml.SafeLoader): + """Apply env-var substitution to string configuration values in a node tree. + + Substitution runs after parsing on configuration values only, so comments + and mapping keys are never candidates (per the configuration spec). For an + unquoted standalone ``${VAR}`` reference the node's type tag is re-resolved + from the substituted value so YAML type coercion still applies (e.g. + ``${LIMIT}`` -> int); quoted or embedded references stay strings. + """ + # Worked example: a node value that makes all three checks below true. + # + # YAML (with environment LIMIT=100): + # attribute_limits: + # attribute_count_limit: ${LIMIT} + # + # The parser hands this function the value node of attribute_count_limit, + # which has value="${LIMIT}", tag=str, style=None (plain/unquoted). Then: + # + # 1. isinstance(node, yaml.ScalarNode) -> True: it is a leaf value. + # 2. node.tag == _YAML_STR_TAG -> True: "${LIMIT}" is not a number or + # bool, so the parser tagged it str. We substitute in place, giving + # node.value = "100". + # 3. node.style is None and _STANDALONE_ENV_REF.match(raw) -> True: + # it was unquoted and the whole value is a single reference. We + # re-resolve the tag: resolve("100") -> int, and construct_document() + # later builds the integer 100 (not the string "100"). + # + # Counter-cases that reach check 3 but stop there: "${LIMIT}" quoted fails + # the style test; x${LIMIT} fails the regex. Both keep the str tag. + if isinstance(node, yaml.ScalarNode): + # A ``ScalarNode`` is a leaf -- one configuration value. Only a + # ``str``-tagged one can hold a ``${VAR}`` reference (nodes the parser + # already typed as int, float, bool or null cannot), so those are the + # only substitution candidates. + if node.tag == _YAML_STR_TAG: + # ``node.value`` is the raw text of this one value (never a key or + # comment). Replace any ${VAR} / ${VAR:-default} / $$ in it. + raw = node.value + node.value = substitute_env_vars(raw) + # The spec says node types are interpreted *after* substitution, + # but only for a value that is exactly one reference and written + # unquoted (``style is None``). Example: ``count: ${N}`` with + # ``N=42`` must yield the int 42, not the string "42". In that case + # re-run YAML's implicit type resolution on the substituted text + # and retag the node. A quoted (``"${N}"``) or embedded + # (``foo-${N}``) value keeps its ``str`` tag and stays a string. + if node.style is None and _STANDALONE_ENV_REF.match(raw): + # ``resolve`` returns the implicit tag (int/float/bool/null/ + # str) YAML would assign to ``node.value``; the ``(True, False)`` + # implicit-flag pair marks it as a plain scalar so that + # type-guessing applies, exactly as during normal parsing. + node.tag = loader.resolve( + yaml.ScalarNode, node.value, (True, False) + ) + elif isinstance(node, yaml.SequenceNode): + # A sequence (list): each item is itself a value node -- recurse. + for item in node.value: + _substitute_env_in_yaml_node(item, loader) + elif isinstance(node, yaml.MappingNode): + # A mapping (dict): recurse into values only; keys are not + # substitution candidates per the spec. + for _key_node, value_node in node.value: + _substitute_env_in_yaml_node(value_node, loader) + + +def _substitute_env_in_json_value(value: Any) -> Any: + """Recursively apply env-var substitution to string values in JSON data. + + JSON has explicit types and no comments, so substitution applies only to + string values (the result stays a string) and mapping keys are left as-is. + """ + if isinstance(value, str): + return substitute_env_vars(value) + if isinstance(value, list): + return [_substitute_env_in_json_value(item) for item in value] + if isinstance(value, dict): + return { + key: _substitute_env_in_json_value(val) + for key, val in value.items() + } + return value + + +def _parse_config_content( + content: str, suffix: str, file_path: str | os.PathLike[str] +) -> Any: + """Parse configuration text and substitute environment variables. + + Parsing happens first, so ``${VAR}`` references in comments and mapping + keys are never substitution candidates; substitution then runs on + configuration values, with YAML node types re-resolved for standalone + references. + + Raises: + ConfigurationError: If the content cannot be parsed or substitution of + a required environment variable fails. + """ + try: + if suffix == ".json": + return _substitute_env_in_json_value(json.loads(content)) + yaml_loader = yaml.SafeLoader(content) + try: + root_node = yaml_loader.get_single_node() + if root_node is None: + return None + _substitute_env_in_yaml_node(root_node, yaml_loader) + return yaml_loader.construct_document(root_node) + finally: + yaml_loader.dispose() + except EnvSubstitutionError as exc: + raise ConfigurationError( + f"Environment variable substitution failed: {exc}" + ) from exc + except yaml.YAMLError as exc: + _logger.exception("Failed to parse YAML from %s", file_path) + raise ConfigurationError(f"Failed to parse YAML: {exc}") from exc + except json.JSONDecodeError as exc: + _logger.exception("Failed to parse JSON from %s", file_path) + raise ConfigurationError(f"Failed to parse JSON: {exc}") from exc + def load_config_file( file_path: str | os.PathLike[str], ) -> OpenTelemetryConfiguration: """Load and parse an OpenTelemetry configuration file. - Supports YAML and JSON formats. Performs environment variable substitution - before parsing. + Supports YAML and JSON formats. Environment variable substitution is + performed after parsing, on configuration values only, so ``${VAR}`` + references in comments or mapping keys are left untouched. Args: file_path: Path to the configuration file (.yaml, .yml, or .json). @@ -109,32 +245,17 @@ def load_config_file( f"Failed to read configuration file: {file_path}" ) from exc - # Perform environment variable substitution - try: - content = substitute_env_vars(content) - except Exception as exc: + # Parse the file, then substitute env vars in configuration values only. + # Parsing first means comments and mapping keys are never substitution + # candidates, and node types are still resolved after substitution. + suffix = path.suffix.lower() + if suffix not in (".yaml", ".yml", ".json"): + _logger.error("Unsupported file format: %s", suffix) raise ConfigurationError( - f"Environment variable substitution failed: {exc}" - ) from exc + f"Unsupported file format: {suffix}. Use .yaml, .yml, or .json" + ) - # Parse based on file extension - suffix = path.suffix.lower() - try: - if suffix in (".yaml", ".yml"): - data = yaml.safe_load(content) - elif suffix == ".json": - data = json.loads(content) - else: - _logger.error("Unsupported file format: %s", suffix) - raise ConfigurationError( - f"Unsupported file format: {suffix}. Use .yaml, .yml, or .json" - ) - except yaml.YAMLError as exc: - _logger.exception("Failed to parse YAML from %s", file_path) - raise ConfigurationError(f"Failed to parse YAML: {exc}") from exc - except json.JSONDecodeError as exc: - _logger.exception("Failed to parse JSON from %s", file_path) - raise ConfigurationError(f"Failed to parse JSON: {exc}") from exc + data = _parse_config_content(content, suffix, file_path) if data is None: _logger.error("Configuration file is empty: %s", file_path) diff --git a/opentelemetry-configuration/tests/file/test_env_substitution.py b/opentelemetry-configuration/tests/file/test_env_substitution.py index 14c79f20fd5..035feb31c66 100644 --- a/opentelemetry-configuration/tests/file/test_env_substitution.py +++ b/opentelemetry-configuration/tests/file/test_env_substitution.py @@ -107,37 +107,22 @@ def test_only_dollar_signs(self): result = substitute_env_vars("$$$$") self.assertEqual(result, "$$") - def test_newline_in_value_prevents_yaml_injection(self): - """Values containing newlines must not inject YAML structure. + def test_newline_in_value_returned_verbatim(self): + """A newline in a value is returned as-is. - Per spec: "It MUST NOT be possible to inject YAML structures by - environment variables." A value like "legit\\nmalicious_key: val" - must be emitted as a quoted scalar, not raw YAML. + Substitution runs per configuration value after parsing, so it does + not escape or quote newlines; YAML injection is prevented structurally + (see the loader tests), not by rewriting the value here. """ - with patch.dict( - os.environ, - {"SERVICE_NAME": "legit-service\nmalicious_key: injected_value"}, - ): - result = substitute_env_vars( - "file_format: '1.0'\nservice_name: ${SERVICE_NAME}" - ) - parsed = yaml.safe_load(result) - self.assertNotIn("malicious_key", parsed) - self.assertIn("legit-service", parsed["service_name"]) - - def test_newline_in_value_preserved_as_literal(self): - """Newline within a value is preserved as a literal newline character.""" with patch.dict(os.environ, {"MULTI": "line1\nline2"}): - result = substitute_env_vars("key: ${MULTI}") - parsed = yaml.safe_load(result) - self.assertEqual(parsed["key"], "line1\nline2") + result = substitute_env_vars("${MULTI}") + self.assertEqual(result, "line1\nline2") - def test_carriage_return_in_value_is_escaped(self): - """Carriage return in value is escaped, not injected.""" + def test_carriage_return_in_value_returned_verbatim(self): + """A carriage return in a value is returned as-is, not escaped.""" with patch.dict(os.environ, {"VAL": "text\r\nmore"}): - result = substitute_env_vars("key: ${VAL}") - parsed = yaml.safe_load(result) - self.assertIsInstance(parsed["key"], str) + result = substitute_env_vars("${VAL}") + self.assertEqual(result, "text\r\nmore") def test_type_coercion_preserved_for_simple_values(self): """Simple values without newlines still undergo YAML type coercion per spec.""" diff --git a/opentelemetry-configuration/tests/file/test_loader.py b/opentelemetry-configuration/tests/file/test_loader.py index c9dcbee8e67..014914b7565 100644 --- a/opentelemetry-configuration/tests/file/test_loader.py +++ b/opentelemetry-configuration/tests/file/test_loader.py @@ -7,6 +7,8 @@ from pathlib import Path from unittest.mock import patch +import yaml + from opentelemetry.configuration._tracer_provider import ( create_tracer_provider, ) @@ -14,6 +16,10 @@ ConfigurationError, load_config_file, ) +from opentelemetry.configuration.file._loader import ( + _substitute_env_in_json_value, + _substitute_env_in_yaml_node, +) from opentelemetry.configuration.models import ( BatchSpanProcessor as BatchSpanProcessorConfig, ) @@ -367,3 +373,125 @@ def test_malformed_version_is_rejected(self): with self.assertRaises(ConfigurationError) as ctx: self._load("not-a-version") self.assertIn("file_format", str(ctx.exception)) + + +class TestEnvVarSubstitutionScope(unittest.TestCase): + """Substitution applies only to configuration values, per the config spec. + + These exercise the YAML node walker directly so the spec's type and + scope rules can be asserted without the JSON schema constraining shape. + """ + + @staticmethod + def _substitute_yaml(text: str): + loader = yaml.SafeLoader(text) + try: + node = loader.get_single_node() + _substitute_env_in_yaml_node(node, loader) + return loader.construct_document(node) + finally: + loader.dispose() + + def test_unquoted_standalone_reference_is_type_coerced(self): + with patch.dict(os.environ, {"N": "42", "FLAG": "true"}): + result = self._substitute_yaml("count: ${N}\nflag: ${FLAG}") + self.assertEqual(result["count"], 42) + self.assertIsInstance(result["count"], int) + self.assertIs(result["flag"], True) + + def test_quoted_reference_stays_string(self): + with patch.dict(os.environ, {"N": "42"}): + result = self._substitute_yaml('count: "${N}"') + self.assertEqual(result["count"], "42") + + def test_embedded_reference_resolves_to_string(self): + with patch.dict(os.environ, {"N": "42"}): + result = self._substitute_yaml("name: svc-${N}") + self.assertEqual(result["name"], "svc-42") + + def test_mapping_key_is_not_substituted(self): + # A ${VAR} in a key position is left verbatim and triggers no lookup, + # so an undefined variable there does not raise. + with patch.dict(os.environ, {}, clear=True): + result = self._substitute_yaml("${UNDEFINED_KEY}: value") + self.assertEqual(result, {"${UNDEFINED_KEY}": "value"}) + + def test_escape_sequence_is_not_a_reference(self): + with patch.dict(os.environ, {}, clear=True): + result = self._substitute_yaml("literal: $${NOT_A_VAR}") + self.assertEqual(result["literal"], "${NOT_A_VAR}") + + def test_value_newline_cannot_inject_mapping_keys(self): + with patch.dict(os.environ, {"VAL": "legit\nmalicious_key: injected"}): + result = self._substitute_yaml("service_name: ${VAL}") + self.assertEqual(list(result), ["service_name"]) + self.assertEqual( + result["service_name"], "legit\nmalicious_key: injected" + ) + + +class TestJsonEnvVarSubstitution(unittest.TestCase): + """JSON substitution touches only string values, not keys or non-strings.""" + + def test_string_values_substituted_keys_untouched(self): + with patch.dict(os.environ, {"V": "resolved"}): + result = _substitute_env_in_json_value( + {"${KEY}": "${V}", "nested": ["${V}", 1, True, None]} + ) + self.assertEqual( + result, + {"${KEY}": "resolved", "nested": ["resolved", 1, True, None]}, + ) + + +class TestEnvVarSubstitutionEndToEnd(unittest.TestCase): + """End-to-end loader behavior for issue #5406.""" + + @staticmethod + def _load_yaml(text: str) -> OpenTelemetryConfiguration: + with tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False + ) as fh: + fh.write(text) + path = fh.name + try: + return load_config_file(path) + finally: + os.unlink(path) + + def test_undefined_variable_in_comment_does_not_crash(self): + # The reported bug: a ${VAR} inside a comment must be ignored, so an + # undefined variable there no longer aborts loading. + text = ( + "file_format: '1.0'\n" + "# documented default uses ${UNDEFINED_VAR} - not substituted\n" + "disabled: false\n" + ) + with patch.dict(os.environ, {}, clear=True): + config = self._load_yaml(text) + self.assertEqual(config.file_format, "1.0") + self.assertIs(config.disabled, False) + + def test_standalone_reference_coerces_type_for_schema(self): + # An integer field populated from ${VAR} must be an int so it passes + # JSON-schema validation. + text = ( + "file_format: '1.0'\n" + "attribute_limits:\n" + " attribute_count_limit: ${LIMIT}\n" + ) + with patch.dict(os.environ, {"LIMIT": "100"}): + config = self._load_yaml(text) + self.assertEqual(config.attribute_limits.attribute_count_limit, 100) + + def test_quoted_reference_for_int_field_fails_schema(self): + # Quoting forces a string, which is invalid for an integer field. + text = ( + "file_format: '1.0'\n" + "attribute_limits:\n" + ' attribute_count_limit: "${LIMIT}"\n' + ) + with patch.dict(os.environ, {"LIMIT": "100"}): + with self.assertRaises(ConfigurationError) as ctx: + self._load_yaml(text) + self.assertIn("schema", str(ctx.exception).lower())