ci: add log retention to example Lambdas

.github/workflows/cloud-tests.yml

@@ -104,6 +104,18 @@ jobs:
           echo "Using custom Lambda endpoint override"
         fi
 
+        PYTEST_FUNCTION_NAME_MAP=$(
+          python packages/aws-durable-execution-sdk-python-examples/scripts/generate_function_name_map.py \
+            --function-name-prefix "$FUNCTION_NAME_PREFIX"
+        )
+
+        # TODO: Remove this one-time cleanup after existing CI stacks adopt the
+        # generated AWS::Logs::LogGroup resources.
+        python packages/aws-durable-execution-sdk-python-examples/scripts/cleanup_unmanaged_log_groups.py \
+          --stack-name "$STACK_NAME" \
+          --function-name-prefix "$FUNCTION_NAME_PREFIX" \
+          --region "$AWS_REGION"
+
         echo "Deploying stack $STACK_NAME with function prefix $FUNCTION_NAME_PREFIX and runtime $PYTHON_RUNTIME"
         sam deploy \
           --template-file .aws-sam/build/template.yaml \
@@ -114,11 +126,6 @@ jobs:
           --region "$AWS_REGION" \
           --parameter-overrides "${PARAMETER_OVERRIDES[@]}"
 
-        PYTEST_FUNCTION_NAME_MAP=$(
-          python packages/aws-durable-execution-sdk-python-examples/scripts/generate_function_name_map.py \
-            --function-name-prefix "$FUNCTION_NAME_PREFIX"
-        )
-
         echo "PYTEST_FUNCTION_NAME_MAP=$PYTEST_FUNCTION_NAME_MAP" >> $GITHUB_ENV
 
     - name: Run Integration Tests

packages/aws-durable-execution-sdk-python-examples/scripts/cleanup_unmanaged_log_groups.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+
+from generate_sam_template import load_catalog, to_logical_id
+
+
+def log_group_resources(function_name_prefix: str) -> list[tuple[str, str]]:
+    """Return generated LogGroup logical ids and physical names."""
+    logical_ids = {
+        to_logical_id(example["handler"]) for example in load_catalog()["examples"]
+    }
+    return [
+        (f"{logical_id}LogGroup", f"/aws/lambda/{function_name_prefix}{logical_id}")
+        for logical_id in sorted(logical_ids)
+    ]
+
+
+def run_aws(args: list[str], region: str) -> subprocess.CompletedProcess[str]:
+    """Run an AWS CLI command and capture its output."""
+    return subprocess.run(
+        ["aws", *args, "--region", region],
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+
+def stack_owns_resource(stack_name: str, logical_resource_id: str, region: str) -> bool:
+    """Return true when the stack already owns the generated log group resource."""
+    result = run_aws(
+        [
+            "cloudformation",
+            "describe-stack-resource",
+            "--stack-name",
+            stack_name,
+            "--logical-resource-id",
+            logical_resource_id,
+        ],
+        region,
+    )
+    if result.returncode == 0:
+        return True
+
+    if "does not exist" in result.stderr:
+        return False
+
+    msg = result.stderr.strip() or result.stdout.strip()
+    raise RuntimeError(msg)
+
+
+def delete_log_group(log_group_name: str, region: str) -> bool:
+    """Delete a log group, ignoring log groups that do not exist."""
+    result = run_aws(
+        ["logs", "delete-log-group", "--log-group-name", log_group_name],
+        region,
+    )
+    if result.returncode == 0:
+        print(f"Deleted unmanaged log group {log_group_name}")
+        return True
+
+    if "ResourceNotFoundException" in result.stderr:
+        print(f"No unmanaged log group found for {log_group_name}")
+        return True
+
+    print(result.stderr, end="")
+    return False
+
+
+def cleanup_unmanaged_log_groups(
+    stack_name: str,
+    function_name_prefix: str,
+    region: str,
+) -> bool:
+    """Remove log groups that would block CloudFormation from owning them."""
+    success = True
+    resources = log_group_resources(function_name_prefix)
+    for logical_resource_id, log_group_name in resources:
+        if stack_owns_resource(stack_name, logical_resource_id, region):
+            print(f"Stack already owns {logical_resource_id}; leaving it in place")
+            continue
+
+        success = delete_log_group(log_group_name, region) and success
+
+    return success
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Delete unmanaged Lambda log groups before SAM owns them"
+    )
+    parser.add_argument("--stack-name", required=True)
+    parser.add_argument("--function-name-prefix", required=True)
+    parser.add_argument("--region", required=True)
+    args = parser.parse_args()
+
+    if not cleanup_unmanaged_log_groups(
+        stack_name=args.stack_name,
+        function_name_prefix=args.function_name_prefix,
+        region=args.region,
+    ):
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

packages/aws-durable-execution-sdk-python-examples/scripts/generate_sam_template.py

@@ -7,6 +7,7 @@
 
 
 DEFAULT_FUNCTION_NAME_PREFIX = "DurablePythonExample-"
+LOG_RETENTION_DAYS = 7
 
 
 def load_catalog() -> dict[str, Any]:
@@ -78,8 +79,9 @@ def build_template(examples: list[dict[str, Any]]) -> dict[str, Any]:
         if "durableConfig" in example:
             properties["DurableConfig"] = example["durableConfig"]
 
-        if "loggingConfig" in example:
-            properties["LoggingConfig"] = example["loggingConfig"]
+        logging_config: dict[str, Any] = dict(example.get("loggingConfig", {}))
+        logging_config["LogGroup"] = {"Ref": f"{logical_id}LogGroup"}
+        properties["LoggingConfig"] = logging_config
 
         if "layers" in example:
             properties["Layers"] = example["layers"]
@@ -92,8 +94,18 @@ def build_template(examples: list[dict[str, Any]]) -> dict[str, Any]:
 
         template["Resources"][logical_id] = {
             "Type": "AWS::Serverless::Function",
+            "DependsOn": [f"{logical_id}LogGroup"],
             "Properties": properties,
         }
+        template["Resources"][f"{logical_id}LogGroup"] = {
+            "Type": "AWS::Logs::LogGroup",
+            "Properties": {
+                "LogGroupName": {
+                    "Fn::Sub": f"/aws/lambda/${{FunctionNamePrefix}}{logical_id}"
+                },
+                "RetentionInDays": LOG_RETENTION_DAYS,
+            },
+        }
 
     return template
 

packages/aws-durable-execution-sdk-python-examples/test/test_log_group_deployment.py

@@ -0,0 +1,125 @@
+import importlib.util
+import sys
+from pathlib import Path
+
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts"
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+
+def load_script(name: str):
+    path = SCRIPTS_DIR / f"{name}.py"
+    spec = importlib.util.spec_from_file_location(name, path)
+    assert spec is not None
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+generate_sam_template = load_script("generate_sam_template")
+cleanup_unmanaged_log_groups = load_script("cleanup_unmanaged_log_groups")
+
+
+def test_build_template_creates_lambda_log_group_with_seven_day_retention():
+    template = generate_sam_template.build_template(
+        [
+            {
+                "handler": "hello_world.handler",
+                "description": "Example handler",
+                "durableConfig": {
+                    "RetentionPeriodInDays": 7,
+                    "ExecutionTimeout": 300,
+                },
+            }
+        ]
+    )
+
+    assert template["Resources"]["HelloWorldLogGroup"] == {
+        "Type": "AWS::Logs::LogGroup",
+        "Properties": {
+            "LogGroupName": {"Fn::Sub": "/aws/lambda/${FunctionNamePrefix}HelloWorld"},
+            "RetentionInDays": 7,
+        },
+    }
+    assert template["Resources"]["HelloWorld"]["DependsOn"] == ["HelloWorldLogGroup"]
+    assert template["Resources"]["HelloWorld"]["Properties"]["LoggingConfig"] == {
+        "LogGroup": {"Ref": "HelloWorldLogGroup"},
+    }
+
+
+def test_build_template_preserves_existing_logging_config():
+    template = generate_sam_template.build_template(
+        [
+            {
+                "handler": "otel_logger_example.handler",
+                "description": "Example handler",
+                "loggingConfig": {
+                    "ApplicationLogLevel": "INFO",
+                    "LogFormat": "JSON",
+                },
+            }
+        ]
+    )
+
+    assert template["Resources"]["OtelLoggerExample"]["Properties"][
+        "LoggingConfig"
+    ] == {
+        "ApplicationLogLevel": "INFO",
+        "LogFormat": "JSON",
+        "LogGroup": {"Ref": "OtelLoggerExampleLogGroup"},
+    }
+
+
+def test_log_group_resources_match_generated_template_names(monkeypatch):
+    monkeypatch.setattr(
+        cleanup_unmanaged_log_groups,
+        "load_catalog",
+        lambda: {
+            "examples": [
+                {"handler": "hello_world.handler"},
+                {"handler": "step_with_name.handler"},
+            ]
+        },
+    )
+
+    assert cleanup_unmanaged_log_groups.log_group_resources("Py313-") == [
+        ("HelloWorldLogGroup", "/aws/lambda/Py313-HelloWorld"),
+        ("StepWithNameLogGroup", "/aws/lambda/Py313-StepWithName"),
+    ]
+
+
+def test_cleanup_skips_stack_owned_log_groups_and_deletes_unmanaged(monkeypatch):
+    deleted_log_groups = []
+
+    monkeypatch.setattr(
+        cleanup_unmanaged_log_groups,
+        "log_group_resources",
+        lambda _prefix: [
+            ("OwnedLogGroup", "/aws/lambda/Py313-Owned"),
+            ("UnmanagedLogGroup", "/aws/lambda/Py313-Unmanaged"),
+        ],
+    )
+    monkeypatch.setattr(
+        cleanup_unmanaged_log_groups,
+        "stack_owns_resource",
+        lambda _stack_name, logical_id, _region: logical_id == "OwnedLogGroup",
+    )
+
+    def fake_delete_log_group(log_group_name, _region):
+        deleted_log_groups.append(log_group_name)
+        return True
+
+    monkeypatch.setattr(
+        cleanup_unmanaged_log_groups,
+        "delete_log_group",
+        fake_delete_log_group,
+    )
+
+    assert cleanup_unmanaged_log_groups.cleanup_unmanaged_log_groups(
+        stack_name="Py313-python-examples",
+        function_name_prefix="Py313-",
+        region="us-west-2",
+    )
+    assert deleted_log_groups == ["/aws/lambda/Py313-Unmanaged"]