Skip to content

Commit b207a52

Browse files
TheodoreSpeaksTheodoreSpeaks
committed
fix(ci): harden Trigger.dev cutover gate — reject stale executions, verify all ECS targets
1 parent 30e1270 commit b207a52

2 files changed

Lines changed: 112 additions & 176 deletions

File tree

‎.github/scripts/wait-for-ecs-cutover.sh‎

Lines changed: 67 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,32 @@
11
#!/usr/bin/env bash
22
# Waits for the ECS blue/green deploy triggered by a specific app image push to
3-
# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded), then exits 0.
3+
# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded on every ECS
4+
# target), then exits 0.
45
#
56
# ECR app images use a floating tag (latest/staging) with no git SHA, so the
67
# only durable key linking this CI push to its ECS deploy is the image DIGEST.
7-
# Correlation: image digest -> CodePipeline execution (AppEcrImage revision) ->
8+
# Correlation: image digest -> CodePipeline execution (ECR_Source revision) ->
89
# Deploy action externalExecutionId (== CodeDeploy deployment id) -> AllowTraffic.
910
#
10-
# Usage: wait-for-ecs-cutover.sh <pipeline-name> <image-digest>
11-
# Requires: awscli v2, configured credentials with codedeploy + codepipeline read.
11+
# The digest alone is ambiguous: a prior run with the same image could match an
12+
# older, already-cutover execution and promote too early. SINCE_EPOCH (the time
13+
# the deploy tag was retagged, i.e. when THIS push's pipeline was triggered)
14+
# disambiguates — only an execution that started at/after the retag is ours.
15+
#
16+
# Usage: wait-for-ecs-cutover.sh <pipeline-name> <image-digest> <since-epoch>
17+
# Requires: awscli v2, python3, credentials with codedeploy + codepipeline read.
1218
set -euo pipefail
1319

1420
PIPELINE="${1:?pipeline name required}"
1521
DIGEST="${2:?image digest required}"
22+
SINCE_EPOCH="${3:?since-epoch (retag time) required}"
1623

1724
POLL_INTERVAL="${POLL_INTERVAL:-15}"
1825
# 70 min covers a prod deploy whose Deploy stage is queued behind a prior
1926
# deploy's ~50-min termination bake before its own traffic shift begins.
2027
OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}"
28+
# Tolerate minor clock skew between the runner (retag time) and CodePipeline.
29+
SINCE_SKEW="${SINCE_SKEW:-120}"
2130

2231
deadline=$(( $(date +%s) + OVERALL_TIMEOUT ))
2332
remaining() { echo $(( deadline - $(date +%s) )); }
@@ -31,21 +40,44 @@ fail_if_expired() {
3140

3241
log "Pipeline: $PIPELINE"
3342
log "Target app image digest: $DIGEST"
43+
log "Requiring execution started at/after epoch $SINCE_EPOCH (minus ${SINCE_SKEW}s skew)"
3444

35-
# Phase A: find the pipeline execution whose ECR source revision matches our
36-
# digest. --max-items bounds the fetch (the CLI otherwise auto-paginates the whole
37-
# execution history); our push is the newest execution, so it's on the first page.
38-
# The revisionId match is done server-side via JMESPath; grep isolates the UUID
39-
# from any trailing pagination-token line in text output.
45+
# Phase A: find the newest pipeline execution whose ECR source revision matches
46+
# our digest AND that started at/after the retag. The since filter rejects a
47+
# stale historical execution reusing the same digest. --max-items bounds the
48+
# fetch (the CLI otherwise auto-paginates the whole history).
4049
EXECUTION_ID=""
4150
while [ -z "$EXECUTION_ID" ]; do
42-
fail_if_expired "pipeline execution matching digest"
43-
EXECUTION_ID=$(aws codepipeline list-pipeline-executions \
51+
fail_if_expired "pipeline execution matching digest since retag"
52+
matches=$(aws codepipeline list-pipeline-executions \
4453
--pipeline-name "$PIPELINE" --max-items 30 \
45-
--query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].pipelineExecutionId" \
46-
--output text 2>/dev/null | tr '\t ' '\n\n' | grep -Em1 '^[0-9a-f-]{36}$' || true)
54+
--query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].[startTime, pipelineExecutionId]" \
55+
--output text 2>/dev/null || true)
56+
EXECUTION_ID=$(printf '%s\n' "$matches" | SINCE="$SINCE_EPOCH" SKEW="$SINCE_SKEW" python3 -c '
57+
import sys, os, datetime
58+
since = float(os.environ["SINCE"]) - float(os.environ["SKEW"])
59+
best_epoch = None
60+
best_id = None
61+
for line in sys.stdin:
62+
parts = line.rstrip("\n").split("\t")
63+
if len(parts) < 2:
64+
continue
65+
ts, eid = parts[0].strip(), parts[1].strip()
66+
if not ts or not eid or "-" not in eid:
67+
continue
68+
try:
69+
epoch = float(ts)
70+
except ValueError:
71+
try:
72+
epoch = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
73+
except ValueError:
74+
continue
75+
if epoch >= since and (best_epoch is None or epoch > best_epoch):
76+
best_epoch, best_id = epoch, eid
77+
print(best_id or "")
78+
' 2>/dev/null || true)
4779
if [ -z "$EXECUTION_ID" ]; then
48-
log "No matching pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)"
80+
log "No matching post-retag pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)"
4981
sleep "$POLL_INTERVAL"
5082
fi
5183
done
@@ -77,9 +109,11 @@ while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; do
77109
done
78110
log "CodeDeploy deployment: $DEPLOYMENT_ID"
79111

80-
# Phase C: wait for the traffic cutover (AllowTraffic lifecycle event Succeeded).
112+
# Phase C: wait for the traffic cutover. Require AllowTraffic == Succeeded on
113+
# EVERY ECS target, so a multi-target deploy can't promote while one target is
114+
# still mid-cutover or failed.
81115
while true; do
82-
fail_if_expired "AllowTraffic (traffic cutover)"
116+
fail_if_expired "AllowTraffic (traffic cutover) on all targets"
83117
dstatus=$(aws deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \
84118
--query 'deploymentInfo.status' --output text 2>/dev/null || true)
85119
case "$dstatus" in
@@ -88,18 +122,25 @@ while true; do
88122
exit 1
89123
;;
90124
esac
91-
target_id=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \
92-
--query 'targetIds[0]' --output text 2>/dev/null || true)
93-
at_status=""
94-
if [ -n "$target_id" ] && [ "$target_id" != "None" ]; then
95-
at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target_id" \
96-
--query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \
97-
--output text 2>/dev/null || true)
98-
if [ "$at_status" = "Succeeded" ]; then
99-
log "Traffic cutover complete (AllowTraffic Succeeded) for $DEPLOYMENT_ID"
125+
target_ids=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \
126+
--query 'targetIds' --output text 2>/dev/null || true)
127+
if [ -n "$target_ids" ] && [ "$target_ids" != "None" ]; then
128+
all_ok=1
129+
ntargets=0
130+
for tid in $target_ids; do
131+
ntargets=$((ntargets + 1))
132+
at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$tid" \
133+
--query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \
134+
--output text 2>/dev/null || true)
135+
if [ "$at_status" != "Succeeded" ]; then
136+
all_ok=0
137+
fi
138+
done
139+
if [ "$ntargets" -gt 0 ] && [ "$all_ok" = "1" ]; then
140+
log "Traffic cutover complete (AllowTraffic Succeeded on all $ntargets target(s)) for $DEPLOYMENT_ID"
100141
exit 0
101142
fi
102143
fi
103-
log "Deployment $DEPLOYMENT_ID status=$dstatus AllowTraffic=${at_status:-pending}; wait ${POLL_INTERVAL}s (remaining $(remaining)s)"
144+
log "Deployment $DEPLOYMENT_ID status=$dstatus; not all targets past AllowTraffic; wait ${POLL_INTERVAL}s (remaining $(remaining)s)"
104145
sleep "$POLL_INTERVAL"
105146
done

0 commit comments

Comments
 (0)