From 9461d320bca647f1c37c88d810474615414dddbd Mon Sep 17 00:00:00 2001 From: hessjc Date: Tue, 2 Jun 2026 18:19:56 +0000 Subject: [PATCH] feat: Support AI Developer Edition connections through the Cloud SQL Auth Proxy Cloud SQL AI Developer Edition instances allow connections through the SqlDataService. With this change and the corresponding chagnes in the Cloud SQL Go Connector, the Auth Proxy will allow users to connect to AIDE instances. Set the --sql-data flag to enable this support. --- .gitignore | 1 + build.sh | 62 +++++++++++++++++--- cmd/root.go | 18 ++++++ cmd/root_test.go | 95 ++++++++++++++++++++++++++++++ docs/cmd/cloud-sql-proxy.md | 2 + go.mod | 2 +- go.sum | 4 +- internal/proxy/proxy.go | 39 +++++++++++-- internal/proxy/proxy_test.go | 109 +++++++++++++++++++++++++++++++++++ 9 files changed, 317 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index de8224085..958b50e44 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ /logs/ .tools test_results.txt +test-results diff --git a/build.sh b/build.sh index efdd6d303..5ad0c43c7 100755 --- a/build.sh +++ b/build.sh @@ -33,12 +33,25 @@ function clean() { ## build - Builds the project without running tests. function build() { - go build -o ./cloud-sql-proxy main.go + local metadata="${1:-}" + local ldflags="" + if [[ -n "$metadata" ]] ; then + ldflags="-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata" + fi + go build -ldflags "$ldflags" -o ./cloud-sql-proxy main.go } ## test - Runs local unit tests. function test() { - go test -v -race -cover -short ./... + get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2' + mkdir -p test-results + local args=( "./..." ) + if [[ "$#" -gt 0 ]] ; then + args=( "$@" ) + fi + go test -v -race -cover -short "${args[@]}" -json \ + | .tools/go-junit-report -iocopy -parser gojson -out test-results/unit.xml \ + | jq -j 'select(.Output) | .Output ' } ## e2e - Runs end-to-end integration tests. @@ -53,7 +66,11 @@ function e2e() { # e2e_ci - Run end-to-end integration tests in the CI system. # This assumes that the secrets in the env vars are already set. function e2e_ci() { - go test -race -v ./... | tee test_results.txt + get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2' + mkdir -p test-results + go test -race -v ./... -json \ + | .tools/go-junit-report -iocopy -parser gojson -out test-results/e2e.xml \ + | jq -j 'select(.Output) | .Output ' } function get_golang_tool() { @@ -227,16 +244,47 @@ function write_e2e_env(){ done # Set IAM User env vars to the local gcloud user - echo "export MYSQL_IAM_USER='${local_user%%@*}'" - echo "export POSTGRES_USER_IAM='$local_user'" + echo "export MYSQL_IAM_USER='$(iam_user_mysql)'" + echo "export POSTGRES_USER_IAM='$(iam_user_pg)'" } > "$1" } +function iam_user_pg() { + # Truncate the suffix `.iam.gserviceaccount.com` if it exists. Otherwise return the email. + local email + local pguser + + email="$(iam_user_email)" + pguser="${email%%.iam.gserviceaccount.com}" + if [[ -n "$pguser" ]] ; then + echo "$pguser" + else + echo "$email" + fi + +} + +function iam_user_mysql() { + # Truncate the part after the @ + local email + local pguser + + email=$(iam_user_email) + mysqluser="${email%%@*}" + echo "$mysqluser" +} + +function iam_user_email() { + gcloud auth list --format json | jq -r '.[] | select (.status == "ACTIVE") | .account' +} + + ## build_image - Builds and pushes the proxy container image using local source. -## Usage: ./build.sh build_image [image-url] +## Usage: ./build.sh build_image [image-url] [metadata] function build_image() { local image_url="${1:-}" + local metadata="${2:-container}" local push_arg="" if [[ -n "$image_url" ]]; then @@ -254,7 +302,7 @@ function build_image() { trap cleanup_build EXIT echo "Building binary locally..." - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=container" -o cloud-sql-proxy + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata" -o cloud-sql-proxy echo "Creating temporary Dockerfile..." cat > Dockerfile.local < 0 && c.conf.Instances[0].SQLDataEnabled != nil { + t.Fatalf("args = %v, want nil, got = %v", tc.args, *c.conf.Instances[0].SQLDataEnabled) + } + return + } + if len(c.conf.Instances) == 0 { + t.Fatal("expected at least one instance") + } + got := c.conf.Instances[0].SQLDataEnabled + if got == nil { + t.Fatalf("args = %v, want = %v, got = nil", tc.args, *tc.want) + } + if *got != *tc.want { + t.Errorf("args = %v, want = %v, got = %v", tc.args, *tc.want, *got) + } + }) + } +} + func TestNewCommandWithErrors(t *testing.T) { tcs := []struct { desc string @@ -1152,6 +1239,14 @@ func TestNewCommandWithErrors(t *testing.T) { desc: "when the iam authn login query param contains multiple values", args: []string{"proj:region:inst?auto-iam-authn=true&auto-iam-authn=false"}, }, + { + desc: "when the sql-data query param contains multiple values", + args: []string{"proj:region:inst?sql-data=true&sql-data=false"}, + }, + { + desc: "when the sql-data query param is bogus", + args: []string{"proj:region:inst?sql-data=nope"}, + }, { desc: "when the iam authn login query param is bogus", args: []string{"proj:region:inst?auto-iam-authn=nope"}, diff --git a/docs/cmd/cloud-sql-proxy.md b/docs/cmd/cloud-sql-proxy.md index 6454113ec..ab5260df2 100644 --- a/docs/cmd/cloud-sql-proxy.md +++ b/docs/cmd/cloud-sql-proxy.md @@ -279,7 +279,9 @@ cloud-sql-proxy INSTANCE_CONNECTION_NAME... [flags] status code. --skip-failed-instance-config If set, the Proxy will skip any instances that are invalid/unreachable ( only applicable to Unix sockets) + --sql-data Enable SQL Data to tunnel through the Cloud SQL Admin API without needing network access to your public or private IP --sqladmin-api-endpoint string API endpoint for all Cloud SQL Admin API requests. (default: https://sqladmin.googleapis.com) + --sqldata-api-endpoint string Override the SQL Data API endpoint -l, --structured-logs Enable structured logging with LogEntry format --telemetry-prefix string Prefix for Cloud Monitoring metrics. --telemetry-project string Enable Cloud Monitoring and Cloud Trace with the provided project ID. diff --git a/go.mod b/go.mod index bdda07c1d..31438ea64 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/GoogleCloudPlatform/cloud-sql-proxy/v2 go 1.25.8 require ( - cloud.google.com/go/cloudsqlconn v1.21.2 + cloud.google.com/go/cloudsqlconn v1.22.0 contrib.go.opencensus.io/exporter/prometheus v0.4.2 contrib.go.opencensus.io/exporter/stackdriver v0.13.14 github.com/coreos/go-systemd/v22 v22.7.0 diff --git a/go.sum b/go.sum index d68b5c98f..b8866253d 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/cloudsqlconn v1.21.2 h1:Iw/3W+6eIB9AChWFF2wlqLwpgaZ+DOX2DBUKKfsSDMI= -cloud.google.com/go/cloudsqlconn v1.21.2/go.mod h1:AXcbXAjdud2Hl6JLe80VHCaOjAsvh8O/JgQnhrRvJl8= +cloud.google.com/go/cloudsqlconn v1.22.0 h1:4+uh5gGbjmFzitCD9owKZhwjBN2U+4sWxD9gJjcpOeE= +cloud.google.com/go/cloudsqlconn v1.22.0/go.mod h1:AXcbXAjdud2Hl6JLe80VHCaOjAsvh8O/JgQnhrRvJl8= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index ff58c552a..5c0043197 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -95,6 +95,8 @@ type InstanceConnConfig struct { // necessary. If set, UnixSocketPath takes precedence over UnixSocket, Addr // and Port. UnixSocketPath string + // SQLDataEnabled enables connections through the SqlDataService for this connection. + SQLDataEnabled *bool // IAMAuthN enables automatic IAM DB Authentication for the instance. // MySQL and Postgres only. If it is nil, the value was not specified. IAMAuthN *bool @@ -200,6 +202,11 @@ type Config struct { // of a request context, e.g., Cloud Run. LazyRefresh bool + // SQLDataEnabled configures the dialer to use the SQL Data API. + SQLDataEnabled bool + // SQLDataEndpoint configures the endpoint of the SQL Data service. + SQLDataEndpoint string + // Instances are configuration for individual instances. Instance // configuration takes precedence over global configuration. Instances []InstanceConnConfig @@ -282,6 +289,9 @@ func dialOptions(c Config, i InstanceConnConfig) []cloudsqlconn.DialOption { if i.IAMAuthN != nil { opts = append(opts, cloudsqlconn.WithDialIAMAuthN(*i.IAMAuthN)) } + if i.SQLDataEnabled != nil && *i.SQLDataEnabled || c.SQLDataEnabled { + opts = append(opts, cloudsqlconn.WithSQLData()) + } switch { // If private IP is enabled at the instance level, or private IP is enabled globally @@ -469,6 +479,10 @@ func (c *Config) DialerOptions(l cloudsql.Logger) ([]cloudsqlconn.Option, error) opts = append(opts, cloudsqlconn.WithLazyRefresh()) } + if c.SQLDataEndpoint != "" { + opts = append(opts, cloudsqlconn.WithSQLDataEndpoint(c.SQLDataEndpoint)) + } + return opts, nil } @@ -564,8 +578,12 @@ func NewClient(ctx context.Context, d cloudsql.Dialer, l cloudsql.Logger, conf * return configureFUSE(c, conf) } + // unless the proxy is in SqlDataEnabled mode, initiate a refresh operation to warm the cache for _, inst := range conf.Instances { - // Initiate refresh operation and warm the cache. + // Skip instances with SqlDataEnabled + if conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled { + continue + } go func(name string) { _, _ = d.EngineVersion(ctx, name) }(inst.Name) } @@ -859,6 +877,10 @@ func (c *Client) newSocketMount(ctx context.Context, conf *Config, pc *portConfi np = inst.Port case conf.Port != 0: np = pc.nextPort() + case conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled: + // Only Postgres is supported by the SqlDataService + // when more engines are supported, this code will need to change. + np = pc.nextDBPort("POSTGRES") default: version, err := c.dialer.EngineVersion(ctx, inst.Name) // Exit if the port is not specified for inactive instance @@ -873,10 +895,17 @@ func (c *Client) newSocketMount(ctx context.Context, conf *Config, pc *portConfi } else { network = "unix" - version, err := c.dialer.EngineVersion(ctx, inst.Name) - if err != nil { - c.logger.Errorf("[%v] could not resolve instance version: %v", inst.Name, err) - return nil, err + var version string + switch { + case conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled: + version = "POSTGRES" + default: + var err error + version, err = c.dialer.EngineVersion(ctx, inst.Name) + if err != nil { + c.logger.Errorf("[%v] could not resolve instance version: %v", inst.Name, err) + return nil, err + } } address, err = newUnixSocketMount(inst, conf.UnixSocket, strings.HasPrefix(version, "POSTGRES")) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 2de5421f5..c529d6a84 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -33,6 +33,11 @@ import ( "github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/internal/proxy" ) +var ( + sd = "proj:region:sd" + sd2 = "proj:region:sd2" +) + var testLogger = log.NewStdLogger(os.Stdout, os.Stdout) type fakeDialer struct { @@ -114,6 +119,12 @@ func createTempDir(t *testing.T) (string, func()) { } } +// pointer returns the address of v and makes it easy to take the address of a +// predeclared identifier. +func pointer[T any](v T) *T { + return &v +} + func TestClientInitialization(t *testing.T) { ctx := context.Background() testDir, cleanup := createTempDir(t) @@ -213,6 +224,47 @@ func TestClientInitialization(t *testing.T) { "127.0.0.1:1434", }, }, + { + desc: "with SQL Data enabled globally defaulting to Postgres port", + in: &proxy.Config{ + Addr: "127.0.0.1", + SQLDataEnabled: true, + Instances: []proxy.InstanceConnConfig{ + {Name: sd}, + {Name: sd2}, + }, + }, + wantTCPAddrs: []string{ + "127.0.0.1:5432", + "127.0.0.1:5433", + }, + }, + { + desc: "with SQL Data enabled per-instance defaulting to Postgres port", + in: &proxy.Config{ + Addr: "127.0.0.1", + Instances: []proxy.InstanceConnConfig{ + {Name: sd, SQLDataEnabled: pointer(true)}, + {Name: sd2, SQLDataEnabled: pointer(true)}, + }, + }, + wantTCPAddrs: []string{ + "127.0.0.1:5432", + "127.0.0.1:5433", + }, + }, + { + desc: "with SQL Data enabled but explicit port", + in: &proxy.Config{ + Addr: "127.0.0.1", + Instances: []proxy.InstanceConnConfig{ + {Name: sd, SQLDataEnabled: pointer(true), Port: 60000}, + }, + }, + wantTCPAddrs: []string{ + "127.0.0.1:60000", + }, + }, { desc: "with a Unix socket", in: &proxy.Config{ @@ -589,6 +641,7 @@ func TestClientNotifiesCallerOnServe(t *testing.T) { if err != nil { t.Fatalf("want error = nil, got = %v", err) } + defer c.Close() done := make(chan struct{}) notify := func() { close(done) } @@ -650,6 +703,62 @@ func TestClientConnCount(t *testing.T) { verifyOpen(t, 1) } +func TestSQLDataWarmup(t *testing.T) { + tcs := []struct { + desc string + in *proxy.Config + want int + }{ + { + desc: "standard warmup", + in: &proxy.Config{ + Instances: []proxy.InstanceConnConfig{{Name: "proj:reg:inst", Port: 30104}}, + }, + want: 1, + }, + { + desc: "warmup skipped globally", + in: &proxy.Config{ + SQLDataEnabled: true, + Instances: []proxy.InstanceConnConfig{{Name: "proj:reg:inst", Port: 30105}}, + }, + want: 0, + }, + { + desc: "warmup skipped per-instance", + in: &proxy.Config{ + Instances: []proxy.InstanceConnConfig{{ + Name: "proj:reg:inst", + Port: 30106, + SQLDataEnabled: pointer(true), + }}, + }, + want: 0, + }, + } + + for _, tc := range tcs { + t.Run(tc.desc, func(t *testing.T) { + d := &fakeDialer{} + c, err := proxy.NewClient(context.Background(), d, testLogger, tc.in, nil) + if err != nil { + t.Fatalf("proxy.NewClient error: %v", err) + } + defer c.Close() + + var got int + for i := 0; i < 10; i++ { + got = d.engineVersionAttempts() + if got == tc.want { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("warmup attempts, want = %v, got = %v", tc.want, got) + }) + } +} + func TestCheckConnections(t *testing.T) { in := &proxy.Config{ Addr: "127.0.0.1",