diff --git a/.gitignore b/.gitignore index de8224085..958b50e44 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ /logs/ .tools test_results.txt +test-results diff --git a/build.sh b/build.sh index efdd6d303..5ad0c43c7 100755 --- a/build.sh +++ b/build.sh @@ -33,12 +33,25 @@ function clean() { ## build - Builds the project without running tests. function build() { - go build -o ./cloud-sql-proxy main.go + local metadata="${1:-}" + local ldflags="" + if [[ -n "$metadata" ]] ; then + ldflags="-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata" + fi + go build -ldflags "$ldflags" -o ./cloud-sql-proxy main.go } ## test - Runs local unit tests. function test() { - go test -v -race -cover -short ./... + get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2' + mkdir -p test-results + local args=( "./..." ) + if [[ "$#" -gt 0 ]] ; then + args=( "$@" ) + fi + go test -v -race -cover -short "${args[@]}" -json \ + | .tools/go-junit-report -iocopy -parser gojson -out test-results/unit.xml \ + | jq -j 'select(.Output) | .Output ' } ## e2e - Runs end-to-end integration tests. @@ -53,7 +66,11 @@ function e2e() { # e2e_ci - Run end-to-end integration tests in the CI system. # This assumes that the secrets in the env vars are already set. function e2e_ci() { - go test -race -v ./... | tee test_results.txt + get_golang_tool 'go-junit-report' 'jstemmer/go-junit-report' 'github.com/jstemmer/go-junit-report/v2' + mkdir -p test-results + go test -race -v ./... -json \ + | .tools/go-junit-report -iocopy -parser gojson -out test-results/e2e.xml \ + | jq -j 'select(.Output) | .Output ' } function get_golang_tool() { @@ -227,16 +244,47 @@ function write_e2e_env(){ done # Set IAM User env vars to the local gcloud user - echo "export MYSQL_IAM_USER='${local_user%%@*}'" - echo "export POSTGRES_USER_IAM='$local_user'" + echo "export MYSQL_IAM_USER='$(iam_user_mysql)'" + echo "export POSTGRES_USER_IAM='$(iam_user_pg)'" } > "$1" } +function iam_user_pg() { + # Truncate the suffix `.iam.gserviceaccount.com` if it exists. Otherwise return the email. + local email + local pguser + + email="$(iam_user_email)" + pguser="${email%%.iam.gserviceaccount.com}" + if [[ -n "$pguser" ]] ; then + echo "$pguser" + else + echo "$email" + fi + +} + +function iam_user_mysql() { + # Truncate the part after the @ + local email + local pguser + + email=$(iam_user_email) + mysqluser="${email%%@*}" + echo "$mysqluser" +} + +function iam_user_email() { + gcloud auth list --format json | jq -r '.[] | select (.status == "ACTIVE") | .account' +} + + ## build_image - Builds and pushes the proxy container image using local source. -## Usage: ./build.sh build_image [image-url] +## Usage: ./build.sh build_image [image-url] [metadata] function build_image() { local image_url="${1:-}" + local metadata="${2:-container}" local push_arg="" if [[ -n "$image_url" ]]; then @@ -254,7 +302,7 @@ function build_image() { trap cleanup_build EXIT echo "Building binary locally..." - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=container" -o cloud-sql-proxy + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cmd.metadataString=$metadata" -o cloud-sql-proxy echo "Creating temporary Dockerfile..." cat > Dockerfile.local < 0 && c.conf.Instances[0].SQLDataEnabled != nil { + t.Fatalf("args = %v, want nil, got = %v", tc.args, *c.conf.Instances[0].SQLDataEnabled) + } + return + } + if len(c.conf.Instances) == 0 { + t.Fatal("expected at least one instance") + } + got := c.conf.Instances[0].SQLDataEnabled + if got == nil { + t.Fatalf("args = %v, want = %v, got = nil", tc.args, *tc.want) + } + if *got != *tc.want { + t.Errorf("args = %v, want = %v, got = %v", tc.args, *tc.want, *got) + } + }) + } +} + func TestNewCommandWithErrors(t *testing.T) { tcs := []struct { desc string @@ -1152,6 +1239,14 @@ func TestNewCommandWithErrors(t *testing.T) { desc: "when the iam authn login query param contains multiple values", args: []string{"proj:region:inst?auto-iam-authn=true&auto-iam-authn=false"}, }, + { + desc: "when the sql-data query param contains multiple values", + args: []string{"proj:region:inst?sql-data=true&sql-data=false"}, + }, + { + desc: "when the sql-data query param is bogus", + args: []string{"proj:region:inst?sql-data=nope"}, + }, { desc: "when the iam authn login query param is bogus", args: []string{"proj:region:inst?auto-iam-authn=nope"}, diff --git a/docs/cmd/cloud-sql-proxy.md b/docs/cmd/cloud-sql-proxy.md index 6454113ec..ab5260df2 100644 --- a/docs/cmd/cloud-sql-proxy.md +++ b/docs/cmd/cloud-sql-proxy.md @@ -279,7 +279,9 @@ cloud-sql-proxy INSTANCE_CONNECTION_NAME... [flags] status code. --skip-failed-instance-config If set, the Proxy will skip any instances that are invalid/unreachable ( only applicable to Unix sockets) + --sql-data Enable SQL Data to tunnel through the Cloud SQL Admin API without needing network access to your public or private IP --sqladmin-api-endpoint string API endpoint for all Cloud SQL Admin API requests. (default: https://sqladmin.googleapis.com) + --sqldata-api-endpoint string Override the SQL Data API endpoint -l, --structured-logs Enable structured logging with LogEntry format --telemetry-prefix string Prefix for Cloud Monitoring metrics. --telemetry-project string Enable Cloud Monitoring and Cloud Trace with the provided project ID. diff --git a/go.mod b/go.mod index bdda07c1d..31438ea64 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/GoogleCloudPlatform/cloud-sql-proxy/v2 go 1.25.8 require ( - cloud.google.com/go/cloudsqlconn v1.21.2 + cloud.google.com/go/cloudsqlconn v1.22.0 contrib.go.opencensus.io/exporter/prometheus v0.4.2 contrib.go.opencensus.io/exporter/stackdriver v0.13.14 github.com/coreos/go-systemd/v22 v22.7.0 diff --git a/go.sum b/go.sum index d68b5c98f..b8866253d 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/cloudsqlconn v1.21.2 h1:Iw/3W+6eIB9AChWFF2wlqLwpgaZ+DOX2DBUKKfsSDMI= -cloud.google.com/go/cloudsqlconn v1.21.2/go.mod h1:AXcbXAjdud2Hl6JLe80VHCaOjAsvh8O/JgQnhrRvJl8= +cloud.google.com/go/cloudsqlconn v1.22.0 h1:4+uh5gGbjmFzitCD9owKZhwjBN2U+4sWxD9gJjcpOeE= +cloud.google.com/go/cloudsqlconn v1.22.0/go.mod h1:AXcbXAjdud2Hl6JLe80VHCaOjAsvh8O/JgQnhrRvJl8= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index ff58c552a..5c0043197 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -95,6 +95,8 @@ type InstanceConnConfig struct { // necessary. If set, UnixSocketPath takes precedence over UnixSocket, Addr // and Port. UnixSocketPath string + // SQLDataEnabled enables connections through the SqlDataService for this connection. + SQLDataEnabled *bool // IAMAuthN enables automatic IAM DB Authentication for the instance. // MySQL and Postgres only. If it is nil, the value was not specified. IAMAuthN *bool @@ -200,6 +202,11 @@ type Config struct { // of a request context, e.g., Cloud Run. LazyRefresh bool + // SQLDataEnabled configures the dialer to use the SQL Data API. + SQLDataEnabled bool + // SQLDataEndpoint configures the endpoint of the SQL Data service. + SQLDataEndpoint string + // Instances are configuration for individual instances. Instance // configuration takes precedence over global configuration. Instances []InstanceConnConfig @@ -282,6 +289,9 @@ func dialOptions(c Config, i InstanceConnConfig) []cloudsqlconn.DialOption { if i.IAMAuthN != nil { opts = append(opts, cloudsqlconn.WithDialIAMAuthN(*i.IAMAuthN)) } + if i.SQLDataEnabled != nil && *i.SQLDataEnabled || c.SQLDataEnabled { + opts = append(opts, cloudsqlconn.WithSQLData()) + } switch { // If private IP is enabled at the instance level, or private IP is enabled globally @@ -469,6 +479,10 @@ func (c *Config) DialerOptions(l cloudsql.Logger) ([]cloudsqlconn.Option, error) opts = append(opts, cloudsqlconn.WithLazyRefresh()) } + if c.SQLDataEndpoint != "" { + opts = append(opts, cloudsqlconn.WithSQLDataEndpoint(c.SQLDataEndpoint)) + } + return opts, nil } @@ -564,8 +578,12 @@ func NewClient(ctx context.Context, d cloudsql.Dialer, l cloudsql.Logger, conf * return configureFUSE(c, conf) } + // unless the proxy is in SqlDataEnabled mode, initiate a refresh operation to warm the cache for _, inst := range conf.Instances { - // Initiate refresh operation and warm the cache. + // Skip instances with SqlDataEnabled + if conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled { + continue + } go func(name string) { _, _ = d.EngineVersion(ctx, name) }(inst.Name) } @@ -859,6 +877,10 @@ func (c *Client) newSocketMount(ctx context.Context, conf *Config, pc *portConfi np = inst.Port case conf.Port != 0: np = pc.nextPort() + case conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled: + // Only Postgres is supported by the SqlDataService + // when more engines are supported, this code will need to change. + np = pc.nextDBPort("POSTGRES") default: version, err := c.dialer.EngineVersion(ctx, inst.Name) // Exit if the port is not specified for inactive instance @@ -873,10 +895,17 @@ func (c *Client) newSocketMount(ctx context.Context, conf *Config, pc *portConfi } else { network = "unix" - version, err := c.dialer.EngineVersion(ctx, inst.Name) - if err != nil { - c.logger.Errorf("[%v] could not resolve instance version: %v", inst.Name, err) - return nil, err + var version string + switch { + case conf.SQLDataEnabled || inst.SQLDataEnabled != nil && *inst.SQLDataEnabled: + version = "POSTGRES" + default: + var err error + version, err = c.dialer.EngineVersion(ctx, inst.Name) + if err != nil { + c.logger.Errorf("[%v] could not resolve instance version: %v", inst.Name, err) + return nil, err + } } address, err = newUnixSocketMount(inst, conf.UnixSocket, strings.HasPrefix(version, "POSTGRES")) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 2de5421f5..c529d6a84 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -33,6 +33,11 @@ import ( "github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/internal/proxy" ) +var ( + sd = "proj:region:sd" + sd2 = "proj:region:sd2" +) + var testLogger = log.NewStdLogger(os.Stdout, os.Stdout) type fakeDialer struct { @@ -114,6 +119,12 @@ func createTempDir(t *testing.T) (string, func()) { } } +// pointer returns the address of v and makes it easy to take the address of a +// predeclared identifier. +func pointer[T any](v T) *T { + return &v +} + func TestClientInitialization(t *testing.T) { ctx := context.Background() testDir, cleanup := createTempDir(t) @@ -213,6 +224,47 @@ func TestClientInitialization(t *testing.T) { "127.0.0.1:1434", }, }, + { + desc: "with SQL Data enabled globally defaulting to Postgres port", + in: &proxy.Config{ + Addr: "127.0.0.1", + SQLDataEnabled: true, + Instances: []proxy.InstanceConnConfig{ + {Name: sd}, + {Name: sd2}, + }, + }, + wantTCPAddrs: []string{ + "127.0.0.1:5432", + "127.0.0.1:5433", + }, + }, + { + desc: "with SQL Data enabled per-instance defaulting to Postgres port", + in: &proxy.Config{ + Addr: "127.0.0.1", + Instances: []proxy.InstanceConnConfig{ + {Name: sd, SQLDataEnabled: pointer(true)}, + {Name: sd2, SQLDataEnabled: pointer(true)}, + }, + }, + wantTCPAddrs: []string{ + "127.0.0.1:5432", + "127.0.0.1:5433", + }, + }, + { + desc: "with SQL Data enabled but explicit port", + in: &proxy.Config{ + Addr: "127.0.0.1", + Instances: []proxy.InstanceConnConfig{ + {Name: sd, SQLDataEnabled: pointer(true), Port: 60000}, + }, + }, + wantTCPAddrs: []string{ + "127.0.0.1:60000", + }, + }, { desc: "with a Unix socket", in: &proxy.Config{ @@ -589,6 +641,7 @@ func TestClientNotifiesCallerOnServe(t *testing.T) { if err != nil { t.Fatalf("want error = nil, got = %v", err) } + defer c.Close() done := make(chan struct{}) notify := func() { close(done) } @@ -650,6 +703,62 @@ func TestClientConnCount(t *testing.T) { verifyOpen(t, 1) } +func TestSQLDataWarmup(t *testing.T) { + tcs := []struct { + desc string + in *proxy.Config + want int + }{ + { + desc: "standard warmup", + in: &proxy.Config{ + Instances: []proxy.InstanceConnConfig{{Name: "proj:reg:inst", Port: 30104}}, + }, + want: 1, + }, + { + desc: "warmup skipped globally", + in: &proxy.Config{ + SQLDataEnabled: true, + Instances: []proxy.InstanceConnConfig{{Name: "proj:reg:inst", Port: 30105}}, + }, + want: 0, + }, + { + desc: "warmup skipped per-instance", + in: &proxy.Config{ + Instances: []proxy.InstanceConnConfig{{ + Name: "proj:reg:inst", + Port: 30106, + SQLDataEnabled: pointer(true), + }}, + }, + want: 0, + }, + } + + for _, tc := range tcs { + t.Run(tc.desc, func(t *testing.T) { + d := &fakeDialer{} + c, err := proxy.NewClient(context.Background(), d, testLogger, tc.in, nil) + if err != nil { + t.Fatalf("proxy.NewClient error: %v", err) + } + defer c.Close() + + var got int + for i := 0; i < 10; i++ { + got = d.engineVersionAttempts() + if got == tc.want { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("warmup attempts, want = %v, got = %v", tc.want, got) + }) + } +} + func TestCheckConnections(t *testing.T) { in := &proxy.Config{ Addr: "127.0.0.1",