diff --git a/cmd/cloudflared/tunnel/cmd.go b/cmd/cloudflared/tunnel/cmd.go index f44922e4a25..fe754bea76b 100644 --- a/cmd/cloudflared/tunnel/cmd.go +++ b/cmd/cloudflared/tunnel/cmd.go @@ -1065,8 +1065,9 @@ func configureProxyFlags(shouldHide bool) []cli.Flag { Hidden: shouldHide, }), altsrc.NewBoolFlag(&cli.BoolFlag{ - Name: ingress.Http2OriginFlag, - Usage: "Enables HTTP/2 origin servers.", + Name: ingress.Http2OriginFlag, + Usage: "Use HTTP/2 to connect to the origin. TLS origins negotiate HTTP/2. " + + "Cleartext HTTP and Unix origins use prior-knowledge h2c, which requires HTTP/2 support with no HTTP/1.1 fallback.", EnvVars: []string{"TUNNEL_ORIGIN_ENABLE_HTTP2"}, Hidden: shouldHide, Value: false, diff --git a/ingress/config.go b/ingress/config.go index 83f893fefe1..b95a4e4444f 100644 --- a/ingress/config.go +++ b/ingress/config.go @@ -328,7 +328,9 @@ type OriginRequestConfig struct { ProxyType string `yaml:"proxyType" json:"proxyType"` // IP rules for the proxy service IPRules []ipaccess.Rule `yaml:"ipRules" json:"ipRules"` - // Attempt to connect to origin with HTTP/2 + // Use HTTP/2 to connect to the origin. TLS origins negotiate HTTP/2. + // Cleartext HTTP and Unix origins use prior-knowledge h2c, which requires + // HTTP/2 support with no HTTP/1.1 fallback. Http2Origin bool `yaml:"http2Origin" json:"http2Origin"` // Access holds all access related configs @@ -544,7 +546,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig } func convertToRawIPRules(ipRules []ipaccess.Rule) []config.IngressIPRule { - result := make([]config.IngressIPRule, 0) + result := make([]config.IngressIPRule, 0, len(ipRules)) for _, r := range ipRules { cidr := r.StringCIDR() diff --git a/ingress/origin_service.go b/ingress/origin_service.go index e13204c5789..da081329fb8 100644 --- a/ingress/origin_service.go +++ b/ingress/origin_service.go @@ -99,7 +99,6 @@ type rawTCPService struct { name string dialer net.Dialer writeTimeout time.Duration - logger *zerolog.Logger } func (o *rawTCPService) String() string { @@ -233,10 +232,14 @@ func (o *helloWorld) start( if err != nil { return errors.Wrap(err, "Cannot start Hello World Server") } - go hello.StartHelloWorldServer(log, helloListener, shutdownC) + go func() { + if err := hello.StartHelloWorldServer(log, helloListener, shutdownC); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error().Err(err).Msg("Hello World server failed") + } + }() o.server = helloListener - o.httpService.url = &url.URL{ + o.url = &url.URL{ Scheme: "https", Host: o.server.Addr().String(), } @@ -343,12 +346,24 @@ func (nrc *NopReadCloser) Close() error { return nil } +func isH2COrigin(service OriginService) bool { + switch service := service.(type) { + case *httpService: + return service.url != nil && service.url.Scheme == "http" + case *unixSocketPath: + return service.scheme == "http" + default: + return false + } +} + func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerolog.Logger) (*http.Transport, error) { originCertPool, err := tlsconfig.LoadOriginCA(cfg.CAPool, log) if err != nil { return nil, errors.Wrap(err, "Error loading cert pool") } + useH2C := cfg.Http2Origin && isH2COrigin(service) httpTransport := http.Transport{ Proxy: http.ProxyFromEnvironment, MaxIdleConns: cfg.KeepAliveConnections, @@ -356,8 +371,17 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol IdleConnTimeout: cfg.KeepAliveTimeout.Duration, TLSHandshakeTimeout: cfg.TLSTimeout.Duration, ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify}, - ForceAttemptHTTP2: cfg.Http2Origin, + TLSClientConfig: &tls.Config{ + RootCAs: originCertPool, + InsecureSkipVerify: cfg.NoTLSVerify, //nolint:gosec // Disabling verification is an explicit origin setting. + }, + ForceAttemptHTTP2: cfg.Http2Origin && !useH2C, + } + if useH2C { + // HTTP/1 must remain disabled for the transport to use prior-knowledge h2c. + protocols := new(http.Protocols) + protocols.SetUnencryptedHTTP2(true) + httpTransport.Protocols = protocols } if _, isHelloWorld := service.(*helloWorld); !isHelloWorld && cfg.OriginServerName != "" { httpTransport.TLSClientConfig.ServerName = cfg.OriginServerName @@ -374,7 +398,6 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol // DialContext depends on which kind of origin is being used. dialContext := dialer.DialContext switch service := service.(type) { - // If this origin is a unix socket, enforce network type "unix". case *unixSocketPath: httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) { diff --git a/ingress/origin_service_posix_test.go b/ingress/origin_service_posix_test.go new file mode 100644 index 00000000000..83c6d6b3a2a --- /dev/null +++ b/ingress/origin_service_posix_test.go @@ -0,0 +1,68 @@ +//go:build !windows + +package ingress + +import ( + "errors" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +func TestUnixSocketH2COrigin(t *testing.T) { + t.Parallel() + + socketFile, err := os.CreateTemp("/tmp", "cloudflared-h2c-*.sock") + require.NoError(t, err) + socketPath := socketFile.Name() + require.NoError(t, socketFile.Close()) + require.NoError(t, os.Remove(socketPath)) + + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + t.Cleanup(func() { + err := os.Remove(socketPath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Errorf("failed to remove Unix socket: %v", err) + } + }) + + protocols := new(http.Protocols) + protocols.SetUnencryptedHTTP2(true) + server := &http.Server{ + ReadHeaderTimeout: time.Second, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Origin-Protocol", r.Proto) + }), + Protocols: protocols, + } + serveErr := make(chan error, 1) + go func() { + serveErr <- server.Serve(listener) + }() + t.Cleanup(func() { + require.NoError(t, server.Close()) + require.ErrorIs(t, <-serveErr, http.ErrServerClosed) + }) + + log := zerolog.Nop() + service := &unixSocketPath{ + path: socketPath, + scheme: "http", + } + require.NoError(t, service.start(&log, nil, OriginRequestConfig{Http2Origin: true})) + t.Cleanup(service.transport.CloseIdleConnections) + + req, err := http.NewRequest(http.MethodGet, "http://localhost", nil) + require.NoError(t, err) + resp, err := service.RoundTrip(req) + require.NoError(t, err) + require.Equal(t, "HTTP/2.0", resp.Header.Get("X-Origin-Protocol")) + require.Equal(t, 2, resp.ProtoMajor) + require.NoError(t, resp.Body.Close()) +} diff --git a/ingress/origin_service_test.go b/ingress/origin_service_test.go index 081b08ae920..bdc3f8add9b 100644 --- a/ingress/origin_service_test.go +++ b/ingress/origin_service_test.go @@ -1,13 +1,19 @@ package ingress import ( + "net/http" + "net/http/httptest" "net/url" "testing" "github.com/stretchr/testify/require" ) +const testOriginProtocolHeader = "X-Test-Origin-Protocol" + func TestAddPortIfMissing(t *testing.T) { + t.Parallel() + testCases := []struct { input string expected string @@ -21,9 +27,234 @@ func TestAddPortIfMissing(t *testing.T) { for _, tc := range testCases { t.Run(tc.input, func(t *testing.T) { - url1, _ := url.Parse(tc.input) + t.Parallel() + + url1, err := url.Parse(tc.input) + require.NoError(t, err) addPortIfMissing(url1, 22) require.Equal(t, tc.expected, url1.Host) }) } } + +func TestIsH2COrigin(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + service OriginService + expected bool + }{ + { + name: "HTTP", + service: &httpService{url: MustParseURL(t, "http://localhost")}, + expected: true, + }, + { + name: "HTTPS", + service: &httpService{url: MustParseURL(t, "https://localhost")}, + }, + { + name: "WebSocket", + service: &httpService{url: MustParseURL(t, "ws://localhost")}, + }, + { + name: "secure WebSocket", + service: &httpService{url: MustParseURL(t, "wss://localhost")}, + }, + { + name: "Unix", + service: &unixSocketPath{path: "/tmp/cloudflared.sock", scheme: "http"}, + expected: true, + }, + { + name: "Unix with TLS", + service: &unixSocketPath{path: "/tmp/cloudflared.sock", scheme: "https"}, + }, + { + name: "HTTP service without URL", + service: &httpService{}, + }, + { + name: "Hello World before configuration", + service: &helloWorld{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, test.expected, isH2COrigin(test.service)) + }) + } +} + +func TestHTTPTransportUsesH2C(t *testing.T) { + t.Parallel() + + origin := newTestOriginServer(t, unencryptedHTTP2Protocols(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(testOriginProtocolHeader, r.Proto) + w.WriteHeader(http.StatusNoContent) + }) + service := newTestHTTPService(t, origin.URL, OriginRequestConfig{Http2Origin: true}) + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + require.NoError(t, err) + resp, err := service.RoundTrip(req) + require.NoError(t, err) + require.Equal(t, http.StatusNoContent, resp.StatusCode) + require.Equal(t, "HTTP/2.0", resp.Header.Get(testOriginProtocolHeader)) + require.Equal(t, 2, resp.ProtoMajor) + require.NoError(t, resp.Body.Close()) +} + +func TestHTTPTransportUsesHTTP1WhenHTTP2OriginIsDisabled(t *testing.T) { + t.Parallel() + + origin := newTestOriginServer(t, http1AndUnencryptedHTTP2Protocols(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(testOriginProtocolHeader, r.Proto) + w.WriteHeader(http.StatusNoContent) + }) + service := newTestHTTPService(t, origin.URL, OriginRequestConfig{}) + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + require.NoError(t, err) + resp, err := service.RoundTrip(req) + require.NoError(t, err) + require.Equal(t, http.StatusNoContent, resp.StatusCode) + require.Equal(t, "HTTP/1.1", resp.Header.Get(testOriginProtocolHeader)) + require.Equal(t, 1, resp.ProtoMajor) + require.NoError(t, resp.Body.Close()) +} + +func TestHTTPTransportDoesNotFallBackFromH2CToHTTP1(t *testing.T) { + t.Parallel() + + origin := newTestOriginServer(t, http1Protocols(), func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + service := newTestHTTPService(t, origin.URL, OriginRequestConfig{Http2Origin: true}) + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + require.NoError(t, err) + resp, err := service.RoundTrip(req) + if resp != nil { + require.NoError(t, resp.Body.Close()) + } + require.Error(t, err) + require.Nil(t, resp) +} + +func TestHTTPTransportUsesHTTP2OverTLS(t *testing.T) { + t.Parallel() + + origin := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(testOriginProtocolHeader, r.Proto) + w.WriteHeader(http.StatusNoContent) + })) + origin.EnableHTTP2 = true + origin.StartTLS() + t.Cleanup(origin.Close) + service := newTestHTTPService(t, origin.URL, OriginRequestConfig{ + Http2Origin: true, + NoTLSVerify: true, + }) + + req, err := http.NewRequest(http.MethodGet, origin.URL, nil) + require.NoError(t, err) + resp, err := service.RoundTrip(req) + require.NoError(t, err) + require.Equal(t, http.StatusNoContent, resp.StatusCode) + require.Equal(t, "HTTP/2.0", resp.Header.Get(testOriginProtocolHeader)) + require.Equal(t, 2, resp.ProtoMajor) + require.NoError(t, resp.Body.Close()) +} + +func TestWebSocketOriginUsesHTTP1WithHTTP2OriginEnabled(t *testing.T) { + t.Parallel() + + origin := newTestOriginServer(t, http1AndUnencryptedHTTP2Protocols(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(testOriginProtocolHeader, r.Proto) + w.Header().Set("Connection", "Upgrade") + w.Header().Set("Upgrade", "websocket") + w.WriteHeader(http.StatusSwitchingProtocols) + }) + originURL := MustParseURL(t, origin.URL) + originURL.Scheme = "ws" + service := newTestHTTPService(t, originURL.String(), OriginRequestConfig{Http2Origin: true}) + + req, err := http.NewRequest(http.MethodGet, "http://eyeball.example", nil) + require.NoError(t, err) + setWebSocketUpgradeHeaders(req) + resp, err := service.RoundTrip(req) + require.NoError(t, err) + require.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode) + require.Equal(t, "HTTP/1.1", resp.Header.Get(testOriginProtocolHeader)) + require.Equal(t, 1, resp.ProtoMajor) + require.NoError(t, resp.Body.Close()) +} + +func TestWebSocketUpgradeDoesNotFallBackFromH2C(t *testing.T) { + t.Parallel() + + origin := newTestOriginServer(t, http1AndUnencryptedHTTP2Protocols(), func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Connection", "Upgrade") + w.Header().Set("Upgrade", "websocket") + w.WriteHeader(http.StatusSwitchingProtocols) + }) + service := newTestHTTPService(t, origin.URL, OriginRequestConfig{Http2Origin: true}) + + req, err := http.NewRequest(http.MethodGet, "http://eyeball.example", nil) + require.NoError(t, err) + setWebSocketUpgradeHeaders(req) + resp, err := service.RoundTrip(req) + if resp != nil { + require.NoError(t, resp.Body.Close()) + } + require.Error(t, err) + require.Nil(t, resp) +} + +func newTestHTTPService(t *testing.T, rawURL string, cfg OriginRequestConfig) *httpService { + t.Helper() + + service := &httpService{url: MustParseURL(t, rawURL)} + require.NoError(t, service.start(TestLogger, nil, cfg)) + service.transport.Proxy = nil + t.Cleanup(service.transport.CloseIdleConnections) + return service +} + +func newTestOriginServer(t *testing.T, protocols *http.Protocols, handler http.HandlerFunc) *httptest.Server { + t.Helper() + + origin := httptest.NewUnstartedServer(handler) + origin.Config.Protocols = protocols + origin.Start() + t.Cleanup(origin.Close) + return origin +} + +func http1Protocols() *http.Protocols { + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + return protocols +} + +func unencryptedHTTP2Protocols() *http.Protocols { + protocols := new(http.Protocols) + protocols.SetUnencryptedHTTP2(true) + return protocols +} + +func http1AndUnencryptedHTTP2Protocols() *http.Protocols { + protocols := http1Protocols() + protocols.SetUnencryptedHTTP2(true) + return protocols +} + +func setWebSocketUpgradeHeaders(req *http.Request) { + req.Header.Set("Connection", "Upgrade") + req.Header.Set("Upgrade", "websocket") +}