Merge pull request #1066 from keybase/strib/909-git-push-speedup-when-local

remote: speed up pushes when the "remote" repo is local

Author: strib

config/config.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strconv"
 
+	"gopkg.in/src-d/go-git.v4/internal/url"
 	format "gopkg.in/src-d/go-git.v4/plumbing/format/config"
 )
 
@@ -399,3 +400,7 @@ func (c *RemoteConfig) marshal() *format.Subsection {
 
 	return c.raw
 }
+
+func (c *RemoteConfig) IsFirstURLLocal() bool {
+	return url.IsLocalEndpoint(c.URLs[0])
+}

internal/url/url.go

@@ -0,0 +1,37 @@
+package url
+
+import (
+	"regexp"
+)
+
+var (
+	isSchemeRegExp   = regexp.MustCompile(`^[^:]+://`)
+	scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P<user>[^@]+)@)?(?P<host>[^:\s]+):(?:(?P<port>[0-9]{1,5})/)?(?P<path>[^\\].*)$`)
+)
+
+// MatchesScheme returns true if the given string matches a URL-like
+// format scheme.
+func MatchesScheme(url string) bool {
+	return isSchemeRegExp.MatchString(url)
+}
+
+// MatchesScpLike returns true if the given string matches an SCP-like
+// format scheme.
+func MatchesScpLike(url string) bool {
+	return scpLikeUrlRegExp.MatchString(url)
+}
+
+// FindScpLikeComponents returns the user, host, port and path of the
+// given SCP-like URL.
+func FindScpLikeComponents(url string) (user, host, port, path string) {
+	m := scpLikeUrlRegExp.FindStringSubmatch(url)
+	return m[1], m[2], m[3], m[4]
+}
+
+// IsLocalEndpoint returns true if the given URL string specifies a
+// local file endpoint.  For example, on a Linux machine,
+// `/home/user/src/go-git` would match as a local endpoint, but
+// `https://github.com/src-d/go-git` would not.
+func IsLocalEndpoint(url string) bool {
+	return !MatchesScheme(url) && !MatchesScpLike(url)
+}

plumbing/revlist/revlist.go

@@ -21,7 +21,20 @@ func Objects(
 	objs,
 	ignore []plumbing.Hash,
 ) ([]plumbing.Hash, error) {
-	ignore, err := objects(s, ignore, nil, true)
+	return ObjectsWithStorageForIgnores(s, s, objs, ignore)
+}
+
+// ObjectsWithStorageForIgnores is the same as Objects, but a
+// secondary storage layer can be provided, to be used to finding the
+// full set of objects to be ignored while finding the reachable
+// objects.  This is useful when the main `s` storage layer is slow
+// and/or remote, while the ignore list is available somewhere local.
+func ObjectsWithStorageForIgnores(
+	s, ignoreStore storer.EncodedObjectStorer,
+	objs,
+	ignore []plumbing.Hash,
+) ([]plumbing.Hash, error) {
+	ignore, err := objects(ignoreStore, ignore, nil, true)
 	if err != nil {
 		return nil, err
 	}
@@ -114,7 +127,6 @@ func reachableObjects(
 	i := object.NewCommitPreorderIter(commit, seen, ignore)
 	pending := make(map[plumbing.Hash]bool)
 	addPendingParents(pending, visited, commit)
-
 	for {
 		commit, err := i.Next()
 		if err == io.EOF {

plumbing/revlist/revlist_test.go

@@ -129,6 +129,32 @@ func (s *RevListSuite) TestRevListObjectsTagObject(c *C) {
 	c.Assert(len(hist), Equals, len(expected))
 }
 
+func (s *RevListSuite) TestRevListObjectsWithStorageForIgnores(c *C) {
+	sto := filesystem.NewStorage(
+		fixtures.ByTag("merge-conflict").One().DotGit(),
+		cache.NewObjectLRUDefault())
+
+	// The "merge-conflict" repo has one extra commit in it, with a
+	// two files modified in two different subdirs.
+	expected := map[string]bool{
+		"1980fcf55330d9d94c34abee5ab734afecf96aba": true, // commit
+		"73d9cf44e9045254346c73f6646b08f9302c8570": true, // root dir
+		"e8435d512a98586bd2e4fcfcdf04101b0bb1b500": true, // go/
+		"257cc5642cb1a054f08cc83f2d943e56fd3ebe99": true, // haskal.hs
+		"d499a1a0b79b7d87a35155afd0c1cce78b37a91c": true, // example.go
+		"d108adc364fb6f21395d011ae2c8a11d96905b0d": true, // haskal/
+	}
+
+	hist, err := ObjectsWithStorageForIgnores(sto, s.Storer, []plumbing.Hash{plumbing.NewHash("1980fcf55330d9d94c34abee5ab734afecf96aba")}, []plumbing.Hash{plumbing.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")})
+	c.Assert(err, IsNil)
+
+	for _, h := range hist {
+		c.Assert(expected[h.String()], Equals, true)
+	}
+
+	c.Assert(len(hist), Equals, len(expected))
+}
+
 // ---
 // | |\
 // | | * b8e471f Creating changelog

plumbing/transport/common.go

@@ -19,10 +19,10 @@ import (
 	"fmt"
 	"io"
 	"net/url"
-	"regexp"
 	"strconv"
 	"strings"
 
+	giturl "gopkg.in/src-d/go-git.v4/internal/url"
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/protocol/packp"
 	"gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability"
@@ -224,34 +224,28 @@ func getPath(u *url.URL) string {
 	return res
 }
 
-var (
-	isSchemeRegExp   = regexp.MustCompile(`^[^:]+://`)
-	scpLikeUrlRegExp = regexp.MustCompile(`^(?:(?P<user>[^@]+)@)?(?P<host>[^:\s]+):(?:(?P<port>[0-9]{1,5})/)?(?P<path>[^\\].*)$`)
-)
-
 func parseSCPLike(endpoint string) (*Endpoint, bool) {
-	if isSchemeRegExp.MatchString(endpoint) || !scpLikeUrlRegExp.MatchString(endpoint) {
+	if giturl.MatchesScheme(endpoint) || !giturl.MatchesScpLike(endpoint) {
 		return nil, false
 	}
 
-	m := scpLikeUrlRegExp.FindStringSubmatch(endpoint)
-
-	port, err := strconv.Atoi(m[3])
+	user, host, portStr, path := giturl.FindScpLikeComponents(endpoint)
+	port, err := strconv.Atoi(portStr)
 	if err != nil {
 		port = 22
 	}
 
 	return &Endpoint{
 		Protocol: "ssh",
-		User:     m[1],
-		Host:     m[2],
+		User:     user,
+		Host:     host,
 		Port:     port,
-		Path:     m[4],
+		Path:     path,
 	}, true
 }
 
 func parseFile(endpoint string) (*Endpoint, bool) {
-	if isSchemeRegExp.MatchString(endpoint) {
+	if giturl.MatchesScheme(endpoint) {
 		return nil, false
 	}
 

remote.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"io"
 
+	"gopkg.in/src-d/go-billy.v4/osfs"
 	"gopkg.in/src-d/go-git.v4/config"
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/go-git.v4/plumbing/protocol/packp"
@@ -18,6 +20,7 @@ import (
 	"gopkg.in/src-d/go-git.v4/plumbing/transport"
 	"gopkg.in/src-d/go-git.v4/plumbing/transport/client"
 	"gopkg.in/src-d/go-git.v4/storage"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 	"gopkg.in/src-d/go-git.v4/storage/memory"
 	"gopkg.in/src-d/go-git.v4/utils/ioutil"
 )
@@ -149,7 +152,17 @@ func (r *Remote) PushContext(ctx context.Context, o *PushOptions) (err error) {
 	var hashesToPush []plumbing.Hash
 	// Avoid the expensive revlist operation if we're only doing deletes.
 	if !allDelete {
-		hashesToPush, err = revlist.Objects(r.s, objects, haves)
+		if r.c.IsFirstURLLocal() {
+			// If we're are pushing to a local repo, it might be much
+			// faster to use a local storage layer to get the commits
+			// to ignore, when calculating the object revlist.
+			localStorer := filesystem.NewStorage(
+				osfs.New(r.c.URLs[0]), cache.NewObjectLRUDefault())
+			hashesToPush, err = revlist.ObjectsWithStorageForIgnores(
+				r.s, localStorer, objects, haves)
+		} else {
+			hashesToPush, err = revlist.Objects(r.s, objects, haves)
+		}
 		if err != nil {
 			return err
 		}