Merge pull request #416 from mcarmonaa/improvement/load-siva-git-depth

cmd/gitbase: load siva files and git indistinctly

Author: ajnavarro

Dockerfile

@@ -36,4 +36,4 @@ CMD gitbase server -v \
     --port=3306 \
     --user="$GITBASE_USER" \
     --password="$GITBASE_PASSWORD" \
-    --git="$GITBASE_REPOS"
+    --directories="$GITBASE_REPOS"

cmd/gitbase/command/server.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"github.com/src-d/gitbase"
 	"github.com/src-d/gitbase/internal/function"
@@ -34,8 +35,10 @@ const (
 // Server represents the `server` command of gitbase cli tool.
 type Server struct {
 	Verbose       bool     `short:"v" description:"Activates the verbose mode"`
-	Git           []string `short:"g" long:"git" description:"Path where the git repositories are located, multiple directories can be defined. Accepts globs."`
-	Siva          []string `long:"siva" description:"Path where the siva repositories are located, multiple directories can be defined. Accepts globs."`
+	Directories   []string `short:"d" long:"directories" description:"Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs."`
+	Depth         int      `long:"depth" default:"1000" description:"load repositories looking at less than <depth> nested subdirectories."`
+	DisableGit    bool     `long:"no-git" description:"disable the load of git standard repositories."`
+	DisableSiva   bool     `long:"no-siva" description:"disable the load of siva files."`
 	Host          string   `long:"host" default:"localhost" description:"Host where the server is going to listen"`
 	Port          int      `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
 	User          string   `short:"u" long:"user" default:"root" description:"User name used for connection"`
@@ -196,52 +199,121 @@ func (c *Server) registerDrivers() error {
 }
 
 func (c *Server) addDirectories() error {
-	if len(c.Git) == 0 && len(c.Siva) == 0 {
-		logrus.Error("At least one git folder or siva folder should be provided.")
+	if len(c.Directories) == 0 {
+		logrus.Error("At least one folder should be provided.")
 	}
 
-	for _, pattern := range c.Git {
-		if err := c.addGitPattern(pattern); err != nil {
-			return err
-		}
+	if c.DisableGit && c.DisableSiva {
+		logrus.Warn("The load of git repositories and siva files are disabled," +
+			" no repository will be added.")
+
+		return nil
 	}
 
-	for _, pattern := range c.Siva {
-		if err := c.addSivaPattern(pattern); err != nil {
+	if c.Depth < 1 {
+		logrus.Warn("--depth flag set to a number less than 1," +
+			" no repository will be added.")
+
+		return nil
+	}
+
+	for _, directory := range c.Directories {
+		if err := c.addDirectory(directory); err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
 
-func (c *Server) addGitPattern(pattern string) error {
-	prefix, matches, err := gitbase.PatternMatches(pattern)
+func (c *Server) addDirectory(directory string) error {
+	matches, err := gitbase.PatternMatches(directory)
 	if err != nil {
 		return err
 	}
 
-	for _, m := range matches {
-		logrus.WithField("dir", m).Debug("git repositories directory added")
-		if err := c.pool.AddDir(prefix, m); err != nil {
-			return err
+	for _, match := range matches {
+		if err := c.addMatch(match); err != nil {
+			logrus.WithFields(logrus.Fields{
+				"path":  match,
+				"error": err,
+			}).Error("path couldn't be inspected")
 		}
 	}
 
 	return nil
 }
 
-func (c *Server) addSivaPattern(pattern string) error {
-	matches, err := filepath.Glob(pattern)
+func (c *Server) addMatch(match string) error {
+	root, err := filepath.Abs(match)
 	if err != nil {
 		return err
 	}
 
-	for _, m := range matches {
-		logrus.WithField("dir", m).Debug("siva repositories directory added")
-		if err := c.pool.AddSivaDir(m); err != nil {
+	initDepth := strings.Count(root, string(os.PathSeparator))
+	return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
 			return err
 		}
+
+		if info.IsDir() {
+			if err := c.addIfGitRepo(path); err != nil {
+				return err
+			}
+
+			depth := strings.Count(path, string(os.PathSeparator)) - initDepth
+			if depth >= c.Depth {
+				return filepath.SkipDir
+			}
+
+			return nil
+		}
+
+		if !c.DisableSiva &&
+			info.Mode().IsRegular() && gitbase.IsSivaFile(info.Name()) {
+			if err := c.pool.AddSivaFile(path); err != nil {
+				logrus.WithFields(logrus.Fields{
+					"path":  path,
+					"error": err,
+				}).Error("repository could not be addded")
+
+				return nil
+			}
+
+			logrus.WithField("path", path).Debug("repository added")
+		}
+
+		return nil
+	})
+}
+
+func (c *Server) addIfGitRepo(path string) error {
+	ok, err := gitbase.IsGitRepo(path)
+	if err != nil {
+		logrus.WithFields(logrus.Fields{
+			"path":  path,
+			"error": err,
+		}).Error("path couldn't be inspected")
+
+		return filepath.SkipDir
+	}
+
+	if ok {
+		if !c.DisableGit {
+			base := filepath.Base(path)
+			if err := c.pool.AddGitWithID(base, path); err != nil {
+				logrus.WithFields(logrus.Fields{
+					"id":    base,
+					"path":  path,
+					"error": err,
+				}).Error("repository could not be added")
+			}
+
+			logrus.WithField("path", path).Debug("repository added")
+		}
+
+		// either the repository is added or not, the path must be skipped
+		return filepath.SkipDir
 	}
 
 	return nil

common_test.go

@@ -38,11 +38,14 @@ func buildSession(t *testing.T, repos fixtures.Fixtures,
 	pool := NewRepositoryPool()
 	for _, fixture := range repos {
 		path := fixture.Worktree().Root()
-		_, err := pool.AddGit(path)
-		if err == nil {
-			_, err := pool.GetRepo(path)
-			require.NoError(err)
-			paths = append(paths, path)
+		ok, err := IsGitRepo(path)
+		require.NoError(err)
+		if ok {
+			if err := pool.AddGit(path); err == nil {
+				_, err := pool.GetRepo(path)
+				require.NoError(err)
+				paths = append(paths, path)
+			}
 		}
 	}
 

docs/using-gitbase/configuration.md

@@ -57,27 +57,27 @@ Usage:
 
 Starts a gitbase server instance
 
-By default when gitbase encounters and error in a repository it
+By default when gitbase encounters an error in a repository it
 stops the query. With GITBASE_SKIP_GIT_ERRORS variable it won't
 complain and just skip those rows or repositories.
 
 Help Options:
-  -h, --help           Show this help message
+  -h, --help             Show this help message
 
 [server command options]
-      -v               Activates the verbose mode
-      -g, --git=       Path where the git repositories are located, multiple directories can
-                       be defined. Accepts globs.
-          --siva=      Path where the siva repositories are located, multiple directories can
-                       be defined. Accepts globs.
-          --host=      Host where the server is going to listen (default: localhost)
-      -p, --port=      Port where the server is going to listen (default: 3306)
-      -u, --user=      User name used for connection (default: root)
-      -P, --password=  Password used for connection
-          --pilosa=    URL to your pilosa server (default: http://localhost:10101)
-                       [$PILOSA_ENDPOINT]
-      -i, --index=     Directory where the gitbase indexes information will be persisted.
-                       (default: /var/lib/gitbase/index) [$GITBASE_INDEX_DIR]
-          --no-squash  Disables the table squashing.
-          --trace      Enables jaeger tracing [$GITBASE_TRACE]
-```
+      -v                 Activates the verbose mode
+      -d, --directories= Path where the git repositories are located (standard and siva), multiple directories can be defined. Accepts globs.
+          --depth=       load repositories looking at less than <depth> nested subdirectories. (default: 1000)
+          --no-git       disable the load of git standard repositories.
+          --no-siva      disable the load of siva files.
+          --host=        Host where the server is going to listen (default: localhost)
+      -p, --port=        Port where the server is going to listen (default: 3306)
+      -u, --user=        User name used for connection (default: root)
+      -P, --password=    Password used for connection
+          --pilosa=      URL to your pilosa server (default: http://localhost:10101) [$PILOSA_ENDPOINT]
+      -i, --index=       Directory where the gitbase indexes information will be persisted. (default: /var/lib/gitbase/index) [$GITBASE_INDEX_DIR]
+          --no-squash    Disables the table squashing.
+          --trace        Enables jaeger tracing [$GITBASE_TRACE]
+      -r, --readonly     Only allow read queries. This disables creating and deleting indexes as well. [$GITBASE_READONLY]
+
+```

docs/using-gitbase/getting-started.md

@@ -16,7 +16,7 @@ For more info about command line arguments, [go here](/docs/using-gitbase/config
 You can start a server by providing a path which contains multiple git repositories `/path/to/repositories` with this command:
 
 ```
-$ gitbase server -v -g /path/to/repositories
+$ gitbase server -v -d /path/to/repositories
 ```
 
 ## Installing from source
@@ -83,4 +83,4 @@ As of MySQL 8.0 [the default authentication method is `caching_sha2_password`](h
 
 ```
 mysql -q -u root -h 127.0.0.1 --default-auth=mysql_native_password
-```
+```

integration_test.go

@@ -33,8 +33,7 @@ func TestIntegration(t *testing.T) {
 	path := fixtures.ByTag("worktree").One().Worktree().Root()
 
 	pool := gitbase.NewRepositoryPool()
-	_, err := pool.AddGitWithID("worktree", path)
-	require.NoError(t, err)
+	require.NoError(t, pool.AddGitWithID("worktree", path))
 
 	testCases := []struct {
 		query  string
@@ -357,8 +356,8 @@ func TestSquashCorrectness(t *testing.T) {
 		`SELECT repository_id, num_files FROM (
 			SELECT COUNT(f.*) num_files, f.repository_id
 			FROM ref_commits r
-			INNER JOIN commit_files cf 
-				ON r.commit_hash = cf.commit_hash 
+			INNER JOIN commit_files cf
+				ON r.commit_hash = cf.commit_hash
 				AND r.repository_id = cf.repository_id
 			INNER JOIN files f
 				ON cf.repository_id = f.repository_id
@@ -414,7 +413,19 @@ func TestMissingHeadRefs(t *testing.T) {
 	)
 
 	pool := gitbase.NewRepositoryPool()
-	require.NoError(pool.AddSivaDir(path))
+	require.NoError(
+		filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if gitbase.IsSivaFile(path) {
+				require.NoError(pool.AddSivaFile(path))
+			}
+
+			return nil
+		}),
+	)
 
 	engine := newBaseEngine()
 

path_utils.go

@@ -1,31 +1,29 @@
 package gitbase
 
 import (
-	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+
+	git "gopkg.in/src-d/go-git.v4"
 )
 
 // RegMatchChars matches a string with a glob expression inside.
 var RegMatchChars = regexp.MustCompile(`(^|[^\\])([*[?])`)
 
-// PatternMatches returns the depth of the fixed part of a patters, the paths
-// matched and any error found.
-func PatternMatches(pattern string) (int, []string, error) {
+// PatternMatches returns the paths matched and any error found.
+func PatternMatches(pattern string) ([]string, error) {
 	abs, err := filepath.Abs(pattern)
 	if err != nil {
-		return 0, nil, err
+		return nil, err
 	}
 
 	matches, err := filepath.Glob(abs)
 	if err != nil {
-		return 0, nil, err
+		return nil, err
 	}
 
-	depth := PatternPrefixDepth(abs)
-
-	return depth, removeDsStore(matches), nil
+	return removeDsStore(matches), nil
 }
 
 func removeDsStore(matches []string) []string {
@@ -38,46 +36,20 @@ func removeDsStore(matches []string) []string {
 	return result
 }
 
-// PatternPrefixDepth returns the number of directories before the first
-// glob expression is found.
-func PatternPrefixDepth(pattern string) int {
-	if pattern == "" {
-		return 0
-	}
-
-	parts := SplitPath(pattern)
-
-	for i, part := range parts {
-		if RegMatchChars.MatchString(part) {
-			return i
+// IsGitRepo checks that the given path is a git repository.
+func IsGitRepo(path string) (bool, error) {
+	if _, err := git.PlainOpen(path); err != nil {
+		if git.ErrRepositoryNotExists == err {
+			return false, nil
 		}
-	}
 
-	return len(parts)
-}
-
-// IDFromPath returns a repository ID from a path stripping a number of
-// directories from it.
-func IDFromPath(prefix int, path string) string {
-	parts := SplitPath(path)
-
-	if prefix >= len(parts) {
-		return path
+		return false, err
 	}
 
-	return filepath.Join(parts[prefix:]...)
+	return true, nil
 }
 
-// SplitPath slices a path in its components.
-func SplitPath(path string) []string {
-	parts := strings.Split(path, string(os.PathSeparator))
-	saneParts := make([]string, 0, len(parts))
-
-	for _, p := range parts {
-		if p != "" {
-			saneParts = append(saneParts, p)
-		}
-	}
-
-	return saneParts
+//IsSivaFile checks that the given file is a siva file.
+func IsSivaFile(file string) bool {
+	return strings.HasSuffix(file, ".siva")
 }