Merge pull request #175 from erizocosmico/feature/blob-content

gitquery: add content column to blobs table

Author: erizocosmico

blobs.go

@@ -1,21 +1,38 @@
 package gitquery
 
 import (
+	"bufio"
 	"io"
+	"io/ioutil"
 
 	"gopkg.in/src-d/go-mysql-server.v0/sql"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 )
 
+const (
+	blobsMaxSizeKey     = "GITQUERY_BLOBS_MAX_SIZE"
+	blobsAllowBinaryKey = "GITQUERY_BLOBS_ALLOW_BINARY"
+
+	b   = 1
+	kib = 1024 * b
+	mib = 1024 * kib
+)
+
+var (
+	blobsAllowBinary = getBoolEnv(blobsAllowBinaryKey, false)
+	blobsMaxSize     = getIntEnv(blobsMaxSizeKey, 5) * mib
+)
+
 type blobsTable struct {
 	pool *RepositoryPool
 }
 
 var blobsSchema = sql.Schema{
 	{Name: "hash", Type: sql.Text, Nullable: false, Source: blobsTableName},
 	{Name: "size", Type: sql.Int64, Nullable: false, Source: blobsTableName},
+	{Name: "content", Type: sql.Blob, Nullable: false, Source: blobsTableName},
 }
 
 var _ sql.PushdownProjectionAndFiltersTable = (*blobsTable)(nil)
@@ -104,7 +121,7 @@ func (i *blobIter) Next() (sql.Row, error) {
 		return nil, err
 	}
 
-	return blobToRow(o), nil
+	return blobToRow(o)
 }
 
 func (i *blobIter) Close() error {
@@ -142,17 +159,75 @@ func (i *blobsByHashIter) Next() (sql.Row, error) {
 			return nil, err
 		}
 
-		return blobToRow(blob), nil
+		return blobToRow(blob)
 	}
 }
 
 func (i *blobsByHashIter) Close() error {
 	return nil
 }
 
-func blobToRow(c *object.Blob) sql.Row {
+func blobToRow(c *object.Blob) (sql.Row, error) {
+	var content []byte
+	var isAllowed = blobsAllowBinary
+	if !isAllowed {
+		ok, err := isBinary(c)
+		if err != nil {
+			return nil, err
+		}
+		isAllowed = !ok
+	}
+
+	if c.Size <= int64(blobsMaxSize) && isAllowed {
+		r, err := c.Reader()
+		if err != nil {
+			return nil, err
+		}
+
+		content, err = ioutil.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return sql.NewRow(
 		c.Hash.String(),
 		c.Size,
-	)
+		content,
+	), nil
+}
+
+const sniffLen = 8000
+
+// isBinary detects if data is a binary value based on:
+// http://git.kernel.org/cgit/git/git.git/tree/xdiff-interface.c?id=HEAD#n198
+func isBinary(blob *object.Blob) (bool, error) {
+	r, err := blob.Reader()
+	if err != nil {
+		return false, err
+	}
+
+	defer r.Close()
+
+	rd := bufio.NewReader(r)
+	var i int
+	for {
+		if i >= sniffLen {
+			return false, nil
+		}
+		i++
+
+		b, err := rd.ReadByte()
+		if err == io.EOF {
+			return false, nil
+		}
+
+		if err != nil {
+			return false, err
+		}
+
+		if b == 0 {
+			return true, nil
+		}
+	}
 }

blobs_test.go

@@ -49,6 +49,50 @@ func TestBlobsTable_RowIter(t *testing.T) {
 	}
 }
 
+func TestBlobsLimit(t *testing.T) {
+	require := require.New(t)
+	session, _, cleanup := setup(t)
+	defer cleanup()
+
+	prev := blobsMaxSize
+	blobsMaxSize = 200000
+	defer func() {
+		blobsMaxSize = prev
+	}()
+
+	table := newBlobsTable(session.Pool)
+	iter, err := table.RowIter(session)
+	require.NoError(err)
+
+	rows, err := sql.RowIterToRows(iter)
+	require.NoError(err)
+
+	expected := []struct {
+		hash  string
+		bytes int64
+		empty bool
+	}{
+		{"32858aad3c383ed1ff0a0f9bdf231d54a00c9e88", 189, false},
+		{"d3ff53e0564a9f87d8e84b6e28e5060e517008aa", 18, false},
+		{"c192bd6a24ea1ab01d78686e417c8bdc7c3d197f", 1072, false},
+		{"7e59600739c96546163833214c36459e324bad0a", 9, false},
+		{"d5c0f4ab811897cadf03aec358ae60d21f91c50d", 76110, true}, // is binary
+		{"880cd14280f4b9b6ed3986d6671f907d7cc2a198", 2780, false},
+		{"49c6bb89b17060d7b4deacb7b338fcc6ea2352a9", 217848, true}, // exceeds threshold
+		{"c8f1d8c61f9da76f4cb49fd86322b6e685dba956", 706, false},
+		{"9a48f23120e880dfbe41f7c9b7b708e9ee62a492", 11488, false},
+		{"9dea2395f5403188298c1dabe8bdafe562c491e3", 78, false},
+	}
+
+	require.Len(rows, len(expected))
+	for i, row := range rows {
+		e := expected[i]
+		require.Equal(e.hash, row[0].(string))
+		require.Equal(e.bytes, row[1].(int64))
+		require.Equal(e.empty, len(row[2].([]byte)) == 0)
+	}
+}
+
 func TestBlobsPushdown(t *testing.T) {
 	require := require.New(t)
 	session, _, cleanup := setup(t)

env.go

@@ -0,0 +1,27 @@
+package gitquery
+
+import (
+	"os"
+	"strconv"
+)
+
+func getIntEnv(key string, defaultValue int) int {
+	val := os.Getenv(key)
+	if val == "" {
+		return defaultValue
+	}
+	v, err := strconv.Atoi(val)
+	if err != nil {
+		return defaultValue
+	}
+	return v
+}
+
+func getBoolEnv(key string, defaultValue bool) bool {
+	_, ok := os.LookupEnv(key)
+	if ok {
+		return true
+	}
+
+	return defaultValue
+}