[Prototype] Blame func (#844)

[Prototype] Blame func

Author: ajnavarro

CHANGELOG.md

@@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Added BLAME function.
+
 ## [0.24.0-rc2] - 2019-10-02
 
 ## Fixed

docs/using-gitbase/examples.md

@@ -200,6 +200,41 @@ The output will be similar to this:
 +-----------------+--------------------------------------+---------------+------------------+--------------------+
 ```
 
+## Get miscelaneous information about lines with a "// TODO" comment in HEAD
+
+```sql
+SELECT repository_id,
+       JSON_UNQUOTE(JSON_EXTRACT(bl, "$.commit")),
+       JSON_UNQUOTE(JSON_EXTRACT(bl, "$.file")),
+       JSON_UNQUOTE(JSON_EXTRACT(bl, "$.linenum")),
+       JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author")),
+       JSON_UNQUOTE(JSON_EXTRACT(bl, "$.text"))
+FROM   (SELECT repository_id,
+               EXPLODE(BLAME(repository_id, commit_hash)) AS bl
+        FROM   ref_commits
+               NATURAL JOIN blobs
+        WHERE  ref_name = 'HEAD'
+               AND NOT IS_BINARY(blob_content)
+        ) as p
+WHERE  JSON_EXTRACT(bl, "$.text") LIKE '%// TODO%';
+```
+
+## Report of authors with more lines authored in HEAD
+
+```sql
+SELECT
+       JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author")),
+       COUNT(JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author")))
+
+FROM   (SELECT EXPLODE(BLAME(repository_id, commit_hash)) AS bl
+        FROM   ref_commits
+               NATURAL JOIN blobs
+        WHERE  ref_name = 'HEAD'
+               AND NOT IS_BINARY(blob_content)
+        ) AS p
+GROUP BY JSON_UNQUOTE(JSON_EXTRACT(bl, "$.author"));
+```
+
 # UAST UDFs Examples
 
 First of all, you should check out the [bblfsh documentation](https://docs.sourced.tech/babelfish) to get yourself familiar with UAST concepts.

docs/using-gitbase/functions.md

@@ -6,19 +6,21 @@ To make some common tasks easier for the user, there are some functions to inter
 
 |     Name     |                                               Description                                                                      |
 |:-------------|:-------------------------------------------------------------------------------------------------------------------------------|
-|`commit_stats(repository_id, [from_commit_hash], to_commit_hash) json`|returns the stats between two commits for a repository. If `from_commit_hash` is empty, it will compare the given `to_commit_hash` with its parent commit. Vendored files stats are not included in the result of this function. This function is more thoroughly explained later in this document.|
+|`blame(repository, commit)`|Returns an array of lines changes and authorship.                                                                 |
 |`commit_file_stats(repository_id, [from_commit_hash], to_commit_hash) json array`|returns an array with the stats of each file in `to_commit_hash` since the given `from_commit_hash`. If `from_commit_hash` is not given, the parent commit will be used. Vendored files stats are not included in the result of this function. This function is more thoroughly explained later in this document.|
-|`is_remote(reference_name)bool`| checks if the given reference name is from a remote one.                                                          |
-|`is_tag(reference_name)bool`| checks if the given reference name is a tag.                                                                         |
-|`is_vendor(file_path)bool`| checks if the given file name is a vendored file.                                                                         |
+|`commit_stats(repository_id, [from_commit_hash], to_commit_hash) json`|returns the stats between two commits for a repository. If `from_commit_hash` is empty, it will compare the given `to_commit_hash` with its parent commit. Vendored files stats are not included in the result of this function. This function is more thoroughly explained later in this document.|
+|`is_remote(reference_name)bool`| checks if the given reference name is from a remote one.                                                         |
+|`is_tag(reference_name)bool`| checks if the given reference name is a tag.                                                                     |
+|`is_vendor(file_path)bool`| checks if the given file name is a vendored file.                                                                  |
 |`language(path, [blob])text`| gets the language of a file given its path and the optional content of the file.                                    |
+|`loc(path, blob) json`| returns a JSON map, containing the lines of code of a file, separated in three categories: Code, Blank and Comment lines. |
 |`uast(blob, [lang, [xpath]]) blob`| returns a node array of UAST nodes in semantic mode.                                                          |
+|`uast_children(blob) blob`| returns a flattened array of the children UAST nodes from each one of the UAST nodes in the given array.              |
+|`uast_extract(blob, key) text array`| extracts information identified by the given key from the uast nodes.                                       |
 |`uast_mode(mode, blob, lang) blob`| returns a node array of UAST nodes specifying its language and mode (semantic, annotated or native).          |
 |`uast_xpath(blob, xpath) blob`| performs an XPath query over the given UAST nodes.                                                                |
-|`uast_extract(blob, key) text array`| extracts information identified by the given key from the uast nodes.                                       |
-|`uast_children(blob) blob`| returns a flattened array of the children UAST nodes from each one of the UAST nodes in the given array.              |
-|`loc(path, blob) json`| returns a JSON map, containing the lines of code of a file, separated in three categories: Code, Blank and Comment lines. |
 |`version() text`| returns the gitbase version in the following format `8.0.11-{GITBASE_VERSION}` for compatibility with MySQL versioning. |
+
 ## Standard functions
 
 These are all functions that are available because they are implemented in `go-mysql-server`, used by gitbase.
@@ -36,10 +38,10 @@ These are all functions that are available because they are implemented in `go-m
 |`CONCAT_WS(sep, ...)`| concatenates any group of fields into a single string. The first argument is the separator for the rest of the arguments. The separator is added between the strings to be concatenated. The separator can be a string, as can the rest of the arguments. If the separator is NULL, the result is NULL.|
 |`CONNECTION_ID()`| returns the current connection ID.|
 |`COUNT(expr)`|  returns a count of the number of non-NULL values of expr in the rows retrieved by a SELECT statement.|
+|`DATE(date)`| returns the date part of the given `date`.|
 |`DATE_ADD(date, interval)`| adds the interval to the given `date`.|
 |`DATE_SUB(date, interval)`| subtracts the interval from the given `date`.|
 |`DAY(date)`| is a synonym for DAYOFMONTH().|
-|`DATE(date)`| returns the date part of the given `date`.|
 |`DAYOFMONTH(date)`| returns the day of the month (0-31).|
 |`DAYOFWEEK(date)`| returns the day of the week of the given `date`.|
 |`DAYOFYEAR(date)`| returns the day of the year of the given `date`.|

go.mod

@@ -31,6 +31,7 @@ require (
 	gopkg.in/src-d/go-billy-siva.v4 v4.6.0
 	gopkg.in/src-d/go-billy.v4 v4.3.2
 	gopkg.in/src-d/go-errors.v1 v1.0.0
+	gopkg.in/src-d/go-git-fixtures.v3 v3.5.0
 	gopkg.in/src-d/go-git.v4 v4.12.0
 	gopkg.in/yaml.v2 v2.2.2
 	vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible

go.sum

@@ -244,7 +244,8 @@ github.com/src-d/enry/v2 v2.0.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj
 github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
 github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
-github.com/src-d/go-borges v0.1.3 h1:EgNqVy6Mw6wT3mx5AIUWjOAJJJQ6yuiDimi8JV/R2hE=
+github.com/src-d/go-borges v0.1.1 h1:URkX6ycDzodt5n6mAF54cdjmmAMSWIyuL12ga/E5/sM=
+github.com/src-d/go-borges v0.1.1/go.mod h1:q/ufK0FXUS0ngH1Crd1TyirdtUM4dl/5ky2Oxiaw4YA=
 github.com/src-d/go-borges v0.1.3/go.mod h1:2lbHENELjpD+6bTOOnrnGEpVPMmO+wBGtYwzvQdr/Zc=
 github.com/src-d/go-git v4.7.0+incompatible h1:IYSSnbAHeKmsfbQFi9ozbid+KNh0bKjlorMfQehQbcE=
 github.com/src-d/go-git v4.7.0+incompatible/go.mod h1:1bQciz+hn0jzPQNsYj0hDFZHLJBdV7gXE2mWhC7EkFk=
@@ -330,7 +331,9 @@ golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
+golang.org/x/sys v0.0.0-20190609082536-301114b31cce/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190618155005-516e3c20635f h1:dHNZYIYdq2QuU6w73vZ/DzesPbVlZVYZTtTZmrnsbQ8=
+golang.org/x/sys v0.0.0-20190618155005-516e3c20635f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -363,12 +366,12 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/src-d/go-billy-siva.v4 v4.6.0 h1:HO5m7lqYewIZ3Otay3IkQg3gFznW8Gy9HIbHWm1mYX0=
+gopkg.in/src-d/go-billy-siva.v4 v4.5.1 h1:+UdpGGmJjANhXwg6TCcTVbACUqsbtX19QvJ9AdeX4ts=
+gopkg.in/src-d/go-billy-siva.v4 v4.5.1/go.mod h1:4wKeCzOCSsdyFeM5+58M6ObU6FM+lZT12p7zm7A+9n0=
 gopkg.in/src-d/go-billy-siva.v4 v4.6.0/go.mod h1:EcgzPxovlWGD+lZFFriUleL3EVZ/SPs6CH2FOE/eooI=
 gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
 gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek=
 gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
-gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
 gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
 gopkg.in/src-d/go-errors.v1 v1.0.0 h1:cooGdZnCjYbeS1zb1s6pVAAimTdKceRrpn7aKOnNIfc=
 gopkg.in/src-d/go-errors.v1 v1.0.0/go.mod h1:q1cBlomlw2FnDBDNGlnh6X0jPihy+QxZfMMNxPCbdYg=
@@ -379,7 +382,8 @@ gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYC
 gopkg.in/src-d/go-git.v4 v4.12.0 h1:CKgvBCJCcdfNnyXPYI4Cp8PaDDAmAPEN0CtfEdEAbd8=
 gopkg.in/src-d/go-git.v4 v4.12.0/go.mod h1:zjlNnzc1Wjn43v3Mtii7RVxiReNP0fIu9npcXKzuNp4=
 gopkg.in/src-d/go-log.v1 v1.0.2/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
-gopkg.in/src-d/go-siva.v1 v1.7.0 h1:igjgSEFweZ2kEfRlGEJH767o8GJRiPWp8JmHDCe0Vdk=
+gopkg.in/src-d/go-siva.v1 v1.5.0 h1:WowvbZTlz0SPoV7WNCGktPSi2yRK78HPyXl7wYqDeHE=
+gopkg.in/src-d/go-siva.v1 v1.5.0/go.mod h1:tk1jnIXawd/PTlRNWdr5V5lC0PttNJmu1fv7wt7IZlw=
 gopkg.in/src-d/go-siva.v1 v1.7.0/go.mod h1:ChxMHSRkICHZ9IbTlG3ihkuG7gc2RZPsIYh7OaXYvic=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE=

integration_test.go

@@ -509,6 +509,18 @@ func TestIntegration(t *testing.T) {
 				{"vendor/foo.go"},
 			},
 		},
+		{
+			`
+				SELECT repository_id, JSON_EXTRACT(bl, "$.author"),
+					   COUNT(bl)
+				FROM (
+					SELECT repository_id, EXPLODE(BLAME(repository_id, commit_hash)) as bl
+					FROM commits
+					WHERE  commit_hash = '918c48b83bd081e863dbe1b80f8998f058cd8294'
+				) as p
+			`,
+			[]sql.Row{{"worktree", "[email protected]", int64(7235)}},
+		},
 	}
 
 	var pid uint64

internal/function/blame.go

@@ -0,0 +1,187 @@
+package function
+
+import (
+	"fmt"
+
+	"github.com/src-d/gitbase"
+	"github.com/src-d/go-mysql-server/sql"
+	"gopkg.in/src-d/go-git.v4"
+
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+type BlameGenerator struct {
+	commit  *object.Commit
+	fIter   *object.FileIter
+	curLine int
+	curFile *object.File
+	lines   []*git.Line
+}
+
+func NewBlameGenerator(c *object.Commit, f *object.FileIter) (*BlameGenerator, error) {
+	return &BlameGenerator{commit: c, fIter: f, curLine: -1}, nil
+}
+
+func (g *BlameGenerator) loadNewFile() error {
+	var err error
+	g.curFile, err = g.fIter.Next()
+	if err != nil {
+		return err
+	}
+
+	result, err := git.Blame(g.commit, g.curFile.Name)
+	if err != nil {
+		return err
+	}
+
+	if len(result.Lines) == 0 {
+		return g.loadNewFile()
+	}
+
+	g.lines = result.Lines
+	g.curLine = 0
+	return nil
+}
+
+func (g *BlameGenerator) Next() (interface{}, error) {
+	if g.curLine == -1 || g.curLine >= len(g.lines) {
+		err := g.loadNewFile()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	l := g.lines[g.curLine]
+	b := BlameLine{
+		Commit:  g.commit.Hash.String(),
+		File:    g.curFile.Name,
+		LineNum: g.curLine,
+		Author:  l.Author,
+		Text:    l.Text,
+	}
+	g.curLine++
+	return b, nil
+}
+
+func (g *BlameGenerator) Close() error {
+	g.fIter.Close()
+	return nil
+}
+
+var _ sql.Generator = (*BlameGenerator)(nil)
+
+type (
+	// Blame implements git-blame function as UDF
+	Blame struct {
+		repo   sql.Expression
+		commit sql.Expression
+	}
+
+	// BlameLine represents each line of git blame's output
+	BlameLine struct {
+		Commit  string `json:"commit"`
+		File    string `json:"file"`
+		LineNum int    `json:"linenum"`
+		Author  string `json:"author"`
+		Text    string `json:"text"`
+	}
+)
+
+// NewBlame constructor
+func NewBlame(repo, commit sql.Expression) sql.Expression {
+	return &Blame{repo, commit}
+}
+
+func (b *Blame) String() string {
+	return fmt.Sprintf("blame(%s, %s)", b.repo, b.commit)
+}
+
+// Type implements the sql.Expression interface
+func (*Blame) Type() sql.Type {
+	return sql.Array(sql.JSON)
+}
+
+func (b *Blame) WithChildren(children ...sql.Expression) (sql.Expression, error) {
+	if len(children) != 2 {
+		return nil, sql.ErrInvalidChildrenNumber.New(b, len(children), 2)
+	}
+
+	return NewBlame(children[0], children[1]), nil
+}
+
+// Children implements the Expression interface.
+func (b *Blame) Children() []sql.Expression {
+	return []sql.Expression{b.repo, b.commit}
+}
+
+// IsNullable implements the Expression interface.
+func (b *Blame) IsNullable() bool {
+	return b.repo.IsNullable() || (b.commit.IsNullable())
+}
+
+// Resolved implements the Expression interface.
+func (b *Blame) Resolved() bool {
+	return b.repo.Resolved() && b.commit.Resolved()
+}
+
+// Eval implements the sql.Expression interface.
+func (b *Blame) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
+	span, ctx := ctx.Span("gitbase.Blame")
+	defer span.Finish()
+
+	repo, err := b.resolveRepo(ctx, row)
+	if err != nil {
+		ctx.Warn(0, err.Error())
+		return nil, nil
+	}
+
+	commit, err := b.resolveCommit(ctx, repo, row)
+	if err != nil {
+		ctx.Warn(0, err.Error())
+		return nil, nil
+	}
+
+	fIter, err := commit.Files()
+	if err != nil {
+		return nil, err
+	}
+
+	bg, err := NewBlameGenerator(commit, fIter)
+	if err != nil {
+		return nil, err
+	}
+
+	return bg, nil
+}
+
+func (b *Blame) resolveCommit(ctx *sql.Context, repo *gitbase.Repository, row sql.Row) (*object.Commit, error) {
+	str, err := exprToString(ctx, b.commit, row)
+	if err != nil {
+		return nil, err
+	}
+
+	commitHash, err := repo.ResolveRevision(plumbing.Revision(str))
+	if err != nil {
+		h := plumbing.NewHash(str)
+		commitHash = &h
+	}
+	to, err := repo.CommitObject(*commitHash)
+	if err != nil {
+		return nil, err
+	}
+
+	return to, nil
+}
+
+func (b *Blame) resolveRepo(ctx *sql.Context, r sql.Row) (*gitbase.Repository, error) {
+	repoID, err := exprToString(ctx, b.repo, r)
+	if err != nil {
+		return nil, err
+	}
+	s, ok := ctx.Session.(*gitbase.Session)
+	if !ok {
+		return nil, gitbase.ErrInvalidGitbaseSession.New(ctx.Session)
+	}
+	return s.Pool.GetRepo(repoID)
+}