Merge pull request #944 from erizocosmico/feature/uast-imports

function: implement uast_imports UDF

Author: ajnavarro

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `uast_imports` function to gather import paths from an UAST.
+
 ## [0.24.0-beta2] - 2019-07-31
 
 ### Changed

docs/using-gitbase/examples.md

@@ -227,6 +227,27 @@ As result, you will get an array showing a list of the retrieved information. Ea
 +-------------------+-----------------------------------------------------------------------------------------------+
 ```
 
+## Extracting all import paths
+
+```sql
+SELECT file_path,
+       uast_imports(uast(blob_content, LANGUAGE(file_path, blob_content))) AS imports
+FROM refs
+NATURAL JOIN commit_files
+NATURAL JOIN blobs
+WHERE ref_name = 'HEAD' AND LANGUAGE(file_path) = 'Go';
+```
+
+As result, you will get an array with an array of import paths for each node in the UAST.
+
+```sh
++-------------------------------------------------------------------------------------------------------------------+
+| file_path        | imports                                                                                        |
++-------------------+-----------------------------------------------------------------------------------------------+
+| _example/main.go | [["fmt","database/sql","github.com/sirupsen/logrus"]]                                          |
++-------------------+-----------------------------------------------------------------------------------------------+
+```
+
 ## Monitor the progress of a query
 
 You can monitor the progress of a gitbase query (either a regular query or an index creation query using `SHOW PROCESSLIST`).

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/bblfsh/go-client/v4 v4.1.0
-	github.com/bblfsh/sdk/v3 v3.1.0
+	github.com/bblfsh/sdk/v3 v3.2.2
 	github.com/gliderlabs/ssh v0.2.0 // indirect
 	github.com/go-kit/kit v0.8.0
 	github.com/go-sql-driver/mysql v1.4.1
@@ -14,7 +14,7 @@ require (
 	github.com/miekg/dns v1.1.1 // indirect
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/prometheus/client_golang v1.0.0
-	github.com/sirupsen/logrus v1.3.0
+	github.com/sirupsen/logrus v1.4.2
 	github.com/src-d/enry/v2 v2.0.0
 	github.com/src-d/go-borges v0.0.0-20190628121335-da12a84d60fd
 	github.com/src-d/go-git v4.7.0+incompatible
@@ -25,7 +25,6 @@ require (
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect
 	go.uber.org/atomic v1.4.0 // indirect
-	golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 // indirect
 	golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
 	golang.org/x/sys v0.0.0-20190618155005-516e3c20635f // indirect

go.sum

@@ -1,4 +1,5 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -33,9 +34,12 @@ github.com/bblfsh/go-client/v4 v4.1.0 h1:z9S5GrUSB0uiipbWauc5fV6D/R0LNBi65dhfP5u
 github.com/bblfsh/go-client/v4 v4.1.0/go.mod h1:UUAG7jrxSr7WHlFL/U8den1kLHfysZWJ9jA1Y0IiQ+0=
 github.com/bblfsh/sdk/v3 v3.1.0 h1:V+4DqGlqbxiAEAU0XlpTjjbq/g0ljfiFoh9jsIkfbtU=
 github.com/bblfsh/sdk/v3 v3.1.0/go.mod h1:juMiu8rP3lYJN1e4neEkSyzNieqiFceZzN4AOo0Rm1Q=
+github.com/bblfsh/sdk/v3 v3.2.2 h1:+Kr5hTK8ZklcjRQgfiMnM6JNI5faN1bsW/JZAHD8kyI=
+github.com/bblfsh/sdk/v3 v3.2.2/go.mod h1:LSY0KJDbK4tQHGIk3rEYX4sAeWgZzNqV1E3Bp30/Nrw=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
@@ -219,6 +223,8 @@ github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjM
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -230,6 +236,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/src-d/enry/v2 v2.0.0 h1:2ADqfDHhroFwL1RGhMS9e15NkEwln8P4AABwVvIdAlo=
 github.com/src-d/enry/v2 v2.0.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028=
+github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
 github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
 github.com/src-d/go-borges v0.0.0-20190628121335-da12a84d60fd h1:jUbtZFWSqGX1DfD2evCwA4y7LIrWV0W8h06sjWZANf0=
@@ -278,8 +285,8 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/xJ2BjNN2jWv7JoyOsMxXUU=
-golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -289,6 +296,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227022144-312bce6e941f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -299,6 +307,7 @@ golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE=
 golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -364,6 +373,7 @@ gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzW
 gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk=
 gopkg.in/src-d/go-git.v4 v4.12.0 h1:CKgvBCJCcdfNnyXPYI4Cp8PaDDAmAPEN0CtfEdEAbd8=
 gopkg.in/src-d/go-git.v4 v4.12.0/go.mod h1:zjlNnzc1Wjn43v3Mtii7RVxiReNP0fIu9npcXKzuNp4=
+gopkg.in/src-d/go-log.v1 v1.0.2/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
 gopkg.in/src-d/go-siva.v1 v1.5.0 h1:WowvbZTlz0SPoV7WNCGktPSi2yRK78HPyXl7wYqDeHE=
 gopkg.in/src-d/go-siva.v1 v1.5.0/go.mod h1:tk1jnIXawd/PTlRNWdr5V5lC0PttNJmu1fv7wt7IZlw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

internal/function/registry.go

@@ -15,5 +15,6 @@ var Functions = []sql.Function{
 	sql.Function2{Name: "uast_xpath", Fn: NewUASTXPath},
 	sql.Function2{Name: "uast_extract", Fn: NewUASTExtract},
 	sql.Function1{Name: "uast_children", Fn: NewUASTChildren},
+	sql.Function1{Name: "uast_imports", Fn: NewUASTImports},
 	sql.Function1{Name: "is_vendor", Fn: NewIsVendor},
 }

internal/function/uast.go

@@ -759,3 +759,70 @@ func getChildren(node nodes.Object) nodes.Array {
 
 	return children
 }
+
+// UASTImports finds the imports in UAST nodes.
+type UASTImports struct {
+	expression.UnaryExpression
+}
+
+// NewUASTImports creates a new UASTImports node.
+func NewUASTImports(child sql.Expression) sql.Expression {
+	return &UASTImports{expression.UnaryExpression{Child: child}}
+}
+
+// Type implements the sql.Expression interface.
+func (f *UASTImports) Type() sql.Type {
+	return sql.Array(sql.Array(sql.Text))
+}
+
+// IsNullable implements the sql.Expression interface.
+func (f *UASTImports) IsNullable() bool { return true }
+
+// Eval implements the sql.Expression interface.
+func (f *UASTImports) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
+	span, ctx := ctx.Span("function.UASTImports")
+	defer span.Finish()
+
+	child, err := f.Child.Eval(ctx, row)
+	if err != nil {
+		return nil, err
+	}
+
+	nodes, err := getNodes(child)
+	if err != nil {
+		return nil, err
+	}
+
+	if nodes == nil {
+		return nil, nil
+	}
+
+	var result = make([]interface{}, nodes.Size())
+	for i := 0; i < nodes.Size(); i++ {
+		node := nodes.ValueAt(i)
+		imports := uast.AllImportPaths(node)
+		nodeImports := make([]interface{}, len(imports))
+		for j, imp := range imports {
+			nodeImports[j] = imp
+		}
+		result[i] = nodeImports
+	}
+
+	return result, nil
+}
+
+func (f *UASTImports) String() string {
+	return fmt.Sprintf("uast_imports(%s)", f.Child)
+}
+
+// Children implements the sql.Expression interface.
+func (f *UASTImports) Children() []sql.Expression { return []sql.Expression{f.Child} }
+
+// WithChildren implements the sql.Expression interface.
+func (f *UASTImports) WithChildren(children ...sql.Expression) (sql.Expression, error) {
+	if len(children) != 1 {
+		return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 1)
+	}
+
+	return NewUASTImports(children[0]), nil
+}

internal/function/uast_test.go

@@ -360,6 +360,47 @@ func TestExtractAnyProp(t *testing.T) {
 	require.Nil(t, props)
 }
 
+func TestUASTImports(t *testing.T) {
+	code := `
+package foo
+
+import "fmt"
+import "github.com/src-d/foo"
+import "gopkg.in/src-d/bar"
+
+func main() {
+	fmt.Println("Hello", foo.Foo, bar.Bar)
+}
+	`
+
+	ctx, cleanup := setup(t)
+	defer cleanup()
+
+	require := require.New(t)
+
+	f, err := NewUAST(
+		expression.NewLiteral([]byte(code), sql.Blob),
+		expression.NewLiteral("Go", sql.Text),
+	)
+	require.NoError(err)
+	result, err := f.Eval(ctx, nil)
+	require.NoError(err)
+	require.NotNil(result)
+
+	result, err = NewUASTImports(expression.NewLiteral(result, sql.Blob)).Eval(ctx, nil)
+	require.NoError(err)
+
+	expected := []interface{}{
+		[]interface{}{
+			"fmt",
+			"github.com/src-d/foo",
+			"gopkg.in/src-d/bar",
+		},
+	}
+
+	require.Equal(expected, result)
+}
+
 func assertUASTBlobs(t *testing.T, ctx *sql.Context, a, b interface{}) {
 	t.Helper()
 	var require = require.New(t)