function: implement uast_imports UDF

Closes #943

After changes to the structure of imports, the way we used to collect
them is no longer valid, so now we're using the uast.AllImportPaths
helper included in the bblfsh sdk to obtain import paths from nodes.

Signed-off-by: Miguel Molina <[email protected]>

Author: erizocosmico

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `uast_imports` function to gather import paths from an UAST.
+
 ## [0.24.0-beta2] - 2019-07-31
 
 ### Changed

docs/using-gitbase/examples.md

@@ -227,6 +227,27 @@ As result, you will get an array showing a list of the retrieved information. Ea
 +-------------------+-----------------------------------------------------------------------------------------------+
 ```
 
+## Extracting all import paths
+
+```sql
+SELECT file_path,
+       uast_imports(uast(blob_content, LANGUAGE(file_path, blob_content))) AS imports
+FROM refs
+NATURAL JOIN commit_files
+NATURAL JOIN blobs
+WHERE ref_name = 'HEAD' AND LANGUAGE(file_path) = 'Go';
+```
+
+As result, you will get an array with an array of import paths for each node in the UAST.
+
+```sh
++-------------------------------------------------------------------------------------------------------------------+
+| file_path        | imports                                                                                        |
++-------------------+-----------------------------------------------------------------------------------------------+
+| _example/main.go | [["fmt","database/sql","github.com/sirupsen/logrus"]]                                          |
++-------------------+-----------------------------------------------------------------------------------------------+
+```
+
 ## Monitor the progress of a query
 
 You can monitor the progress of a gitbase query (either a regular query or an index creation query using `SHOW PROCESSLIST`).

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/bblfsh/go-client/v4 v4.1.0
-	github.com/bblfsh/sdk/v3 v3.1.0
+	github.com/bblfsh/sdk/v3 v3.2.2
 	github.com/gliderlabs/ssh v0.2.0 // indirect
 	github.com/go-kit/kit v0.8.0
 	github.com/go-sql-driver/mysql v1.4.1
@@ -14,7 +14,7 @@ require (
 	github.com/miekg/dns v1.1.1 // indirect
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/prometheus/client_golang v1.0.0
-	github.com/sirupsen/logrus v1.3.0
+	github.com/sirupsen/logrus v1.4.2
 	github.com/src-d/enry/v2 v2.0.0
 	github.com/src-d/go-borges v0.0.0-20190628121335-da12a84d60fd
 	github.com/src-d/go-git v4.7.0+incompatible
@@ -25,7 +25,6 @@ require (
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect
 	go.uber.org/atomic v1.4.0 // indirect
-	golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 // indirect
 	golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
 	golang.org/x/sys v0.0.0-20190618155005-516e3c20635f // indirect

go.sum

@@ -1,4 +1,5 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -33,9 +34,12 @@ github.com/bblfsh/go-client/v4 v4.1.0 h1:z9S5GrUSB0uiipbWauc5fV6D/R0LNBi65dhfP5u
 github.com/bblfsh/go-client/v4 v4.1.0/go.mod h1:UUAG7jrxSr7WHlFL/U8den1kLHfysZWJ9jA1Y0IiQ+0=
 github.com/bblfsh/sdk/v3 v3.1.0 h1:V+4DqGlqbxiAEAU0XlpTjjbq/g0ljfiFoh9jsIkfbtU=
 github.com/bblfsh/sdk/v3 v3.1.0/go.mod h1:juMiu8rP3lYJN1e4neEkSyzNieqiFceZzN4AOo0Rm1Q=
+github.com/bblfsh/sdk/v3 v3.2.2 h1:+Kr5hTK8ZklcjRQgfiMnM6JNI5faN1bsW/JZAHD8kyI=
+github.com/bblfsh/sdk/v3 v3.2.2/go.mod h1:LSY0KJDbK4tQHGIk3rEYX4sAeWgZzNqV1E3Bp30/Nrw=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
@@ -219,6 +223,8 @@ github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjM
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -230,6 +236,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/src-d/enry/v2 v2.0.0 h1:2ADqfDHhroFwL1RGhMS9e15NkEwln8P4AABwVvIdAlo=
 github.com/src-d/enry/v2 v2.0.0/go.mod h1:qQeCMRwzMF3ckeGr+h0tJLdxXnq+NVZsIDMELj0t028=
+github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
 github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
 github.com/src-d/go-borges v0.0.0-20190628121335-da12a84d60fd h1:jUbtZFWSqGX1DfD2evCwA4y7LIrWV0W8h06sjWZANf0=
@@ -278,8 +285,8 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/xJ2BjNN2jWv7JoyOsMxXUU=
-golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -289,6 +296,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227022144-312bce6e941f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -299,6 +307,7 @@ golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE=
 golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -364,6 +373,7 @@ gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzW
 gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk=
 gopkg.in/src-d/go-git.v4 v4.12.0 h1:CKgvBCJCcdfNnyXPYI4Cp8PaDDAmAPEN0CtfEdEAbd8=
 gopkg.in/src-d/go-git.v4 v4.12.0/go.mod h1:zjlNnzc1Wjn43v3Mtii7RVxiReNP0fIu9npcXKzuNp4=
+gopkg.in/src-d/go-log.v1 v1.0.2/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
 gopkg.in/src-d/go-siva.v1 v1.5.0 h1:WowvbZTlz0SPoV7WNCGktPSi2yRK78HPyXl7wYqDeHE=
 gopkg.in/src-d/go-siva.v1 v1.5.0/go.mod h1:tk1jnIXawd/PTlRNWdr5V5lC0PttNJmu1fv7wt7IZlw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

internal/function/registry.go

@@ -15,5 +15,6 @@ var Functions = []sql.Function{
 	sql.Function2{Name: "uast_xpath", Fn: NewUASTXPath},
 	sql.Function2{Name: "uast_extract", Fn: NewUASTExtract},
 	sql.Function1{Name: "uast_children", Fn: NewUASTChildren},
+	sql.Function1{Name: "uast_imports", Fn: NewUASTImports},
 	sql.Function1{Name: "is_vendor", Fn: NewIsVendor},
 }

internal/function/uast.go

@@ -759,3 +759,70 @@ func getChildren(node nodes.Object) nodes.Array {
 
 	return children
 }
+
+// UASTImports finds the imports in UAST nodes.
+type UASTImports struct {
+	expression.UnaryExpression
+}
+
+// NewUASTImports creates a new UASTImports node.
+func NewUASTImports(child sql.Expression) sql.Expression {
+	return &UASTImports{expression.UnaryExpression{Child: child}}
+}
+
+// Type implements the sql.Expression interface.
+func (f *UASTImports) Type() sql.Type {
+	return sql.Array(sql.Array(sql.Text))
+}
+
+// IsNullable implements the sql.Expression interface.
+func (f *UASTImports) IsNullable() bool { return true }
+
+// Eval implements the sql.Expression interface.
+func (f *UASTImports) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
+	span, ctx := ctx.Span("function.UASTImports")
+	defer span.Finish()
+
+	child, err := f.Child.Eval(ctx, row)
+	if err != nil {
+		return nil, err
+	}
+
+	nodes, err := getNodes(child)
+	if err != nil {
+		return nil, err
+	}
+
+	if nodes == nil {
+		return nil, nil
+	}
+
+	var result = make([]interface{}, nodes.Size())
+	for i := 0; i < nodes.Size(); i++ {
+		node := nodes.ValueAt(i)
+		imports := uast.AllImportPaths(node)
+		nodeImports := make([]interface{}, len(imports))
+		for j, imp := range imports {
+			nodeImports[j] = imp
+		}
+		result[i] = nodeImports
+	}
+
+	return result, nil
+}
+
+func (f *UASTImports) String() string {
+	return fmt.Sprintf("uast_imports(%s)", f.Child)
+}
+
+// Children implements the sql.Expression interface.
+func (f *UASTImports) Children() []sql.Expression { return []sql.Expression{f.Child} }
+
+// WithChildren implements the sql.Expression interface.
+func (f *UASTImports) WithChildren(children ...sql.Expression) (sql.Expression, error) {
+	if len(children) != 1 {
+		return nil, sql.ErrInvalidChildrenNumber.New(f, len(children), 1)
+	}
+
+	return NewUASTImports(children[0]), nil
+}

internal/function/uast_test.go

@@ -360,6 +360,47 @@ func TestExtractAnyProp(t *testing.T) {
 	require.Nil(t, props)
 }
 
+func TestUASTImports(t *testing.T) {
+	code := `
+package foo
+
+import "fmt"
+import "github.com/src-d/foo"
+import "gopkg.in/src-d/bar"
+
+func main() {
+	fmt.Println("Hello", foo.Foo, bar.Bar)
+}
+	`
+
+	ctx, cleanup := setup(t)
+	defer cleanup()
+
+	require := require.New(t)
+
+	f, err := NewUAST(
+		expression.NewLiteral([]byte(code), sql.Blob),
+		expression.NewLiteral("Go", sql.Text),
+	)
+	require.NoError(err)
+	result, err := f.Eval(ctx, nil)
+	require.NoError(err)
+	require.NotNil(result)
+
+	result, err = NewUASTImports(expression.NewLiteral(result, sql.Blob)).Eval(ctx, nil)
+	require.NoError(err)
+
+	expected := []interface{}{
+		[]interface{}{
+			"fmt",
+			"github.com/src-d/foo",
+			"gopkg.in/src-d/bar",
+		},
+	}
+
+	require.Equal(expected, result)
+}
+
 func assertUASTBlobs(t *testing.T, ctx *sql.Context, a, b interface{}) {
 	t.Helper()
 	var require = require.New(t)