Daniel Whitenack - Go For Data Science

Author: dwhitena

DanielWhitenack-GoForDataScience/code/counts/counts.go

@@ -0,0 +1,146 @@
+// All material is licensed under the Apache License Version 2.0, January 2004
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// This example demonstrates how to train a regression model in Go.  The example
+// also prints out formatted results and saves two plot: (i) a plot of the raw input
+// data, and (ii) a plot of the trained function overlaid on the raw input data.
+// The input data is data about Go github repositories gathered via getrepos.go.
+package main
+
+import (
+	"bytes"
+	"encoding/csv"
+	"log"
+	"sort"
+	"time"
+
+	"github.com/gonum/plot"
+	"github.com/gonum/plot/plotter"
+	"github.com/gonum/plot/plotutil"
+	"github.com/gonum/plot/vg"
+	"github.com/pachyderm/pachyderm/src/client"
+	"github.com/pkg/errors"
+)
+
+func main() {
+
+	// Aggregate the counts of created repos per day over all days.
+	counts, err := prepareCountData("repodata")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Create and save the plot showing the time series of
+	// observed daily created repo counts.
+	xys := preparePlotData(counts)
+	if err = makePlots(xys); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// prepareCountData transforms the dataset into a series of
+// daily count values for plotting.
+func prepareCountData(dataSet string) ([][]int, error) {
+
+	// Create a map to store the daily counts of created repos.
+	countMap := make(map[int]int)
+
+	// Get the data set we stored in pachyderm.
+	data, err := getDataSet(dataSet, "master", "godata")
+	if err != nil {
+		return [][]int{}, errors.Wrap(err, "Could not get data from pachyderm")
+	}
+
+	// Extract the records from the data.
+	reader := csv.NewReader(bytes.NewReader(data.Bytes()))
+	reader.FieldsPerRecord = -1
+	records, err := reader.ReadAll()
+	if err != nil {
+		return [][]int{}, errors.Wrap(err, "Could not read in data records.")
+	}
+
+	// Create a map of daily created repos where the keys are the days and
+	// the values are the counts of created repos on that day.
+	startTime := time.Date(2013, time.January, 1, 0, 0, 0, 0, time.UTC)
+	layout := "2006-01-02 15:04:05"
+	for _, each := range records {
+		t, err := time.Parse(layout, each[2][0:19])
+		if err != nil {
+			return [][]int{}, errors.Wrap(err, "Could not parse timestamps")
+		}
+		interval := int(t.Sub(startTime).Hours() / 24.0)
+		countMap[interval]++
+	}
+
+	// Sort the day values which is required for plotting.
+	var keys []int
+	for k := range countMap {
+		keys = append(keys, k)
+	}
+	sort.Ints(keys)
+	var sortedCounts [][]int
+	for _, k := range keys {
+		sortedCounts = append(sortedCounts, []int{k, countMap[k]})
+	}
+
+	return sortedCounts, nil
+}
+
+// getDataSet gets a previously stored dataset from pachyderm data versioning.
+func getDataSet(dataSet, branch, repoName string) (bytes.Buffer, error) {
+
+	// Open a connection to pachyderm running on localhost.
+	c, err := client.NewFromAddress("localhost:30650")
+	if err != nil {
+		return bytes.Buffer{}, errors.Wrap(err, "Could not connect to Pachyderm")
+	}
+
+	// Read the latest commit of filename to the given repoName.
+	var buffer bytes.Buffer
+	if err := c.GetFile(repoName, branch, dataSet, 0, 0, "", nil, &buffer); err != nil {
+		return buffer, errors.Wrap(err, "Could not retrieve pachyderm file")
+	}
+
+	return buffer, nil
+}
+
+// preparePlotData prepares the raw input data for plotting.
+func preparePlotData(counts [][]int) plotter.XYs {
+	pts := make(plotter.XYs, len(counts))
+	var i int
+
+	for _, count := range counts {
+		pts[i].X = float64(count[0])
+		pts[i].Y = float64(count[1])
+		i++
+	}
+
+	return pts
+}
+
+// makePlots creates and saves the first of our plots showing the raw input data.
+func makePlots(xys plotter.XYs) error {
+
+	// Create a new plot.
+	p, err := plot.New()
+	if err != nil {
+		return errors.Wrap(err, "Could not create plot object")
+	}
+
+	// Label the new plot.
+	p.Title.Text = "Daily Counts of Go Repos Created"
+	p.X.Label.Text = "Days from Jan. 1, 2013"
+	p.Y.Label.Text = "Count"
+
+	// Add the prepared points to the plot.
+	if err = plotutil.AddLinePoints(p, "Counts", xys); err != nil {
+		return errors.Wrap(err, "Could not add lines to plot")
+	}
+
+	// Save the plot to a PNG file.
+	if err := p.Save(7*vg.Inch, 4*vg.Inch, "countseries.png"); err != nil {
+		return errors.Wrap(err, "Could not output plot")
+	}
+
+	return nil
+}

DanielWhitenack-GoForDataScience/code/getrepos/getrepos.go

@@ -0,0 +1,133 @@
+// All material is licensed under the Apache License Version 2.0, January 2004
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// This example demonstrates how to scrape data about Go github repositories
+// using the github.com/google/go-github/github package.
+package main
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"time"
+
+	"github.com/google/go-github/github"
+	"github.com/pkg/errors"
+)
+
+func main() {
+
+	// Capture the date from which we will start scraping github data.
+	t1 := time.Date(2013, time.January, 1, 0, 0, 0, 0, time.UTC)
+
+	// Pull the github data from t1 to now.
+	if err := queryFromStartTime(t1); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// queryFromStartTime queries github for all 2 day time ranges of repo create
+// dates from a start time until now.
+func queryFromStartTime(t1 time.Time) error {
+
+	// Create a new github client.
+	client := github.NewClient(nil)
+
+	// Loop over 2 days periods from the start time until now such that
+	// the number of github repos we return is less than the 10 pages X
+	// 100 results limit of the github API.
+	for t1.Unix() < time.Now().Unix() {
+
+		// Create the github query string.
+		t2 := t1.Add(time.Hour * 24 * 2)
+		tString := fmt.Sprintf("\"%d-%02d-%02d .. %d-%02d-%02d\"",
+			t1.Year(), t1.Month(), t1.Day(),
+			t2.Year(), t2.Month(), t2.Day())
+		query := fmt.Sprintf("language:Go created:" + tString)
+
+		// Query github with our new query.
+		err := clientQuery(client, query)
+		if err != nil {
+			errors.Wrap(err, "Could not search Github repos")
+		}
+
+		// Increment the start time of the 2 day chunk.
+		t1 = t1.Add(time.Hour * 24 * 2)
+	}
+
+	return nil
+}
+
+// clientQuery executes github queries and searches over all pages of a result
+// set parsing results.
+func clientQuery(gh *github.Client, query string) error {
+
+	// Set the github search page we are currently searching.
+	page := 1
+
+	// Set the maximum page of github results for a given query.
+	maxPage := math.MaxInt32
+
+	// Set the github search options for the query.
+	opts := github.SearchOptions{
+		Sort:  "stars",
+		Order: "desc",
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+
+	// Loop over pages in the github results so we can gather all
+	// of the repositories in the search results.
+	for page <= maxPage {
+
+		// We can utilized the Search.Repositories method to Query github
+		// for a particular page of results.
+		opts.Page = page
+		result, response, err := gh.Search.Repositories(query, &opts)
+		if err != nil {
+			return errors.Wrap(err, "Could not search Github result pages")
+		}
+
+		// Wait for the results.
+		Wait(response)
+		maxPage = response.LastPage
+
+		// Then we loop over the repositories in the search results.
+		for _, repo := range result.Repositories {
+
+			// Extract the data of interest.
+			name := *repo.FullName
+			updatedAt := repo.UpdatedAt.String()
+			createdAt := repo.CreatedAt.String()
+			forks := *repo.ForksCount
+			issues := *repo.OpenIssuesCount
+			stars := *repo.StargazersCount
+			size := *repo.Size
+
+			// Print out the results for now.  However, this can be
+			// redirected to a CSV file if desired.
+			fmt.Printf("%s,%s,%s,%d,%d,%d,%d\n",
+				name, updatedAt, createdAt, forks, issues, stars, size)
+		}
+
+		// Sleep for 10 seconds to stay within Github's rate limiting
+		// constraints, then it increments the page and query again.
+		time.Sleep(time.Second * 10)
+		page++
+	}
+
+	return nil
+}
+
+// Wait waits to make sure we return the full github response.
+func Wait(response *github.Response) {
+	if response != nil && response.Remaining <= 1 {
+		gap := time.Duration(response.Reset.Local().Unix() - time.Now().Unix())
+		sleep := gap * time.Second
+		if sleep < 0 {
+			sleep = -sleep
+		}
+		time.Sleep(sleep)
+	}
+}