initial commit

Author: rvagg

.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+config.json

.jshintrc

@@ -0,0 +1,61 @@
+{
+    "predef": [ ]
+  , "bitwise": false
+  , "camelcase": false
+  , "curly": false
+  , "eqeqeq": false
+  , "forin": false
+  , "immed": false
+  , "latedef": false
+  , "newcap": true
+  , "noarg": true
+  , "noempty": true
+  , "nonew": true
+  , "plusplus": false
+  , "quotmark": true
+  , "regexp": false
+  , "undef": true
+  , "unused": true
+  , "strict": false
+  , "trailing": true
+  , "maxlen": 120
+  , "asi": true
+  , "boss": true
+  , "debug": true
+  , "eqnull": true
+  , "es5": true
+  , "esnext": true
+  , "evil": true
+  , "expr": true
+  , "funcscope": false
+  , "globalstrict": false
+  , "iterator": false
+  , "lastsemic": true
+  , "laxbreak": true
+  , "laxcomma": true
+  , "loopfunc": true
+  , "multistr": false
+  , "onecase": false
+  , "proto": false
+  , "regexdash": false
+  , "scripturl": true
+  , "smarttabs": false
+  , "shadow": false
+  , "sub": true
+  , "supernew": false
+  , "validthis": true
+  , "browser": true
+  , "couch": false
+  , "devel": false
+  , "dojo": false
+  , "mootools": false
+  , "node": true
+  , "nonstandard": true
+  , "prototypejs": false
+  , "rhino": false
+  , "worker": true
+  , "wsh": false
+  , "nomen": false
+  , "onevar": true
+  , "passfail": false
+}

LICENSE

@@ -0,0 +1,39 @@
+Copyright 2013, Rod Vagg (the "Original Author")
+All rights reserved.
+
+MIT +no-false-attribs License
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+Distributions of all or part of the Software intended to be used
+by the recipients as they would use the unmodified Software,
+containing modifications that substantially alter, remove, or
+disable functionality of the Software, outside of the documented
+configuration mechanisms provided by the Software, shall be
+modified such that the Original Author's bug reporting email
+addresses and urls are either replaced with the contact information
+of the parties responsible for the changes, or removed entirely.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+
+Except where noted, this license applies to any and all software
+programs and associated documentation files created by the
+Original Author, when distributed with the Software.

fetch-github-data.js

@@ -0,0 +1,29 @@
+const GITHUB_USER_API_URL = 'https://api.github.com/users/{user}'
+
+const request = require('request')
+
+var requestPool = { maxSockets: 20 }
+
+function fetchGithubData (githubToken, user, callback) {
+
+  var opts = {
+          url     : GITHUB_USER_API_URL.replace('{user}', user)
+        , headers : {
+              authorization : 'token ' + githubToken
+            , 'user-agent'  : 'npm user data fetcher <https://github.com/polyhack/>'
+          }
+        , json    : true
+        , pool    : requestPool
+      }
+
+    , handle = function (err, response, body) {
+        if (err)
+          return callback('Error requesting repo data from GitHub for ' + user + ': ' + err)
+
+        callback(null, body)
+      }
+
+  request(opts, handle)
+}
+
+module.exports = fetchGithubData

filter-aussies.js

@@ -0,0 +1,17 @@
+const AU_LOCATION_REGEX =
+        /\Wau(s|st)?\W|australia|straya|hobart|sydney|melbourne|brisbane|perth|darwin|adelaide|canberra|\W(nsw|vic|qld|new south wales|victoria|queensland|western australia|northern territory|south australia|tasmania)\W/i
+      // secondary guess, does your blog UI end with .au?
+    , AU_BLOG_REGEX     = /\.au$/i
+
+function isAussie (user) {
+  if (!user) return false
+  if (user.location && AU_LOCATION_REGEX.test(user.location)) return true
+  if (user.blog && AU_BLOG_REGEX.test(user.blog)) return true
+  return false
+}
+
+function filterAussies (users) {
+  return users.filter(isAussie)
+}
+
+module.exports = filterAussies

index.js

@@ -0,0 +1,88 @@
+const GITHUB_CALLS_PER_HOUR = 4500
+
+const fs              = require('fs')
+    , async           = require('async')
+    , rateLimit       = require('function-rate-limit')
+    , loadNpmData     = require('./load-npm-data')
+    , filterAussies   = require('./filter-aussies')
+    , fetchGithubData = rateLimit(
+          GITHUB_CALLS_PER_HOUR
+        , 1000 * 60 * 60
+        , require('./fetch-github-data')
+      )
+    , config          = require('./config')
+
+/* ./config.json needs to look like this:
+{
+    "allPackagesOutput"  : "/path/to/allpackages.json"
+  , "repositoriesOutput" : "/path/to/repositories.json"
+  , "githubOutput"       : "/path/to/githubusers.json"
+  , "githubAuthToken"    : "yourgithubauthtoken"
+}
+
+where githubAuthToken can be obtained with something like:
+curl -i -u <your_username> -d '{"scopes": ["repo"]}' https://api.github.com/authorizations
+(not that it needs "repo" scope)
+*/
+
+function write (location, data) {
+  fs.writeFile(
+      location
+    , JSON.stringify(data, null, 2)
+    , function (err) {
+        if (err)
+          console.error(err)
+        console.log('Wrote', location)
+      }
+  )
+}
+
+function githubUsers (repositories) {
+  var users   = []
+
+  repositories.forEach(function (repo) {
+    if (users.indexOf(repo.githubUser) == -1)
+      users.push(repo.githubUser)
+  })
+
+  return users
+}
+
+function fetchUsers (users, callback) {
+  async.map(
+      users
+    , function (user, callback) {
+        fetchGithubData(config.githubAuthToken, user, function (err, data) {
+          if (err || !user) {
+            console.error(err || 'No user data for ' + user)
+            return callback()
+          }
+
+          if (user.message) {
+            console.error('GitHub:', user.message)
+            return callback()
+          }
+
+          callback(null, data)
+        })
+      }
+    , callback
+  )
+}
+
+loadNpmData(function (err, data) {
+  if (err)
+    return console.error(err)
+
+  write(config.allPackagesOutput, data.allPackages)
+  write(config.repositoriesOutput, data.repositories)
+
+  fetchUsers(githubUsers(data.repositories), function (err, data) {
+    if (err)
+      return console.error(err)
+
+    write(config.githubOutput, data)
+
+    write(config.aussieOutput, filterAussies(data))
+  })
+})

load-npm-data.js

@@ -0,0 +1,48 @@
+// if your GitHub location field matches this then we'll guess you're Aussie
+const GITHUB_REPO_REGEX = /github.com[:\/]([\.\-\w]+)\/([^$\/\.]+)/
+
+const npm = require('npm')
+
+function matchGitHubRepo (npmPackage, repo) {
+  var match = repo
+    && typeof repo.url == 'string'
+    && repo.url.match(GITHUB_REPO_REGEX)
+
+  return match && {
+      githubUser : match[1]
+    , githubRepo : match[2]
+    , npmPackage : npmPackage
+  }
+}
+
+// load the list of all npm libs with 'repo' pointing to GitHub
+function loadNpmData (callback) {
+  var repositories = []
+    , allPackages  = []
+
+  npm.load(function (err) {
+    if (err) return callback(err)
+
+    npm.registry.get('/-/all', function (err, data) {
+      if (err) return callback(err)
+
+      Object.keys(data).forEach(function (k) {
+        var repo = matchGitHubRepo(data[k].name, data[k].repository)
+        if (repo)
+          repositories.push(repo)
+
+        allPackages.push({
+            name        : data[k].name
+          , maintainers : (data[k].maintainers || []).map(function (m) { return m.name })
+          , githubUser  : repo ? repo.githubUser : null
+          , githubRepo  : repo ? repo.githubRepo : null
+          , description : data[k].description
+        })
+      })
+
+      callback(null, { repositories: repositories, allPackages: allPackages })
+    })
+  })
+}
+
+module.exports = loadNpmData

package.json

@@ -0,0 +1,16 @@
+{
+  "name": "npm-github-data-collector",
+  "version": "0.0.0",
+  "description": "",
+  "main": "index.js",
+  "repository": "",
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "npm": "~1.2.18",
+    "async": "~0.2.7",
+    "request": "~2.20.0",
+    "function-rate-limit": "0.0.1"
+  },
+  "private": true
+}