Clone your starred GitHub repositories via Node.js script

GitHub API makes it easy to access lists of things, such as starred repositories which is used in this example.

The idea is to clone all those repositories which the current user (me) has starred, thus marking them something to keep an ey on. To accomplish this, some steps are needed:

Get list of starred repositories
Clone each of them

Not that many steps after all…

Get a list of starred repositories

As stated in the GitHub API documentation, the following request fetches the starred repositories of the currently authenticated user.

GET /user/starred

Thus using it with password based Basic Authentication, it could be used like this:

curl -u paazmaya https://api.github.com/user/starred \
    -o github-paazmaya-starred.json

In order to get your own stars, substitute paazmaya with your own GitHub username. The results are saved in a JSON file called github-paazmaya-starred.json.

The resulting JSON is an array, in which each object represents a given starred repository. By default they are ordered descending, meaning that the most recently starred repository will be listed on the top.

For example today I starred FontCustom/fontcustom, thus is the first item. A snippet shown below.

[
  {
    "id": 6118575,
    "name": "fontcustom",
    "full_name": "FontCustom/fontcustom",
    "owner": {
      "login": "FontCustom",
      "id": 3375656,
      "avatar_url": "https://identicons.github.com/cb82e67691313959335bfed01089f85c.png",
      "gravatar_id": null,
      "url": "https://api.github.com/users/FontCustom",
      "html_url": "https://github.com/FontCustom",
      "followers_url": "https://api.github.com/users/FontCustom/followers",
      "following_url": "https://api.github.com/users/FontCustom/following{/other_user}",
      "gists_url": "https://api.github.com/users/FontCustom/gists{/gist_id}",
      "starred_url": "https://api.github.com/users/FontCustom/starred{/owner}{/repo}",
      "subscriptions_url": "https://api.github.com/users/FontCustom/subscriptions",
      "organizations_url": "https://api.github.com/users/FontCustom/orgs",
      "repos_url": "https://api.github.com/users/FontCustom/repos",
      "events_url": "https://api.github.com/users/FontCustom/events{/privacy}",
      "received_events_url": "https://api.github.com/users/FontCustom/received_events",
      "type": "Organization",
      "site_admin": false
    },
    "private": false,
    "html_url": "https://github.com/FontCustom/fontcustom",
    "description": "Generate custom icon webfonts from the comfort of the command line.",
    "fork": false,
    "url": "https://api.github.com/repos/FontCustom/fontcustom",
    "forks_url": "https://api.github.com/repos/FontCustom/fontcustom/forks",
    "keys_url": "https://api.github.com/repos/FontCustom/fontcustom/keys{/key_id}",
    "collaborators_url": "https://api.github.com/repos/FontCustom/fontcustom/collaborators{/collaborator}",
    "teams_url": "https://api.github.com/repos/FontCustom/fontcustom/teams",
    "hooks_url": "https://api.github.com/repos/FontCustom/fontcustom/hooks",
    "issue_events_url": "https://api.github.com/repos/FontCustom/fontcustom/issues/events{/number}",
    "events_url": "https://api.github.com/repos/FontCustom/fontcustom/events",
    "assignees_url": "https://api.github.com/repos/FontCustom/fontcustom/assignees{/user}",
    "branches_url": "https://api.github.com/repos/FontCustom/fontcustom/branches{/branch}",
    "tags_url": "https://api.github.com/repos/FontCustom/fontcustom/tags",
    "blobs_url": "https://api.github.com/repos/FontCustom/fontcustom/git/blobs{/sha}",
    "git_tags_url": "https://api.github.com/repos/FontCustom/fontcustom/git/tags{/sha}",
    "git_refs_url": "https://api.github.com/repos/FontCustom/fontcustom/git/refs{/sha}",
    "trees_url": "https://api.github.com/repos/FontCustom/fontcustom/git/trees{/sha}",
    "statuses_url": "https://api.github.com/repos/FontCustom/fontcustom/statuses/{sha}",
    "languages_url": "https://api.github.com/repos/FontCustom/fontcustom/languages",
    "stargazers_url": "https://api.github.com/repos/FontCustom/fontcustom/stargazers",
    "contributors_url": "https://api.github.com/repos/FontCustom/fontcustom/contributors",
    "subscribers_url": "https://api.github.com/repos/FontCustom/fontcustom/subscribers",
    "subscription_url": "https://api.github.com/repos/FontCustom/fontcustom/subscription",
    "commits_url": "https://api.github.com/repos/FontCustom/fontcustom/commits{/sha}",
    "git_commits_url": "https://api.github.com/repos/FontCustom/fontcustom/git/commits{/sha}",
    "comments_url": "https://api.github.com/repos/FontCustom/fontcustom/comments{/number}",
    "issue_comment_url": "https://api.github.com/repos/FontCustom/fontcustom/issues/comments/{number}",
    "contents_url": "https://api.github.com/repos/FontCustom/fontcustom/contents/{+path}",
    "compare_url": "https://api.github.com/repos/FontCustom/fontcustom/compare/{base}...{head}",
    "merges_url": "https://api.github.com/repos/FontCustom/fontcustom/merges",
    "archive_url": "https://api.github.com/repos/FontCustom/fontcustom/{archive_format}{/ref}",
    "downloads_url": "https://api.github.com/repos/FontCustom/fontcustom/downloads",
    "issues_url": "https://api.github.com/repos/FontCustom/fontcustom/issues{/number}",
    "pulls_url": "https://api.github.com/repos/FontCustom/fontcustom/pulls{/number}",
    "milestones_url": "https://api.github.com/repos/FontCustom/fontcustom/milestones{/number}",
    "notifications_url": "https://api.github.com/repos/FontCustom/fontcustom/notifications{?since,all,participating}",
    "labels_url": "https://api.github.com/repos/FontCustom/fontcustom/labels{/name}",
    "releases_url": "https://api.github.com/repos/FontCustom/fontcustom/releases{/id}",
    "created_at": "2012-10-08T02:14:42Z",
    "updated_at": "2014-02-20T11:14:40Z",
    "pushed_at": "2014-02-20T11:14:40Z",
    "git_url": "git://github.com/FontCustom/fontcustom.git",
    "ssh_url": "git@github.com:FontCustom/fontcustom.git",
    "clone_url": "https://github.com/FontCustom/fontcustom.git",
    "svn_url": "https://github.com/FontCustom/fontcustom",
    "homepage": "https://fontcustom.com",
    "size": 2707,
    "stargazers_count": 1742,
    "watchers_count": 1742,
    "language": "Ruby",
    "has_issues": true,
    "has_downloads": true,
    "has_wiki": true,
    "forks_count": 188,
    "mirror_url": null,
    "open_issues_count": 32,
    "forks": 188,
    "open_issues": 32,
    "watchers": 1742,
    "default_branch": "master",
    "master_branch": "master",
    "permissions": {
      "admin": false,
      "push": false,
      "pull": true
    }
  },
  {
    "id": 14489011,
    "name": "grunt-font-optimizer",
    "full_name": "ActivearkJWT/grunt-font-optimizer",

  ...

The properties git_url and clone_url are of interest, as they are the ones mainly used for cloning.

To find all of them would be to use a regular expression to find all of them, or with native JavaScript method JSON.parse() via Node.js.

The regular expression approach for finding all lines that have the required clone_url:

/"clone_url": "(\S+)"/g

A minimal Node.js script could look like, calling it clone-starred.js:

var fs = require('fs');

var raw = fs.readFileSync('github-paazmaya-starred.json', { encoding: 'utf8' });
var data = JSON.parse(raw);

data.forEach(function (item) {
  console.log(item.clone_url);
});

The output of running node clone-starred.js could be something similar to:

https://github.com/FontCustom/fontcustom.git
https://github.com/ActivearkJWT/grunt-font-optimizer.git
https://github.com/getify/You-Dont-Know-JS.git
https://github.com/enyo/dropzone.git
https://github.com/DavidDurman/joint.git
https://github.com/jh3y/progre-c-ss.git
https://github.com/mathiasbynens/regenerate.git
https://github.com/sindresorhus/pageres.git
https://github.com/bower/bower.json-spec.git
https://github.com/mangini/gdocs2md.git
https://github.com/joyent/node.git
https://github.com/dropbox/dropbox-sdk-php.git
https://github.com/beatfactor/nightwatch.git
https://github.com/tobiasahlin/SpinKit.git
https://github.com/taye/interact.js.git
https://github.com/israelidanny/veinjs.git
https://github.com/darcyclarke/Front-end-Developer-Interview-Questions.git
https://github.com/selendroid/selendroid.git
https://github.com/shootaroo/jit-grunt.git
https://github.com/btford/zone.js.git
https://github.com/mrmrs/colors.git
https://github.com/spy-js/spy-js.git
https://github.com/josh/css-explain.git
https://github.com/josh/jquery-selector-set.git
https://github.com/ebrehault/resurrectio.git
https://github.com/bfirsh/needle.git
https://github.com/python-imaging/Pillow.git
https://github.com/ai/autoprefixer.git
https://github.com/leereilly/emoji.git
https://github.com/stephenharris/Event-Organiser.git

Now this list only some of the hundreds of repositories that I have starred during the last four years, what could be the problem?

Due to Pagination, the default request returns only the 30 first items. The maximum number of items is 100 is the given (per_page) parameter is used.

The original request can be rewritten to handle pagination. Please note that the URL should be quoted in order to avoid command line confusion:

curl -u paazmaya "https://api.github.com/user/starred?per_page=100&page=1" \
    -o github-paazmaya-starred-1.json

This would give the first 100 items and by incrementing the page parameter additional results can be retrieved.

How to get the total amount of pages available?

You could just keep incrementing the page parameter until the result becomes empty, but there exists another solution. The returned headers contain links of which one points to the last page.

By adding the -i parameter to the curl command, it will include the incoming header information in the resulting output file. However this would break the JSON parsing method. Another parameter for handling incoming headers is -D which saves them in a different file.

Yet another update to the command line:

curl -u paazmaya "https://api.github.com/user/starred?per_page=100&page=1" \
    -o github-paazmaya-starred-1.json -D headers.txt

The header could look something similar to, stating that there are three pages available:

Link: <https://api.github.com/user/starred?per_page=100&page=3>; rel="next",
  <https://api.github.com/user/starred?per_page=100&page=3>; rel="last",
  <https://api.github.com/user/starred?per_page=100&page=1>; rel="first",
  <https://api.github.com/user/starred?per_page=100&page=1>; rel="prev"

In case your results are already in the last page, for example you have less than 100 stars, the next and last links are missing.

Clone them

Since all the JSON data has been fetched, the previously created Node.js script can be used as a base to trigger Git client for cloning.

Steps that needs to be taken into account:

Multiple result pages needs to be combined
Simultaneously cloning would use too many resources, thus one cloning process at a time

The previously used script would become, in complete form:

var fs = require('fs');

var pageCount = 3;
var cloneUrls = [];

for (var i = 1; i <= pageCount; ++i) {
  var raw = fs.readFileSync(
    'github-paazmaya-starred-' + i + '.json',
    { encoding: 'utf8' }
  );
  var data = JSON.parse(raw);
  data.forEach(function (item) {
    cloneUrls.push(item.clone_url);
  });
}

console.log('Amount of starred repositories: ' + cloneUrls.length);

var exec = require('child_process').exec;

var next = function (index) {
  if (index >= cloneUrls.length) {
    return;
  }
  var url = cloneUrls[index];
  console.log(index + '\t' + url);

  exec('git clone ' + url, function (error, stdout, stderr) {
    console.log('stdout: ' + stdout);
    console.log('stderr: ' + stderr);
    if (error !== null) {
      console.log('exec error: ' + error);
    }
    next(++index);
  });
};

next(0);

After running the above script, the current folder should be filled with clones of your starred repositories, assuming git was found in the path.