Create a Node JS application for Downloading sources from GitHub

Lucas Jellema 2

My objective: create a Node application to download sources from a repository on GitHub. I want to use this application to read a simple package.json-like file (that describes which reusable components (from which GitHub repositories) the application has dependencies on) and download all required resources from GitHub and store them in the local file system. This by itself may not seem very useful. However, this is a stepping stone on the road to a facility to trigger run time update of appliation components triggered by GitHub WebHook triggers.

I am making use of the Octokit Node JS library to interact with the REST APIs of GitHub. The code I have created will:

  • fetch the meta-data for all items in the root folder of a GitHub Repo (at the tip of a specific branch, or at a specific tag or commit identifier)
  • iterate over all items:
    • download the contents of the item if it is a file and create a local file with the content (and cater for large files and for binary files)
    • create a local directory for each item in the GitHub repo that is a diectory, then recursively process the contents of the directory on GitHub

An example of the code in action:

A randomly selected GitHub repo (at


The local target directory is empty at the beginning of the action:


Run the code:


And the content is downloaded and written locally:


Note: the code could easily provide an execution report with details such as file size, download, last change date etc. It is currently very straightforward. Note: the gitToken is something you need to get hold of yourself in the GitHub dashboard: . Without a token, the code will still work, but you will be bound to the GitHub rate limit (of about 60 requests per hour).

const octokit = require('@octokit/rest')() 
const fs = require('fs');

var gitToken = "YourToken"

    type: 'token',
    token: gitToken

var targetProjectRoot = "C:/data/target/" 
var github = { "owner": "lucasjellema", "repo": "WebAppIframe2ADFSynchronize", "branch": "master" }

downloadGitHubRepo(github, targetProjectRoot)

async function downloadGitHubRepo(github, targetDirectory) {
    console.log(`Installing GitHub Repo ${github.owner}\\${github.repo}`)
    var repo = github.repo;
    var path = ''
    var owner = github.owner
    var ref = github.commit ? github.commit : (github.tag ? github.tag : (github.branch ? github.branch : 'master'))
    processGithubDirectory(owner, repo, ref, path, path, targetDirectory)

// let's assume that if the name ends with one of these extensions, we are dealing with a binary file:
const binaryExtensions = ['png', 'jpg', 'tiff', 'wav', 'mp3', 'doc', 'pdf']
var maxSize = 1000000;
function processGithubDirectory(owner, repo, ref, path, sourceRoot, targetRoot) {
    octokit.repos.getContent({ "owner": owner, "repo": repo, "path": path, "ref": ref })
        .then(result => {
            var targetDir = targetRoot + path
            // check if targetDir exists 
   => {
                if (item.type == "dir") {
                    processGithubDirectory(owner, repo, ref, item.path, sourceRoot, targetRoot)
                } // if directory
                if (item.type == "file") {
                    if (item.size > maxSize) {
                        var sha = item.sha
                        octokit.gitdata.getBlob({ "owner": owner, "repo": repo, "sha": item.sha }
                        ).then(result => {
                            var target = `${targetRoot + item.path}`
                                , Buffer.from(, 'base64').toString('utf8'), function (err, data) { })
                            .catch((error) => { console.log("ERROR BIGGA" + error) })
                    }// if bigga
                    octokit.repos.getContent({ "owner": owner, "repo": repo, "path": item.path, "ref": ref })
                        .then(result => {
                            var target = `${targetRoot + item.path}`
                            if (binaryExtensions.includes(item.path.slice(-3))) {
                                    , Buffer.from(, 'base64'), function (err, data) { reportFile(item, target) })
                            } else
                                    , Buffer.from(, 'base64').toString('utf8'), function (err, data) { if (!err) reportFile(item, target); else console.log('Fuotje ' + err) })

                        .catch((error) => { console.log("ERROR " + error) })
                }// if file
        }).catch((error) => { console.log("ERROR XXX" + error) })

function reportFile(item, target) {
    console.log(`- installed ${} (${item.size} bytes )in ${target}`)

function checkDirectorySync(directory) {
    try {
    } catch (e) {
        console.log("Created directory: " + directory)


Octokit REST API Node JS library: 

API Documentation for Octokit:

2 thoughts on “Create a Node JS application for Downloading sources from GitHub

  1. Looks like this needs to be updated since octokit.authenticate() is deprecated and no longer works.

    1. Hi Brian,

      Thanks for letting us – and other readers – know about this. I am afraid I will not have time for an update of this article in the foreseeable future.

      kind regards

Comments are closed.

Next Post

Node & Express application to proxy HTTP requests – simply forwarding the response to the original caller

The requirement is simple: a Node JS application that receives HTTP requests and forwards (some of) them to other hosts and subsequently the returns the responses it receives to the original caller. This can be used in many situations – to ensure all resources loaded in a web application come […]
%d bloggers like this: