0 
0 My objective: create a Node application to download sources from a repository on GitHub. I want to use this application to read a simple package.json-like file (that describes which reusable components (from which GitHub repositories) the application has dependencies on) and download all required resources from GitHub and store them in the local file system. This by itself may not seem very useful. However, this is a stepping stone on the road to a facility to trigger run time update of appliation components triggered by GitHub WebHook triggers.
I am making use of the Octokit Node JS library to interact with the REST APIs of GitHub. The code I have created will:
- fetch the meta-data for all items in the root folder of a GitHub Repo (at the tip of a specific branch, or at a specific tag or commit identifier)
- iterate over all items:
- download the contents of the item if it is a file and create a local file with the content (and cater for large files and for binary files)
- create a local directory for each item in the GitHub repo that is a diectory, then recursively process the contents of the directory on GitHub
An example of the code in action:
A randomly selected GitHub repo (at https://github.com/lucasjellema/WebAppIframe2ADFSynchronize):
The local target directory is empty at the beginning of the action:
Run the code:
And the content is downloaded and written locally:
Note: the code could easily provide an execution report with details such as file size, download, last change date etc. It is currently very straightforward. Note: the gitToken is something you need to get hold of yourself in the GitHub dashboard: https://github.com/settings/tokens . Without a token, the code will still work, but you will be bound to the GitHub rate limit (of about 60 requests per hour).
const octokit = require('@octokit/rest')()
const fs = require('fs');
var gitToken = "YourToken"
octokit.authenticate({
type: 'token',
token: gitToken
})
var targetProjectRoot = "C:/data/target/"
var github = { "owner": "lucasjellema", "repo": "WebAppIframe2ADFSynchronize", "branch": "master" }
downloadGitHubRepo(github, targetProjectRoot)
async function downloadGitHubRepo(github, targetDirectory) {
console.log(`Installing GitHub Repo ${github.owner}\\${github.repo}`)
var repo = github.repo;
var path = ''
var owner = github.owner
var ref = github.commit ? github.commit : (github.tag ? github.tag : (github.branch ? github.branch : 'master'))
processGithubDirectory(owner, repo, ref, path, path, targetDirectory)
}
// let's assume that if the name ends with one of these extensions, we are dealing with a binary file:
const binaryExtensions = ['png', 'jpg', 'tiff', 'wav', 'mp3', 'doc', 'pdf']
var maxSize = 1000000;
function processGithubDirectory(owner, repo, ref, path, sourceRoot, targetRoot) {
octokit.repos.getContent({ "owner": owner, "repo": repo, "path": path, "ref": ref })
.then(result => {
var targetDir = targetRoot + path
// check if targetDir exists
checkDirectorySync(targetDir)
result.data.forEach(item => {
if (item.type == "dir") {
processGithubDirectory(owner, repo, ref, item.path, sourceRoot, targetRoot)
} // if directory
if (item.type == "file") {
if (item.size > maxSize) {
var sha = item.sha
octokit.gitdata.getBlob({ "owner": owner, "repo": repo, "sha": item.sha }
).then(result => {
var target = `${targetRoot + item.path}`
fs.writeFile(target
, Buffer.from(result.data.content, 'base64').toString('utf8'), function (err, data) { })
})
.catch((error) => { console.log("ERROR BIGGA" + error) })
return;
}// if bigga
octokit.repos.getContent({ "owner": owner, "repo": repo, "path": item.path, "ref": ref })
.then(result => {
var target = `${targetRoot + item.path}`
if (binaryExtensions.includes(item.path.slice(-3))) {
fs.writeFile(target
, Buffer.from(result.data.content, 'base64'), function (err, data) { reportFile(item, target) })
} else
fs.writeFile(target
, Buffer.from(result.data.content, 'base64').toString('utf8'), function (err, data) { if (!err) reportFile(item, target); else console.log('Fuotje ' + err) })
})
.catch((error) => { console.log("ERROR " + error) })
}// if file
})
}).catch((error) => { console.log("ERROR XXX" + error) })
}//processGithubDirectory
function reportFile(item, target) {
console.log(`- installed ${item.name} (${item.size} bytes )in ${target}`)
}
function checkDirectorySync(directory) {
try {
fs.statSync(directory);
} catch (e) {
fs.mkdirSync(directory);
console.log("Created directory: " + directory)
}
}
Resources
Octokit REST API Node JS library: https://github.com/octokit/rest.js
API Documentation for Octokit: https://octokit.github.io/rest.js/#api-Repos-getContent
Related posts:
Node.js run from GitHub in Generic Docker Container backed by Dockerized Redis Cache 
Generic Docker Container Image for running and live reloading a Node application based on a GitHub Repo 
REST API on Node.js and Express for data retrieved from Oracle Database with node-oracledb Database Driver running on Application Container Cloud 
Oracle NoSQL Database 4.x and the Node.js driver 3.x 
Handle a GitHub Push Event from a Web Hook Trigger in a Node application
Looks like this needs to be updated since octokit.authenticate() is deprecated and no longer works.
Hi Brian,
Thanks for letting us – and other readers – know about this. I am afraid I will not have time for an update of this article in the foreseeable future.
kind regards
Lucas