From 1fe0698149a0b9b8e455d560d7b57f11c8bdd6df Mon Sep 17 00:00:00 2001 From: mguegan Date: Sat, 9 Aug 2014 09:15:20 +0200 Subject: [PATCH 1/9] Try to be compatible with the version 2 of IMGAPI * see https://images.joyent.com/docs/ --- server.js | 70 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/server.js b/server.js index 7514e14..27bf896 100755 --- a/server.js +++ b/server.js @@ -3,13 +3,14 @@ var restify = require('restify'); var path = require('path'); var fs = require('fs'); +var crypto = require('crypto'); /* read in configuration */ var config = require('./config'); var serve_dir; -if (config["serve_dir"]){ - serve_dir = config["serve_dir"]; +if (config['serve_dir']){ + serve_dir = config['serve_dir']; } else { serve_dir = process.cwd(); @@ -24,14 +25,14 @@ function process_manifest(req, uuid) { var manifest; try { manifest = require(path.join(serve_dir, uuid + '/manifest')); - var url_prefix = config.prefix + req.header('Host') + config.suffix + "/datasets/" + uuid + "/"; + var url_prefix = config.prefix + req.header('Host') + config.suffix + '/images/' + uuid + '/'; for (entry in manifest.files) { manifest.files[entry].url = url_prefix + manifest.files[entry].path }; return manifest; } catch(err) { - req.log.error("Failed to parse manifest for " + uuid + " error: " + err); + req.log.error('Failed to parse manifest for ' + uuid + ' error: ' + err); return false; } } @@ -47,9 +48,9 @@ function alldatasets(req, res, next) { return; } else { - res.header("Access-Control-Allow-Headers", "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token"); - res.header("Access-Control-Allow-Methods", "PUT, GET, POST, DELETE, OPTIONS"); - res.header("Access-Control-Allow-Origin", "*"); + res.header('Access-Control-Allow-Headers', 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'); + res.header('Access-Control-Allow-Methods', 'PUT, GET, POST, DELETE, OPTIONS'); + res.header('Access-Control-Allow-Origin', '*'); for (entry in dirlist) { if (fs.existsSync(path.join(serve_dir, dirlist[entry] + '/manifest.json'))) { var manifest = process_manifest(req, dirlist[entry]); @@ -59,7 +60,7 @@ function alldatasets(req, res, next) { }; }; res.send(everything); - req.log.info("served up /datasets"); + req.log.info('served up /images'); }; }); return next(); @@ -75,23 +76,38 @@ function manifest(req, res, next) { return next(); } res.send(process_manifest(req, req.params.id)); - req.log.info("served up /datasets/" + req.params.id); + req.log.info('served up /images/' + req.params.id); return next(); } +/* + * Generate a MD5 checksum of the dataset + */ +function process_dataset(filename) { + var hash = crypto.createHash('md5'); + var stream = fs.createReadStream(filename); + + stream.on('data', function (data) { hash.update(data); }); + stream.on('end', function () { + return hash.digest('base64'); + }); +} + /* * Serve up the requested image file */ function imagefile(req, res, next) { - var filename = path.join(serve_dir, req.params.id + '/' + req.params.path); - req.log.info("serving up /datasets/" + req.params.id + '/' + req.params.path); + var filename = path.join(serve_dir, req.params.id + '/file'); + var hash = process_dataset(filename); + req.log.info('serving up /images/' + req.params.id + '/file'); fs.exists(filename, function (exists) { if (!exists) { res.send(404, '404 Not Found'); return; } else { - res.header("Content-Type", "application/octet-stream"); - res.header("Content-Length", fs.statSync(filename).size); + res.header('Content-Type', 'application/octet-stream'); + res.header('Content-Length', fs.statSync(filename).size); + res.header('Content-MD5', hash); var stream = fs.createReadStream(filename, { bufferSize: 64 * 1024 }); stream.pipe(res); } @@ -103,8 +119,12 @@ function imagefile(req, res, next) { * Implement ping */ function ping(req, res, next) { - res.send({"ping":"pong"}); - req.log.info("served up /ping"); + res.send({ + 'ping':'pong', + 'version':'1.2.0', + 'imgapi': true + }); + req.log.info('served up /ping'); return next(); } @@ -112,16 +132,16 @@ function ping(req, res, next) { * Serve / */ function slash(req, res, next) { - var accept = req.header("Accept"); - if (accept && (accept.search("application/xhtml+xml") != -1 || accept.search("text/html") != -1)) { - var stream = fs.createReadStream(serve_dir + "/index.html", { bufferSize: 64 * 1024 }); + var accept = req.header('Accept'); + if (accept && (accept.search('application/xhtml+xml') != -1 || accept.search('text/html') != -1)) { + var stream = fs.createReadStream(serve_dir + '/index.html', { bufferSize: 64 * 1024 }); stream.pipe(res); } else { - res.header("Content-Type", "application/json"); - var stream = fs.createReadStream(serve_dir + "/index.json", { bufferSize: 64 * 1024 }); + res.header('Content-Type', 'application/json'); + var stream = fs.createReadStream(serve_dir + '/index.json', { bufferSize: 64 * 1024 }); stream.pipe(res); } - req.log.info("served up /"); + req.log.info('served up /'); return next(); } @@ -141,15 +161,15 @@ var server = restify.createServer({ var server = restify.createServer(); -setup_routes(server, '/datasets', alldatasets); -setup_routes(server, '/datasets/:id', manifest); -setup_routes(server, '/datasets/:id/:path', imagefile); +setup_routes(server, '/images', alldatasets); +setup_routes(server, '/images/:id', manifest); +setup_routes(server, '/images/:id/file', imagefile); setup_routes(server, '/ping', ping); setup_routes(server, '/', slash); server.log.level(config.loglevel); server.listen(config.listen_port, function() { - if ( typeof config.listen_port == "string") { + if ( typeof config.listen_port == 'string') { console.log('listening on unix socket: %s', config.listen_port); } else { From 69a27e1dac3b594b56c3db06dc3342595115ddbf Mon Sep 17 00:00:00 2001 From: mguegan Date: Sat, 9 Aug 2014 10:02:31 +0200 Subject: [PATCH 2/9] Wait on md5sum before sending the dataset --- server.js | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/server.js b/server.js index 27bf896..6e06018 100755 --- a/server.js +++ b/server.js @@ -80,19 +80,6 @@ function manifest(req, res, next) { return next(); } -/* - * Generate a MD5 checksum of the dataset - */ -function process_dataset(filename) { - var hash = crypto.createHash('md5'); - var stream = fs.createReadStream(filename); - - stream.on('data', function (data) { hash.update(data); }); - stream.on('end', function () { - return hash.digest('base64'); - }); -} - /* * Serve up the requested image file */ @@ -107,9 +94,15 @@ function imagefile(req, res, next) { } else { res.header('Content-Type', 'application/octet-stream'); res.header('Content-Length', fs.statSync(filename).size); - res.header('Content-MD5', hash); - var stream = fs.createReadStream(filename, { bufferSize: 64 * 1024 }); - stream.pipe(res); + // process the file two times : one for md5hash and the last one for + // sending it. + var md5sum = fs.createReadStream(filename, { bufferSize: 64 * 1024 }); + md5sum.on('data', function (data) { hash.update(data); }); + md5sum.on('end', function () { + res.header('Content-MD5', hash.digest('base64')); + var stream = fs.createReadStream(filename, { bufferSize: 64 * 1024 }); + stream.pipe(res); + }); } }); return next(); From 44b4d68e2fc1dfec43bc4570d7cd3c0f65b38886 Mon Sep 17 00:00:00 2001 From: mguegan Date: Sat, 9 Aug 2014 10:09:16 +0200 Subject: [PATCH 3/9] Fix last commit cleanup --- server.js | 1 - 1 file changed, 1 deletion(-) diff --git a/server.js b/server.js index 6e06018..3ed3f56 100755 --- a/server.js +++ b/server.js @@ -85,7 +85,6 @@ function manifest(req, res, next) { */ function imagefile(req, res, next) { var filename = path.join(serve_dir, req.params.id + '/file'); - var hash = process_dataset(filename); req.log.info('serving up /images/' + req.params.id + '/file'); fs.exists(filename, function (exists) { if (!exists) { From 15caad1b1b183da407dc579b3a95b3b190526b74 Mon Sep 17 00:00:00 2001 From: mguegan Date: Sat, 9 Aug 2014 10:11:55 +0200 Subject: [PATCH 4/9] Forgot hash definition --- server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server.js b/server.js index 3ed3f56..26d9445 100755 --- a/server.js +++ b/server.js @@ -85,6 +85,7 @@ function manifest(req, res, next) { */ function imagefile(req, res, next) { var filename = path.join(serve_dir, req.params.id + '/file'); + var hash = crypto.createHash('md5'); req.log.info('serving up /images/' + req.params.id + '/file'); fs.exists(filename, function (exists) { if (!exists) { From 2922d56e7a563c70634eb873c1ed31ec226fb1b6 Mon Sep 17 00:00:00 2001 From: mguegan Date: Sat, 9 Aug 2014 10:15:56 +0200 Subject: [PATCH 5/9] Fix indent --- server.js | 164 +++++++++++++++++++++++++++--------------------------- 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/server.js b/server.js index 26d9445..07fa377 100755 --- a/server.js +++ b/server.js @@ -10,10 +10,10 @@ var config = require('./config'); var serve_dir; if (config['serve_dir']){ - serve_dir = config['serve_dir']; + serve_dir = config['serve_dir']; } else { - serve_dir = process.cwd(); + serve_dir = process.cwd(); } /* @@ -22,78 +22,78 @@ else { * Adjust config.json if these urls are coming out wrong */ function process_manifest(req, uuid) { - var manifest; - try { - manifest = require(path.join(serve_dir, uuid + '/manifest')); + var manifest; + try { + manifest = require(path.join(serve_dir, uuid + '/manifest')); var url_prefix = config.prefix + req.header('Host') + config.suffix + '/images/' + uuid + '/'; - for (entry in manifest.files) { - manifest.files[entry].url = url_prefix + manifest.files[entry].path - }; - return manifest; - } - catch(err) { - req.log.error('Failed to parse manifest for ' + uuid + ' error: ' + err); - return false; - } + for (entry in manifest.files) { + manifest.files[entry].url = url_prefix + manifest.files[entry].path + }; + return manifest; + } + catch(err) { + req.log.error('Failed to parse manifest for ' + uuid + ' error: ' + err); + return false; + } } /* * Smoosh together all manifests into an array and return it */ function alldatasets(req, res, next) { - var everything = []; - fs.readdir(serve_dir, function (err, dirlist) { - if (err) { - res.send(500, 'Internal Server Error'); - return; - } - else { - res.header('Access-Control-Allow-Headers', 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'); - res.header('Access-Control-Allow-Methods', 'PUT, GET, POST, DELETE, OPTIONS'); - res.header('Access-Control-Allow-Origin', '*'); - for (entry in dirlist) { - if (fs.existsSync(path.join(serve_dir, dirlist[entry] + '/manifest.json'))) { - var manifest = process_manifest(req, dirlist[entry]); - if ( manifest ) { - everything.push(manifest); - }; - }; - }; - res.send(everything); - req.log.info('served up /images'); - }; - }); - return next(); + var everything = []; + fs.readdir(serve_dir, function (err, dirlist) { + if (err) { + res.send(500, 'Internal Server Error'); + return; + } + else { + res.header('Access-Control-Allow-Headers', 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'); + res.header('Access-Control-Allow-Methods', 'PUT, GET, POST, DELETE, OPTIONS'); + res.header('Access-Control-Allow-Origin', '*'); + for (entry in dirlist) { + if (fs.existsSync(path.join(serve_dir, dirlist[entry] + '/manifest.json'))) { + var manifest = process_manifest(req, dirlist[entry]); + if ( manifest ) { + everything.push(manifest); + }; + }; + }; + res.send(everything); + req.log.info('served up /images'); + }; + }); + return next(); } /* * Process and return the requested manifest */ function manifest(req, res, next) { - var manifest = process_manifest(req, req.params.id); - if (!manifest) { - res.send(404, '404 Not Found'); - return next(); - } - res.send(process_manifest(req, req.params.id)); - req.log.info('served up /images/' + req.params.id); - return next(); + var manifest = process_manifest(req, req.params.id); + if (!manifest) { + res.send(404, '404 Not Found'); + return next(); + } + res.send(process_manifest(req, req.params.id)); + req.log.info('served up /images/' + req.params.id); + return next(); } /* * Serve up the requested image file */ function imagefile(req, res, next) { - var filename = path.join(serve_dir, req.params.id + '/file'); + var filename = path.join(serve_dir, req.params.id + '/file'); var hash = crypto.createHash('md5'); - req.log.info('serving up /images/' + req.params.id + '/file'); - fs.exists(filename, function (exists) { - if (!exists) { - res.send(404, '404 Not Found'); - return; - } else { - res.header('Content-Type', 'application/octet-stream'); - res.header('Content-Length', fs.statSync(filename).size); + req.log.info('serving up /images/' + req.params.id + '/file'); + fs.exists(filename, function (exists) { + if (!exists) { + res.send(404, '404 Not Found'); + return; + } else { + res.header('Content-Type', 'application/octet-stream'); + res.header('Content-Length', fs.statSync(filename).size); // process the file two times : one for md5hash and the last one for // sending it. var md5sum = fs.createReadStream(filename, { bufferSize: 64 * 1024 }); @@ -103,53 +103,53 @@ function imagefile(req, res, next) { var stream = fs.createReadStream(filename, { bufferSize: 64 * 1024 }); stream.pipe(res); }); - } - }); - return next(); + } + }); + return next(); } /* * Implement ping */ function ping(req, res, next) { - res.send({ + res.send({ 'ping':'pong', 'version':'1.2.0', 'imgapi': true }); - req.log.info('served up /ping'); - return next(); + req.log.info('served up /ping'); + return next(); } /* * Serve / */ function slash(req, res, next) { - var accept = req.header('Accept'); - if (accept && (accept.search('application/xhtml+xml') != -1 || accept.search('text/html') != -1)) { - var stream = fs.createReadStream(serve_dir + '/index.html', { bufferSize: 64 * 1024 }); - stream.pipe(res); - } else { - res.header('Content-Type', 'application/json'); - var stream = fs.createReadStream(serve_dir + '/index.json', { bufferSize: 64 * 1024 }); - stream.pipe(res); - } - req.log.info('served up /'); - return next(); + var accept = req.header('Accept'); + if (accept && (accept.search('application/xhtml+xml') != -1 || accept.search('text/html') != -1)) { + var stream = fs.createReadStream(serve_dir + '/index.html', { bufferSize: 64 * 1024 }); + stream.pipe(res); + } else { + res.header('Content-Type', 'application/json'); + var stream = fs.createReadStream(serve_dir + '/index.json', { bufferSize: 64 * 1024 }); + stream.pipe(res); + } + req.log.info('served up /'); + return next(); } /* * route creation helper */ function setup_routes(server, route, handler) { - server.get(route, handler); - server.head(route, handler); + server.get(route, handler); + server.head(route, handler); } /* node restify rocks! */ var server = restify.createServer({ - name: 'dsapi', - version: '0.1.1' + name: 'dsapi', + version: '0.1.1' }); var server = restify.createServer(); @@ -162,11 +162,11 @@ setup_routes(server, '/', slash); server.log.level(config.loglevel); server.listen(config.listen_port, function() { - if ( typeof config.listen_port == 'string') { - console.log('listening on unix socket: %s', config.listen_port); - } - else { - console.log('%s listening at %s', server.name, server.url); - } - console.log('serving from %s', serve_dir); + if ( typeof config.listen_port == 'string') { + console.log('listening on unix socket: %s', config.listen_port); + } + else { + console.log('%s listening at %s', server.name, server.url); + } + console.log('serving from %s', serve_dir); }); From 22639a4e2e9df83871041ca693f50a2a6e3bd929 Mon Sep 17 00:00:00 2001 From: mguegan Date: Sun, 19 Apr 2015 19:27:20 +0200 Subject: [PATCH 6/9] imgadm is now using accept-version header --- server.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server.js b/server.js index 07fa377..573c68d 100755 --- a/server.js +++ b/server.js @@ -142,14 +142,14 @@ function slash(req, res, next) { * route creation helper */ function setup_routes(server, route, handler) { - server.get(route, handler); - server.head(route, handler); + server.get({path: route, version: '2.0.0'}, handler); + server.head({path: route, version: '2.0.0'}, handler); } /* node restify rocks! */ var server = restify.createServer({ name: 'dsapi', - version: '0.1.1' + version: '2.0.0' }); var server = restify.createServer(); From f0c85b3ec4de2ff62bb724b2578eef477261913e Mon Sep 17 00:00:00 2001 From: mguegan Date: Sun, 19 Apr 2015 19:35:21 +0200 Subject: [PATCH 7/9] Version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ff9d669..4751c60 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "smartos-image-server" , "description": "Serve SmartOS images" - , "version": "0.1.0" + , "version": "0.1.2" , "bin": { "image-server": "./server.js" } From b3a8fefb68aee86e84012ddd35e0a01b828244c5 Mon Sep 17 00:00:00 2001 From: mguegan Date: Sun, 19 Apr 2015 19:43:51 +0200 Subject: [PATCH 8/9] Add nodemon config file (clean restarting) --- nodemon.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 nodemon.json diff --git a/nodemon.json b/nodemon.json new file mode 100644 index 0000000..b289ea2 --- /dev/null +++ b/nodemon.json @@ -0,0 +1,7 @@ +{ + "restartable": "rs", + "verbose": true, + "events": { + "restart": "rm /var/tmp/image-server.sock" + } +} From 6e361ef5064c31d7e32621f8adc95c9294ea6cd5 Mon Sep 17 00:00:00 2001 From: mguegan Date: Sun, 19 Apr 2015 21:33:55 +0200 Subject: [PATCH 9/9] Add SIGINT/SIGTERM handler (ready for PM2 management) --- server.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/server.js b/server.js index 573c68d..0e97d2c 100755 --- a/server.js +++ b/server.js @@ -170,3 +170,24 @@ server.listen(config.listen_port, function() { } console.log('serving from %s', serve_dir); }); + +/* + * handling exit signals + */ +// Ctrl+C +process.once('SIGINT', function() { + // synchronous call of fs.unlink + // ensure we are deleting socket before exiting + fs.unlinkSync(config.listen_port); + console.log('exiting..'); + process.exit(0); +}); + +// Used by PM2 (restart) +process.once('SIGTERM', function() { + // synchronous call of fs.unlink + // ensure we are deleting socket before exiting + fs.unlinkSync(config.listen_port); + console.log('exiting..'); + process.exit(0); +});