From f599105dc36df6a869262e9046d7438aba18b20c Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 10:37:26 +0100 Subject: [PATCH 01/13] Add all primary logic for .NET Core support --- .gitignore | 14 +++ bin/microcule | 8 +- .../hello-world/dotnet-hello/Program.cs | 12 ++ .../dotnet-hello/dotnet-hello.csproj | 8 ++ .../compileServiceCode/dotnet/index.js | 104 ++++++++++++++++++ lib/plugins/compile/compileServiceMappings.js | 4 +- lib/plugins/compile/index.js | 3 + lib/plugins/spawn/index.js | 6 +- lib/requireServiceSync.js | 1 + 9 files changed, 152 insertions(+), 8 deletions(-) create mode 100755 examples/services/hello-world/dotnet-hello/Program.cs create mode 100755 examples/services/hello-world/dotnet-hello/dotnet-hello.csproj create mode 100644 lib/plugins/compile/compileServiceCode/dotnet/index.js diff --git a/.gitignore b/.gitignore index 08cb4a8..3ca2406 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,17 @@ node_modules/ __pycache__/ *.pyc *.pyo + + +# .NET Core build assets +tmp/dotnet/ +release/dotnet/ +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]eleases/ +x64/ +x86/ +build/ +bld/ +[Bb]in/ +[Oo]bj/ \ No newline at end of file diff --git a/bin/microcule b/bin/microcule index dfaf439..8487553 100755 --- a/bin/microcule +++ b/bin/microcule @@ -71,7 +71,7 @@ if (typeof argv.l !== "undefined" || typeof argv.language !== "undefined") { } if (stdin.isTTY) { - // console.log('ignoring will not run in TTY mode') + console.log('ignoring will not run in TTY mode') // helps with peformance of STDIN tool http = require('resource-http'); startServer(_service); @@ -116,7 +116,7 @@ function startSTDIN (service) { var output = Writable(); output._write = function (chunk, enc, next) { - // console.log('wirtint out',chunk.toString()); + console.log('wirtint out',chunk.toString()); process.stdout.write(chunk); next(); }; @@ -153,7 +153,7 @@ function startSTDIN (service) { config: config, log: console.log })(stdin, output, function(){ - // console.log("COMPLETED STREAMING SERVICE") + console.log("COMPLETED STREAMING SERVICE") }); } else { var result = ''; @@ -164,7 +164,7 @@ function startSTDIN (service) { STDIN: result } }; - // console.log('spawning service'.yellow) + console.log('spawning service'.yellow) microcule.plugins.spawn({ bin: service.bin, argv: service.argv, diff --git a/examples/services/hello-world/dotnet-hello/Program.cs b/examples/services/hello-world/dotnet-hello/Program.cs new file mode 100755 index 0000000..a4a4206 --- /dev/null +++ b/examples/services/hello-world/dotnet-hello/Program.cs @@ -0,0 +1,12 @@ +using System; + +namespace dotnet_hello +{ + class Program + { + static void Main(string[] args) + { + Console.WriteLine("Hello World from .NET Core!"); + } + } +} diff --git a/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj b/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj new file mode 100755 index 0000000..87e649a --- /dev/null +++ b/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj @@ -0,0 +1,8 @@ + + + + Exe + netcoreapp1.1 + + + diff --git a/lib/plugins/compile/compileServiceCode/dotnet/index.js b/lib/plugins/compile/compileServiceCode/dotnet/index.js new file mode 100644 index 0000000..f2f450d --- /dev/null +++ b/lib/plugins/compile/compileServiceCode/dotnet/index.js @@ -0,0 +1,104 @@ +var fs = require('fs'); +var spawn = require('child_process').spawn; +var crypto = require('crypto'); +var path = require('path'); +var mkdirp = require('mkdirp'); +var npc = require('ncp').ncp; + +module.exports = function (req, res, cb) { + + var service = req.service; + + var hash = service.sha1; + + + + + +/* +Target: .NET Core 1.1 + +Uasge: +- Standard .NET Core console app created via `dotnet new console -n ` +- To execute microcule .//Program.cs + +TODO: +- copy all files to tmp (sha as folder name) +- Copy the folder contents to a folder using the a hash of the service ? +- dotnet restore in the folder +- dotnet build -f 1.1 -c release -o -v minimal +- dotnet run , and probably pass arguments in + +*/ + // first, we need to write the file to a temporary location on the disk + // it could be possible to bypass writing to the disk entirely here, but it will be easier to debug user-scripts, + // and also interact with various compilers without having to look up special use cases for stdin / argv code parsing + var outputPath = service.buildDir + '/' + hash; + console.log('Copy code to', outputPath) + fs.writeFile(outputPath, service.code, function (err, result) { + + if (err) { + return cb(err); + } + console.log('wrote file with sucess!'); + console.log('starting compiler dotnet', 'dotnet', [outputPath]); + + var compiler = spawn('dotnet build', [outputPath], { + cwd: service.releaseDir // ensure binary compiles to target directory + }); + + var stderr = '', stdout = ''; + + compiler.on('error', function(data){ + console.log('error', data) + cb(data); + }); + + /* + vm.stdin.error + vm.exit + vm.stdout.end + vm.stderr + */ + + compiler.on('data', function(data){ + console.log('got data', data) + }); + + compiler.on('close', function(data){ + console.log('close', data) + //cb(null, 'close') + }); + + compiler.stdin.on('error', function(data){ + console.log('stdin.error', data); + // cb(null, 'close') + }); + + compiler.stdout.on('data', function(data){ + console.log('stdout', data); + stdout += data.toString(); + //cb(null, 'close') + }); + + compiler.stderr.on('data', function(data){ + // console.log('stderr', data); + stderr += data.toString(); + //cb(null, 'close') + }); + + compiler.on('exit', function(data){ + console.log('exit', data) + var result = { + tmpSourceFile: outputPath, + bin: service.releaseDir + '/' + hash, + stderr: stderr, + stdout: stdout, + exitCode: data + }; + cb(null, result); + }); + + }); + +}; \ No newline at end of file diff --git a/lib/plugins/compile/compileServiceMappings.js b/lib/plugins/compile/compileServiceMappings.js index d0c5b6b..fd58521 100644 --- a/lib/plugins/compile/compileServiceMappings.js +++ b/lib/plugins/compile/compileServiceMappings.js @@ -3,5 +3,7 @@ module.exports = { "gcc": require('./compileServiceCode/gcc'), "go": require('./compileServiceCode/golang'), "rust": require('./compileServiceCode/rust'), - "r": require('./compileServiceCode/r') + "r": require('./compileServiceCode/r'), + "dotnet": require('./compileServiceCode/dotnet') + }; \ No newline at end of file diff --git a/lib/plugins/compile/index.js b/lib/plugins/compile/index.js index 4952010..1a530df 100644 --- a/lib/plugins/compile/index.js +++ b/lib/plugins/compile/index.js @@ -67,6 +67,9 @@ module.exports = function compile (config) { binLocation = _service.buildDir + '/' + sha + '/' + 'hook.class'; } + console.log(_service.language); + + // TODO: check that sha matches as well as language? potential issue here with same content files, but diffirent target languages // we will assume that if the binary file exists, it has completed it's build step and is ready to be run fs.stat(binLocation, function (err, _stat) { diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index 052e6e2..46fab85 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -119,7 +119,7 @@ module['exports'] = function spawnService (service) { service.presenter = service.presenterSource || service.presenter; // if targetLanguage is a compiled / static langauge, we must first compile the source code - // console.log(_service, compileService) + console.log(_service, compileService) if (typeof compileService[_service.language] === "function") { // console.log('found compiled service!', _service); _compile(input, output, function (err, _build) { @@ -139,7 +139,7 @@ module['exports'] = function spawnService (service) { // instead of attempting to spawn the service, we need to pipe the error back to the client return output.json(_build); } - // console.log('attempting to spawn', service); + console.log('attempting to spawn', service); _service.bin = _build.bin; // Remark: Java is an edge case now, as its both compiled than interpreted if (_service.language === 'java') { @@ -162,7 +162,7 @@ module['exports'] = function spawnService (service) { function _spawnService (input, output) { - // console.log('running spawn service'.blue, input.url) + console.log('running spawn service'.blue, input.url) // use passed in config if its defined, if not will default to ./config folder if (typeof service.config === "object") { config = service.config; diff --git a/lib/requireServiceSync.js b/lib/requireServiceSync.js index 3091283..662688d 100644 --- a/lib/requireServiceSync.js +++ b/lib/requireServiceSync.js @@ -86,6 +86,7 @@ function loadService (p, type) { var extentions = { ".c": "gcc", ".coffee": "coffee-script", + ".cs": "dotnet", ".go": "go", ".java": "java", ".js": "javascript", From 04e81475ba9b63118819284e9a3ed89851db1978 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 10:46:46 +0100 Subject: [PATCH 02/13] Add NCP module which will be used for file copying for .NET Core compile logic. --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index be9e886..ee6faf1 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,8 @@ "stream-buffers": "^3.0.1", "through2": "^2.0.1", "tree-kill": "^1.1.0", - "view": "1.0.0" + "view": "1.0.0", + "ncp":"^2.0.0" }, "devDependencies": { "basic-auth": "^1.1.0", From 3d508fcc35b3f283026254415eddebf2fc153743 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 12:10:48 +0100 Subject: [PATCH 03/13] Doc: improve documentation --- bin/microcule | 2 ++ lib/requireServiceSync.js | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/bin/microcule b/bin/microcule index 8487553..fda3731 100755 --- a/bin/microcule +++ b/bin/microcule @@ -16,6 +16,8 @@ config.watch = argv.w || argv.watch || config.watch; config.stream = argv.s || argv.stream || config.stream; // default to releasing compiled binaries to current working directory of `microcule` binary + +// Doesn't seem to pick up `releaseDir` from config config.releaseDir = config.releaseDir || process.cwd(); if (typeof argv.v !== "undefined" || typeof argv.version !== "undefined") { diff --git a/lib/requireServiceSync.js b/lib/requireServiceSync.js index 662688d..5555fea 100644 --- a/lib/requireServiceSync.js +++ b/lib/requireServiceSync.js @@ -8,6 +8,10 @@ function extToLang (ext) { } // same as requireService, only with sync interface ( useful for requiring services as packages ) +// Desciption: +// Params: +// opts: object {path: string} fully qualified path to code being executed. Folder or file including ext. +// module['exports'] = function requireServiceSync (opts) { var service = {}; From 7ade771d8fd79b0a14566144904f1bb2ce6bfd62 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 15:52:50 +0100 Subject: [PATCH 04/13] Add: dotent restore and build --- bin/microcule | 6 +- .../compileServiceCode/dotnet/index.js | 209 ++++++++++++------ lib/plugins/compile/index.js | 15 +- lib/plugins/spawn/index.js | 4 +- lib/requireServiceSync.js | 11 +- 5 files changed, 165 insertions(+), 80 deletions(-) diff --git a/bin/microcule b/bin/microcule index fda3731..c6b4563 100755 --- a/bin/microcule +++ b/bin/microcule @@ -73,7 +73,7 @@ if (typeof argv.l !== "undefined" || typeof argv.language !== "undefined") { } if (stdin.isTTY) { - console.log('ignoring will not run in TTY mode') + //console.log('ignoring will not run in TTY mode') // helps with peformance of STDIN tool http = require('resource-http'); startServer(_service); @@ -152,6 +152,7 @@ function startSTDIN (service) { presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, + sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(stdin, output, function(){ @@ -176,6 +177,7 @@ function startSTDIN (service) { presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, + sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(stdin, output, function(){ @@ -259,6 +261,7 @@ function startServer (_service) { presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, + sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(req, output, function(){ @@ -277,6 +280,7 @@ function startServer (_service) { presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, + sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(req, res, function(){ diff --git a/lib/plugins/compile/compileServiceCode/dotnet/index.js b/lib/plugins/compile/compileServiceCode/dotnet/index.js index f2f450d..63b666f 100644 --- a/lib/plugins/compile/compileServiceCode/dotnet/index.js +++ b/lib/plugins/compile/compileServiceCode/dotnet/index.js @@ -2,103 +2,172 @@ var fs = require('fs'); var spawn = require('child_process').spawn; var crypto = require('crypto'); var path = require('path'); -var mkdirp = require('mkdirp'); -var npc = require('ncp').ncp; +var ncp = require('ncp').ncp; +var shasum = require('shasum'); +var path = require('path'); module.exports = function (req, res, cb) { var service = req.service; + // TODO: override hash logic here to include the csproj contents. + // by default handled in compile/index.js and only used the code file var hash = service.sha1; + //var sha = shasum(_service.code); + //_service.sha1 = sha; + //console.log('what is sha1', sha); + + + /* + Target: .NET Core 1.1 + + Uasge: + - Standard .NET Core console app created via `dotnet new console -n ` + - To execute microcule .//Program.cs + + TODO: + - copy all files to the temp build location under folder that's the hash of the Program.cs + *.csproj content. + -- check if a folder by the has exists to decide if a rebuild is required. + - dotnet restore in the build folder + - dotnet build -f 1.1 -c release -o -v minimal + + - dotnet run , and probably pass arguments in + -- Not sure where this goes. + -- might need to create a spawn/generateCommandLineArguments module + + + */ + //console.log('buildDir: ' + service.buildDir); + //console.log('releaseDir: ' + service.releaseDir); + //console.log('code: ' + service.releaseDir); + + + + // release is explicityly defaulted to the root of the microcule binary. Not sure why given there is a release folder. + // This may will break if the releaseDir default is overriden in config + //service.releaseDir = service.releaseDir + '/release'; + + // first, we need to write the file to a temporary location on the disk + // it could be possible to bypass writing to the disk entirely here, but it will be easier to debug user-scripts, + // and also interact with various compilers without having to look up special use cases for stdin / argv code parsing + console.dir(service, { depth: null, colors: true }); -/* -Target: .NET Core 1.1 + var tmpBuildDir = service.buildDir + '/dotnet/' + hash; + var buildOutputDir = service.releaseDir + '/' + hash; + service.sourceCodeFilePath = '/Users/janaka.abeywardhana/code-projects/microcule/examples/services/hello-world/dotnet-hello/Program.cs'; + console.log('codefilepath = ', service.sourceCodeFilePath); -Uasge: -- Standard .NET Core console app created via `dotnet new console -n ` -- To execute microcule .//Program.cs + var sourceCodeDir = path.dirname(service.sourceCodeFilePath); -TODO: -- copy all files to tmp (sha as folder name) -- Copy the folder contents to a folder using the a hash of the service ? -- dotnet restore in the folder -- dotnet build -f 1.1 -c release -o -v minimal -- dotnet run , and probably pass arguments in -*/ - // first, we need to write the file to a temporary location on the disk - // it could be possible to bypass writing to the disk entirely here, but it will be easier to debug user-scripts, - // and also interact with various compilers without having to look up special use cases for stdin / argv code parsing - var outputPath = service.buildDir + '/' + hash; - console.log('Copy code to', outputPath) - fs.writeFile(outputPath, service.code, function (err, result) { + //console.log(''); - if (err) { - return cb(err); + + //If a dir with the hash exists then it's not new code, let's (re)build. + if (!fs.existsSync(buildOutputDir)) { + fs.mkdirSync(buildOutputDir); + if (!fs.existsSync(tmpBuildDir)) { + fs.mkdirSync(tmpBuildDir); } - console.log('wrote file with sucess!'); - console.log('starting compiler dotnet', 'dotnet', [outputPath]); - var compiler = spawn('dotnet build', [outputPath], { - cwd: service.releaseDir // ensure binary compiles to target directory - }); + console.log('Copy from ', sourceCodeDir) + console.log('Copying code to', tmpBuildDir); - var stderr = '', stdout = ''; + ncp.limit = 5; // concurrent copies - compiler.on('error', function(data){ - console.log('error', data) - cb(data); - }); + ncp(sourceCodeDir, tmpBuildDir, function (err) { - /* - vm.stdin.error - vm.exit - vm.stdout.end - vm.stderr - */ + if (err) { + return cb(err); + } + console.log('Successfully copied code to temp build dir'); + console.log('starting dependency restore', 'dotnet restore'); - compiler.on('data', function(data){ - console.log('got data', data) - }); + var restore = spawn('dotnet', ['restore'], { + cwd: tmpBuildDir // ensure binary compiles to target directory + }); - compiler.on('close', function(data){ - console.log('close', data) - //cb(null, 'close') - }); + var stderr = '', stdout = ''; - compiler.stdin.on('error', function(data){ - console.log('stdin.error', data); - // cb(null, 'close') - }); + restore.on('error', function (data) { + console.log('error', data) + cb(data); + }); - compiler.stdout.on('data', function(data){ - console.log('stdout', data); - stdout += data.toString(); - //cb(null, 'close') - }); - compiler.stderr.on('data', function(data){ - // console.log('stderr', data); - stderr += data.toString(); - //cb(null, 'close') - }); + restore.on('exit', function (data) { + console.log('exit', data); + + + + + + console.log('starting compiler dotnet', 'dotnet build -c release -v minimal -o ', buildOutputDir); + + // command syntax: dotnet build -f 1.1 -c release -o -v minimal + var compiler = spawn('dotnet', ['build', '-c', 'release', '-v','minimal', '-o', buildOutputDir], { + cwd: tmpBuildDir // ensure binary compiles to target directory + }); + + var stderr = '', stdout = ''; + + compiler.on('error', function (data) { + console.log('error', data) + cb(data); + }); + + /* + vm.stdin.error + vm.exit + vm.stdout.end + vm.stderr + */ + + compiler.on('data', function (data) { + console.log('got data', data) + }); + + compiler.on('close', function (data) { + console.log('close', data) + //cb(null, 'close') + }); + + compiler.stdin.on('error', function (data) { + console.log('stdin.error', data); + // cb(null, 'close') + }); + + compiler.stdout.on('data', function (data) { + console.log('stdout', data); + stdout += data.toString(); + //cb(null, 'close') + }); + + compiler.stderr.on('data', function (data) { + // console.log('stderr', data); + stderr += data.toString(); + //cb(null, 'close') + }); + + compiler.on('exit', function (data) { + console.log('exit', data) + var result = { + tmpSourceFile: tmpBuildDir, + bin: buildOutputDir, + stderr: stderr, + stdout: stdout, + exitCode: data + }; + cb(null, result); + }); - compiler.on('exit', function(data){ - console.log('exit', data) - var result = { - tmpSourceFile: outputPath, - bin: service.releaseDir + '/' + hash, - stderr: stderr, - stdout: stdout, - exitCode: data - }; - cb(null, result); + }); }); - }); + } }; \ No newline at end of file diff --git a/lib/plugins/compile/index.js b/lib/plugins/compile/index.js index 1a530df..e246bb6 100644 --- a/lib/plugins/compile/index.js +++ b/lib/plugins/compile/index.js @@ -14,7 +14,10 @@ module.exports = function compile (config) { // if no releaseDir is specified, will default to ./microcule/release directory config.releaseDir = config.releaseDir || path.resolve(__dirname + '/../../../release'); - console.log('using compile config', config) + //console.log('using compile config', config) + + console.log('in compile()'); + config.errorHandler = config.errorHandler || function (err, next) { if (err.message === "Bad credentials") { @@ -52,10 +55,11 @@ module.exports = function compile (config) { // create shasum1 of service source code contents var sha = shasum(_service.code); _service.sha1 = sha; - console.log('what is sha1', sha); + //console.log('what is sha1', sha); return function compileMiddleware (req, res, next) { req.service = _service; + console.log('in compileMiddleware()'); // check if binary of same sha1 already exists ( has been previously compiled ) // if compiled service already exists, simply continue with spawn and defined binary path @@ -67,7 +71,12 @@ module.exports = function compile (config) { binLocation = _service.buildDir + '/' + sha + '/' + 'hook.class'; } - console.log(_service.language); + // this should really be in the language specific compiler logic + if (_service.language === 'dotnet') { + binLocation = _service.releaseDir + '/dotnet/' + sha + } + + //console.log(_service.language); // TODO: check that sha matches as well as language? potential issue here with same content files, but diffirent target languages diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index 46fab85..585c851 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -119,9 +119,9 @@ module['exports'] = function spawnService (service) { service.presenter = service.presenterSource || service.presenter; // if targetLanguage is a compiled / static langauge, we must first compile the source code - console.log(_service, compileService) + //console.log( compileService) if (typeof compileService[_service.language] === "function") { - // console.log('found compiled service!', _service); + console.log('found compiled service!', _service.language); _compile(input, output, function (err, _build) { // [x] service.bin // service.argv diff --git a/lib/requireServiceSync.js b/lib/requireServiceSync.js index 5555fea..e7f5b74 100644 --- a/lib/requireServiceSync.js +++ b/lib/requireServiceSync.js @@ -7,8 +7,8 @@ function extToLang (ext) { return lang; } -// same as requireService, only with sync interface ( useful for requiring services as packages ) -// Desciption: +// +// Desciption: This hydrates the `service` object. Same as requireService, only with sync interface ( useful for requiring services as packages ) // Params: // opts: object {path: string} fully qualified path to code being executed. Folder or file including ext. // @@ -40,10 +40,11 @@ module['exports'] = function requireServiceSync (opts) { }; function loadService (p, type) { - +console.log('in loadService()') var service = { name: p, - owner: "examples" + owner: "examples", + sourceCodeFilePath: p }; var _service = { @@ -145,5 +146,7 @@ function loadService (p, type) { } catch (err) { // console.log(err); } + + return service; } \ No newline at end of file From b13cb9cfafe8333f72bdc6a60ce7e0da00cb9abd Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 17:53:26 +0100 Subject: [PATCH 05/13] Fix: happy path working for .NET Core. --- examples/express-compiled-languages.js | 2 +- .../compile/compileServiceCode/dotnet/index.js | 2 +- lib/plugins/compile/index.js | 9 +++++++-- lib/plugins/spawn/index.js | 15 +++++++++++++-- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/examples/express-compiled-languages.js b/examples/express-compiled-languages.js index a62bbcf..a21c27e 100644 --- a/examples/express-compiled-languages.js +++ b/examples/express-compiled-languages.js @@ -10,7 +10,7 @@ var service = { var spawn = microcule.plugins.spawn(service); app.use(function(req, res, next){ - console.log('attempting to spawn', service) + console.log('[express-compiled-languages.js] attempting to spawn ', service) spawn(req, res, next); }); diff --git a/lib/plugins/compile/compileServiceCode/dotnet/index.js b/lib/plugins/compile/compileServiceCode/dotnet/index.js index 63b666f..e0f9f3a 100644 --- a/lib/plugins/compile/compileServiceCode/dotnet/index.js +++ b/lib/plugins/compile/compileServiceCode/dotnet/index.js @@ -57,7 +57,7 @@ module.exports = function (req, res, cb) { console.dir(service, { depth: null, colors: true }); var tmpBuildDir = service.buildDir + '/dotnet/' + hash; - var buildOutputDir = service.releaseDir + '/' + hash; + var buildOutputDir = service.releaseDir + '/dotnet/' + hash; service.sourceCodeFilePath = '/Users/janaka.abeywardhana/code-projects/microcule/examples/services/hello-world/dotnet-hello/Program.cs'; console.log('codefilepath = ', service.sourceCodeFilePath); diff --git a/lib/plugins/compile/index.js b/lib/plugins/compile/index.js index e246bb6..db625c6 100644 --- a/lib/plugins/compile/index.js +++ b/lib/plugins/compile/index.js @@ -65,6 +65,7 @@ module.exports = function compile (config) { // if compiled service already exists, simply continue with spawn and defined binary path var binLocation = _service.releaseDir + '/' + sha; + var tmpBuidDir = _service.buildDir + '/' + sha; // java requires a fixed class name, so we must create a unique directory instead if (_service.language === 'java') { @@ -73,7 +74,8 @@ module.exports = function compile (config) { // this should really be in the language specific compiler logic if (_service.language === 'dotnet') { - binLocation = _service.releaseDir + '/dotnet/' + sha + binLocation = _service.releaseDir + '/dotnet/' + sha; + tmpBuidDir = _service.buildDir + '/dotnet/' + sha; } //console.log(_service.language); @@ -125,9 +127,12 @@ module.exports = function compile (config) { // we find the file, attempt to execute it // if the stat returned ( a file ) then use that path instead of compiling a new one console.log('using compiled version of service', binLocation); + + var result = { bin: binLocation, - buildDir: _service.buildDir + '/' + sha, + buildDir: tmpBuidDir, + tmpSourceFile: tmpBuidDir, sha1: sha, compiledFresh: false, foundCompiledCache: true, diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index 585c851..0716878 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -139,7 +139,8 @@ module['exports'] = function spawnService (service) { // instead of attempting to spawn the service, we need to pipe the error back to the client return output.json(_build); } - console.log('attempting to spawn', service); + console.log('attempting to spawn'); + console.dir(_build); _service.bin = _build.bin; // Remark: Java is an edge case now, as its both compiled than interpreted if (_service.language === 'java') { @@ -154,6 +155,12 @@ module['exports'] = function spawnService (service) { _service.argv = [_build.tmpSourceFile]; //_service.cwd = '/Users/a/dev/stackvana/microcule/tmp/'; } + + if (_service.language === 'dotnet'){ + _service.bin = 'dotnet'; + _service.argv = ['run', '-p', _build.buildDir + '/dotnet-hello.csproj']; //TODO implemented dynamically getting the csproj file name + _service.cwd = _build.bin; + } _spawnService(input, output); }); } else { @@ -375,7 +382,8 @@ module['exports'] = function spawnService (service) { preprocessCommandLineArguments(); // process.cwd(), - // console.log('spawning', targetBinary, 'in', _service.cwd, 'with', binaryArgs) + console.log('spawning'.red, targetBinary, 'in', _service.cwd, 'with', binaryArgs); + console.log(targetBinary, binaryArgs); vm = spawn(targetBinary, binaryArgs, { stdio: ['pipe', 'pipe', 'pipe', 'pipe'], cwd: _service.cwd }); /* @@ -388,6 +396,9 @@ module['exports'] = function spawnService (service) { // see: https://github.com/mhart/epipebomb/blob/master/epipebomb.js // see: https://github.com/nodejs/node/issues/947 // see: https://github.com/nodejs/node/pull/9470 + + /Users/janaka.abeywardhana/code-projects/microcule/tmp/dotnet/a07f72f1aadf192732b74e3814ee40d4b05fd693/dotnet-hello.csproj + /Users/janaka.abeywardhana/code-projects/microcule/tmp/dotnet/a07f72f1aadf192732b74e3814ee40d4b05fd693/dotnet-hello.csprj } // if not specific EPIPE issue, log the error console.log(err.message); From 25265f3a412998cedbfdda4cd1e6a4afaab4b694 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 18:06:29 +0100 Subject: [PATCH 06/13] Update: .NET Core example code --- examples/services/hello-world/dotnet-hello/Program.cs | 2 +- examples/services/hello-world/dotnet-hello/dotnet-hello.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/services/hello-world/dotnet-hello/Program.cs b/examples/services/hello-world/dotnet-hello/Program.cs index a4a4206..177c7b1 100755 --- a/examples/services/hello-world/dotnet-hello/Program.cs +++ b/examples/services/hello-world/dotnet-hello/Program.cs @@ -6,7 +6,7 @@ class Program { static void Main(string[] args) { - Console.WriteLine("Hello World from .NET Core!"); + Console.WriteLine("Hello World from .NET Core!!!"); } } } diff --git a/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj b/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj index 87e649a..1215d7a 100755 --- a/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj +++ b/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj @@ -4,5 +4,5 @@ Exe netcoreapp1.1 - + From f68b89b56ecefcc133bd968b4c37317eba3a6274 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 18:07:53 +0100 Subject: [PATCH 07/13] Fix: inconsistent path values when running previously compiled code vs compiling and running. --- config/index.js | 4 +++- lib/plugins/compile/compileServiceCode/dotnet/index.js | 5 +++-- lib/plugins/compile/index.js | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/config/index.js b/config/index.js index 8684793..71cb614 100644 --- a/config/index.js +++ b/config/index.js @@ -10,10 +10,12 @@ module.exports = { ca: [fs.readFileSync(__dirname + '/ssl/ca-crt.pem').toString()], sslRequired: true, // redirects all http traffic to https, optional onlySSL: false // will only start https server with no unprotected http interface, optional + }, SERVICE_MAX_TIMEOUT: 10000, messages: { childProcessSpawnError: require('./messages/childProcessSpawnError'), serviceExecutionTimeout: require('./messages/serviceExecutionTimeout') - } + }, + releaseDir: process.cwd() + '/release' }; \ No newline at end of file diff --git a/lib/plugins/compile/compileServiceCode/dotnet/index.js b/lib/plugins/compile/compileServiceCode/dotnet/index.js index e0f9f3a..9eae5be 100644 --- a/lib/plugins/compile/compileServiceCode/dotnet/index.js +++ b/lib/plugins/compile/compileServiceCode/dotnet/index.js @@ -38,8 +38,8 @@ module.exports = function (req, res, cb) { */ - //console.log('buildDir: ' + service.buildDir); - //console.log('releaseDir: ' + service.releaseDir); + console.log('buildDir: ' + service.buildDir); + console.log('releaseDir: ' + service.releaseDir); //console.log('code: ' + service.releaseDir); @@ -158,6 +158,7 @@ module.exports = function (req, res, cb) { var result = { tmpSourceFile: tmpBuildDir, bin: buildOutputDir, + buildDir: tmpBuildDir, stderr: stderr, stdout: stdout, exitCode: data diff --git a/lib/plugins/compile/index.js b/lib/plugins/compile/index.js index db625c6..7a3479f 100644 --- a/lib/plugins/compile/index.js +++ b/lib/plugins/compile/index.js @@ -75,7 +75,7 @@ module.exports = function compile (config) { // this should really be in the language specific compiler logic if (_service.language === 'dotnet') { binLocation = _service.releaseDir + '/dotnet/' + sha; - tmpBuidDir = _service.buildDir + '/dotnet/' + sha; + tmpBuildDir = _service.buildDir + '/dotnet/' + sha; } //console.log(_service.language); @@ -131,8 +131,8 @@ module.exports = function compile (config) { var result = { bin: binLocation, - buildDir: tmpBuidDir, - tmpSourceFile: tmpBuidDir, + buildDir: tmpBuildDir, + tmpSourceFile: tmpBuildDir, sha1: sha, compiledFresh: false, foundCompiledCache: true, From aaece82c3c5679b619b68801975db36dfcdaa391 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 19:20:42 +0100 Subject: [PATCH 08/13] Add: looping through and echoing all args to the .NET Core example --- examples/services/hello-world/dotnet-hello/Program.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/services/hello-world/dotnet-hello/Program.cs b/examples/services/hello-world/dotnet-hello/Program.cs index 177c7b1..65ce358 100755 --- a/examples/services/hello-world/dotnet-hello/Program.cs +++ b/examples/services/hello-world/dotnet-hello/Program.cs @@ -7,6 +7,12 @@ class Program static void Main(string[] args) { Console.WriteLine("Hello World from .NET Core!!!"); + + foreach (var arg in args) + { + Console.WriteLine("param: {0}", arg); + } + } } } From 4d41486948d00a74301a7e465eb6e1bea8e92e3e Mon Sep 17 00:00:00 2001 From: Janaka A Date: Sat, 26 Aug 2017 19:21:18 +0100 Subject: [PATCH 09/13] Add: support for arguments to .NET Core --- bin/microcule | 4 +- .../dotnet/index.js | 22 ++++++++++ lib/plugins/spawn/index.js | 43 ++++++++++--------- 3 files changed, 46 insertions(+), 23 deletions(-) create mode 100644 lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js diff --git a/bin/microcule b/bin/microcule index c6b4563..90c9717 100755 --- a/bin/microcule +++ b/bin/microcule @@ -15,9 +15,7 @@ config.http.port = argv.p || argv.port || config.http.port; config.watch = argv.w || argv.watch || config.watch; config.stream = argv.s || argv.stream || config.stream; -// default to releasing compiled binaries to current working directory of `microcule` binary - -// Doesn't seem to pick up `releaseDir` from config +// If no config, default to releasing compiled binaries to current working directory of `microcule` binary config.releaseDir = config.releaseDir || process.cwd(); if (typeof argv.v !== "undefined" || typeof argv.version !== "undefined") { diff --git a/lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js b/lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js new file mode 100644 index 0000000..2503b65 --- /dev/null +++ b/lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js @@ -0,0 +1,22 @@ +/* +This args array returned from here is used as paramters when spawning `dotnet run` +Just pass service and env back along with the `-p` dotnet run parameter. +For dotnet we don't want to pass the code as a string, that's for lame lanuages ;) +*/ +module['exports'] = function generategccArguments (service, env) { + + + + var dotnetArgv = []; + + + dotnetArgv = [ + 'run', + '-p', service.buildDir, + '--s', JSON.stringify(service), // important: two dashes `--` for params that are passed into the app. + '--e', JSON.stringify(env), + ]; + + return dotnetArgv; + +} \ No newline at end of file diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index 0716878..35a52a9 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -18,6 +18,7 @@ var generateArguments = { go: require('./generateCommandLineArguments/golang'), bash: require('./generateCommandLineArguments/bash'), clisp: require('./generateCommandLineArguments/clisp'), + dotnet: require('./generateCommandLineArguments/dotnet'), lua: require('./generateCommandLineArguments/lua'), ocaml: require('./generateCommandLineArguments/ocaml'), perl: require('./generateCommandLineArguments/perl'), @@ -158,7 +159,8 @@ module['exports'] = function spawnService (service) { if (_service.language === 'dotnet'){ _service.bin = 'dotnet'; - _service.argv = ['run', '-p', _build.buildDir + '/dotnet-hello.csproj']; //TODO implemented dynamically getting the csproj file name + _service.buildDir = _build.buildDir + '/dotnet-hello.csproj'; //needed in commandlineArgs generator + //_service.argv = ['run', '-p', _build.buildDir + '/dotnet-hello.csproj']; //TODO implemented dynamically getting the csproj file name _service.cwd = _build.bin; } _spawnService(input, output); @@ -319,11 +321,11 @@ module['exports'] = function spawnService (service) { var vm; var binaries = { + "babel": "micro-node", "bash": "micro-bash", "clisp": "micro-clisp", "coffee-script": "micro-node", "coffee": "micro-node", - "babel": "micro-node", "es7": "micro-node", // legacy name, renamed to "babel" "lua": "micro-lua", "javascript": "micro-node", @@ -383,7 +385,7 @@ module['exports'] = function spawnService (service) { // process.cwd(), console.log('spawning'.red, targetBinary, 'in', _service.cwd, 'with', binaryArgs); - console.log(targetBinary, binaryArgs); + //console.log(targetBinary, binaryArgs); vm = spawn(targetBinary, binaryArgs, { stdio: ['pipe', 'pipe', 'pipe', 'pipe'], cwd: _service.cwd }); /* @@ -493,7 +495,8 @@ module['exports'] = function spawnService (service) { vm.stdout.on('end', function (data) { status.stdoutEnded = true; status.pipe3ended = true; - // console.log('vm.stdout.end', status); + console.log(data); + //console.log('vm.stdout.end', status); if (!status.checkingRegistry && !status.ended && !status.erroring) { //status.ended = true; // Remark: The vm's STDOUT has ended ( spawned service has completed ), @@ -511,20 +514,20 @@ module['exports'] = function spawnService (service) { if (vm.stdin) { - /* - input.on('end', function(){ - if (!status.pipe3ended) { - pipe3.write(Buffer('input.end')); - pipe3.write(Buffer('\n')); - } - }); - - input.on('close', function(){ - if (!status.pipe3ended) { - pipe3.write(Buffer('input.close')); - pipe3.write(Buffer('\n')); - } - }); + + // input.on('end', function(){ + // if (!status.pipe3ended) { + // pipe3.write(Buffer('input.end')); + // pipe3.write(Buffer('\n')); + // } + // }); + + // input.on('close', function(){ + // if (!status.pipe3ended) { + // pipe3.write(Buffer('input.close')); + // pipe3.write(Buffer('\n')); + // } + // }); vm.stdin.on('end', function (data) { input.emit('end') @@ -533,11 +536,11 @@ module['exports'] = function spawnService (service) { vm.stdin.on('close', function (data) { input.emit('close') }); - */ + vm.stdin.on('error', function (data) { status.stdinError = true; - // console.log('vm.stdin.error', status, data); + console.log('vm.stdin.error', status, data); // do nothing with this error? // without this error handler, `run-remote-service` will experience an uncaught stream error, // this is bad, because we lose the error stack with the uncaught stream error From 7e301ef163b1c7efab8ca18686184e0ffe72c340 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Mon, 28 Aug 2017 20:41:32 +0100 Subject: [PATCH 10/13] Add: dotnet re-compile when either code file or csproj changes. --- bin/microcule | 25 ++++--- .../compileServiceCode/dotnet/index.js | 52 +++++---------- lib/plugins/compile/index.js | 65 +++++++++++++------ .../dotnet/index.js | 19 +++--- lib/plugins/spawn/index.js | 15 +++-- lib/requireServiceSync.js | 7 +- 6 files changed, 104 insertions(+), 79 deletions(-) diff --git a/bin/microcule b/bin/microcule index 90c9717..fd7a896 100755 --- a/bin/microcule +++ b/bin/microcule @@ -19,8 +19,8 @@ config.stream = argv.s || argv.stream || config.stream; config.releaseDir = config.releaseDir || process.cwd(); if (typeof argv.v !== "undefined" || typeof argv.version !== "undefined") { - var pkg = require('../package.json') - console.log(pkg.version) + var pkg = require('../package.json'); + console.log(pkg.version); process.exit(); } @@ -48,6 +48,10 @@ var _service; try { _service = requireServiceSync({ path: servicePath }); + + //console.log('#### microcule: requireServiceSync() ######'); + //console.dir(_service); + } catch (err) { if (err.code === 'ENOENT') { // if we could not find the service, attempt to see if it's a globally available binary @@ -109,6 +113,7 @@ function startSTDIN (service) { mode: 'stream' }; + var Readable = require('stream').Readable; var Writable = require('stream').Writable; @@ -145,12 +150,12 @@ function startSTDIN (service) { bin: service.bin, argv: service.argv, code: service.code, + originalCodeFilePath: service.originalCodeFilePath, schema: service.schema, view: service.view, presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, - sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(stdin, output, function(){ @@ -170,12 +175,12 @@ function startSTDIN (service) { bin: service.bin, argv: service.argv, code: service.code, + originalCodeFilePath: service.originalCodeFilePath, schema: service.schema, view: service.view, presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, - sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(stdin, output, function(){ @@ -210,7 +215,7 @@ function startServer (_service) { console.log('using presenter for service') } - console.log(_service.language + ' microcule started at: http://' + addr.address + ":" + addr.port); + console.log(_service.language + ' microcule started at: http://'.blue + addr.address + ":" + addr.port); // Remark: Will automatically map the process.env `microcule` was spawned in to the service.env of the spawned function config.env = process.env; @@ -235,9 +240,11 @@ function startServer (_service) { spawnService(__service); }); */ + console.log('1'.red); spawnService(requireServiceSync({ path: servicePath })); } else { - spawnService(_service) + console.log('2'.red); + spawnService(_service); } function spawnService (service) { @@ -250,16 +257,17 @@ function startServer (_service) { if (err) { return next(err); } + console.log('3'.red); microcule.plugins.spawn({ bin: service.bin, argv: service.argv, code: service.code, + originalCodeFilePath: service.originalCodeFilePath, schema: service.schema, view: service.view, presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, - sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(req, output, function(){ @@ -269,16 +277,17 @@ function startServer (_service) { }); }); } else { + console.log('4'.red); microcule.plugins.spawn({ bin: service.bin, argv: service.argv, code: service.code, + originalCodeFilePath: service.originalCodeFilePath, schema: service.schema, view: service.view, presenter: service.presenter, releaseDir: config.releaseDir, language: service.language, - sourceCodeFilePath: service.sourceCodeFilePath, config: config, log: console.log })(req, res, function(){ diff --git a/lib/plugins/compile/compileServiceCode/dotnet/index.js b/lib/plugins/compile/compileServiceCode/dotnet/index.js index 9eae5be..5ec8f20 100644 --- a/lib/plugins/compile/compileServiceCode/dotnet/index.js +++ b/lib/plugins/compile/compileServiceCode/dotnet/index.js @@ -7,37 +7,22 @@ var shasum = require('shasum'); var path = require('path'); module.exports = function (req, res, cb) { - +console.log('compileServiceCode.dotnet(). just executed'.yellow); var service = req.service; // TODO: override hash logic here to include the csproj contents. // by default handled in compile/index.js and only used the code file var hash = service.sha1; - //var sha = shasum(_service.code); - //_service.sha1 = sha; - //console.log('what is sha1', sha); - - + console.log('inside compileServiceCode.dotnet()'.yellow) + console.dir(service); /* Target: .NET Core 1.1 - Uasge: - Standard .NET Core console app created via `dotnet new console -n ` - To execute microcule .//Program.cs - - TODO: - - copy all files to the temp build location under folder that's the hash of the Program.cs + *.csproj content. - -- check if a folder by the has exists to decide if a rebuild is required. - - dotnet restore in the build folder - - dotnet build -f 1.1 -c release -o -v minimal - - - dotnet run , and probably pass arguments in - -- Not sure where this goes. - -- might need to create a spawn/generateCommandLineArguments module - - */ + console.log('buildDir: ' + service.buildDir); console.log('releaseDir: ' + service.releaseDir); //console.log('code: ' + service.releaseDir); @@ -54,18 +39,14 @@ module.exports = function (req, res, cb) { // and also interact with various compilers without having to look up special use cases for stdin / argv code parsing - console.dir(service, { depth: null, colors: true }); + //console.dir(service, { depth: null, colors: true }); var tmpBuildDir = service.buildDir + '/dotnet/' + hash; var buildOutputDir = service.releaseDir + '/dotnet/' + hash; - service.sourceCodeFilePath = '/Users/janaka.abeywardhana/code-projects/microcule/examples/services/hello-world/dotnet-hello/Program.cs'; - console.log('codefilepath = ', service.sourceCodeFilePath); - - var sourceCodeDir = path.dirname(service.sourceCodeFilePath); + //service.sourceCodeFilePath = '/Users/janaka.abeywardhana/code-projects/microcule/examples/services/hello-world/dotnet-hello/Program.cs'; + //console.log('codefilepath = '.yellow, service.originalCodeFilePath); - //console.log(''); - //If a dir with the hash exists then it's not new code, let's (re)build. if (!fs.existsSync(buildOutputDir)) { @@ -73,13 +54,15 @@ module.exports = function (req, res, cb) { if (!fs.existsSync(tmpBuildDir)) { fs.mkdirSync(tmpBuildDir); } + } + - console.log('Copy from ', sourceCodeDir) - console.log('Copying code to', tmpBuildDir); + console.log('Copy from ', service.originalSourceCodeDir, ' to ', tmpBuildDir); + ncp.limit = 5; // concurrent copies - ncp(sourceCodeDir, tmpBuildDir, function (err) { + ncp(service.originalSourceCodeDir, tmpBuildDir, function (err) { if (err) { return cb(err); @@ -94,7 +77,7 @@ module.exports = function (req, res, cb) { var stderr = '', stdout = ''; restore.on('error', function (data) { - console.log('error', data) + console.log('error', data); cb(data); }); @@ -102,10 +85,6 @@ module.exports = function (req, res, cb) { restore.on('exit', function (data) { console.log('exit', data); - - - - console.log('starting compiler dotnet', 'dotnet build -c release -v minimal -o ', buildOutputDir); // command syntax: dotnet build -f 1.1 -c release -o -v minimal @@ -154,11 +133,12 @@ module.exports = function (req, res, cb) { }); compiler.on('exit', function (data) { - console.log('exit', data) + console.log('exit', data); var result = { tmpSourceFile: tmpBuildDir, bin: buildOutputDir, buildDir: tmpBuildDir, + originalCodeFilePath: service.originalCodeFilePath, stderr: stderr, stdout: stdout, exitCode: data @@ -169,6 +149,6 @@ module.exports = function (req, res, cb) { }); }); - } + }; \ No newline at end of file diff --git a/lib/plugins/compile/index.js b/lib/plugins/compile/index.js index 7a3479f..3444ef2 100644 --- a/lib/plugins/compile/index.js +++ b/lib/plugins/compile/index.js @@ -14,10 +14,6 @@ module.exports = function compile (config) { // if no releaseDir is specified, will default to ./microcule/release directory config.releaseDir = config.releaseDir || path.resolve(__dirname + '/../../../release'); - //console.log('using compile config', config) - - console.log('in compile()'); - config.errorHandler = config.errorHandler || function (err, next) { if (err.message === "Bad credentials") { @@ -52,39 +48,69 @@ module.exports = function compile (config) { throw new Error('invalid language choice: ' + _service.language); } - // create shasum1 of service source code contents - var sha = shasum(_service.code); + + var sha = ''; + if (_service.language === 'dotnet') { + // we want to checksum the code file and the project file to detect changes. + _service.originalCodeFilePath = config.service.originalCodeFilePath; + _service.originalSourceCodeDir = path.dirname(_service.originalCodeFilePath); + + fs.readdirSync(_service.originalSourceCodeDir).forEach(function(fileName){ + //console.log(fileName); + var ext = path.extname(fileName); + //console.log(ext); + if (ext === '.csproj'){ + _service.dotnetCsproj = fileName; + } + }); + if (!_service.dotnetCsproj) { + // TODO: throw an error here + console.log('error: *.csproj file not found in dir ', _service.originalSourceCodeDir); + } + + var csprojFilePath = _service.originalSourceCodeDir + "/" + _service.dotnetCsproj; + //console.log(csprojFilePath); + var csprojCode = fs.readFileSync(path.resolve(csprojFilePath).toString()); + sha = shasum(_service.code + csprojCode); + } + else { + // Default: create shasum1 of service source code contents + shasum(_service.code); + } + _service.sha1 = sha; - //console.log('what is sha1', sha); + console.log('sha1: ', sha); + //console.log('##### shaum() #####'); + //console.dir(config); + return function compileMiddleware (req, res, next) { req.service = _service; - console.log('in compileMiddleware()'); - - // check if binary of same sha1 already exists ( has been previously compiled ) - // if compiled service already exists, simply continue with spawn and defined binary path - + var binLocation = _service.releaseDir + '/' + sha; var tmpBuidDir = _service.buildDir + '/' + sha; + // ### Langugae specific compile meta data overrides ### + // java requires a fixed class name, so we must create a unique directory instead if (_service.language === 'java') { binLocation = _service.buildDir + '/' + sha + '/' + 'hook.class'; } - // this should really be in the language specific compiler logic + if (_service.language === 'dotnet') { binLocation = _service.releaseDir + '/dotnet/' + sha; tmpBuildDir = _service.buildDir + '/dotnet/' + sha; } - //console.log(_service.language); - + // ### check if binary of same sha1 already exists ( has been previously compiled ) ### + // if compiled service already exists, simply continue with spawn and defined binary path + // TODO: check that sha matches as well as language? potential issue here with same content files, but diffirent target languages // we will assume that if the binary file exists, it has completed it's build step and is ready to be run fs.stat(binLocation, function (err, _stat) { - console.log('back from stat', err, _stat); + console.log('back from stat! Result: ', err, _stat); if (err) { // could not find the file, attempt to compile it console.log('could not find file, so lets compile it'.red); @@ -99,6 +125,7 @@ module.exports = function compile (config) { if (err) { return res.end(err.message); } + if (result === 0) { // TODO: null result means no build status, create a new build status provider.set('/builds/' + sha, { status: 'building' }, function (err, result){ @@ -126,7 +153,7 @@ module.exports = function compile (config) { } else { // we find the file, attempt to execute it // if the stat returned ( a file ) then use that path instead of compiling a new one - console.log('using compiled version of service', binLocation); + console.log('using compiled version of code in ', binLocation); var result = { @@ -143,5 +170,5 @@ module.exports = function compile (config) { } }); - } -} \ No newline at end of file + }; +}; \ No newline at end of file diff --git a/lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js b/lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js index 2503b65..3c1dfa6 100644 --- a/lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js +++ b/lib/plugins/spawn/generateCommandLineArguments/dotnet/index.js @@ -2,21 +2,22 @@ This args array returned from here is used as paramters when spawning `dotnet run` Just pass service and env back along with the `-p` dotnet run parameter. For dotnet we don't want to pass the code as a string, that's for lame lanuages ;) -*/ -module['exports'] = function generategccArguments (service, env) { - +Other languages been implemented such that the JSON is flattened here and passed as individual params. +That's a waste of time. We can use a JSON parser in the. - var dotnetArgv = []; +May be there's a case for pulling out some key paramas here suchs as the querystring params collection. +*/ +module['exports'] = function generategccArguments(service, env) { + var dotnetArgv = []; - dotnetArgv = [ - 'run', + dotnetArgv = [ + 'run', '-p', service.buildDir, '--s', JSON.stringify(service), // important: two dashes `--` for params that are passed into the app. '--e', JSON.stringify(env), - ]; + ]; - return dotnetArgv; - + return dotnetArgv; } \ No newline at end of file diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index 35a52a9..975bf54 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -48,6 +48,8 @@ module['exports'] = function spawnService (service) { // so we want to make sure it only has properties that will seralize well / should be available to microservice scope var _service = {}; + _service.originalCodeFilePath = service.originalCodeFilePath; + // if a binary path has been passed in, assume we are spawning an already compiled application // since the application is already compiled, we don't need to know it's language or source code @@ -82,6 +84,9 @@ module['exports'] = function spawnService (service) { // probably not a good idea to attempt to confifigure compile plugin inside of spawnServiceMiddleware() handler var _compile; if (typeof compileService[_service.language] === "function") { + + //console.log('#### spawn._complile() ####'); + //console.log(_service); _compile = compile({ service: _service, releaseDir: config.releaseDir @@ -122,7 +127,8 @@ module['exports'] = function spawnService (service) { // if targetLanguage is a compiled / static langauge, we must first compile the source code //console.log( compileService) if (typeof compileService[_service.language] === "function") { - console.log('found compiled service!', _service.language); + console.log('found compile service for ', _service.language); + _compile(input, output, function (err, _build) { // [x] service.bin // service.argv @@ -171,7 +177,7 @@ module['exports'] = function spawnService (service) { function _spawnService (input, output) { - console.log('running spawn service'.blue, input.url) + //console.log('running spawn service'.blue, input.url) // use passed in config if its defined, if not will default to ./config folder if (typeof service.config === "object") { config = service.config; @@ -210,6 +216,7 @@ module['exports'] = function spawnService (service) { // if so, we must compile / check the caching options for the compiled service code if (typeof transpileService[targetLanguage] === "function") { // service.code = service.code || ""; + var md5 = checksum(_service.code); if (typeof transpileCache[md5] === "undefined") { @@ -384,7 +391,7 @@ module['exports'] = function spawnService (service) { preprocessCommandLineArguments(); // process.cwd(), - console.log('spawning'.red, targetBinary, 'in', _service.cwd, 'with', binaryArgs); + //console.log('spawning'.red, targetBinary, 'in', _service.cwd, 'with', binaryArgs); //console.log(targetBinary, binaryArgs); vm = spawn(targetBinary, binaryArgs, { stdio: ['pipe', 'pipe', 'pipe', 'pipe'], cwd: _service.cwd }); @@ -495,7 +502,7 @@ module['exports'] = function spawnService (service) { vm.stdout.on('end', function (data) { status.stdoutEnded = true; status.pipe3ended = true; - console.log(data); + //console.log(data); //console.log('vm.stdout.end', status); if (!status.checkingRegistry && !status.ended && !status.erroring) { //status.ended = true; diff --git a/lib/requireServiceSync.js b/lib/requireServiceSync.js index e7f5b74..a388cb5 100644 --- a/lib/requireServiceSync.js +++ b/lib/requireServiceSync.js @@ -40,11 +40,11 @@ module['exports'] = function requireServiceSync (opts) { }; function loadService (p, type) { -console.log('in loadService()') +//console.log('in loadService()') var service = { name: p, owner: "examples", - sourceCodeFilePath: p + originalCodeFilePath: p }; var _service = { @@ -147,6 +147,7 @@ console.log('in loadService()') // console.log(err); } - + //console.log('##### inside requireServiceSync() ####'); + //console.dir(service); return service; } \ No newline at end of file From 56cf5c96665f4dca0a0fa245563e016ef8f37341 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Mon, 28 Aug 2017 20:59:35 +0100 Subject: [PATCH 11/13] Refactor: clean up some console logging --- .../hello-world/dotnet-hello/dotnet-hello.csproj | 1 - .../compile/compileServiceCode/dotnet/index.js | 16 ++++++++-------- lib/plugins/spawn/index.js | 3 +++ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj b/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj index 1215d7a..97a38c1 100755 --- a/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj +++ b/examples/services/hello-world/dotnet-hello/dotnet-hello.csproj @@ -4,5 +4,4 @@ Exe netcoreapp1.1 - diff --git a/lib/plugins/compile/compileServiceCode/dotnet/index.js b/lib/plugins/compile/compileServiceCode/dotnet/index.js index 5ec8f20..99e8578 100644 --- a/lib/plugins/compile/compileServiceCode/dotnet/index.js +++ b/lib/plugins/compile/compileServiceCode/dotnet/index.js @@ -17,17 +17,16 @@ console.log('compileServiceCode.dotnet(). just executed'.yellow); console.log('inside compileServiceCode.dotnet()'.yellow) console.dir(service); /* - Target: .NET Core 1.1 + Supported Target: .NET Core 1.1 + .NET Core 2.0 (untested) Uasge: - Standard .NET Core console app created via `dotnet new console -n ` - - To execute microcule .//Program.cs + - To execute run `microcule .//Program.cs` */ - console.log('buildDir: ' + service.buildDir); - console.log('releaseDir: ' + service.releaseDir); - //console.log('code: ' + service.releaseDir); - - + //console.log('buildDir: ' + service.buildDir); + //console.log('releaseDir: ' + service.releaseDir); + // release is explicityly defaulted to the root of the microcule binary. Not sure why given there is a release folder. // This may will break if the releaseDir default is overriden in config @@ -48,7 +47,8 @@ console.log('compileServiceCode.dotnet(). just executed'.yellow); - //If a dir with the hash exists then it's not new code, let's (re)build. + // Create the folders we need. Because put code in a folders based on the hash we need to create folders it code changes. + // Logic for deciding if we needed re-build is in the shared complile logic. if (!fs.existsSync(buildOutputDir)) { fs.mkdirSync(buildOutputDir); if (!fs.existsSync(tmpBuildDir)) { diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index 975bf54..414ccfb 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -609,12 +609,15 @@ module['exports'] = function spawnService (service) { // The service has ended but the VM end event may not have fired, we should attempt to end response endResponse(); } + }); if (vm.stdin) { input.pipe(vm.stdin); } + + } } From 559cfde6fd9b78f16de48ffd8858e38c933ab0af Mon Sep 17 00:00:00 2001 From: Janaka A Date: Mon, 28 Aug 2017 21:30:01 +0100 Subject: [PATCH 12/13] Fix: replace hardcoded csproj name with dynamically obtained one. --- lib/plugins/compile/compileServiceCode/dotnet/index.js | 1 + lib/plugins/compile/index.js | 5 +++++ lib/plugins/spawn/index.js | 7 ++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/plugins/compile/compileServiceCode/dotnet/index.js b/lib/plugins/compile/compileServiceCode/dotnet/index.js index 99e8578..37e23ed 100644 --- a/lib/plugins/compile/compileServiceCode/dotnet/index.js +++ b/lib/plugins/compile/compileServiceCode/dotnet/index.js @@ -139,6 +139,7 @@ console.log('compileServiceCode.dotnet(). just executed'.yellow); bin: buildOutputDir, buildDir: tmpBuildDir, originalCodeFilePath: service.originalCodeFilePath, + dotnetCsproj: service.dotnetCsproj, stderr: stderr, stdout: stdout, exitCode: data diff --git a/lib/plugins/compile/index.js b/lib/plugins/compile/index.js index 3444ef2..b90568e 100644 --- a/lib/plugins/compile/index.js +++ b/lib/plugins/compile/index.js @@ -166,6 +166,11 @@ module.exports = function compile (config) { stderr: '', stdout: '' }; + + if (_service.language === 'dotnet') { + result.dotnetCsproj = _service.dotnetCsproj; + } + next(null, result); } diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index 414ccfb..bdfa602 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -127,7 +127,7 @@ module['exports'] = function spawnService (service) { // if targetLanguage is a compiled / static langauge, we must first compile the source code //console.log( compileService) if (typeof compileService[_service.language] === "function") { - console.log('found compile service for ', _service.language); + console.log('found compile service for '.yellow, _service.language); _compile(input, output, function (err, _build) { // [x] service.bin @@ -146,7 +146,8 @@ module['exports'] = function spawnService (service) { // instead of attempting to spawn the service, we need to pipe the error back to the client return output.json(_build); } - console.log('attempting to spawn'); + console.log('Compiler result: '.yellow); + //console.log('### spawnServiceMiddleware._compile() ###') console.dir(_build); _service.bin = _build.bin; // Remark: Java is an edge case now, as its both compiled than interpreted @@ -165,7 +166,7 @@ module['exports'] = function spawnService (service) { if (_service.language === 'dotnet'){ _service.bin = 'dotnet'; - _service.buildDir = _build.buildDir + '/dotnet-hello.csproj'; //needed in commandlineArgs generator + _service.buildDir = _build.buildDir + '/' + _build.dotnetCsproj; //needed in commandlineArgs generator //_service.argv = ['run', '-p', _build.buildDir + '/dotnet-hello.csproj']; //TODO implemented dynamically getting the csproj file name _service.cwd = _build.bin; } From ad613a3e0fcc79408a97f65fddaa3aba909ba126 Mon Sep 17 00:00:00 2001 From: Janaka A Date: Mon, 28 Aug 2017 21:34:21 +0100 Subject: [PATCH 13/13] Update readme.md with dotnet --- ReadMe.md | 1 + lib/plugins/spawn/index.js | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ReadMe.md b/ReadMe.md index 56af35d..2c73e97 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -28,6 +28,7 @@ see: [100+ Working Service Examples](https://github.com/stackvana/microcule-exam - coffee-script - common lisp - bash + - dotnet (.NET Core) - lua - golang - ocaml diff --git a/lib/plugins/spawn/index.js b/lib/plugins/spawn/index.js index bdfa602..6bc8ac8 100644 --- a/lib/plugins/spawn/index.js +++ b/lib/plugins/spawn/index.js @@ -167,7 +167,6 @@ module['exports'] = function spawnService (service) { if (_service.language === 'dotnet'){ _service.bin = 'dotnet'; _service.buildDir = _build.buildDir + '/' + _build.dotnetCsproj; //needed in commandlineArgs generator - //_service.argv = ['run', '-p', _build.buildDir + '/dotnet-hello.csproj']; //TODO implemented dynamically getting the csproj file name _service.cwd = _build.bin; } _spawnService(input, output);