diff --git a/benchmark/_cli.js b/benchmark/_cli.js new file mode 100644 index 00000000000000..be2f7ffff83624 --- /dev/null +++ b/benchmark/_cli.js @@ -0,0 +1,99 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +// Create an object of all benchmark scripts +const benchmarks = {}; +fs.readdirSync(__dirname) + .filter(function(name) { + return fs.statSync(path.resolve(__dirname, name)).isDirectory(); + }) + .forEach(function(category) { + benchmarks[category] = fs.readdirSync(path.resolve(__dirname, category)) + .filter((filename) => filename[0] !== '.' && filename[0] !== '_'); + }); + +function CLI(usage, settings) { + if (!(this instanceof CLI)) return new CLI(usage, settings); + + if (process.argv.length < 3) { + this.abort(usage); // abort will exit the process + } + + this.usage = usage; + this.optional = {}; + this.items = []; + + for (const argName of settings.arrayArgs) { + this.optional[argName] = []; + } + + let currentOptional = null; + let mode = 'both'; // possible states are: [both, option, item] + + for (const arg of process.argv.slice(2)) { + if (arg === '--') { + // Only items can follow -- + mode = 'item'; + } else if (['both', 'option'].includes(mode) && arg[0] === '-') { + // Optional arguments declaration + + if (arg[1] === '-') { + currentOptional = arg.slice(2); + } else { + currentOptional = arg.slice(1); + } + + // Default the value to true + if (!settings.arrayArgs.includes(currentOptional)) { + this.optional[currentOptional] = true; + } + + // expect the next value to be option related (either -- or the value) + mode = 'option'; + } else if (mode === 'option') { + // Optional arguments value + + if (settings.arrayArgs.includes(currentOptional)) { + this.optional[currentOptional].push(arg); + } else { + this.optional[currentOptional] = arg; + } + + // the next value can be either an option or an item + mode = 'both'; + } else if (['both', 'item'].includes(mode)) { + // item arguments + this.items.push(arg); + + // the next value must be an item + mode = 'item'; + } else { + // Bad case, abort + this.abort(usage); + return; + } + } +} +module.exports = CLI; + +CLI.prototype.abort = function(msg) { + console.error(msg); + process.exit(1); +}; + +CLI.prototype.benchmarks = function() { + const paths = []; + const filter = this.optional.filter || false; + + for (const category of this.items) { + for (const scripts of benchmarks[category]) { + if (filter && scripts.lastIndexOf(filter) === -1) continue; + + paths.push(path.join(category, scripts)); + } + } + + return paths; +}; diff --git a/benchmark/common.js b/benchmark/common.js index 3744a4420a09cb..669a4c642b2bfd 100644 --- a/benchmark/common.js +++ b/benchmark/common.js @@ -1,201 +1,227 @@ 'use strict'; -var assert = require('assert'); -var fs = require('fs'); -var path = require('path'); -var child_process = require('child_process'); - -var outputFormat = process.env.OUTPUT_FORMAT || - (+process.env.NODE_BENCH_SILENT ? 'silent' : false) || - 'default'; - -// verify outputFormat -if (['default', 'csv', 'silent'].indexOf(outputFormat) == -1) { - throw new Error('OUTPUT_FORMAT set to invalid value'); -} -exports.PORT = process.env.PORT || 12346; +const child_process = require('child_process'); -function hasWrk() { - var result = child_process.spawnSync('wrk', ['-h']); - if (result.error && result.error.code === 'ENOENT') { - console.error('Couldn\'t locate `wrk` which is needed for running ' + - 'benchmarks. Check benchmark/README.md for further instructions.'); - process.exit(-1); - } -} +// The port used by servers and wrk +exports.PORT = process.env.PORT || 12346; exports.createBenchmark = function(fn, options) { return new Benchmark(fn, options); }; function Benchmark(fn, options) { - this.fn = fn; - this.options = options; - this.config = parseOpts(options); - this._name = require.main.filename.split(/benchmark[\/\\]/).pop(); - this._start = [0, 0]; + this.name = require.main.filename.slice(__dirname.length + 1); + this.options = this._parseArgs(process.argv.slice(2), options); + this.queue = this._queue(this.options); + this.config = this.queue[0]; + + this._time = [0, 0]; // holds process.hrtime value this._started = false; - var self = this; + // this._run will use fork() to create a new process for each configuration + // combination. + if (process.env.hasOwnProperty('NODE_RUN_BENCHMARK_FN')) { + process.nextTick(() => fn(this.config)); + } else { + process.nextTick(() => this._run()); + } +} - process.nextTick(function() { - self._run(); - }); +Benchmark.prototype._parseArgs = function(argv, options) { + const cliOptions = Object.assign({}, options); + + // Parse configuarion arguments + for (const arg of argv) { + const match = arg.match(/^(.+?)=([\s\S]*)$/); + if (!match || !match[1]) { + console.error('bad argument: ' + arg); + process.exit(1); + } + + // Infer the type from the options object and parse accordingly + const isNumber = typeof options[match[1]][0] === 'number'; + const value = isNumber ? +match[2] : match[2]; + + cliOptions[match[1]] = [value]; + } + + return cliOptions; +}; + +Benchmark.prototype._queue = function(options) { + const queue = []; + const keys = Object.keys(options); + + // Perform a depth-first walk though all options to genereate a + // configuration list that contains all combinations. + function recursive(keyIndex, prevConfig) { + const key = keys[keyIndex]; + const values = options[key]; + const type = typeof values[0]; + + for (const value of values) { + if (typeof value !== 'number' && typeof value !== 'string') { + throw new TypeError(`configuration "${key}" had type ${typeof value}`); + } + if (typeof value !== type) { + // This is a requirement for being able to consistently and predictably + // parse CLI provided configuration values. + throw new TypeError(`configuration "${key}" has mixed types`); + } + + const currConfig = Object.assign({ [key]: value }, prevConfig); + + if (keyIndex + 1 < keys.length) { + recursive(keyIndex + 1, currConfig); + } else { + queue.push(currConfig); + } + } + } + + if (keys.length > 0) { + recursive(0, {}); + } else { + queue.push({}); + } + + return queue; +}; + +function hasWrk() { + const result = child_process.spawnSync('wrk', ['-h']); + if (result.error && result.error.code === 'ENOENT') { + console.error('Couldn\'t locate `wrk` which is needed for running ' + + 'benchmarks. Check benchmark/README.md for further instructions.'); + process.exit(1); + } } // benchmark an http server. -Benchmark.prototype.http = function(p, args, cb) { +const WRK_REGEXP = /Requests\/sec:[ \t]+([0-9\.]+)/; +Benchmark.prototype.http = function(urlPath, args, cb) { hasWrk(); - var self = this; - var regexp = /Requests\/sec:[ \t]+([0-9\.]+)/; - var url = 'http://127.0.0.1:' + exports.PORT + p; - - args = args.concat(url); + const self = this; - var out = ''; - var child = child_process.spawn('wrk', args); + const urlFull = 'http://127.0.0.1:' + exports.PORT + urlPath; + args = args.concat(urlFull); - child.stdout.setEncoding('utf8'); + const childStart = process.hrtime(); + const child = child_process.spawn('wrk', args); + child.stderr.pipe(process.stderr); - child.stdout.on('data', function(chunk) { - out += chunk; - }); + // Collect stdout + let stdout = ''; + child.stdout.on('data', (chunk) => stdout += chunk.toString()); - child.on('close', function(code) { - if (cb) - cb(code); + child.once('close', function(code) { + const elapsed = process.hrtime(childStart); + if (cb) cb(code); if (code) { console.error('wrk failed with ' + code); process.exit(code); } - var match = out.match(regexp); - var qps = match && +match[1]; - if (!qps) { - console.error('%j', out); - console.error('wrk produced strange output'); + + // Extract requests pr second and check for odd results + const match = stdout.match(WRK_REGEXP); + if (!match || match.length <= 1) { + console.error('wrk produced strange output:'); + console.error(stdout); process.exit(1); } - self.report(+qps); + + // Report rate + self.report(+match[1], elapsed); }); }; Benchmark.prototype._run = function() { - if (this.config) - return this.fn(this.config); - - // some options weren't set. - // run with all combinations - var main = require.main.filename; - var options = this.options; - - var queue = Object.keys(options).reduce(function(set, key) { - var vals = options[key]; - assert(Array.isArray(vals)); - - // match each item in the set with each item in the list - var newSet = new Array(set.length * vals.length); - var j = 0; - set.forEach(function(s) { - vals.forEach(function(val) { - if (typeof val !== 'number' && typeof val !== 'string') { - throw new TypeError(`configuration "${key}" had type ${typeof val}`); - } - - newSet[j++] = s.concat(key + '=' + val); - }); - }); - return newSet; - }, [[main]]); - - // output csv heading - if (outputFormat == 'csv') - console.log('filename,' + Object.keys(options).join(',') + ',result'); - - var node = process.execPath; - var i = 0; - function run() { - var argv = queue[i++]; - if (!argv) - return; - argv = process.execArgv.concat(argv); - var child = child_process.spawn(node, argv, { stdio: 'inherit' }); - child.on('close', function(code, signal) { - if (code) - console.error('child process exited with code ' + code); - else - run(); - }); - } - run(); -}; + const self = this; + + (function recursive(queueIndex) { + const config = self.queue[queueIndex]; -function parseOpts(options) { - // verify that there's an option provided for each of the options - // if they're not *all* specified, then we return null. - var keys = Object.keys(options); - var num = keys.length; - var conf = {}; - for (var i = 2; i < process.argv.length; i++) { - var match = process.argv[i].match(/^(.+?)=([\s\S]*)$/); - if (!match || !match[1] || !options[match[1]]) { - return null; - } else { - conf[match[1]] = match[2]; - num--; + // set NODE_RUN_BENCHMARK_FN to indicate that the child shouldn't construct + // a configuration queue, but just execute the benchmark function. + const childEnv = Object.assign({}, process.env); + childEnv.NODE_RUN_BENCHMARK_FN = ''; + + // Create configuration arguments + const childArgs = []; + for (const key of Object.keys(config)) { + childArgs.push(`${key}=${config[key]}`); } - } - // still go ahead and set whatever WAS set, if it was. - if (num !== 0) { - Object.keys(conf).forEach(function(k) { - options[k] = [conf[k]]; + + const child = child_process.fork(require.main.filename, childArgs, { + env: childEnv }); - } - return num === 0 ? conf : null; -} + child.on('message', sendResult); + child.on('close', function(code) { + if (code) { + process.exit(code); + return; + } + + if (queueIndex + 1 < self.queue.length) { + recursive(queueIndex + 1); + } + }); + })(0); +}; Benchmark.prototype.start = function() { if (this._started) throw new Error('Called start more than once in a single benchmark'); this._started = true; - this._start = process.hrtime(); + this._time = process.hrtime(); }; Benchmark.prototype.end = function(operations) { - var elapsed = process.hrtime(this._start); + // get elapsed time now and do error checking later for accuracy. + const elapsed = process.hrtime(this._time); - if (!this._started) + if (!this._started) { throw new Error('called end without start'); - if (typeof operations !== 'number') + } + if (typeof operations !== 'number') { throw new Error('called end() without specifying operation count'); + } - var time = elapsed[0] + elapsed[1] / 1e9; - var rate = operations / time; - this.report(rate); + const time = elapsed[0] + elapsed[1] / 1e9; + const rate = operations / time; + this.report(rate, elapsed); }; -Benchmark.prototype.report = function(value) { - var heading = this.getHeading(); +function formatResult(data) { + // Construct confiuration string, " A=a, B=b, ..." + let conf = ''; + for (const key of Object.keys(data.conf)) { + conf += ' ' + key + '=' + JSON.stringify(data.conf[key]); + } - if (outputFormat == 'default') - console.log('%s: %s', heading, value.toFixed(5)); - else if (outputFormat == 'csv') - console.log('%s,%s', heading, value.toFixed(5)); -}; + return `${data.name}${conf}: ${data.rate}`; +} -Benchmark.prototype.getHeading = function() { - var conf = this.config; - - if (outputFormat == 'default') { - return this._name + ' ' + Object.keys(conf).map(function(key) { - return key + '=' + JSON.stringify('' + conf[key]); - }).join(' '); - } else if (outputFormat == 'csv') { - return this._name + ',' + Object.keys(conf).map(function(key) { - return JSON.stringify('' + conf[key]); - }).join(','); +function sendResult(data) { + if (process.send) { + // If forked, report by process send + process.send(data); + } else { + // Otherwise report by stdout + console.log(formatResult(data)); } +} +exports.sendResult = sendResult; + +Benchmark.prototype.report = function(rate, elapsed) { + sendResult({ + name: this.name, + conf: this.config, + rate: rate, + time: elapsed[0] + elapsed[1] / 1e9 + }); }; exports.v8ForceOptimization = function(method, ...args) { diff --git a/benchmark/http/http_server_for_chunky_client.js b/benchmark/http/http_server_for_chunky_client.js index d85e15bcbed4e9..fade895aa07fdd 100644 --- a/benchmark/http/http_server_for_chunky_client.js +++ b/benchmark/http/http_server_for_chunky_client.js @@ -3,8 +3,8 @@ var path = require('path'); var http = require('http'); var fs = require('fs'); -var spawn = require('child_process').spawn; -require('../common.js'); +var fork = require('child_process').fork; +var common = require('../common.js'); var test = require('../../test/common.js'); var pep = path.dirname(process.argv[1]) + '/_chunky_http_client.js'; var PIPE = test.PIPE; @@ -30,25 +30,10 @@ server.on('error', function(err) { throw new Error('server error: ' + err); }); -try { - var child; - - server.listen(PIPE); - - child = spawn(process.execPath, [pep], { }); - - child.on('error', function(err) { - throw new Error('spawn error: ' + err); - }); - - child.stdout.pipe(process.stdout); - child.stderr.pipe(process.stderr); - - child.on('close', function(exitCode) { - server.close(); - }); - -} catch (e) { - throw new Error('error: ' + e); -} +server.listen(PIPE); +var child = fork(pep, process.argv.slice(2)); +child.on('message', common.sendResult); +child.on('close', function() { + server.close(); +}); diff --git a/benchmark/misc/v8-bench.js b/benchmark/misc/v8-bench.js index 0b9a5139ba23cf..9c0448a510d471 100644 --- a/benchmark/misc/v8-bench.js +++ b/benchmark/misc/v8-bench.js @@ -3,21 +3,49 @@ var fs = require('fs'); var path = require('path'); var vm = require('vm'); +var common = require('../common.js'); var dir = path.join(__dirname, '..', '..', 'deps', 'v8', 'benchmarks'); -global.print = function(s) { - if (s === '----') return; - console.log('misc/v8_bench.js %s', s); -}; - -global.load = function(filename) { +function load(filename, inGlobal) { var source = fs.readFileSync(path.join(dir, filename), 'utf8'); - // deps/v8/benchmarks/regexp.js breaks console.log() because it clobbers - // the RegExp global, Restore the original when the script is done. - var $RegExp = global.RegExp; - vm.runInThisContext(source, { filename: filename }); - global.RegExp = $RegExp; -}; + if (!inGlobal) source = '(function () {' + source + '\n})()'; + vm.runInThisContext(source, { filename: 'v8/bechmark/' + filename }); +} + +load('base.js', true); +load('richards.js'); +load('deltablue.js'); +load('crypto.js'); +load('raytrace.js'); +load('earley-boyer.js'); +load('regexp.js'); +load('splay.js'); +load('navier-stokes.js'); -global.load('run.js'); +const times = {}; +global.BenchmarkSuite.RunSuites({ + NotifyStart: function(name) { + times[name] = process.hrtime(); + }, + NotifyResult: function(name, result) { + const elapsed = process.hrtime(times[name]); + common.sendResult({ + name: name, + conf: {}, + rate: result, + time: elapsed[0] + elapsed[1] / 1e9 + }); + }, + NotifyError: function(name, error) { + console.error(name + ': ' + error); + }, + NotifyScore: function(score) { + common.sendResult({ + name: 'Score (version ' + global.BenchmarkSuite.version + ')', + conf: {}, + rate: score, + time: 0 + }); + } +}); diff --git a/benchmark/run.js b/benchmark/run.js index ad590ea34a8952..756a7408bbbf37 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -1,63 +1,51 @@ 'use strict'; -const fs = require('fs'); const path = require('path'); -const child_process = require('child_process'); - -var outputFormat = process.env.OUTPUT_FORMAT || - (+process.env.NODE_BENCH_SILENT ? 'silent' : false) || - 'default'; - -// If this is the main module, then run the benchmarks -if (module === require.main) { - var type = process.argv[2]; - var testFilter = process.argv[3]; - if (!type) { - console.error('usage:\n ./node benchmark/run.js [testFilter]'); - process.exit(1); - } - - var dir = path.join(__dirname, type); - var tests = fs.readdirSync(dir); - - if (testFilter) { - var filteredTests = tests.filter(function(item) { - if (item.lastIndexOf(testFilter) >= 0) { - return item; - } - }); - - if (filteredTests.length === 0) { - console.error('%s is not found in \n %j', testFilter, tests); - return; - } - tests = filteredTests; - } - - runBenchmarks(); +const fork = require('child_process').fork; +const CLI = require('./_cli.js'); + +const cli = CLI(`usage: ./node run.js [options] [--] ... + Run each benchmark in the directory a single time, more than one + directory can be specified. + + --filter pattern string to filter benchmark scripts + --set variable=value set benchmark variable (can be repeated) +`, { + arrayArgs: ['set'] +}); +const benchmarks = cli.benchmarks(); + +if (benchmarks.length === 0) { + console.error('no benchmarks found'); + process.exit(1); } -function runBenchmarks() { - var test = tests.shift(); - if (!test) - return; +(function recursive(i) { + const filename = benchmarks[i]; + const child = fork(path.resolve(__dirname, filename), cli.optional.set); - if (test.match(/^[\._]/)) - return process.nextTick(runBenchmarks); + console.log(); + console.log(filename); - if (outputFormat == 'default') - console.error(type + '/' + test); + child.on('message', function(data) { + // Construct configuration string, " A=a, B=b, ..." + let conf = ''; + for (const key of Object.keys(data.conf)) { + conf += ' ' + key + '=' + JSON.stringify(data.conf[key]); + } - test = path.resolve(dir, test); + console.log(`${data.name}${conf}: ${data.rate}`); + }); - var a = (process.execArgv || []).concat(test); - var child = child_process.spawn(process.execPath, a, { stdio: 'inherit' }); - child.on('close', function(code) { + child.once('close', function(code) { if (code) { process.exit(code); - } else { - console.log(''); - runBenchmarks(); + return; + } + + // If there are more benchmarks execute the next + if (i + 1 < benchmarks.length) { + recursive(i + 1); } }); -} +})(0);