SIGN IN SIGN UP

A data visualization and analytics component, especially well-suited for large and/or streaming datasets.

0 0 4 C++
2019-12-07 00:15:42 -05:00
/*******************************************************************************
2019-10-04 15:18:57 -04:00
*
* Copyright (c) 2017, the Perspective Authors.
*
* This file is part of the Perspective library, distributed under the terms of
* the Apache License 2.0. The full license can be found in the LICENSE file.
*
*/
require("dotenv").config({path: "./.perspectiverc"});
process.env.FORCE_COLOR = true;
2019-10-04 15:18:57 -04:00
const execSync = require("child_process").execSync;
const _path = require("path");
const fs = require("fs");
const rimraf = require("rimraf");
2021-01-22 14:07:32 -05:00
const {promisify} = require("util");
2020-02-01 21:57:39 -05:00
const isWin = process.platform === "win32";
2019-10-04 15:18:57 -04:00
2019-12-07 00:15:42 -05:00
/*******************************************************************************
*
* Private
*/
2019-10-04 15:18:57 -04:00
function rimraf_err(e) {
if (e) {
console.error(e.message);
process.exit(1);
}
}
2019-12-06 23:30:20 -05:00
function depath(strings, ...args) {
if (Array.isArray(strings)) {
strings = strings.map((x, i) => x + (args[i] || "")).join("");
2019-10-04 15:18:57 -04:00
}
2019-12-06 23:30:20 -05:00
strings = strings.split("/");
if (strings[0] === "") {
strings = strings.slice(1);
strings[0] = "/" + strings[0];
}
return strings;
}
2019-10-04 15:18:57 -04:00
2019-12-07 00:15:42 -05:00
function cut_last(f) {
let x = f.split(" ");
return x.slice(0, x.length - 1).join(" ");
}
function cut_first(f) {
2021-09-12 14:08:32 -04:00
return f.split(" ").slice(1).join(" ");
2019-12-07 00:15:42 -05:00
}
2021-09-12 14:08:32 -04:00
const execute_throw = (cmd) => {
if (process.argv.indexOf("--debug") > -1) {
2022-03-28 18:38:14 -04:00
process.stdout.write(`$ ${cmd}\n`);
}
let env = {...process.env};
if (!!process.env.PSP_DEBUG || process.argv.indexOf("--debug") >= 0) {
env.PSP_DEBUG = 1;
}
2022-03-28 18:38:14 -04:00
execSync(cmd, {stdio: "inherit", env});
};
2021-09-12 14:08:32 -04:00
const execute = (cmd) => {
2019-12-07 00:15:42 -05:00
try {
execute_throw(cmd);
2019-12-07 00:15:42 -05:00
} catch (e) {
console.error("\n" + e.message);
2019-12-07 00:15:42 -05:00
process.exit(1);
}
};
2021-09-12 14:08:32 -04:00
const execute_return = async (cmd) => {
2021-01-22 14:07:32 -05:00
if (process.argv.indexOf("--debug") > -1) {
console.log(`$ ${cmd}`);
}
const ex = promisify(require("child_process").exec);
return await ex(cmd);
};
2021-01-22 14:07:32 -05:00
2019-12-07 00:15:42 -05:00
/*******************************************************************************
*
* Public
*/
/**
* Calls `path.join` on the result of splitting the input string by the default
* path delimiter `/`, which allows writing simpler path statements that will
* still be cross platform. Can be used as an template literal.
*
* @param {string} path a `/` encoded path.
* @returns {string} A system-correct path
* @example
* console.assert(path`a/b/c` === `a\\b\\c`) // on Windows
*/
2019-12-06 23:30:20 -05:00
exports.path = function path(strings, ...args) {
return _path.join(...depath(strings, ...args));
2019-10-04 15:18:57 -04:00
};
2019-12-07 00:15:42 -05:00
/**
* Like `path`, but uses `path.resolve` to get the absolute path, carefully
* preserving leading delimiter. Can be used as an template literal.
*
* @param {string} path a relative `/` encoded path.
* @returns {string} A system-correct absolute path
* @example
* console.assert(path`a/b/c` === `${process.cwd()}\\a\\b\\v`) // on Windows
*/
2019-12-06 23:30:20 -05:00
const resolve = (exports.resolve = function resolve(strings, ...args) {
2020-09-10 05:19:27 -04:00
return _path.resolve(...depath(strings, ...args)).replace(/\\/g, "\\");
2019-12-06 23:30:20 -05:00
});
2019-12-07 00:15:42 -05:00
/**
* Calls `join` on each of the input path arguments, then `rimraf`s the path if
* it exists. Can be used as an template literal, and can also take multiple
* arguments call itself in sequence.
*
* @param {string} [path] a `/` encode path.
* @example
* clean`a/b/c`; // Cleans this dir
* clean(path`a/b/c`, path`d/e/f`); // Cleans both dirs
*/
2019-10-04 15:18:57 -04:00
exports.clean = function clean(...dirs) {
2019-12-06 23:30:20 -05:00
if (Array.isArray(dirs[0])) {
const dir = exports.path(...dirs);
2019-10-04 15:18:57 -04:00
if (fs.existsSync(dir)) {
rimraf(dir, rimraf_err);
}
2019-12-06 23:30:20 -05:00
} else {
for (let dir of dirs) {
dir = exports.path([dir]);
if (fs.existsSync(dir)) {
rimraf(dir, rimraf_err);
}
}
2019-10-04 15:18:57 -04:00
}
};
2019-12-07 00:15:42 -05:00
/**
* For working with shell commands, `bash` knows how to remove consecutive
* text from strings when arguments are "falsey", which makes mapping flags to
* JS expressions a breeze. Can be used as a template literal.
*
* @param {string} expression a bash command to be templated.
* @returns {string} A command with the missing argument's flags removed.
* @example
* console.assert(
* bash`run -t${1} -u"${undefined}" task`,
* `run -t1 task`
* );
*/
2019-12-06 23:30:20 -05:00
const bash = (exports.bash = function bash(strings, ...args) {
2019-10-04 15:18:57 -04:00
let terms = [];
2019-12-06 23:30:20 -05:00
if (strings.length === 1) {
return strings[0];
}
for (let i = 0; i < strings.length - 1; i++) {
const arg = args[i];
2019-12-11 12:48:05 -05:00
const start = terms.length === 0 ? strings[i] : terms.pop();
2019-12-06 23:30:20 -05:00
if (arg === undefined || arg !== arg || arg === false) {
terms = [...terms, cut_last(start), " ", cut_first(strings[i + 1])];
} else if (Array.isArray(arg)) {
terms = [...terms, start, arg.join(" "), strings[i + 1]];
} else {
terms = [...terms, start, arg, strings[i + 1]];
2019-10-04 15:18:57 -04:00
}
}
2019-12-06 23:30:20 -05:00
return terms
.join("")
.replace(/[ \t\n]+/g, " ")
2019-12-06 23:30:20 -05:00
.trim();
});
2019-12-07 00:15:42 -05:00
/**
* Just like `bash, but executes the command immediately. Will log if the
* `--debug` flag is used to build. If an error is encountered, it is logged
* and the child process will exit with error code 1.
2019-12-07 00:15:42 -05:00
*
* @param {string} expression a bash command to be templated.
* @returns {string} A command with the missing argument's flags removed.
* @example
* execute`run -t${1} -u"${undefined}" task`;
*/
2021-09-12 14:08:32 -04:00
exports.execute = (strings, ...args) =>
execute(Array.isArray(strings) ? bash(strings, ...args) : strings);
/**
* Just like `execute`, except it throws anddoes not exit the child process
* if the command throws an error.
*
* @param {string} expression a bash command to be templated.
* @returns {string} A command with the missing argument's flags removed.
* @example
* execute`run -t${1} -u"${undefined}" task`;
*/
2021-09-12 14:08:32 -04:00
exports.execute_throw = (strings, ...args) =>
execute_throw(Array.isArray(strings) ? bash(strings, ...args) : strings);
2019-12-07 00:15:42 -05:00
2021-01-22 14:07:32 -05:00
/**
* Just like `execute`, except it will return the output of the command
* it runs.
*
* @async
* @param {string} expression a bash command to be templated.
* @returns {string} A command with the missing argument's flags removed.
* @example
* execute`run -t${1} -u"${undefined}" task`;
*/
2021-09-12 14:08:32 -04:00
exports.execute_return = async (strings, ...args) =>
await execute_return(
Array.isArray(strings) ? bash(strings, ...args) : strings
);
2021-01-22 14:07:32 -05:00
2019-12-07 00:15:42 -05:00
/**
* Returns the value after this command-line flag, or `true` if it is the last
* arg. This makes it easy to null-pun for boolean flags, and capture the
* argument for argument-providing flags, and respect quotes and parens, in
* one function. Can be used as a template literal - not sure why, 2 less
* characters?
*
* @param {string} flag The command line flag name. Returns all arguments if
* this param is `undefined`.
* @returns {string} The next argument after this flag in the command args, or
* `true.
* @example
* console.assert(getarg`--debug`);
*/
2021-09-12 14:08:32 -04:00
const getarg = (exports.getarg = function (flag, ...args) {
2019-12-07 00:15:42 -05:00
if (Array.isArray(flag)) {
flag = flag.map((x, i) => x + (args[i] || "")).join("");
}
const argv = process.argv.slice(2);
if (flag) {
const index = argv.indexOf(flag);
if (index > -1) {
const next = argv[index + 1];
if (next) {
return next;
} else {
return true;
}
2019-12-06 23:30:20 -05:00
}
2019-12-07 00:15:42 -05:00
} else {
return argv
2021-09-12 14:08:32 -04:00
.map(function (arg) {
2019-12-07 00:15:42 -05:00
return "'" + arg.replace(/'/g, "'\\''") + "'";
})
.join(" ");
2019-12-06 23:30:20 -05:00
}
2019-12-07 00:15:42 -05:00
});
/**
* A `bash` expression for running commands in Docker images
*
* @param {string} image The Docker image name.
* @returns {string} A command for invoking this docker image.
* @example
* execute`${docker()} echo "Hello from Docker"`;
*/
exports.docker = function docker(image = "puppeteer") {
console.log(`-- Creating perspective/${image} docker image`);
const IS_WRITE = getarg("--write") || process.env.WRITE_TESTS;
const CPUS = parseInt(process.env.PSP_CPU_COUNT);
2019-12-11 12:48:05 -05:00
const PACKAGE = process.env.PACKAGE;
2019-12-07 00:15:42 -05:00
const CWD = process.cwd();
2020-02-10 10:03:35 -05:00
const IS_CI = getarg("--ci");
2019-12-11 12:48:05 -05:00
const IS_MANYLINUX = image.indexOf("manylinux") > -1 ? true : false;
const IMAGE = `perspective/${image}`;
2019-12-11 12:48:05 -05:00
let env_vars = bash`-eWRITE_TESTS=${IS_WRITE} \
-ePACKAGE="${PACKAGE}"`;
let flags = IS_CI ? bash`--rm` : bash`--rm -it`;
2020-02-10 10:03:35 -05:00
2019-12-10 21:48:45 +00:00
if (IS_MANYLINUX) {
console.log(`-- Using manylinux build`);
2020-02-10 10:03:35 -05:00
env_vars += bash` -ePSP_MANYLINUX=1 `;
2019-12-10 21:48:45 +00:00
}
return bash`docker run \
2020-02-10 10:03:35 -05:00
${flags} \
2019-12-10 21:48:45 +00:00
${env_vars} \
-v${CWD}:/usr/src/app/perspective \
2019-12-07 00:15:42 -05:00
-w /usr/src/app/perspective --shm-size=2g -u root \
2019-12-11 12:48:05 -05:00
--cpus="${CPUS}.0" ${IMAGE}`;
2019-12-06 23:30:20 -05:00
};
2019-12-10 21:48:45 +00:00
/**
* Get the docker image to use for the given image/python combination
*
* @param {string} image The Docker image name.
* @param {string} python The python version requested
* @returns {string} The docker image to use
*/
exports.python_image = function python_image(image = "", python = "") {
2021-09-12 14:08:32 -04:00
console.log(
`-- Getting image for image: '${image}' and python: '${python}'`
);
2019-12-10 21:48:45 +00:00
if (python == "python2") {
if (image == "manylinux2014") {
2019-12-10 21:48:45 +00:00
throw "Python2 not supported for manylinux2014";
} else {
return "manylinux2010";
2019-12-10 21:48:45 +00:00
}
}
return `${image}`;
2019-12-10 21:48:45 +00:00
};
2019-12-07 00:15:42 -05:00
/*******************************************************************************
*
* Tests
*/
2019-12-06 23:30:20 -05:00
2019-12-07 00:15:42 -05:00
function run_suite(tests) {
for (const [actual, expected] of tests) {
2021-09-12 14:08:32 -04:00
console.assert(
actual === expected,
`"${actual}" received, expected: "${expected}"`
);
2019-12-07 00:15:42 -05:00
}
}
2020-09-10 05:19:27 -04:00
if (false) {
if (isWin) {
run_suite([
[resolve`a/b/c`, `${process.cwd()}\\a\\b\\c`],
2021-09-12 14:08:32 -04:00
[
resolve`${__dirname}/../cpp/perspective`,
`${process.cwd()}\\cpp\\perspective`,
],
[
resolve`${__dirname}/../python/perspective/dist`,
_path.resolve(__dirname, "..", "python", "perspective", "dist"),
],
[
resolve`${__dirname}/../cpp/perspective`,
_path.resolve(__dirname, "..", "cpp", "perspective"),
],
[
resolve`${__dirname}/../cmake`,
_path.resolve(__dirname, "..", "cmake"),
],
[
resolve`${resolve`${__dirname}/../python/perspective/dist`}/cmake`,
_path.resolve(
_path.resolve(
__dirname,
"..",
"python",
"perspective",
"dist"
),
"cmake"
),
],
[
resolve`${resolve`${__dirname}/../python/perspective/dist`}/obj`,
_path.resolve(
_path.resolve(
__dirname,
"..",
"python",
"perspective",
"dist"
),
"obj"
),
],
2020-09-10 05:19:27 -04:00
]);
} else {
run_suite([
[resolve`a/b/c`, `${process.cwd()}/a/b/c`],
2021-09-12 14:08:32 -04:00
[
resolve`${__dirname}/../cpp/perspective`,
`${process.cwd()}/cpp/perspective`,
],
[
resolve`${__dirname}/../python/perspective/dist`,
_path.resolve(__dirname, "..", "python", "perspective", "dist"),
],
[
resolve`${__dirname}/../cpp/perspective`,
_path.resolve(__dirname, "..", "cpp", "perspective"),
],
[
resolve`${__dirname}/../cmake`,
_path.resolve(__dirname, "..", "cmake"),
],
[
resolve`${resolve`${__dirname}/../python/perspective/dist`}/cmake`,
_path.resolve(
_path.resolve(
__dirname,
"..",
"python",
"perspective",
"dist"
),
"cmake"
),
],
[
resolve`${resolve`${__dirname}/../python/perspective/dist`}/obj`,
_path.resolve(
_path.resolve(
__dirname,
"..",
"python",
"perspective",
"dist"
),
"obj"
),
],
2020-09-10 05:19:27 -04:00
]);
}
2020-02-01 21:57:39 -05:00
run_suite([
2020-09-10 05:19:27 -04:00
[bash`run -t${1}`, `run -t1`],
[bash`run -t${undefined}`, `run`],
[bash`run -t${true}`, `run -ttrue`],
[bash`run -t${false}`, `run`],
[bash`run -t${1} task`, `run -t1 task`],
[bash`run -t${undefined} task`, `run task`],
[bash`run -t="${1}"`, `run -t="1"`],
[bash`run -t="${undefined}"`, `run`],
[bash`run -t="${1}" task`, `run -t="1" task`],
[bash`run -t="${undefined}" task`, `run task`],
[bash`run -t${1} -u${2} task`, `run -t1 -u2 task`],
[bash`run -t${1} -u${undefined} task`, `run -t1 task`],
[bash`run -t${undefined} -u${2} task`, `run -u2 task`],
[bash`run -t${undefined} -u${undefined} task`, `run task`],
[bash`run -t"${undefined}" -u"${undefined}" task`, `run task`],
[bash`run "${undefined}" task`, `run task`],
[bash`run ${undefined} task`, `run task`],
[bash`TEST=${undefined} run`, `run`],
[bash`TEST=${1} run`, `TEST=1 run`],
[bash`TEST=${1}`, `TEST=1`],
[bash`TEST=${undefined}`, ``],
[bash`this is a test`, `this is a test`],
[bash`this is a test `, `this is a test `],
2021-09-12 14:08:32 -04:00
[bash`--test="${undefined}.0" ${1}`, `1`],
2020-02-01 21:57:39 -05:00
]);
}