/*eslint-env node*/ import { writeFileSync, copyFileSync, readFileSync, existsSync } from "fs"; import { readFile, writeFile } from "fs/promises"; import { join, basename, resolve, dirname } from "path"; import { exec, execSync } from "child_process"; import fetch from "node-fetch"; import { createRequire } from "module"; import gulp from "gulp"; import gulpTap from "gulp-tap"; import gulpZip from "gulp-zip"; import gulpRename from "gulp-rename"; import gulpReplace from "gulp-replace"; import { globby } from "globby"; import open from "open"; import { rimraf } from "rimraf"; import { mkdirp } from "mkdirp"; import mergeStream from "merge-stream"; import streamToPromise from "stream-to-promise"; import karma from "karma"; import yargs from "yargs"; import typeScript from "typescript"; import { build as esbuild } from "esbuild"; import { createInstrumenter } from "istanbul-lib-instrument"; import { buildCesium, buildEngine, buildWidgets, bundleWorkers, glslToJavaScript, createCombinedSpecList, createJsHintOptions, defaultESBuildOptions, } from "./scripts/build.js"; // Determines the scope of the workspace packages. If the scope is set to cesium, the workspaces should be @cesium/engine. // This should match the scope of the dependencies of the root level package.json. const scope = "cesium"; const require = createRequire(import.meta.url); const packageJson = require("./package.json"); let version = packageJson.version; if (/\.0$/.test(version)) { version = version.substring(0, version.length - 2); } const karmaConfigFile = resolve("./Specs/karma.conf.cjs"); const devDeployUrl = "https://ci-builds.cesium.com/cesium/"; const isProduction = process.env.PROD; //Gulp doesn't seem to have a way to get the currently running tasks for setting //per-task variables. We use the command line argument here to detect which task is being run. const taskName = process.argv[2]; const noDevelopmentGallery = taskName === "release" || taskName === "makeZip" || taskName === "websiteRelease"; const argv = yargs(process.argv).argv; const verbose = argv.verbose; const sourceFiles = [ "packages/engine/Source/**/*.js", "!packages/engine/Source/*.js", "packages/widgets/Source/**/*.js", "!packages/widgets/Source/*.js", "!packages/engine/Source/Shaders/**", "!packages/engine/Source/ThirdParty/Workers/**", "!packages/engine/Source/ThirdParty/google-earth-dbroot-parser.js", "!packages/engine/Source/ThirdParty/_*", ]; const watchedSpecFiles = [ "packages/engine/Specs/**/*Spec.js", "!packages/engine/Specs/SpecList.js", "packages/widgets/Specs/**/*Spec.js", "!packages/widgets/Specs/SpecList.js", "Specs/*.js", "!Specs/SpecList.js", "Specs/TestWorkers/*.js", ]; const shaderFiles = [ "packages/engine/Source/Shaders/**/*.glsl", "packages/engine/Source/ThirdParty/Shaders/*.glsl", ]; // Print an esbuild warning function printBuildWarning({ location, text }) { const { column, file, line, lineText, suggestion } = location; let message = `\n > ${file}:${line}:${column}: warning: ${text} ${lineText} `; if (suggestion && suggestion !== "") { message += `\n${suggestion}`; } console.log(message); } // Ignore `eval` warnings in third-party code we don't have control over function handleBuildWarnings(result) { for (const warning of result.warnings) { if ( !warning.location.file.includes("protobufjs.js") && !warning.location.file.includes("Build/Cesium") ) { printBuildWarning(warning); } } } export async function build() { // Configure build options from command line arguments. const minify = argv.minify ?? false; const removePragmas = argv.pragmas ?? false; const sourcemap = argv.sourcemap ?? true; const node = argv.node ?? true; const buildOptions = { development: !noDevelopmentGallery, iife: true, minify: minify, removePragmas: removePragmas, sourcemap: sourcemap, node: node, }; // Configure build target. const workspace = argv.workspace ? argv.workspace : undefined; if (workspace === `@${scope}/engine`) { return buildEngine(buildOptions); } else if (workspace === `@${scope}/widgets`) { return buildWidgets(buildOptions); } await buildEngine(buildOptions); await buildWidgets(buildOptions); await buildCesium(buildOptions); } export default build; export const buildWatch = gulp.series(build, async function () { const minify = argv.minify ? argv.minify : false; const removePragmas = argv.pragmas ? argv.pragmas : false; const sourcemap = argv.sourcemap ? argv.sourcemap : true; const outputDirectory = join("Build", `Cesium${!minify ? "Unminified" : ""}`); const bundles = await buildCesium({ minify: minify, path: outputDirectory, removePragmas: removePragmas, sourcemap: sourcemap, incremental: true, }); const esm = bundles.esm; const cjs = bundles.node; const iife = bundles.iife; const specs = bundles.specs; gulp.watch(shaderFiles, async () => { glslToJavaScript(minify, "Build/minifyShaders.state", "engine"); await esm.rebuild(); if (iife) { await iife.rebuild(); } if (cjs) { await cjs.rebuild(); } }); gulp.watch( [ ...sourceFiles, // Shader results are generated in the previous watch task; no need to rebuild twice "!Source/Shaders/**", ], async () => { createJsHintOptions(); await esm.rebuild(); if (iife) { await iife.rebuild(); } if (cjs) { await cjs.rebuild(); } await bundleWorkers({ minify: minify, path: outputDirectory, removePragmas: removePragmas, sourcemap: sourcemap, }); } ); gulp.watch( watchedSpecFiles, { events: ["add", "unlink"], }, async () => { createCombinedSpecList(); await specs.rebuild(); } ); gulp.watch( watchedSpecFiles, { events: ["change"], }, async () => { await specs.rebuild(); } ); process.on("SIGINT", () => { // Free up resources esm.dispose(); if (iife) { iife.dispose(); } if (cjs) { cjs.dispose(); } specs.dispose(); // eslint-disable-next-line n/no-process-exit process.exit(0); }); }); export async function buildTs() { let workspaces; if (argv.workspace && !Array.isArray(argv.workspace)) { workspaces = [argv.workspace]; } else if (argv.workspace) { workspaces = argv.workspace; } else { workspaces = packageJson.workspaces; } // Generate types for passed packages in order. const importModules = {}; for (const workspace of workspaces) { const directory = workspace .replace(`@${scope}/`, "") .replace(`packages/`, ""); const workspaceModules = await generateTypeScriptDefinitions( directory, `packages/${directory}/index.d.ts`, `packages/${directory}/tsd-conf.json`, // The engine package needs additional processing for its enum strings directory === "engine" ? processEngineSource : undefined, // Handle engine's module naming exceptions directory === "engine" ? processEngineModules : undefined, importModules ); importModules[directory] = workspaceModules; } if (argv.workspace) { return; } // Generate types for CesiumJS. await createTypeScriptDefinitions(); } export function buildApps() { return Promise.all([buildCesiumViewer(), buildSandcastle()]); } const filesToClean = [ "Source/Cesium.js", "Source/Shaders/**/*.js", "Source/ThirdParty/Shaders/*.js", "Source/**/*.d.ts", "Specs/SpecList.js", "Specs/jasmine/**", "Apps/Sandcastle/jsHintOptions.js", "Apps/Sandcastle/gallery/gallery-index.js", "Apps/Sandcastle/templates/bucket.css", "Cesium-*.zip", "cesium-*.tgz", "packages/**/*.tgz", ]; export async function clean() { await rimraf("Build"); const files = await globby(filesToClean); return Promise.all(files.map((file) => rimraf(file))); } async function clocSource() { let cmdLine; //Run cloc on primary Source files only const source = new Promise(function (resolve, reject) { cmdLine = "npx cloc" + " --quiet --progress-rate=0" + " packages/engine/Source/ packages/widgets/Source --exclude-dir=Assets,ThirdParty,Workers"; exec(cmdLine, function (error, stdout, stderr) { if (error) { console.log(stderr); return reject(error); } console.log("Source:"); console.log(stdout); resolve(); }); }); //If running cloc on source succeeded, also run it on the tests. await source; return new Promise(function (resolve, reject) { cmdLine = "npx cloc" + " --quiet --progress-rate=0" + " Specs/ packages/engine/Specs packages/widget/Specs --exclude-dir=Data --not-match-f=SpecList.js --not-match-f=.eslintrc.json"; exec(cmdLine, function (error, stdout, stderr) { if (error) { console.log(stderr); return reject(error); } console.log("Specs:"); console.log(stdout); resolve(); }); }); } export async function prepare() { // Copy Draco3D files from node_modules into Source copyFileSync( "node_modules/draco3d/draco_decoder.wasm", "packages/engine/Source/ThirdParty/draco_decoder.wasm" ); // Copy pako and zip.js worker files to Source/ThirdParty copyFileSync( "node_modules/pako/dist/pako_inflate.min.js", "packages/engine/Source/ThirdParty/Workers/pako_inflate.min.js" ); copyFileSync( "node_modules/pako/dist/pako_deflate.min.js", "packages/engine/Source/ThirdParty/Workers/pako_deflate.min.js" ); copyFileSync( "node_modules/@zip.js/zip.js/dist/z-worker-pako.js", "packages/engine/Source/ThirdParty/Workers/z-worker-pako.js" ); // Copy prism.js and prism.css files into Tools copyFileSync( "node_modules/prismjs/prism.js", "Tools/jsdoc/cesium_template/static/javascript/prism.js" ); copyFileSync( "node_modules/prismjs/themes/prism.min.css", "Tools/jsdoc/cesium_template/static/styles/prism.css" ); // Copy jasmine runner files into Specs const files = await globby([ "node_modules/jasmine-core/lib/jasmine-core", "!node_modules/jasmine-core/lib/jasmine-core/example", ]); const stream = gulp.src(files).pipe(gulp.dest("Specs/jasmine")); return streamToPromise(stream); } export const cloc = gulp.series(clean, clocSource); //Builds the documentation export function buildDocs() { const generatePrivateDocumentation = argv.private ? "--private" : ""; execSync( `npx jsdoc --configure Tools/jsdoc/conf.json --pedantic ${generatePrivateDocumentation}`, { stdio: "inherit", env: Object.assign({}, process.env, { CESIUM_VERSION: version, CESIUM_PACKAGES: packageJson.workspaces, }), } ); const stream = gulp .src("Documentation/Images/**") .pipe(gulp.dest("Build/Documentation/Images")); return streamToPromise(stream); } export async function buildDocsWatch() { await buildDocs(); console.log("Listening for changes in documentation..."); return gulp.watch(sourceFiles, buildDocs); } function combineForSandcastle() { const outputDirectory = join("Build", "Sandcastle", "CesiumUnminified"); return buildCesium({ development: false, minify: false, removePragmas: false, node: false, outputDirectory: outputDirectory, }); } export const websiteRelease = gulp.series( buildEngine, buildWidgets, function () { return buildCesium({ development: false, minify: false, removePragmas: false, node: false, }); }, combineForSandcastle, buildDocs ); export const buildRelease = gulp.series( buildEngine, buildWidgets, // Generate Build/CesiumUnminified function () { return buildCesium({ minify: false, removePragmas: false, node: true, sourcemap: false, }); }, // Generate Build/Cesium function () { return buildCesium({ development: false, minify: true, removePragmas: true, node: true, sourcemap: false, }); } ); export const release = gulp.series( buildRelease, gulp.parallel(buildTs, buildDocs) ); /** * Removes scripts from package.json files to ensure that * they still work when run from within the ZIP file. * * @param {string} packageJsonPath The path to the package.json. * @returns {WritableStream} A stream that writes to the updated package.json file. */ async function pruneScriptsForZip(packageJsonPath) { // Read the contents of the file. const contents = await readFile(packageJsonPath); const contentsJson = JSON.parse(contents); const scripts = contentsJson.scripts; // Remove prepare step from package.json to avoid running "prepare" an extra time. delete scripts.prepare; // Remove build and transform tasks since they do not function as intended from within the release zip delete scripts.build; delete scripts["build-release"]; delete scripts["build-watch"]; delete scripts["build-ts"]; delete scripts["build-third-party"]; delete scripts["build-apps"]; delete scripts.clean; delete scripts.cloc; delete scripts["build-docs"]; delete scripts["build-docs-watch"]; delete scripts["make-zip"]; delete scripts.release; delete scripts.prettier; // Remove deploy tasks delete scripts["deploy-status"]; delete scripts["deploy-set-version"]; delete scripts["website-release"]; // Set server tasks to use production flag scripts["start"] = "node server.js --production"; scripts["start-public"] = "node server.js --public --production"; scripts["start-public"] = "node server.js --public --production"; scripts["test"] = "gulp test --production"; scripts["test-all"] = "gulp test --all --production"; scripts["test-webgl"] = "gulp test --include WebGL --production"; scripts["test-non-webgl"] = "gulp test --exclude WebGL --production"; scripts["test-webgl-validation"] = "gulp test --webglValidation --production"; scripts["test-webgl-stub"] = "gulp test --webglStub --production"; scripts["test-release"] = "gulp test --release --production"; // Write to a temporary package.json file. const noPreparePackageJson = join( dirname(packageJsonPath), "Build/package.noprepare.json" ); await writeFile(noPreparePackageJson, JSON.stringify(contentsJson, null, 2)); return gulp.src(noPreparePackageJson).pipe(gulpRename(packageJsonPath)); } export const postversion = async function () { const workspace = argv.workspace; if (!workspace) { return; } const directory = workspace.replaceAll(`@${scope}/`, ``); const workspacePackageJson = require(`./packages/${directory}/package.json`); const version = workspacePackageJson.version; // Iterate through all package JSONs that may depend on the updated package and // update the version of the updated workspace. const packageJsons = await globby([ "./package.json", "./packages/*/package.json", ]); const promises = packageJsons.map(async (packageJsonPath) => { // Ensure that we don't check the updated workspace itself. if (basename(dirname(packageJsonPath)) === directory) { return; } // Ensure that we only update workspaces where the dependency to the updated workspace already exists. const packageJson = require(packageJsonPath); if (!Object.hasOwn(packageJson.dependencies, workspace)) { console.log( `Skipping update for ${workspace} as it is not a dependency.` ); return; } // Update the version for the updated workspace. packageJson.dependencies[workspace] = `^${version}`; await writeFile(packageJsonPath, JSON.stringify(packageJson, undefined, 2)); }); return Promise.all(promises); }; export const makeZip = gulp.series(release, async function () { //For now we regenerate the JS glsl to force it to be unminified in the release zip //See https://github.com/CesiumGS/cesium/pull/3106#discussion_r42793558 for discussion. await glslToJavaScript(false, "Build/minifyShaders.state", "engine"); const packageJsonSrc = await pruneScriptsForZip("package.json"); const enginePackageJsonSrc = await pruneScriptsForZip( "packages/engine/package.json" ); const widgetsPackageJsonSrc = await pruneScriptsForZip( "packages/widgets/package.json" ); const builtSrc = gulp.src( [ "Build/Cesium/**", "Build/CesiumUnminified/**", "Build/Documentation/**", "Build/Specs/**", "!Build/Specs/e2e/**", "!Build/InlineWorkers.js", "Build/package.json", "packages/engine/Build/**", "packages/widgets/Build/**", "!packages/engine/Build/Specs/**", "!packages/widgets/Build/Specs/**", "!packages/engine/Build/minifyShaders.state", "!packages/engine/Build/package.noprepare.json", "!packages/widgets/Build/package.noprepare.json", ], { base: ".", } ); const staticSrc = gulp.src( [ "Apps/**", "Apps/**/.eslintrc.json", "Apps/Sandcastle/.jshintrc", "!Apps/Sandcastle/gallery/development/**", "packages/engine/index.js", "packages/engine/index.d.ts", "packages/engine/LICENSE.md", "packages/engine/README.md", "packages/engine/Source/**", "!packages/engine/.gitignore", "packages/widgets/index.js", "packages/widgets/index.d.ts", "packages/widgets/LICENSE.md", "packages/widgets/README.md", "packages/widgets/Source/**", "!packages/widgets/.gitignore", "Source/**", "Source/**/.eslintrc.json", "Specs/**", "!Specs/e2e/*-snapshots/**", "Specs/**/.eslintrc.json", "ThirdParty/**", "favicon.ico", ".eslintignore", ".eslintrc.json", ".prettierignore", "scripts/**", "gulpfile.js", "server.js", "index.cjs", "LICENSE.md", "CHANGES.md", "README.md", "web.config", ], { base: ".", } ); const indexSrc = gulp .src("index.release.html") .pipe(gulpRename("index.html")); return streamToPromise( mergeStream( packageJsonSrc, enginePackageJsonSrc, widgetsPackageJsonSrc, builtSrc, staticSrc, indexSrc ) .pipe( gulpTap(function (file) { // Work around an issue with gulp-zip where archives generated on Windows do // not properly have their directory executable mode set. // see https://github.com/sindresorhus/gulp-zip/issues/64#issuecomment-205324031 if (file.isDirectory()) { file.stat.mode = parseInt("40777", 8); } }) ) .pipe(gulpZip(`Cesium-${version}.zip`)) .pipe(gulp.dest(".")) .on("finish", function () { rimraf.sync("./Build/package.noprepare.json"); rimraf.sync("./packages/engine/Build/package.noprepare.json"); rimraf.sync("./packages/widgets/Build/package.noprepare.json"); }) ); }); export async function deploySetVersion() { const buildVersion = argv.buildVersion; if (buildVersion) { // NPM versions can only contain alphanumeric and hyphen characters packageJson.version += `-${buildVersion.replace(/[^[0-9A-Za-z-]/g, "")}`; return writeFile("package.json", JSON.stringify(packageJson, undefined, 2)); } } export async function deployStatus() { const status = argv.status; const message = argv.message; const deployUrl = `${devDeployUrl + process.env.BRANCH}/`; const zipUrl = `${deployUrl}Cesium-${version}.zip`; const npmUrl = `${deployUrl}cesium-${version}.tgz`; const coverageUrl = `${ devDeployUrl + process.env.BRANCH }/Build/Coverage/index.html`; return Promise.all([ setStatus(status, deployUrl, message, "deployment"), setStatus(status, zipUrl, message, "zip file"), setStatus(status, npmUrl, message, "npm package"), setStatus(status, coverageUrl, message, "coverage results"), ]); } async function setStatus(state, targetUrl, description, context) { // skip if the environment does not have the token if (!process.env.GITHUB_TOKEN) { return; } const body = { state: state, target_url: targetUrl, description: description, context: context, }; const response = await fetch( `https://api.github.com/repos/${process.env.GITHUB_REPO}/statuses/${process.env.GITHUB_SHA}`, { method: "post", body: JSON.stringify(body), headers: { "Content-Type": "application/json", Authorization: `token ${process.env.GITHUB_TOKEN}`, "User-Agent": "Cesium", }, } ); const result = await response.json(); return result; } /** * Generates coverage report. * * @param {object} options An object with the following properties: * @param {string} options.outputDirectory The output directory for the generated build artifacts. * @param {string} options.coverageDirectory The path where the coverage reports should be saved to. * @param {string} options.specList The path to the spec list for the package. * @param {RegExp} options.filter The filter for finding which files should be instrumented. * @param {boolean} [options.webglStub=false] True if WebGL stub should be used when running tests. * @param {boolean} [options.suppressPassed=false] True if output should be suppressed for tests that pass. * @param {boolean} [options.failTaskOnError=false] True if the gulp task should fail on errors in the tests. * @param {string} options.workspace The name of the workspace, if any. */ export async function runCoverage(options) { const webglStub = options.webglStub ?? false; const suppressPassed = options.suppressPassed ?? false; const failTaskOnError = options.failTaskOnError ?? false; const workspace = options.workspace; const folders = []; let browsers = ["Chrome"]; if (argv.browsers) { browsers = argv.browsers.split(","); } const instrumenter = createInstrumenter({ esModules: true, }); // Setup plugin to use instrumenter on source files. const instrumentPlugin = { name: "instrument", setup: (build) => { build.onLoad( { filter: options.filter, }, async (args) => { const source = await readFile(args.path, { encoding: "utf8" }); try { const generatedCode = instrumenter.instrumentSync( source, args.path ); return { contents: generatedCode }; } catch (e) { return { errors: { text: e.message, }, }; } } ); }, }; const karmaBundle = join(options.outputDirectory, "karma-main.js"); await esbuild({ entryPoints: ["Specs/karma-main.js"], bundle: true, sourcemap: true, format: "esm", target: "es2020", outfile: karmaBundle, logLevel: "error", // print errors immediately, and collect warnings so we can filter out known ones }); // Generate instrumented bundle for Specs. const specListBundle = join(options.outputDirectory, "SpecList.js"); await esbuild({ entryPoints: [options.specList], bundle: true, sourcemap: true, format: "esm", target: "es2020", outfile: specListBundle, plugins: [instrumentPlugin], logLevel: "error", // print errors immediately, and collect warnings so we can filter out known ones }); let files = [ { pattern: karmaBundle, included: true, type: "module", }, { pattern: specListBundle, included: true, type: "module", }, // Static assets are always served from the shared/combined folders. { pattern: "Specs/Data/**", included: false }, { pattern: "Specs/TestWorkers/**/*.wasm", included: false }, { pattern: "Build/CesiumUnminified/**", included: false }, { pattern: "Build/Specs/TestWorkers/**.js", included: false }, ]; let proxies; if (workspace) { // Setup files and proxies for the engine package first, since it is the lowest level dependency. files = [ { pattern: karmaBundle, included: true, type: "module", }, { pattern: specListBundle, included: true, type: "module", }, { pattern: "Specs/Data/**", included: false }, { pattern: "Specs/TestWorkers/**/*.wasm", included: false }, { pattern: "packages/engine/Build/Workers/**", included: false }, { pattern: "packages/engine/Source/Assets/**", included: false }, { pattern: "packages/engine/Source/ThirdParty/**", included: false }, { pattern: "packages/engine/Source/Widget/*.css", included: false }, { pattern: "Build/Specs/TestWorkers/**.js", included: false }, ]; proxies = { "/base/Build/CesiumUnminified/Assets/": "/base/packages/engine/Source/Assets/", "/base/Build/CesiumUnminified/ThirdParty/": "/base/packages/engine/Source/ThirdParty/", "/base/Build/CesiumUnminified/Widgets/CesiumWidget/": "/base/packages/engine/Source/Widget/", "/base/Build/CesiumUnminified/Workers/": "/base/packages/engine/Build/Workers/", }; } // Setup Karma config. const config = await karma.config.parseConfig( karmaConfigFile, { configFile: karmaConfigFile, browsers: browsers, specReporter: { suppressErrorSummary: false, suppressFailed: false, suppressPassed: suppressPassed, suppressSkipped: true, }, files: files, proxies: proxies, reporters: ["spec", "coverage"], coverageReporter: { dir: options.coverageDirectory, subdir: function (browserName) { folders.push(browserName); return browserName; }, includeAllSources: true, }, client: { captureConsole: false, args: [ undefined, undefined, undefined, undefined, undefined, webglStub, undefined, ], }, }, { promiseConfig: true, throwErrors: true } ); return new Promise((resolve, reject) => { const server = new karma.Server(config, function doneCallback(e) { let html = "