feat(core): optimise cache restoration and storing

e2e/js/src/js.spec.ts

@@ -35,7 +35,7 @@ describe('js e2e', () => {
     });
   }, 120000);
 
-  it('xxxshould create libs and apps with js executors (--compiler=tsc)', async () => {
+  it('should create libs and apps with js executors (--compiler=tsc)', async () => {
     const scope = newProject();
     const lib = uniq('lib');
     runCLI(`generate @nrwl/js:lib ${lib} --buildable --compiler=tsc`);

packages/cli/lib/is_ci.ts

@@ -0,0 +1,17 @@
+export function isCI() {
+  return (
+    process.env.CI === 'true' ||
+    process.env.TF_BUILD === 'true' ||
+    process.env['bamboo.buildKey'] ||
+    process.env.BUILDKITE === 'true' ||
+    process.env.CIRCLECI === 'true' ||
+    process.env.CIRRUS_CI === 'true' ||
+    process.env.CODEBUILD_BUILD_ID ||
+    process.env.GITHUB_ACTIONS === 'true' ||
+    process.env.GITLAB_CI ||
+    process.env.HEROKU_TEST_RUN_ID ||
+    process.env.BUILD_ID ||
+    process.env.TEAMCITY_VERSION ||
+    process.env.TRAVIS === 'true'
+  );
+}

packages/cli/lib/output.ts

@@ -4,6 +4,7 @@
  * should be copied here if necessary.
  */
 import * as chalk from 'chalk';
+import { isCI } from './is_ci';
 
 export interface CLIErrorMessageConfig {
   title: string;
@@ -34,7 +35,7 @@ export interface CLISuccessMessageConfig {
 /**
  * Automatically disable styling applied by chalk if CI=true
  */
-if (process.env.CI === 'true') {
+if (isCI()) {
   (chalk as any).Level = 0;
 }
 

packages/workspace/src/core/project-graph/daemon/server/watcher.ts

@@ -11,9 +11,9 @@ import type { AsyncSubscription, Event } from '@parcel/watcher';
 import { readFileSync } from 'fs';
 import { join, relative } from 'path';
 import { FULL_OS_SOCKET_PATH } from '../socket-utils';
-import { serverLogger } from '@nrwl/workspace/src/core/project-graph/daemon/server/logger';
 import { handleServerProcessTermination } from '@nrwl/workspace/src/core/project-graph/daemon/server/shutdown-utils';
 import { Server } from 'net';
+import ignore from 'ignore';
 
 /**
  * This configures the files and directories which we always want to ignore as part of file watching
@@ -59,6 +59,17 @@ export type SubscribeToWorkspaceChangesCallback = (
   changeEvents: Event[] | null
 ) => Promise<void>;
 
+function configureIgnoreObject() {
+  const ig = ignore();
+  try {
+    ig.add(readFileSync(`${appRootPath}/.gitignore`, 'utf-8'));
+  } catch {}
+  try {
+    ig.add(readFileSync(`${appRootPath}/.nxignore`, 'utf-8'));
+  } catch {}
+  return ig;
+}
+
 export async function subscribeToWorkspaceChanges(
   server: Server,
   cb: SubscribeToWorkspaceChangesCallback
@@ -69,6 +80,7 @@ export async function subscribeToWorkspaceChanges(
    * executed by packages which do not have its necessary native binaries available.
    */
   const watcher = await import('@parcel/watcher');
+  const ignoreObj = configureIgnoreObject();
 
   const subscription = await watcher.subscribe(
     appRootPath,
@@ -105,7 +117,13 @@ export async function subscribeToWorkspaceChanges(
         });
       }
 
-      cb(null, workspaceRelativeEvents);
+      const nonIgnoredEvents = workspaceRelativeEvents
+        .filter(({ path }) => !!path)
+        .filter(({ path }) => !ignoreObj.ignores(path));
+
+      if (nonIgnoredEvents && nonIgnoredEvents.length > 0) {
+        cb(null, nonIgnoredEvents);
+      }
     },
     {
       ignore: getIgnoredGlobs(),

packages/workspace/src/core/project-graph/project-graph.ts

@@ -3,6 +3,7 @@ import { ProjectGraphCache, readCache } from '../nx-deps/nx-deps-cache';
 import { buildProjectGraph } from './build-project-graph';
 import { readNxJson, workspaceFileName } from '../file-utils';
 import { output } from '../../utilities/output';
+import { isCI } from '../../utilities/is_ci';
 
 /**
  * Synchronously reads the latest cached copy of the workspace's ProjectGraph.
@@ -59,8 +60,9 @@ export async function createProjectGraphAsync(
   // option=true,env=false => no daemon
   // option=false,env=undefined => no daemon
   // option=false,env=false => no daemon
+  // option=true,env=undefined,ci => no daemon
 
-  // option=true,env=undefined => daemon
+  // option=true,env=undefined,!ci => daemon
   // option=true,env=true => daemon
   // option=false,env=true => daemon
   try {
@@ -68,7 +70,8 @@ export async function createProjectGraphAsync(
       (useDaemonProcessOption === undefined && env === undefined) ||
       (useDaemonProcessOption === true && env === 'false') ||
       (useDaemonProcessOption === false && env === undefined) ||
-      (useDaemonProcessOption === false && env === 'false')
+      (useDaemonProcessOption === false && env === 'false') ||
+      (useDaemonProcessOption === true && env === undefined && isCI())
     ) {
       return projectGraphAdapter(
         '5.0',

packages/workspace/src/tasks-runner/cache.ts

@@ -13,7 +13,7 @@ import {
 } from 'fs-extra';
 import { dirname, join, resolve, sep } from 'path';
 import { DefaultTasksRunnerOptions } from './default-tasks-runner';
-import { spawn } from 'child_process';
+import { spawn, exec } from 'child_process';
 import { cacheDirectory } from '../utilities/cache-directory';
 
 const util = require('util');
@@ -35,6 +35,7 @@ export class Cache {
   cachePath = this.createCacheDir();
   terminalOutputsDir = this.createTerminalOutputsDir();
   latestOutputsHashesDir = this.ensureLatestOutputsHashesDir();
+  useFsExtraToCopyAndRemove = false;
 
   constructor(private readonly options: DefaultTasksRunnerOptions) {}
 
@@ -89,7 +90,7 @@ export class Cache {
 
       // might be left overs from partially-completed cache invocations
       await remove(tdCommit);
-      await remove(td);
+      await this.remove(td);
 
       await mkdir(td);
       await writeFile(
@@ -103,12 +104,9 @@ export class Cache {
           const src = join(this.root, f);
           if (await existsAsync(src)) {
             const cached = join(td, 'outputs', f);
-            // Ensure parent directory is created if src is a file
+            const directory = resolve(cached, '..');
-            const isFile = (await lstatAsync(src)).isFile();
-            const directory = isFile ? resolve(cached, '..') : cached;
             await ensureDir(directory);
-
+            await this.copy(src, directory);
-            await copy(src, cached);
           }
         })
       );
@@ -143,14 +141,11 @@ export class Cache {
         outputs.map(async (f) => {
           const cached = join(cachedResult.outputsPath, f);
           if (await existsAsync(cached)) {
-            const isFile = (await lstatAsync(cached)).isFile();
             const src = join(this.root, f);
-            await remove(src);
+            await this.remove(src);
-
+            const directory = resolve(src, '..');
-            // Ensure parent directory is created if src is a file
-            const directory = isFile ? resolve(src, '..') : src;
             await ensureDir(directory);
-            await copy(cached, src);
+            await this.copy(cached, directory);
           }
         })
       );
@@ -187,6 +182,40 @@ export class Cache {
     );
   }
 
+  private copy(src: string, directory: string) {
+    if (this.useFsExtraToCopyAndRemove) {
+      return copy(src, directory);
+    }
+
+    return new Promise((res, rej) => {
+      exec(`cp -a "${src}" "${directory}"`, (error) => {
+        if (!error) {
+          res(null);
+        } else {
+          this.useFsExtraToCopyAndRemove = true;
+          copy(src, directory).then(res, rej);
+        }
+      });
+    });
+  }
+
+  private remove(folder: string) {
+    if (this.useFsExtraToCopyAndRemove) {
+      return remove(folder);
+    }
+
+    return new Promise((res, rej) => {
+      exec(`rm -rf "${folder}"`, (error) => {
+        if (!error) {
+          res(null);
+        } else {
+          this.useFsExtraToCopyAndRemove = true;
+          remove(folder).then(res, rej);
+        }
+      });
+    });
+  }
+
   private async recordOutputsHash(
     outputs: string[],
     hash: string

packages/workspace/src/utilities/is_ci.ts

@@ -0,0 +1,17 @@
+export function isCI() {
+  return (
+    process.env.CI === 'true' ||
+    process.env.TF_BUILD === 'true' ||
+    process.env['bamboo.buildKey'] ||
+    process.env.BUILDKITE === 'true' ||
+    process.env.CIRCLECI === 'true' ||
+    process.env.CIRRUS_CI === 'true' ||
+    process.env.CODEBUILD_BUILD_ID ||
+    process.env.GITHUB_ACTIONS === 'true' ||
+    process.env.GITLAB_CI ||
+    process.env.HEROKU_TEST_RUN_ID ||
+    process.env.BUILD_ID ||
+    process.env.TEAMCITY_VERSION ||
+    process.env.TRAVIS === 'true'
+  );
+}

packages/workspace/src/utilities/output.ts

@@ -1,4 +1,5 @@
 import * as chalk from 'chalk';
+import { isCI } from './is_ci';
 
 export interface CLIErrorMessageConfig {
   title: string;
@@ -31,7 +32,7 @@ export enum TaskCacheStatus {
 /**
  * Automatically disable styling applied by chalk if CI=true
  */
-if (process.env.CI === 'true') {
+if (isCI()) {
   (chalk as any).level = 0;
 }