From c4967fa462d79e05f1c0eb536e02f5fe5e1abd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Leosvel=20P=C3=A9rez=20Espinosa?= Date: Fri, 16 Apr 2021 15:35:08 +0100 Subject: [PATCH] cleanup(core): copy from cache only when needed --- e2e/workspace/src/workspace.test.ts | 50 ++++++++--- packages/workspace/src/tasks-runner/cache.ts | 82 ++++++++++++++++++- .../src/tasks-runner/task-orchestrator.ts | 28 +++++-- packages/workspace/src/utilities/output.ts | 15 +++- 4 files changed, 151 insertions(+), 24 deletions(-) diff --git a/e2e/workspace/src/workspace.test.ts b/e2e/workspace/src/workspace.test.ts index f665929633..7632f96a88 100644 --- a/e2e/workspace/src/workspace.test.ts +++ b/e2e/workspace/src/workspace.test.ts @@ -15,6 +15,7 @@ import { updateFile, workspaceConfigName, } from '@nrwl/e2e/utils'; +import { TaskCacheStatus } from '@nrwl/workspace/src/utilities/output'; describe('run-one', () => { let proj: string; @@ -637,7 +638,7 @@ describe('cache', () => { }); const outputWithBuildApp2Cached = runCLI(`affected:build ${files}`); expect(outputWithBuildApp2Cached).toContain('read the output from cache'); - expectCached(outputWithBuildApp2Cached, [myapp2]); + expectMatchedOutput(outputWithBuildApp2Cached, [myapp2]); // touch package.json // -------------------------------------------- @@ -651,13 +652,17 @@ describe('cache', () => { // build individual project with caching const individualBuildWithCache = runCLI(`build ${myapp1}`); - expect(individualBuildWithCache).toContain('from cache'); + expect(individualBuildWithCache).toContain( + TaskCacheStatus.MatchedExistingOutput + ); // skip caching when building individual projects const individualBuildWithSkippedCache = runCLI( `build ${myapp1} --skip-nx-cache` ); - expect(individualBuildWithSkippedCache).not.toContain('from cache'); + expect(individualBuildWithSkippedCache).not.toContain( + TaskCacheStatus.MatchedExistingOutput + ); // run lint with caching // -------------------------------------------- @@ -668,7 +673,7 @@ describe('cache', () => { expect(outputWithBothLintTasksCached).toContain( 'read the output from cache' ); - expectCached(outputWithBothLintTasksCached, [ + expectMatchedOutput(outputWithBothLintTasksCached, [ myapp1, myapp2, `${myapp1}-e2e`, @@ -691,13 +696,13 @@ describe('cache', () => { silenceError: true, env: { ...process.env, NX_CACHE_FAILURES: 'true' }, }); - expect(failingRun).not.toContain('[retrieved from cache]'); + expect(failingRun).not.toContain(TaskCacheStatus.RetrievedFromCache); const cachedFailingRun = runCLI(`lint ${myapp1}`, { silenceError: true, env: { ...process.env, NX_CACHE_FAILURES: 'true' }, }); - expect(cachedFailingRun).toContain('[retrieved from cache]'); + expect(cachedFailingRun).toContain(TaskCacheStatus.MatchedExistingOutput); // run without caching // -------------------------------------------- @@ -771,19 +776,38 @@ describe('cache', () => { actualOutput: string, expectedCachedProjects: string[] ) { - const cachedProjects = []; + expectProjectMatchTaskCacheStatus(actualOutput, expectedCachedProjects); + } + + function expectMatchedOutput( + actualOutput: string, + expectedMatchedOutputProjects: string[] + ) { + expectProjectMatchTaskCacheStatus( + actualOutput, + expectedMatchedOutputProjects, + TaskCacheStatus.MatchedExistingOutput + ); + } + + function expectProjectMatchTaskCacheStatus( + actualOutput: string, + expectedProjects: string[], + cacheStatus: TaskCacheStatus = TaskCacheStatus.RetrievedFromCache + ) { + const matchingProjects = []; const lines = actualOutput.split('\n'); - lines.forEach((s, i) => { + lines.forEach((s) => { if (s.startsWith(`> nx run`)) { const projectName = s.split(`> nx run `)[1].split(':')[0].trim(); - if (s.indexOf('from cache') > -1) { - cachedProjects.push(projectName); + if (s.indexOf(cacheStatus) > -1) { + matchingProjects.push(projectName); } } }); - cachedProjects.sort((a, b) => a.localeCompare(b)); - expectedCachedProjects.sort((a, b) => a.localeCompare(b)); - expect(cachedProjects).toEqual(expectedCachedProjects); + matchingProjects.sort((a, b) => a.localeCompare(b)); + expectedProjects.sort((a, b) => a.localeCompare(b)); + expect(matchingProjects).toEqual(expectedProjects); } }); diff --git a/packages/workspace/src/tasks-runner/cache.ts b/packages/workspace/src/tasks-runner/cache.ts index b8f47485fe..97ea7179cb 100644 --- a/packages/workspace/src/tasks-runner/cache.ts +++ b/packages/workspace/src/tasks-runner/cache.ts @@ -6,12 +6,14 @@ import { readFileSync, writeFileSync, lstatSync, + unlinkSync, } from 'fs'; import { join, resolve } from 'path'; import * as fsExtra from 'fs-extra'; import { DefaultTasksRunnerOptions } from './default-tasks-runner'; import { spawn } from 'child_process'; import { cacheDirectory } from '../utilities/cache-directory'; +import { writeToFile } from '../utilities/fileutils'; export type CachedResult = { terminalOutput: string; @@ -42,6 +44,7 @@ export class Cache { root = appRootPath; cachePath = this.createCacheDir(); terminalOutputsDir = this.createTerminalOutputsDir(); + latestTasksHashesDir = this.ensureLatestTasksHashesDir(); cacheConfig = new CacheConfig(this.options); constructor(private readonly options: DefaultTasksRunnerOptions) {} @@ -136,12 +139,12 @@ export class Cache { } copyFilesFromCache( - hash: string, - cachedResult: CachedResult, + taskWithCachedResult: TaskWithCachedResult, outputs: string[] ) { + this.removeRecordedTaskHash(taskWithCachedResult.task, outputs); outputs.forEach((f) => { - const cached = join(cachedResult.outputsPath, f); + const cached = join(taskWithCachedResult.cachedResult.outputsPath, f); if (existsSync(cached)) { const isFile = lstatSync(cached).isFile(); const src = join(this.root, f); @@ -154,6 +157,7 @@ export class Cache { fsExtra.copySync(cached, src); } }); + this.recordTaskHash(taskWithCachedResult.task, outputs); } temporaryOutputPath(task: Task) { @@ -164,6 +168,72 @@ export class Cache { } } + removeRecordedTaskHash(task: Task, outputs: string[]): void { + if (outputs.length === 0) { + return; + } + + const hashFile = this.getFileNameWithLatestRecordedHashForTask(task); + try { + unlinkSync(hashFile); + } catch (e) {} + } + + recordTaskHash(task: Task, outputs: string[]): void { + if (outputs.length === 0) { + return; + } + + const hashFile = this.getFileNameWithLatestRecordedHashForTask(task); + writeToFile(hashFile, task.hash); + } + + shouldCopyOutputsFromCache( + taskWithCachedResult: TaskWithCachedResult, + outputs: string[] + ): boolean { + if (outputs.length === 0) { + return false; + } + + if ( + this.getLatestRecordedHashForTask(taskWithCachedResult.task) !== + taskWithCachedResult.task.hash + ) { + return true; + } + + return this.isAnyOutputMissing(taskWithCachedResult.cachedResult, outputs); + } + + private getLatestRecordedHashForTask(task: Task): string | null { + try { + return readFileSync( + this.getFileNameWithLatestRecordedHashForTask(task) + ).toString(); + } catch (e) { + return null; + } + } + + private isAnyOutputMissing( + cachedResult: CachedResult, + outputs: string[] + ): boolean { + return outputs.some( + (output) => + existsSync(join(cachedResult.outputsPath, output)) && + !existsSync(join(this.root, output)) + ); + } + + private getFileNameWithLatestRecordedHashForTask(task: Task): string { + return join( + this.latestTasksHashesDir, + `${task.target.project}-${task.target.target}.hash` + ); + } + private getFromLocalDir(task: Task) { const tdCommit = join(this.cachePath, `${task.hash}.commit`); const td = join(this.cachePath, task.hash); @@ -199,4 +269,10 @@ export class Cache { fsExtra.ensureDirSync(path); return path; } + + private ensureLatestTasksHashesDir() { + const path = join(this.cachePath, 'latestTasksHashes'); + fsExtra.ensureDirSync(path); + return path; + } } diff --git a/packages/workspace/src/tasks-runner/task-orchestrator.ts b/packages/workspace/src/tasks-runner/task-orchestrator.ts index de11b788f5..2f8a836993 100644 --- a/packages/workspace/src/tasks-runner/task-orchestrator.ts +++ b/packages/workspace/src/tasks-runner/task-orchestrator.ts @@ -4,7 +4,7 @@ import * as dotenv from 'dotenv'; import * as fs from 'fs'; import { ProjectGraph } from '../core/project-graph'; import { appRootPath } from '../utilities/app-root'; -import { output } from '../utilities/output'; +import { output, TaskCacheStatus } from '../utilities/output'; import { Cache, TaskWithCachedResult } from './cache'; import { DefaultTasksRunnerOptions } from './default-tasks-runner'; import { AffectedEventType, Task } from './tasks-runner'; @@ -115,18 +115,29 @@ export class TaskOrchestrator { tasks.forEach((t) => { this.options.lifeCycle.startTask(t.task); + const outputs = getOutputs(this.projectGraph.nodes, t.task); + const shouldCopyOutputsFromCache = this.cache.shouldCopyOutputsFromCache( + t, + outputs + ); + if (shouldCopyOutputsFromCache) { + this.cache.copyFilesFromCache(t, outputs); + } + if ( !this.initiatingProject || this.initiatingProject === t.task.target.project ) { const args = this.getCommandArgs(t.task); - output.logCommand(`nx ${args.join(' ')}`, true); + output.logCommand( + `nx ${args.join(' ')}`, + shouldCopyOutputsFromCache + ? TaskCacheStatus.RetrievedFromCache + : TaskCacheStatus.MatchedExistingOutput + ); process.stdout.write(t.cachedResult.terminalOutput); } - const outputs = getOutputs(this.projectGraph.nodes, t.task); - this.cache.copyFilesFromCache(t.task.hash, t.cachedResult, outputs); - this.options.lifeCycle.endTask(t.task, t.cachedResult.code); }); @@ -175,6 +186,7 @@ export class TaskOrchestrator { if (forwardOutput) { output.logCommand(commandLine); } + this.cache.removeRecordedTaskHash(task, taskOutputs); const p = fork(this.getCommand(), args, { stdio: ['inherit', 'pipe', 'pipe', 'ipc'], env, @@ -211,6 +223,7 @@ export class TaskOrchestrator { this.cache .put(task, terminalOutput, taskOutputs, code) .then(() => { + this.cache.recordTaskHash(task, taskOutputs); this.options.lifeCycle.endTask(task, code); res(code); }) @@ -218,10 +231,12 @@ export class TaskOrchestrator { rej(e); }); } else { + this.cache.recordTaskHash(task, taskOutputs); this.options.lifeCycle.endTask(task, code); res(code); } } else { + this.cache.recordTaskHash(task, taskOutputs); this.options.lifeCycle.endTask(task, code); res(code); } @@ -252,6 +267,7 @@ export class TaskOrchestrator { if (forwardOutput) { output.logCommand(commandLine); } + this.cache.removeRecordedTaskHash(task, taskOutputs); const p = fork(this.getCommand(), args, { stdio: ['inherit', 'inherit', 'inherit', 'ipc'], env, @@ -277,6 +293,7 @@ export class TaskOrchestrator { this.cache .put(task, this.readTerminalOutput(outputPath), taskOutputs, code) .then(() => { + this.cache.recordTaskHash(task, taskOutputs); this.options.lifeCycle.endTask(task, code); res(code); }) @@ -284,6 +301,7 @@ export class TaskOrchestrator { rej(e); }); } else { + this.cache.recordTaskHash(task, taskOutputs); this.options.lifeCycle.endTask(task, code); res(code); } diff --git a/packages/workspace/src/utilities/output.ts b/packages/workspace/src/utilities/output.ts index 01398095f9..3e56b93ccb 100644 --- a/packages/workspace/src/utilities/output.ts +++ b/packages/workspace/src/utilities/output.ts @@ -22,6 +22,12 @@ export interface CLISuccessMessageConfig { bodyLines?: string[]; } +export enum TaskCacheStatus { + NoCache = '[no cache]', + MatchedExistingOutput = '[existing outputs match the cache, left as is]', + RetrievedFromCache = '[retrieved from cache]', +} + /** * Automatically disable styling applied by chalk if CI=true */ @@ -177,13 +183,16 @@ class CLIOutput { this.addNewline(); } - logCommand(message: string, isCached: boolean = false) { + logCommand( + message: string, + cacheStatus: TaskCacheStatus = TaskCacheStatus.NoCache + ) { this.addNewline(); this.writeToStdOut(chalk.bold(`> ${message} `)); - if (isCached) { - this.writeToStdOut(chalk.bold.grey(`[retrieved from cache]`)); + if (cacheStatus !== TaskCacheStatus.NoCache) { + this.writeToStdOut(chalk.bold.grey(cacheStatus)); } this.addNewline();