miel.truyen/nx
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

feat(core): remove parcel/watcher (#19751)

Browse Source
This commit is contained in:
Jonathan Cammisuli 2023-10-20 17:20:00 -04:00 committed by GitHub
parent 39423322c1
commit f19d1db881
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 61 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/nx/package.json
View File

@ -33,7 +33,6 @@
},
"homepage": "https://nx.dev",
"dependencies": {
"@parcel/watcher": "2.0.4",
"@yarnpkg/lockfile": "^1.1.0",
"@yarnpkg/parsers": "3.0.0-rc.46",
"@zkochan/js-yaml": "0.0.6",

6
packages/nx/src/daemon/client/client.ts
View File

@ -366,12 +366,6 @@ export class DaemonClient {
this._out = await open(DAEMON_OUTPUT_LOG_FILE, 'a');
this._err = await open(DAEMON_OUTPUT_LOG_FILE, 'a');
if (this.nxJson.tasksRunnerOptions?.default?.options?.useParcelWatcher) {
DAEMON_ENV_SETTINGS['NX_NATIVE_WATCHER'] = 'false';
} else {
DAEMON_ENV_SETTINGS['NX_NATIVE_WATCHER'] = 'true';
}
const backgroundProcess = spawn(
process.execPath,
[join(__dirname, '../server/start.js')],

15
packages/nx/src/daemon/server/outputs-tracking.spec.ts
View File

@ -1,3 +1,4 @@
import { EventType } from '../../native';
import {
_outputsHashesMatch,
_recordOutputsHash,
@ -19,7 +20,7 @@ describe('outputs tracking', () => {
it('should invalidate output when it is exact match', () => {
_recordOutputsHash(['dist/app/app1'], '123');
processFileChangesInOutputs(
[{ path: 'dist/app/app1', type: 'update' }],
[{ path: 'dist/app/app1', type: EventType.update }],
now
);
expect(recordedHash('dist/app/app1')).toBeUndefined();
@ -28,7 +29,7 @@ describe('outputs tracking', () => {
it('should invalidate output when it is a child', () => {
_recordOutputsHash(['dist/app/app1'], '123');
processFileChangesInOutputs(
[{ path: 'dist/app/app1/child', type: 'update' }],
[{ path: 'dist/app/app1/child', type: EventType.update }],
now
);
expect(recordedHash('dist/app/app1')).toBeUndefined();
@ -36,13 +37,19 @@ describe('outputs tracking', () => {
it('should invalidate output when it is a parent', () => {
_recordOutputsHash(['dist/app/app1'], '123');
processFileChangesInOutputs([{ path: 'dist/app', type: 'update' }], now);
processFileChangesInOutputs(
[{ path: 'dist/app', type: EventType.update }],
now
);
expect(recordedHash('dist/app/app1')).toBeUndefined();
});
it('should not invalidate anything when no match', () => {
_recordOutputsHash(['dist/app/app1'], '123');
processFileChangesInOutputs([{ path: 'dist/app2', type: 'update' }], now);
processFileChangesInOutputs(
[{ path: 'dist/app2', type: EventType.update }],
now
);
expect(recordedHash('dist/app/app1')).toEqual('123');
});
});

10
packages/nx/src/daemon/server/outputs-tracking.ts
View File

@ -1,9 +1,7 @@
import { lstat } from 'fs-extra';
import { dirname, join } from 'path';
import { workspaceRoot } from '../../utils/workspace-root';
import { dirname } from 'path';
import { WatchEvent, getFilesForOutputs } from '../../native';
import { collapseExpandedOutputs } from '../../utils/collapse-expanded-outputs';
import type { Event } from '@parcel/watcher';
import { getFilesForOutputs } from '../../native';
import { workspaceRoot } from '../../utils/workspace-root';
let disabled = false;
@ -66,7 +64,7 @@ async function normalizeOutputs(outputs: string[]) {
}
export function processFileChangesInOutputs(
changeEvents: Event[],
changeEvents: WatchEvent[],
now: number = undefined
) {
if (!now) {

29
packages/nx/src/daemon/server/project-graph-incremental-recomputation.ts
View File

@ -140,37 +140,10 @@ function computeWorkspaceConfigHash(
return hashArray(projectConfigurationStrings);
}
/**
* Temporary work around to handle nested gitignores. The parcel file watcher doesn't handle them well,
* so we need to filter them out here.
*
* TODO(Cammisuli): remove after 16.4 - Rust watcher handles nested gitignores
*/
function filterUpdatedFiles(files: string[]) {
if (files.length === 0 || process.env.NX_NATIVE_WATCHER === 'true') {
return files;
}
try {
const quoted = files.map((f) => '"' + f + '"');
const ignored = execSync(`git check-ignore ${quoted.join(' ')}`, {
windowsHide: true,
})
.toString()
.split('\n');
return files.filter((f) => ignored.indexOf(f) === -1);
} catch (e) {
// none of the files were ignored
return files;
}
}
async function processCollectedUpdatedAndDeletedFiles() {
try {
performance.mark('hash-watched-changes-start');
const updatedFiles = filterUpdatedFiles([
...collectedUpdatedFiles.values(),
]);
const updatedFiles = [...collectedUpdatedFiles.values()];
const deletedFiles = [...collectedDeletedFiles.values()];
let updatedFileHashes = updateFilesInContext(updatedFiles, deletedFiles);
performance.mark('hash-watched-changes-end');

139
packages/nx/src/daemon/server/server.ts
View File

@ -1,65 +1,57 @@
import { workspaceRoot } from '../../utils/workspace-root';
import { existsSync, statSync } from 'fs';
import { createServer, Server, Socket } from 'net';
import { join } from 'path';
import { PerformanceObserver } from 'perf_hooks';
import { hashArray } from '../../hasher/file-hasher';
import { hashFile } from '../../native';
import { consumeMessagesFromSocket } from '../../utils/consume-messages-from-socket';
import { readJsonFile } from '../../utils/fileutils';
import { PackageJson } from '../../utils/package-json';
import { nxVersion } from '../../utils/versions';
import { setupWorkspaceContext } from '../../utils/workspace-context';
import { workspaceRoot } from '../../utils/workspace-root';
import { writeDaemonJsonProcessCache } from '../cache';
import {
FULL_OS_SOCKET_PATH,
isWindows,
killSocketOrPath,
} from '../socket-utils';
import {
registeredFileWatcherSockets,
removeRegisteredFileWatcherSocket,
} from './file-watching/file-watcher-sockets';
import { handleHashTasks } from './handle-hash-tasks';
import {
handleOutputsHashesMatch,
handleRecordOutputsHash,
} from './handle-outputs-tracking';
import { handleProcessInBackground } from './handle-process-in-background';
import { handleRequestFileData } from './handle-request-file-data';
import { handleRequestProjectGraph } from './handle-request-project-graph';
import { handleRequestShutdown } from './handle-request-shutdown';
import { serverLogger } from './logger';
import {
getOutputsWatcherSubscription,
disableOutputsTracking,
processFileChangesInOutputs,
} from './outputs-tracking';
import { addUpdatedAndDeletedFiles } from './project-graph-incremental-recomputation';
import {
getOutputWatcherInstance,
getSourceWatcherSubscription,
getWatcherInstance,
handleServerProcessTermination,
resetInactivityTimeout,
respondToClient,
respondWithErrorAndExit,
SERVER_INACTIVITY_TIMEOUT_MS,
storeOutputsWatcherSubscription,
storeOutputWatcherInstance,
storeProcessJsonSubscription,
storeSourceWatcherSubscription,
storeWatcherInstance,
} from './shutdown-utils';
import {
convertChangeEventsToLogMessage,
subscribeToOutputsChanges,
subscribeToWorkspaceChanges,
FileWatcherCallback,
subscribeToServerProcessJsonChanges,
watchWorkspace,
watchOutputFiles,
watchWorkspace,
} from './watcher';
import { addUpdatedAndDeletedFiles } from './project-graph-incremental-recomputation';
import { existsSync, statSync } from 'fs';
import { handleRequestProjectGraph } from './handle-request-project-graph';
import { handleProcessInBackground } from './handle-process-in-background';
import {
handleOutputsHashesMatch,
handleRecordOutputsHash,
} from './handle-outputs-tracking';
import { consumeMessagesFromSocket } from '../../utils/consume-messages-from-socket';
import {
disableOutputsTracking,
processFileChangesInOutputs,
} from './outputs-tracking';
import { handleRequestShutdown } from './handle-request-shutdown';
import {
registeredFileWatcherSockets,
removeRegisteredFileWatcherSocket,
} from './file-watching/file-watcher-sockets';
import { nxVersion } from '../../utils/versions';
import { readJsonFile } from '../../utils/fileutils';
import { PackageJson } from '../../utils/package-json';
import { getDaemonProcessIdSync, writeDaemonJsonProcessCache } from '../cache';
import { handleHashTasks } from './handle-hash-tasks';
import { hashArray } from '../../hasher/file-hasher';
import { handleRequestFileData } from './handle-request-file-data';
import { setupWorkspaceContext } from '../../utils/workspace-context';
import { hashFile } from '../../native';
let performanceObserver: PerformanceObserver | undefined;
let workspaceWatcherError: Error | undefined;
@ -109,7 +101,6 @@ const server = createServer(async (socket) => {
});
});
registerProcessTerminationListeners();
registerProcessServerJsonTracking();
async function handleMessage(socket, data: string) {
if (workspaceWatcherError) {
@ -239,23 +230,6 @@ function registerProcessTerminationListeners() {
);
}
async function registerProcessServerJsonTracking() {
if (useNativeWatcher()) {
return;
}
storeProcessJsonSubscription(
await subscribeToServerProcessJsonChanges(async () => {
if (getDaemonProcessIdSync() !== process.pid) {
await handleServerProcessTermination({
server,
reason: 'this process is no longer the current daemon',
});
}
})
);
}
let existingLockHash: string | undefined;
function daemonIsOutdated(): boolean {
@ -419,46 +393,20 @@ export async function startServer(): Promise<Server> {
// this triggers the storage of the lock file hash
daemonIsOutdated();
if (useNativeWatcher()) {
if (!getWatcherInstance()) {
storeWatcherInstance(
await watchWorkspace(server, handleWorkspaceChanges)
);
if (!getWatcherInstance()) {
storeWatcherInstance(
await watchWorkspace(server, handleWorkspaceChanges)
);
serverLogger.watcherLog(
`Subscribed to changes within: ${workspaceRoot} (native)`
);
}
serverLogger.watcherLog(
`Subscribed to changes within: ${workspaceRoot} (native)`
);
}
if (!getOutputWatcherInstance()) {
storeOutputWatcherInstance(
await watchOutputFiles(handleOutputsChanges)
);
}
} else {
if (!getSourceWatcherSubscription()) {
storeSourceWatcherSubscription(
await subscribeToWorkspaceChanges(
server,
handleWorkspaceChanges
)
);
serverLogger.watcherLog(
`Subscribed to changes within: ${workspaceRoot}`
);
}
// temporary disable outputs tracking on linux
const outputsTrackingIsEnabled = process.platform != 'linux';
if (outputsTrackingIsEnabled) {
if (!getOutputsWatcherSubscription()) {
storeOutputsWatcherSubscription(
await subscribeToOutputsChanges(handleOutputsChanges)
);
}
} else {
disableOutputsTracking();
}
if (!getOutputWatcherInstance()) {
storeOutputWatcherInstance(
await watchOutputFiles(handleOutputsChanges)
);
}
return resolve(server);
@ -471,8 +419,3 @@ export async function startServer(): Promise<Server> {
}
});
}
// TODO(cammisuli): remove with nx 16.6 (only our watcher will be supported)
function useNativeWatcher() {
return process.env.NX_NATIVE_WATCHER === 'true';
}

44
packages/nx/src/daemon/server/shutdown-utils.ts
View File

@ -2,37 +2,11 @@ import { workspaceRoot } from '../../utils/workspace-root';
import type { Server, Socket } from 'net';
import { serverLogger } from './logger';
import { serializeResult } from '../socket-utils';
import type { AsyncSubscription } from '@parcel/watcher';
import { deleteDaemonJsonProcessCache } from '../cache';
import type { Watcher } from '../../native';
export const SERVER_INACTIVITY_TIMEOUT_MS = 10800000 as const; // 10800000 ms = 3 hours
let sourceWatcherSubscription: AsyncSubscription | undefined;
let outputsWatcherSubscription: AsyncSubscription | undefined;
export function getSourceWatcherSubscription() {
return sourceWatcherSubscription;
}
export function storeSourceWatcherSubscription(s: AsyncSubscription) {
sourceWatcherSubscription = s;
}
export function getOutputsWatcherSubscription() {
return outputsWatcherSubscription;
}
export function storeOutputsWatcherSubscription(s: AsyncSubscription) {
outputsWatcherSubscription = s;
}
let processJsonSubscription: AsyncSubscription | undefined;
export function storeProcessJsonSubscription(s: AsyncSubscription) {
processJsonSubscription = s;
}
let watcherInstance: Watcher | undefined;
export function storeWatcherInstance(instance: Watcher) {
watcherInstance = instance;
@ -61,24 +35,6 @@ export async function handleServerProcessTermination({
try {
server.close();
deleteDaemonJsonProcessCache();
if (sourceWatcherSubscription) {
await sourceWatcherSubscription.unsubscribe();
serverLogger.watcherLog(
`Unsubscribed from changes within: ${workspaceRoot} (sources)`
);
}
if (outputsWatcherSubscription) {
await outputsWatcherSubscription.unsubscribe();
serverLogger.watcherLog(
`Unsubscribed from changes within: ${workspaceRoot} (outputs)`
);
}
if (processJsonSubscription) {
await processJsonSubscription.unsubscribe();
serverLogger.watcherLog(
`Unsubscribed from changes within: ${workspaceRoot} (server-process.json)`
);
}
if (watcherInstance) {
await watcherInstance.stop();

127
packages/nx/src/daemon/server/watcher.ts
View File

@ -1,12 +1,4 @@
/**
* In addition to its native performance, another great advantage of `@parcel/watcher` is that it will
* automatically take advantage of Facebook's watchman tool (https://facebook.github.io/watchman/) if
* the user has it installed (but not require it if they don't).
*
* See https://github.com/parcel-bundler/watcher for more details.
*/
import { workspaceRoot } from '../../utils/workspace-root';
import type { AsyncSubscription, Event } from '@parcel/watcher';
import { dirname, relative } from 'path';
import { FULL_OS_SOCKET_PATH } from '../socket-utils';
import { handleServerProcessTermination } from './shutdown-utils';
@ -25,34 +17,9 @@ const ALWAYS_IGNORE = [...getAlwaysIgnore(workspaceRoot), FULL_OS_SOCKET_PATH];
export type FileWatcherCallback = (
err: Error | string | null,
changeEvents: Event[] | WatchEvent[] | null
changeEvents: WatchEvent[] | null
) => Promise<void>;
export async function subscribeToOutputsChanges(
cb: FileWatcherCallback
): Promise<AsyncSubscription> {
const watcher = await import('@parcel/watcher');
return await watcher.subscribe(
workspaceRoot,
(err, events) => {
if (err) {
return cb(err, null);
} else {
const workspaceRelativeEvents: Event[] = [];
for (const event of events) {
const workspaceRelativeEvent: Event = {
type: event.type,
path: normalizePath(relative(workspaceRoot, event.path)),
};
workspaceRelativeEvents.push(workspaceRelativeEvent);
}
cb(null, workspaceRelativeEvents);
}
},
watcherOptions([...ALWAYS_IGNORE])
);
}
export async function watchWorkspace(server: Server, cb: FileWatcherCallback) {
const { Watcher } = await import('../../native');
@ -115,88 +82,14 @@ export async function watchOutputFiles(cb: FileWatcherCallback) {
return watcher;
}
export async function subscribeToWorkspaceChanges(
server: Server,
cb: FileWatcherCallback
): Promise<AsyncSubscription> {
/**
* The imports and exports of @nx/workspace are somewhat messy and far reaching across the repo (and beyond),
* and so it is much safer for us to lazily load here `@parcel/watcher` so that its inclusion is not inadvertently
* executed by packages which do not have its necessary native binaries available.
*/
const watcher = await import('@parcel/watcher');
const ignoreObj = getIgnoreObject();
return await watcher.subscribe(
workspaceRoot,
(err, events) => {
if (err) {
return cb(err, null);
}
let hasIgnoreFileUpdate = false;
// Most of our utilities (ignore, hashing etc) require unix-style workspace relative paths
const workspaceRelativeEvents: Event[] = [];
for (const event of events) {
const workspaceRelativeEvent: Event = {
type: event.type,
path: normalizePath(relative(workspaceRoot, event.path)),
};
if (
workspaceRelativeEvent.path.endsWith('.gitignore') ||
workspaceRelativeEvent.path === '.nxignore'
) {
hasIgnoreFileUpdate = true;
}
workspaceRelativeEvents.push(workspaceRelativeEvent);
}
// If the ignore files themselves have changed we need to dynamically update our cached ignoreGlobs
if (hasIgnoreFileUpdate) {
handleServerProcessTermination({
server,
reason: 'Stopping the daemon the set of ignored files changed.',
});
}
const nonIgnoredEvents = workspaceRelativeEvents
.filter(({ path }) => !!path)
.filter(({ path }) => !ignoreObj.ignores(path));
if (nonIgnoredEvents && nonIgnoredEvents.length > 0) {
cb(null, nonIgnoredEvents);
}
},
watcherOptions(getIgnoredGlobs(workspaceRoot))
);
}
// TODO: When we update @parcel/watcher to a version that handles negation globs, then this can be folded into the workspace watcher
export async function subscribeToServerProcessJsonChanges(
cb: () => void
): Promise<AsyncSubscription> {
const watcher = await import('@parcel/watcher');
return await watcher.subscribe(
dirname(serverProcessJsonPath),
(err, events) => {
for (const event of events) {
if (event.path === serverProcessJsonPath) {
cb();
}
}
},
watcherOptions([])
);
}
/**
* NOTE: An event type of "create" will also apply to the case where the user has restored
* an original version of a file after modifying/deleting it by using git, so we adjust
* our log language accordingly.
*/
export function convertChangeEventsToLogMessage(changeEvents: Event[]): string {
export function convertChangeEventsToLogMessage(
changeEvents: WatchEvent[]
): string {
// If only a single file was changed, show the information inline
if (changeEvents.length === 1) {
const { path, type } = changeEvents[0];
@ -234,15 +127,3 @@ export function convertChangeEventsToLogMessage(changeEvents: Event[]): string {
return `${numCreatedOrRestoredFiles} file(s) created or restored, ${numModifiedFiles} file(s) modified, ${numDeletedFiles} file(s) deleted`;
}
function watcherOptions(ignore: string[]) {
const options: import('@parcel/watcher').Options = {
ignore,
};
if (platform() === 'win32') {
options.backend = 'windows';
}
return options;
}