feat(core): update ci-workflow generator (#21141)

docs/shared/core-features/distribute-task-execution.md

@@ -46,7 +46,7 @@ If you have a new workspace, you can generate the CI configuration as follows:
 nx generate @nx/workspace:ci-workflow --ci=github
 ```
 
-The `--ci` flag can be `github`, `circleci` or `azure`.
+The `--ci` flag can be `github`, `circleci`, `azure`, `gitlab`, or `bitbucket`.
 
 For existing workspaces you would probably want to adjust your configuration by hand. See below for examples.
 

packages/workspace/src/generators/ci-workflow/ci-workflow.spec.ts

@@ -1,5 +1,6 @@
 import {
   NxJsonConfiguration,
+  PackageManager,
   readJson,
   Tree,
   updateJson,
@@ -7,6 +8,24 @@ import {
 } from '@nx/devkit';
 import { createTreeWithEmptyWorkspace } from '@nx/devkit/testing';
 import { ciWorkflowGenerator } from './ci-workflow';
+import { vol } from 'memfs';
+
+jest.mock('@nx/devkit', () => ({
+  ...jest.requireActual<any>('@nx/devkit'),
+  workspaceRoot: '/root',
+}));
+
+jest.mock('fs', () => {
+  const memFs = require('memfs').fs;
+  const actualFs = jest.requireActual<any>('fs');
+  return {
+    ...jest.requireActual<any>('fs'),
+    existsSync: (p) =>
+      p.endsWith('yarn.lock') || p.endsWith('pnpm-lock.yaml')
+        ? memFs.existsSync(p)
+        : actualFs.existsSync(p),
+  };
+});
 
 describe('CI Workflow generator', () => {
   let tree: Tree;
@@ -15,15 +34,22 @@ describe('CI Workflow generator', () => {
     tree = createTreeWithEmptyWorkspace({ layout: 'apps-libs' });
   });
 
-  ['npm', 'yarn', 'pnpm'].forEach((packageManager) => {
+  afterEach(() => {
+    vol.reset();
+  });
+
+  ['npm', 'yarn', 'pnpm'].forEach((packageManager: PackageManager) => {
     describe(`with ${packageManager}`, () => {
       beforeEach(() => {
-        jest.mock('@nx/devkit', () => ({
+        let fileSys;
-          ...jest.requireActual<any>('@nx/devkit'),
+        if (packageManager === 'yarn') {
-          detectPackageManager: jest
+          fileSys = { 'yarn.lock': '' };
-            .fn()
+        } else if (packageManager === 'pnpm') {
-            .mockImplementation(() => packageManager),
+          fileSys = { 'pnpm-lock.yaml': '' };
-        }));
+        } else {
+          fileSys = { 'package-lock.json': '' };
+        }
+        vol.fromJSON(fileSys, '');
       });
 
       it('should generate github CI config', async () => {

packages/workspace/src/generators/ci-workflow/ci-workflow.ts

@@ -7,6 +7,7 @@ import {
   NxJsonConfiguration,
   formatFiles,
   writeJson,
+  detectPackageManager,
 } from '@nx/devkit';
 import { deduceDefaultBase } from '../../utilities/default-base';
 import { join } from 'path';
@@ -42,6 +43,7 @@ interface Substitutes {
   mainBranch: string;
   workflowName: string;
   workflowFileName: string;
+  packageManager: string;
   packageManagerInstall: string;
   packageManagerPrefix: string;
   tmpl: '';
@@ -51,11 +53,13 @@ function normalizeOptions(options: Schema): Substitutes {
   const { name: workflowName, fileName: workflowFileName } = names(
     options.name
   );
+  const packageManager = detectPackageManager();
   const { exec: packageManagerPrefix, ciInstall: packageManagerInstall } =
-    getPackageManagerCommand();
+    getPackageManagerCommand(packageManager);
   return {
     workflowName,
     workflowFileName,
+    packageManager,
     packageManagerInstall,
     packageManagerPrefix,
     mainBranch: deduceDefaultBase(),

packages/workspace/src/generators/ci-workflow/files/azure/azure-pipelines.yml__tmpl__

@@ -7,7 +7,6 @@ pr:
 
 variables:
   CI: 'true'
-  NX_CLOUD_DISTRIBUTED_EXECUTION: 'true'
   ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
     NX_BRANCH: $(System.PullRequest.PullRequestNumber)
     TARGET_BRANCH: $[replace(variables['System.PullRequest.TargetBranch'],'refs/heads/','origin/')]
@@ -18,41 +17,15 @@ variables:
   HEAD_SHA: $(git rev-parse HEAD)
 
 jobs:
-  - job: agents
-    strategy:
-      parallel: 3
-    displayName: Nx Cloud Agent
-    pool:
-      vmImage: 'ubuntu-latest'
-    steps:
-      <% if(packageManagerPrefix == 'pnpm exec'){ %>
-      - script: npm install --prefix=$HOME/.local -g pnpm@8.2.0
-        displayName: Install PNPM
-      <% } %>- script: <%= packageManagerInstall %>
-        displayName: NPM Install Dependencies
-      - script: npx nx-cloud start-agent
-        displayName: Start Nx-Cloud agent
-
   - job: main
-    displayName: Nx Cloud Main
     pool:
       vmImage: 'ubuntu-latest'
     steps:
-      <% if(packageManagerPrefix == 'pnpm exec'){ %>
+      <% if(packageManager == 'pnpm'){ %>
-      - script: npm install --prefix=$HOME/.local -g pnpm@8.2.0
+      - script: npm install --prefix=$HOME/.local -g pnpm@8
         displayName: Install PNPM
       <% } %>- script: <%= packageManagerInstall %>
-        displayName: NPM Install Dependencies
+      # uncomment to enable task distribution
-      - script: <%= packageManagerPrefix %> nx-cloud start-ci-run --stop-agents-after="build" --agent-count=3
+      # - script: <%= packageManagerPrefix %> nx-cloud start-ci-run --distributes-on="5 linux-medium"
-        displayName: Start CI run
+      - script: <%= packageManagerPrefix %> nx-cloud record -- nx format:check --base=$(BASE_SHA) --head=$(HEAD_SHA)
-      - script: <%= packageManagerPrefix %> nx-cloud record -- <%= packageManagerPrefix %> nx format:check --base=$(BASE_SHA) --head=$(HEAD_SHA)
+      - script: <%= packageManagerPrefix %> nx affected --base=$(BASE_SHA) --head=$(HEAD_SHA) -t=lint,test,build --parallel=3
-        displayName: Check format
-      - script: <%= packageManagerPrefix %> nx affected --base=$(BASE_SHA) --head=$(HEAD_SHA) --target=lint --parallel=3
-        displayName: Run lint
-      - script: <%= packageManagerPrefix %> nx affected --base=$(BASE_SHA) --head=$(HEAD_SHA) --target=test --parallel=3 --ci --code-coverage
-        displayName: Run test
-      - script: <%= packageManagerPrefix %> nx affected --base=$(BASE_SHA) --head=$(HEAD_SHA) --target=build --parallel=3
-        displayName: Run build
-      - script: <%= packageManagerPrefix %> nx-cloud stop-all-agents
-        condition: always()
-        displayName: Stop all Nx-Cloud agents

packages/workspace/src/generators/ci-workflow/files/bitbucket-pipelines/bitbucket-pipelines.yml__tmpl__

@@ -1,20 +1,8 @@
-image: node:16.18
+image: node:20
 
 clone:
   depth: full
 
-definitions:
-  steps:
-    - step: &agent
-        name: Agent
-        script:
-          - export NX_BRANCH=$BITBUCKET_PR_ID
-          <% if(packageManagerPrefix == 'pnpm exec'){ %>
-          - npm install --prefix=$HOME/.local -g pnpm@8.2.0
-          <% } %>
-          - <%= packageManagerInstall %>
-          - <%= packageManagerPrefix %> nx-cloud start-agent
-
 pipelines:
   pull-requests:
     '**':
@@ -23,14 +11,11 @@ pipelines:
               name: <%= workflowName %>
               script:
                 - export NX_BRANCH=$BITBUCKET_PR_ID
-                <% if(packageManagerPrefix == 'pnpm exec'){ %>
+                <% if(packageManager == 'pnpm'){ %>
-                - npm install --prefix=$HOME/.local -g pnpm@8.2.0
+                - npm install --prefix=$HOME/.local -g pnpm@8
                 <% } %>
                 - <%= packageManagerInstall %>
-                - <%= packageManagerPrefix %> nx-cloud start-ci-run --stop-agents-after="build" --agent-count=3
+                # uncomment to enable task distribution
-                - <%= packageManagerPrefix %> nx-cloud record -- <%= packageManagerPrefix %> nx format:check
+                # - <%= packageManagerPrefix %> nx-cloud start-ci-run --distributes-on="5 linux-medium"
-                - <%= packageManagerPrefix %> nx affected --target=lint & <%= packageManagerPrefix %> nx affected --target=test & <%= packageManagerPrefix %> nx affected --target=build
+                - <%= packageManagerPrefix %> nx-cloud record -- nx format:check
-                - <%= packageManagerPrefix %> nx-cloud stop-all-agents
+                - <%= packageManagerPrefix %> nx affected -t=lint,test,build --parallel=3
-          - step: *agent
-          - step: *agent
-          - step: *agent

packages/workspace/src/generators/ci-workflow/files/circleci/.circleci/config.yml__tmpl__

@@ -1,73 +1,29 @@
 version: 2.1
 
 orbs:
-  nx: nrwl/nx@1.6.1
+  nx: nrwl/nx@1.6.2
 
 jobs:
-  agent:
-    docker:
-      - image: cimg/node:lts-browsers
-    parameters:
-      ordinal:
-        type: integer
-    steps:
-      - checkout
-      <% if(packageManagerPrefix == 'pnpm exec'){ %>
-      - run:
-          name: Install PNPM
-          command: npm install --prefix=$HOME/.local -g pnpm@8.2.0
-      <% } %>- run:
-          name: Install dependencies
-          command: <%= packageManagerInstall %>
-      - run:
-          name: Start the agent << parameters.ordinal >>
-          command: <%= packageManagerPrefix %> nx-cloud start-agent
-          no_output_timeout: 60m
   main:
     docker:
       - image: cimg/node:lts-browsers
-    environment:
-      NX_CLOUD_DISTRIBUTED_EXECUTION: 'true'
     steps:
       - checkout
-      <% if(packageManagerPrefix == 'pnpm exec'){ %>
+      <% if(packageManager == 'pnpm'){ %>
       - run:
           name: Install PNPM
-          command: npm install --prefix=$HOME/.local -g pnpm@8.2.0
+          command: npm install --prefix=$HOME/.local -g pnpm@8
-      <% } %>- run:
+      <% } %>- run: <%= packageManagerInstall %>
-          name: Install dependencies
-          command: <%= packageManagerInstall %>
       - nx/set-shas:
           main-branch-name: '<%= mainBranch %>'
-      - run:
+      # uncomment to enable task distribution
-          name: Initialize the Nx Cloud distributed CI run
+      # - run: <%= packageManagerPrefix %> nx-cloud start-ci-run --distributes-on="5 linux-medium"
-          command: <%= packageManagerPrefix %> nx-cloud start-ci-run --stop-agents-after="build" --agent-count=3
+      - run: <%= packageManagerPrefix %> nx-cloud record -- nx format:check --base=$NX_BASE --head=$NX_HEAD
-      - run:
+      - run: <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD -t=lint,test,build --parallel=3
-          name: Check format
-          command: <%= packageManagerPrefix %> nx-cloud record -- <%= packageManagerPrefix %> nx format:check --base=$NX_BASE --head=$NX_HEAD
-      - run:
-          name: Run lint
-          command: <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD --target=lint --parallel=3
-      - run:
-          name: Run test
-          command: <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD --target=test --parallel=3 --ci --code-coverage
-      - run:
-          name: Run build
-          command: <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD --target=build --parallel=3
-      - run:
-          name: Stop all agents
-          command: <%= packageManagerPrefix %> nx-cloud stop-all-agents
-          when: always
 
 workflows:
   version: 2
 
   <%= workflowFileName %>:
     jobs:
-      - agent:
+      - main
-          name: Nx Cloud Agent << matrix.ordinal >>
-          matrix:
-            parameters:
-              ordinal: [1, 2, 3]
-      - main:
-          name: Nx Cloud Main

packages/workspace/src/generators/ci-workflow/files/github/.github/workflows/__workflowFileName__.yml__tmpl__

@@ -6,29 +6,31 @@ on:
       - <%= mainBranch %>
   pull_request:
 
-# Needed for nx-set-shas within nx-cloud-main.yml, when run on the <%= mainBranch %> branch
 permissions:
   actions: read
   contents: read
 
 jobs:
   main:
-    name: Nx Cloud - Main Job
+    runs-on: ubuntu-latest
-    uses: nrwl/ci/.github/workflows/nx-cloud-main.yml@v0.13.1
+    steps:
-    with:
+      - uses: actions/checkout@v4
-      main-branch-name: <%= mainBranch %>
+        with:
-      number-of-agents: 3
+          fetch-depth: 0
-      init-commands: |
+      <% if(packageManager == 'pnpm'){ %>
-        <%= packageManagerPrefix %> nx-cloud start-ci-run --stop-agents-after="build" --agent-count=3
+      - uses: pnpm/action-setup@v2
-      parallel-commands: |
+        with:
-        <%= packageManagerPrefix %> nx-cloud record -- <%= packageManagerPrefix %> nx format:check
+          version: 8
-      parallel-commands-on-agents: |
+      <% } %># Cache node_modules
-        <%= packageManagerPrefix %> nx affected --target=lint --parallel=3
+      - uses: actions/setup-node@v3
-        <%= packageManagerPrefix %> nx affected --target=test --parallel=3 --ci --code-coverage
+        with:
-        <%= packageManagerPrefix %> nx affected --target=build --parallel=3
+          node-version: 20
+          cache: '<%= packageManager %>'
+      - run: <%= packageManagerInstall %>
+      - uses: nrwl/nx-set-shas@v4
 
-  agents:
+      # uncomment to enable task distribution
-    name: Nx Cloud - Agents
+      # - run: <%= packageManagerPrefix %> nx-cloud start-ci-run --distributes-on="5 linux-medium"
-    uses: nrwl/ci/.github/workflows/nx-cloud-agents.yml@v0.13.1
+
-    with:
+      - run: <%= packageManagerPrefix %> nx-cloud record -- nx format:check
-      number-of-agents: 3
+      - run: <%= packageManagerPrefix %> nx affected -t=lint,test,build --parallel=3

packages/workspace/src/generators/ci-workflow/files/gitlab/.gitlab-ci.yml__tmpl__

@@ -1,47 +1,21 @@
-image: node:18
+image: node:20
 variables:
   CI: 'true'
 
-# Creating template for DTE agents
+# Main job
-.dte-agent:
+<%= workflowName %>:
-  interruptible: true
-  script:
-    <% if(packageManagerPrefix == 'pnpm exec'){ %>
-    - npm install --prefix=$HOME/.local -g pnpm@8.2.0
-    <% } %>
-    - <%= packageManagerInstall %>
-    - <%= packageManagerPrefix %> nx-cloud start-agent
-
-# Creating template for a job running DTE (orchestrator)
-.base-pipeline:
   interruptible: true
   only:
     - main
     - merge_requests
-  before_script:
+  script:
-    <% if(packageManagerPrefix == 'pnpm exec'){ %>
+    <% if(packageManager == 'pnpm'){ %>
-    - npm install --prefix=$HOME/.local -g pnpm@8.2.0
+    - npm install --prefix=$HOME/.local -g pnpm@8
     <% } %>
     - <%= packageManagerInstall %>
     - NX_HEAD=$CI_COMMIT_SHA
     - NX_BASE=${CI_MERGE_REQUEST_DIFF_BASE_SHA:-$CI_COMMIT_BEFORE_SHA}
-
+    # uncomment to enable task distribution
-# Main job running DTE
+    # - <%= packageManagerPrefix %> nx-cloud start-ci-run --distributes-on="5 linux-medium"
-<%= workflowName %>:
+    - <%= packageManagerPrefix %> nx-cloud record -- nx format:check --base=$NX_BASE --head=$NX_HEAD
-  stage: affected
+    - <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD -t=lint,test,build --parallel=3
-  extends: .base-pipeline
-  script:
-    - <%= packageManagerPrefix %> nx-cloud start-ci-run --stop-agents-after="build"
-    - <%= packageManagerPrefix %> nx-cloud record -- <%= packageManagerPrefix %> nx format:check --base=$NX_BASE --head=$NX_HEAD
-    - <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD --target=lint --parallel=3 & <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD --target=test --parallel=3 --ci --code-coverage & <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD --target=e2e --parallel=3 --ci --code-coverage & <%= packageManagerPrefix %> nx affected --base=$NX_BASE --head=$NX_HEAD --target=build --parallel=3
-
-# Create as many agents as you want
-nx-dte-agent1:
-  extends: .dte-agent
-  stage: affected
-nx-dte-agent2:
-  extends: .dte-agent
-  stage: affected
-nx-dte-agent3:
-  extends: .dte-agent
-  stage: affected