name: E2E matrix

on:
  schedule:
    - cron: "0 5 * * *"
  workflow_dispatch:
    inputs:
      debug_enabled:
        type: boolean
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false

env:
  CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress

permissions: { }
jobs:
  preinstall:
    if: ${{ github.repository_owner == 'nrwl' }}
    runs-on: ${{ matrix.os }}
    timeout-minutes: 20
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
          # - windows-latest Windows fails to build gradle wrapper which always runs when we build nx.
          ## https://staging.nx.app/runs/LgD4vxGn8w?utm_source=pull-request&utm_medium=comment
        node_version:
          - 20
          - 22
          # - 23
        exclude:
          # run just node v20 on macos and windows
          - os: macos-latest
            node_version: 22
          # - os: macos-latest
          #   node_version: 23
          # - os: windows-latest TODO (emily): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this.
            # node_version: 22
          # - os: windows-latest TODO (emily): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this.
          #   node_version: 23

    name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }})
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: tree:0

      - uses: pnpm/action-setup@v4
        name: Install pnpm
        with:
          version: 10.11.1
          run_install: false

      - name: Set node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node_version }}
          cache: 'pnpm'

      - name: Get pnpm store directory
        id: pnpm-cache
        run: echo "path=$(pnpm store path)" >> $GITHUB_OUTPUT

      - name: Cache pnpm store
        uses: actions/cache@v4
        with:
          path: ${{ steps.pnpm-cache.outputs.path }}
          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-

      - name: Ensure Python setuptools Installed on Macos
        if: ${{ matrix.os == 'macos-latest' }}
        id: brew-install-python-setuptools
        run: brew install python-setuptools

      - name: Install packages
        run: pnpm install --frozen-lockfile

      - name: Homebrew cache directory path
        if: ${{ matrix.os == 'macos-latest' }}
        id: homebrew-cache-dir-path
        run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT

      - name: Cache Homebrew
        if: ${{ matrix.os == 'macos-latest' }}
        uses: actions/cache@v4
        with:
          lookup-only: true
          path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
          key: brew-${{ matrix.node_version }}
          restore-keys: |
            brew-

      - name: Cache Cypress
        id: cache-cypress
        uses: actions/cache@v4
        with:
          lookup-only: true
          path: '${{ github.workspace }}/.cypress'
          key: ${{ runner.os }}-cypress

      - name: Install Cypress
        if: steps.cache-cypress.outputs.cache-hit != 'true'
        run: npx cypress install

  prepare-matrix:
    name: Prepare matrix combinations
    if: ${{ github.repository_owner == 'nrwl' }}
    timeout-minutes: 5
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.process-json.outputs.MATRIX }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: tree:0

      - name: Process matrix data
        id: process-json
        run:
          echo "MATRIX=$(npx tsx .github/workflows/nightly/process-matrix.ts | jq -c .)" >> $GITHUB_OUTPUT

  e2e:
    if: ${{ github.repository_owner == 'nrwl' }}
    needs:
      - preinstall
      - prepare-matrix
    permissions:
      contents: read
    runs-on: ${{ matrix.os }}
    timeout-minutes: 150     # <- cap each job to 150 minutes
    strategy:
      matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} # Load matrix from previous job
      fail-fast: false

    name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: tree:0

      - name: Prepare dir for output
        run: mkdir -p outputs

      - uses: pnpm/action-setup@v4
        name: Install pnpm
        with:
          version: 10.11.1
          run_install: false

      - name: Use Node.js ${{ matrix.node_version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node_version }}
          cache: 'pnpm'

      - name: Install Rust
        if: ${{ matrix.os != 'windows-latest' }}
        run: |
          curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y
          source "$HOME/.cargo/env"
          rustup toolchain install 1.70.0

      - name: Load Cargo Env
        if: ${{ matrix.os != 'windows-latest' }}
        run: echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
  
      - name: Install bun
        if: ${{ matrix.os != 'windows-latest' }}
        uses: oven-sh/setup-bun@v1
        with:
          bun-version: latest

      - name: Install packages
        run: pnpm install --frozen-lockfile

      - name: Cleanup
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          # Workaround to provide additional free space for testing.
          # https://github.com/actions/virtual-environments/issues/2840
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          sudo apt-get install lsof
          echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p

      - name: Homebrew cache directory path
        if: ${{ matrix.os == 'macos-latest' }}
        id: homebrew-cache-dir-path
        run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT

      - name: Cache Homebrew
        if: ${{ matrix.os == 'macos-latest' }}
        uses: actions/cache@v4
        with:
          path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
          key: brew-${{ matrix.node_version }}
          restore-keys: |
            brew-

      - name: Cache Cypress
        id: cache-cypress
        uses: actions/cache@v4
        with:
          path: '${{ github.workspace }}/.cypress'
          key: ${{ runner.os }}-cypress

      - name: Install Cypress
        if: steps.cache-cypress.outputs.cache-hit != 'true'
        run: npx cypress install

      - name: Configure Detox Environment, Install applesimutils
        if: ${{ matrix.os == 'macos-latest' }}
        run: |
          # Ensure Xcode command line tools are installed and configured
          xcode-select --print-path || sudo xcode-select --reset
          sudo xcode-select -s /Applications/Xcode.app
          
          # Install or update applesimutils with error handling
          if ! brew list applesimutils &>/dev/null; then
            echo "Installing applesimutils..."
            HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null
            HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null || {
              echo "Failed to install applesimutils, retrying with update..."
              brew update
              HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils
            }
          else
            echo "Updating applesimutils..."
            HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade applesimutils || true
          fi
          
          # Verify applesimutils installation
          applesimutils --version || (echo "applesimutils installation failed" && exit 1)
          
          # Configure environment for M-series Mac
          echo "DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer" >> $GITHUB_ENV
          echo "PLATFORM_NAME=iOS Simulator" >> $GITHUB_ENV
          
          # Set additional environment variables for better debugging
          echo "DETOX_DISABLE_TELEMETRY=1" >> $GITHUB_ENV
          echo "DETOX_LOG_LEVEL=trace" >> $GITHUB_ENV
          
          # Verify Xcode installation
          xcodebuild -version
          
        timeout-minutes: 10
        continue-on-error: false

      - name: Reset iOS Simulators
        if: ${{ matrix.os == 'macos-latest' }}
        id: reset-simulators
        run: |
          echo "Resetting iOS Simulators..."
          
          # Kill simulator processes
          sudo killall -9 com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
          killall "Simulator" 2>/dev/null || true
          killall "iOS Simulator" 2>/dev/null || true
          
          # Wait for processes to terminate
          sleep 3
          
          # Shutdown and erase all simulators (ignore failures)
          xcrun simctl shutdown all 2>/dev/null || true
          sleep 5
          xcrun simctl erase all 2>/dev/null || true
          
          # If erase failed, try the nuclear option
          if xcrun simctl list devices | grep -q "Booted" 2>/dev/null; then
            echo "Standard reset failed, using nuclear option..."
            rm -rf ~/Library/Developer/CoreSimulator/Devices/* 2>/dev/null || true
            launchctl remove com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
            sleep 3
          fi
          
          # Clean up additional directories
          rm -rf ~/Library/Developer/CoreSimulator/Caches/* 2>/dev/null || true
          rm -rf ~/Library/Logs/CoreSimulator/* 2>/dev/null || true
          rm -rf ~/Library/Developer/Xcode/DerivedData/* 2>/dev/null || true
          
          echo "Simulator reset completed"
        timeout-minutes: 5
        continue-on-error: true

      - name: Verify Simulator Reset
        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
        run: |
          # Verify CoreSimulator service restarted
          pgrep -fl "CoreSimulator" || (echo "CoreSimulator service not running" && exit 1)
          
          # Verify simulator runtime paths exist and are writable
          test -d ~/Library/Developer/CoreSimulator/Devices || (echo "Simulator devices directory missing" && exit 1)
          touch ~/Library/Developer/CoreSimulator/Devices/test || (echo "Simulator devices directory not writable" && exit 1)
          rm ~/Library/Developer/CoreSimulator/Devices/test
        timeout-minutes: 5

      - name: Diagnose Simulator Reset Failure
        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'failure' }}
        run: |
          echo "Simulator reset failed. Collecting diagnostic information..."
          xcrun simctl list
          echo "Checking simulator logs..."
          ls -la ~/Library/Logs/CoreSimulator/ || echo "No simulator logs found"

      - name: Configure git metadata (needed for lerna smoke tests)
        if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
        run: |
          git config --global user.email test@test.com
          git config --global user.name "Test Test"

      - name: Set starting timestamp
        if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
        id: before-e2e
        shell: bash
        run: |
          echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Run e2e tests with pnpm (Linux/Windows)
        id: e2e-run-pnpm
        if: ${{ matrix.os != 'macos-latest' }}
        run: pnpm nx run ${{ matrix.project }}:e2e-local
        shell: bash
        timeout-minutes: ${{ matrix.os_timeout }}
        env:
          NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
          NX_DAEMON: 'true'
          NX_PERF_LOGGING: 'false'
          NX_E2E_VERBOSE_LOGGING: 'true'
          NX_NATIVE_LOGGING: 'false'
          NX_E2E_RUN_E2E: 'true'
          NX_CLOUD_NO_TIMEOUTS: 'true'
          NX_E2E_SKIP_CLEANUP: 'true'
          NODE_OPTIONS: --max_old_space_size=8192
          SELECTED_PM: ${{ matrix.package_manager }}
          npm_config_registry: http://localhost:4872
          YARN_REGISTRY: http://localhost:4872
          CI: true
  
      - name: Run e2e tests with npm (macOS)
        id: e2e-run-npm
        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
        run: |
          # Run the tests
          if [[ "${{ matrix.project }}" == "e2e-detox" ]] || [[ "${{ matrix.project }}" == "e2e-react-native" ]] || [[ "${{ matrix.project }}" == "e2e-expo" ]]; then
            NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-macos-local
          else
            NX_E2E_VERBOSE_DEBUG=1 pnpm nx run ${{ matrix.project }}:e2e-local
          fi
        
        env:
          NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
          NX_PERF_LOGGING: 'false'
          NX_CI_EXECUTION_ENV: 'macos'
          NX_E2E_VERBOSE_LOGGING: 'true'
          NX_NATIVE_LOGGING: 'false'
          NX_E2E_RUN_E2E: 'true'
          NX_E2E_SKIP_CLEANUP: 'true'
          NODE_OPTIONS: --max_old_space_size=8192
          SELECTED_PM: 'npm'
          npm_config_registry: http://localhost:4872
          YARN_REGISTRY: http://localhost:4872
          DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer'
          CI: true

      - name: Save matrix config in file
        if: ${{ always() }}
        id: save-matrix
        shell: bash
        run: |
          before=${{ steps.before-e2e.outputs.timestamp }}
          now=$(date +%s)
          delta=$(($now - $before))
          
          # Determine the outcome based on which step ran
          outcome="${{ matrix.os == 'macos-latest' && steps.e2e-run-npm.outcome || steps.e2e-run-pnpm.outcome }}"
          
          matrix=$((
            echo '${{ toJSON(matrix) }}'
          ) | jq --argjson delta $delta -c '. + { "status": "'"$outcome"'", "duration": $delta }')
          echo "$matrix" > 'outputs/matrix.json'

      - name: Upload matrix config
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: ${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }}
          overwrite: true
          if-no-files-found: 'ignore'
          path: 'outputs/matrix.json'

      - name: Setup tmate session
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
        uses: mxschmitt/action-tmate@v3.8
        timeout-minutes: 15
        with:
          sudo: ${{ matrix.os != 'windows-latest' }} # disable sudo for windows debugging

  process-result:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    runs-on: ubuntu-latest
    needs: e2e
    timeout-minutes: 10
    outputs:
      message: ${{ steps.process-json.outputs.slack_message }}
      proj_duration: ${{ steps.process-json.outputs.slack_proj_duration }}
      pm_duration: ${{ steps.process-json.outputs.slack_pm_duration }}
      codeowners: ${{ steps.process-json.outputs.codeowners }}
      has_golden_failures: ${{ steps.process-json.outputs.has_golden_failures }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: tree:0

      - name: Prepare dir for output
        run: mkdir -p outputs

      - name: Load outputs
        uses: actions/download-artifact@v4
        with:
          path: outputs

      - name: Join and stringify matrix configs
        id: combine-json
        run: |
          combined=$(jq -sc . outputs/*/matrix.json)
          echo "combined=$combined" >> $GITHUB_OUTPUT

      - name: Process results with TypeScript script
        id: process-json
        run: |
          echo '${{ steps.combine-json.outputs.combined }}' | npx tsx .github/workflows/nightly/process-result.ts

  report-failure:
    if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'true' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    name: Report failure
    timeout-minutes: 10
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: 'failure'
          message_format: '${{ needs.process-result.outputs.message }}'
          notification_title: 'Golden Test Failure'
          footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
          mention_groups: ${{ needs.process-result.outputs.codeowners }}
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-success:
    if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'false' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    name: Report status
    timeout-minutes: 10
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: 'success'
          message_format: '${{ needs.process-result.outputs.message }}'
          notification_title: '✅ Golden Tests: All Passed!'
          footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-pm-time:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    timeout-minutes: 10
    name: Report duration per package manager
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: 'skipped'
          message_format: '${{ needs.process-result.outputs.pm_duration }}'
          notification_title: '⌛ Total duration per package manager (ubuntu only)'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-proj-time:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    timeout-minutes: 10
    name: Report duration per project
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: 'skipped'
          message_format: '${{ needs.process-result.outputs.proj_duration }}'
          notification_title: '⌛ E2E Project duration stats'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}