4525ce6a84
Fixes #948 This PR does the following: 1. Changes the trigger for the PR comment updater from the cron method to workflow_run, triggered on completion of the test cases. This should remove the delay between the completion of the performance tests and the updating of the corresponding comment in the PR. 2. Separates the integrity check and performance check into separate workflows and allows them to run in parallel. This will allow the project to take advantage of multiple CI runners if they're available (which appears to be the case). In addition, this fixes an issue with post-merge undeleted/updated branches on PRs. The current "cron" method causes the CI to run the pr-update job over and over forever, unnecessarily. As described in github/docs#799, and the [github docs](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#workflow_run), a `workflow_run` trigger will only fire when the workflow file is on the main branch. Thus, this change will not fire the PR updater on this PR. Thus there's no way to test this working properly without merging onto master and then testing on one of the other PRs.
247 lines
11 KiB
YAML
247 lines
11 KiB
YAML
name: Update PR comments
|
|
|
|
on:
|
|
workflow_run:
|
|
workflows: ["OpenMapTiles Performance CI"]
|
|
types: [completed]
|
|
|
|
jobs:
|
|
update_PRs:
|
|
runs-on: ubuntu-latest
|
|
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
|
|
|
steps:
|
|
- name: main
|
|
env:
|
|
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
|
WORKFLOW_NAME: "OpenMapTiles CI"
|
|
# the name of the artifact whose content comment published by PR. Must have a single markdown file inside.
|
|
MSG_ARTIFACT_NAME: "pr_message"
|
|
# How far back to look for finished runs, in minutes.
|
|
# Set to 10-20 minutes higher than cron's job frequency set above.
|
|
IGNORE_RUNS_OLDER_THAN: 80
|
|
# How far back to look for updated pull requests, in minutes.
|
|
# Should be bigger than IGNORE_RUNS_OLDER_THAN by the maximum time a pull request jobs may take
|
|
IGNORE_PRS_OLDER_THAN: 80
|
|
run: |
|
|
#
|
|
# Strategy:
|
|
# * get all recently updated pull requests
|
|
# * get all recent workflow runs
|
|
# * match pull requests and their current SHA with the last workflow run for the same SHA
|
|
# * for each found match of <pull-request-number> and <workflow-run-id> :
|
|
# * download artifact from the workflow run -- expects a single file with markdown content
|
|
# * look through existing PR comments to see if we have posted a comment before
|
|
# (uses a hidden magical header to identify our comment)
|
|
# * either create or update the comment with the new text (if changed)
|
|
#
|
|
|
|
export GITHUB_API="https://api.github.com/repos/$GITHUB_REPOSITORY"
|
|
export COMMENT_MAGIC_HEADER='<!--'" Do not edit. This comment will be auto-updated with artifact '$MSG_ARTIFACT_NAME' created by action '$WORKFLOW_NAME' -->"
|
|
|
|
# A useful wrapper around CURL
|
|
crl() {
|
|
curl --silent --show-error --location --retry 1 "${@:2}" \
|
|
-H "Accept: application/vnd.github.antiope-preview+json, application/vnd.github.v3+json" \
|
|
"$1"
|
|
}
|
|
|
|
auth_crl() {
|
|
crl "$1" -H "authorization: Bearer $GITHUB_TOKEN" "${@:2}"
|
|
}
|
|
|
|
#
|
|
# Parse current pull requests
|
|
#
|
|
|
|
# Get all pull requests, most recently updated first
|
|
# (this way we don't need to page through all of them)
|
|
# Filter out PRs that are older than $IGNORE_PRS_OLDER_THAN minutes
|
|
# Result is an object, mapping a "key" to the pull request number:
|
|
# {
|
|
# "nyurik/openmaptiles/nyurik-patch-1/4953dd2370b9988a7832d090b5e47b3cd867f594": 6,
|
|
# ...
|
|
# }
|
|
PULL_REQUESTS_RAW="$( crl "$GITHUB_API/pulls?sort=updated&direction=desc" )"
|
|
if ! PULL_REQUESTS="$(jq --arg IGNORE_PRS_OLDER_THAN "$IGNORE_PRS_OLDER_THAN" '
|
|
map(
|
|
# Only select unlocked pull requests updated within last $IGNORE_PRS_OLDER_THAN minutes
|
|
select(.locked==false
|
|
and (now - (.updated_at|fromdate)) / 60 < ($IGNORE_PRS_OLDER_THAN | tonumber))
|
|
# Prepare for "from_entries" by creating a key/value object
|
|
# The key is a combination of repository name, branch name, and latest SHA
|
|
| { key: (.head.repo.full_name + "/" + .head.ref + "/" + .head.sha), value: .number }
|
|
)
|
|
| from_entries
|
|
' <( echo "$PULL_REQUESTS_RAW" ) )"; then
|
|
|
|
echo "Error parsing pull requests"
|
|
echo "$PULL_REQUESTS_RAW"
|
|
exit 1
|
|
fi
|
|
|
|
# Count how many pull requests we should process, and exit early if there are none
|
|
PR_COUNT="$(jq 'length' <( echo "$PULL_REQUESTS" ) )"
|
|
if [ "$PR_COUNT" -eq 0 ]; then
|
|
echo "There are no pull requests updated in the last $IGNORE_PRS_OLDER_THAN minutes. Exiting."
|
|
exit
|
|
else
|
|
echo "$PR_COUNT pull requests have been updated in the last $IGNORE_PRS_OLDER_THAN minutes"
|
|
echo "$PULL_REQUESTS" | jq -r 'keys|.[]|" * " + .'
|
|
fi
|
|
|
|
|
|
|
|
#
|
|
# Resolve workflow name into workflow ID
|
|
#
|
|
WORKFLOW_ID="$(crl "$GITHUB_API/actions/workflows" \
|
|
| jq --arg WORKFLOW_NAME "$WORKFLOW_NAME" '
|
|
.workflows[] | select(.name == $WORKFLOW_NAME) | .id
|
|
')"
|
|
if [ -z "$WORKFLOW_ID" ]; then
|
|
echo "Unable to find workflow '$WORKFLOW_NAME' in $GITHUB_REPOSITORY"
|
|
exit 1
|
|
else
|
|
echo "Resolved workflow '$WORKFLOW_NAME' to ID $WORKFLOW_ID"
|
|
fi
|
|
|
|
|
|
|
|
#
|
|
# Match pull requests with the workflow runs
|
|
#
|
|
|
|
# Get all workflow runs that were triggered by pull requests
|
|
WORKFLOW_PR_RUNS="$(crl "$GITHUB_API/actions/workflows/${WORKFLOW_ID}/runs?event=pull_request")"
|
|
|
|
# For each workflow run, match it with the pull request to get the PR number
|
|
# A match is based on "source repository + branch + SHA" key
|
|
# In rare cases (e.g. force push to an older revision), there could be more than one match
|
|
# for a given PR number, so just use the most recent one.
|
|
# Result is a table (list of lists) - each row with PR number, JOB ID, and the above key
|
|
PR_JOB_MAP="$(jq --arg IGNORE_RUNS_OLDER_THAN "$IGNORE_RUNS_OLDER_THAN" '
|
|
# second input is the pull request map - use it to lookup PR numbers
|
|
input as $PULL_REQUESTS
|
|
| .workflow_runs
|
|
| map(
|
|
# Create a new object with the relevant values
|
|
{
|
|
id,
|
|
updated_at,
|
|
# create lookup key based on source repository + branch + SHA
|
|
key: (.head_repository.full_name + "/" + .head_branch + "/" + .head_sha),
|
|
# was this a successful run?
|
|
# do not include .conclusion=="success" because errors could also post messages
|
|
success: (.status=="completed")
|
|
}
|
|
# lookup PR number from $PULL_REQUESTS using the above key
|
|
| . += { pr_number: $PULL_REQUESTS[.key] }
|
|
# Remove runs that were not in the list of the PRs
|
|
| select(.pr_number)
|
|
)
|
|
# Keep just the most recent run per pull request
|
|
| group_by(.pr_number)
|
|
| map(
|
|
sort_by(.updated_at)
|
|
| last
|
|
# If the most recent run did not succeed, or if the run is too old, ignore it
|
|
| select(.success and (now - (.updated_at|fromdate)) / 60 < ($IGNORE_RUNS_OLDER_THAN | tonumber))
|
|
# Keep just the pull request number mapping to run ID
|
|
| { pr_number, id, key }
|
|
)
|
|
' <( echo "$WORKFLOW_PR_RUNS" ) <( echo "$PULL_REQUESTS" ) )"
|
|
|
|
|
|
# Count how many jobs we should process, and exit early if there are none
|
|
JOBS_COUNT="$(jq 'length' <( echo "$PR_JOB_MAP" ) )"
|
|
if [ "$JOBS_COUNT" -eq 0 ]; then
|
|
echo "There are no recent workflow job runs in the last $IGNORE_RUNS_OLDER_THAN minutes. Exiting."
|
|
exit
|
|
else
|
|
echo "$JOBS_COUNT '$WORKFLOW_NAME' jobs have been updated in the last $IGNORE_RUNS_OLDER_THAN minutes"
|
|
echo "$PR_JOB_MAP" | jq -r '.[] | " * PR #\(.pr_number) Job #\(.id) -- \(.key) "'
|
|
fi
|
|
|
|
#
|
|
# Iterate over the found pairs of PR number + run ID, and update them all
|
|
#
|
|
echo "$PR_JOB_MAP" | jq -r '.[] | [ .pr_number, .id, .key ] | @sh' | \
|
|
while read -r PR_NUMBER RUN_ID RUN_KEY; do
|
|
|
|
echo "Processing '$WORKFLOW_NAME' run #$RUN_ID for pull request #$PR_NUMBER $RUN_KEY..."
|
|
ARTIFACTS="$(crl "$GITHUB_API/actions/runs/$RUN_ID/artifacts")"
|
|
|
|
# Find the artifact download URL for the artifact with the expected name
|
|
ARTIFACT_URL="$(jq -r --arg MSG_ARTIFACT_NAME "$MSG_ARTIFACT_NAME" '
|
|
.artifacts
|
|
| map(select(.name == $MSG_ARTIFACT_NAME and .expired == false))
|
|
| first
|
|
| .archive_download_url
|
|
| select(.!=null)
|
|
' <( echo "$ARTIFACTS" ) )"
|
|
|
|
if [ -z "$ARTIFACT_URL" ]; then
|
|
echo "Unable to find an artifact named '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
|
|
continue
|
|
fi
|
|
|
|
echo "Downloading artifact $ARTIFACT_URL (assuming single text file per artifact)..."
|
|
if ! MESSAGE="$(auth_crl "$ARTIFACT_URL" | gunzip)"; then
|
|
echo "Unable to download or parse message from artifact '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
|
|
continue
|
|
fi
|
|
if [ -z "$MESSAGE" ]; then
|
|
echo "Empty message in artifact '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
|
|
continue
|
|
fi
|
|
|
|
# Create a message body by appending a magic header
|
|
# and stripping any starting and ending whitespace from the original message
|
|
MESSAGE_BODY="$(jq -n \
|
|
--arg COMMENT_MAGIC_HEADER "$COMMENT_MAGIC_HEADER" \
|
|
--arg MESSAGE "$MESSAGE" \
|
|
'{ body: ($COMMENT_MAGIC_HEADER + "\n" + ($MESSAGE | sub( "^[\\s\\p{Cc}]+"; "" ) | sub( "[\\s\\p{Cc}]+$"; "" ))) }' \
|
|
)"
|
|
|
|
EXISTING_PR_COMMENTS="$(crl "$GITHUB_API/issues/$PR_NUMBER/comments")"
|
|
|
|
# Get the comment URL for the first comment that begins with the magic header, or empty string
|
|
OLD_COMMENT="$(jq --arg COMMENT_MAGIC_HEADER "$COMMENT_MAGIC_HEADER" '
|
|
map(select(.body | startswith($COMMENT_MAGIC_HEADER)))
|
|
| first
|
|
| select(.!=null)
|
|
' <( echo "$EXISTING_PR_COMMENTS" ) )"
|
|
|
|
if [ -z "$OLD_COMMENT" ]; then
|
|
COMMENT_HTML_URL="$(auth_crl "$GITHUB_API/issues/$PR_NUMBER/comments" \
|
|
-X POST \
|
|
-H "Content-Type: application/json" \
|
|
--data "$MESSAGE_BODY" \
|
|
| jq -r '.html_url' )"
|
|
COMMENT_INFO="New comment $COMMENT_HTML_URL was created"
|
|
else
|
|
# Make sure the content of the message has changed
|
|
COMMENT_URL="$(jq -r '
|
|
(input | .body) as $body
|
|
| select(.body | . != $body)
|
|
| .url
|
|
' <( echo "$OLD_COMMENT" ) <( echo "$MESSAGE_BODY" ) )"
|
|
|
|
if [ -z "$COMMENT_URL" ]; then
|
|
echo "The message has already been posted from artifact '$MSG_ARTIFACT_NAME' in workflow $RUN_ID (PR #$PR_NUMBER), skipping..."
|
|
continue
|
|
fi
|
|
|
|
COMMENT_HTML_URL="$(auth_crl "$COMMENT_URL" \
|
|
-X PATCH \
|
|
-H "Content-Type: application/json" \
|
|
--data "$MESSAGE_BODY" \
|
|
| jq -r '.html_url' )"
|
|
|
|
COMMENT_INFO="Existing comment $COMMENT_HTML_URL was updated"
|
|
fi
|
|
|
|
echo "$COMMENT_INFO from workflow $WORKFLOW_NAME #$RUN_ID"
|
|
done
|