Github-Actions: Send e-mail on Cirrus cron failure
This repository has a number of automaticly triggered branch-level testing enabled. However, other than remembering to go look at a specific WebUI, there is no way for anybody to notice if/when these jobs fail. This commit introduces a github-action workflow which runs periodically, checking for failed cron-triggered Cirrus-CI jobs. When it finds any, it formats a simple report for e-mail delivery. The list of destination addresses is configurable at any time by merging changes to a simple CSV file. Signed-off-by: Chris Evich <cevich@redhat.com>
This commit is contained in:
Chris Evich
parent
4434bd7978
commit
887f88c490
|
|
@ -0,0 +1,116 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
# Intended to be executed from a github action workflow step.
|
||||
# Outputs the Cirrus cron names and IDs of any failed builds
|
||||
|
||||
err() {
|
||||
# Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions
|
||||
echo "::error file=${BASH_SOURCE[0]},line=${BASH_LINENO[0]}::${1:-No error message given}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
_errfmt="Expecting %s value to not be empty"
|
||||
if [[ -z "$GITHUB_REPOSITORY" ]]; then
|
||||
err $(printf "$_errfmt" "\$GITHUB_REPOSITORY")
|
||||
elif [[ -z "$NAME_ID_FILEPATH" ]]; then
|
||||
err $(printf "$_errfmt" "\$NAME_ID_FILEPATH")
|
||||
fi
|
||||
|
||||
mkdir -p artifacts
|
||||
cat > ./artifacts/query_raw.json << "EOF"
|
||||
{"query":"
|
||||
query CronNameStatus($owner: String!, $repo: String!) {
|
||||
githubRepository(owner: $owner, name: $repo) {
|
||||
cronSettings {
|
||||
name
|
||||
lastInvocationBuild {
|
||||
id
|
||||
status
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
",
|
||||
"variables":"{
|
||||
\"owner\": \"@@OWNER@@\",
|
||||
\"repo\": \"@@REPO@@\"
|
||||
}"}
|
||||
EOF
|
||||
# Makes for easier copy/pasting query to/from
|
||||
# https://cirrus-ci.com/explorer
|
||||
owner=$(cut -d '/' -f 1 <<<"$GITHUB_REPOSITORY")
|
||||
repo=$(cut -d '/' -f 2 <<<"$GITHUB_REPOSITORY")
|
||||
sed -i -r -e "s/@@OWNER@@/$owner/g" -e "s/@@REPO@@/$repo/g" ./artifacts/query_raw.json
|
||||
|
||||
echo "::group::Posting GraphQL Query"
|
||||
# Easier to debug in error-reply when query is compacted
|
||||
tr -d '\n' < ./artifacts/query_raw.json | tr -s ' ' | tee ./artifacts/query.json | \
|
||||
jq --indent 4 --color-output .
|
||||
|
||||
if grep -q '@@' ./artifacts/query.json; then
|
||||
err "Found unreplaced substitution token in raw query JSON"
|
||||
fi
|
||||
curl \
|
||||
--request POST \
|
||||
--silent \
|
||||
--location \
|
||||
--header 'content-type: application/json' \
|
||||
--url 'https://api.cirrus-ci.com/graphql' \
|
||||
--data @./artifacts/query.json \
|
||||
--output ./artifacts/reply.json
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::Received GraphQL Reply"
|
||||
jq --indent 4 --color-output . <./artifacts/reply.json || \
|
||||
cat ./artifacts/reply.json
|
||||
echo "::endgroup::"
|
||||
|
||||
# Desireable to catch non-JSON encoded errors in reply.
|
||||
if grep -qi 'error' ./artifacts/reply.json; then
|
||||
err "Found the word 'error' in reply"
|
||||
fi
|
||||
|
||||
# e.x. reply.json
|
||||
# {
|
||||
# "data": {
|
||||
# "githubRepository": {
|
||||
# "cronSettings": [
|
||||
# {
|
||||
# "name": "Keepalive_v2.0",
|
||||
# "lastInvocationBuild": {
|
||||
# "id": "5776050544181248",
|
||||
# "status": "EXECUTING"
|
||||
# }
|
||||
# },
|
||||
# {
|
||||
# "name": "Keepalive_v1.9",
|
||||
# "lastInvocationBuild": {
|
||||
# "id": "5962921081569280",
|
||||
# "status": "COMPLETED"
|
||||
# }
|
||||
# },
|
||||
# {
|
||||
# "name": "Keepalive_v2.0.5-rhel",
|
||||
# "lastInvocationBuild": {
|
||||
# "id": "5003065549914112",
|
||||
# "status": "FAILED"
|
||||
# }
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
_filt='.data.githubRepository.cronSettings | map(select(.lastInvocationBuild.status=="FAILED") | { name:.name, id:.lastInvocationBuild.id} | join(" ")) | join("\n")'
|
||||
jq --raw-output "$_filt" ./artifacts/reply.json > "$NAME_ID_FILEPATH"
|
||||
|
||||
echo "<Cron Name> <Failed Build ID>"
|
||||
cat "$NAME_ID_FILEPATH"
|
||||
|
||||
# Don't rely on a newline present for zero/one output line, always count words
|
||||
records=$(wc --words "$NAME_ID_FILEPATH" | cut -d ' ' -f 1)
|
||||
# Always two words per record
|
||||
failures=$((records/2))
|
||||
echo "::set-output name=failures::$failures"
|
||||
echo "Total failed Cirrus-CI cron builds: $failures"
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
---
|
||||
|
||||
# Format Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
|
||||
|
||||
# Required to un-FUBAR default ${{github.workflow}} value
|
||||
name: check_cirrus_cron
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Assume cirrus cron jobs runs at least once per day
|
||||
- cron: '59 23 * * *'
|
||||
# Debug: Allow triggering job manually in github-actions WebUI
|
||||
workflow_dispatch: {}
|
||||
|
||||
env:
|
||||
# Debug-mode can reveal secrets, only enable by a secret value.
|
||||
# Ref: https://help.github.com/en/actions/configuring-and-managing-workflows/managing-a-workflow-run#enabling-step-debug-logging
|
||||
ACTIONS_STEP_DEBUG: '${{ secrets.ACTIONS_STEP_DEBUG }}'
|
||||
# File with CSV listing of zero or more e-mail addresses for delivery
|
||||
# of daily failure notice e-mails.
|
||||
FAILMAILCSV: './contrib/cirrus/cron-fail_addrs.csv'
|
||||
# Filename for table of cron-name to build-id data
|
||||
# (must be in $GITHUB_WORKSPACE/artifacts/)
|
||||
NAME_ID_FILEPATH: './artifacts/name_id.txt'
|
||||
|
||||
jobs:
|
||||
cron_failures:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
ref: master
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get failed cron names and Build IDs
|
||||
id: cron
|
||||
run: './.github/actions/${{ github.workflow }}/${{ github.job }}.sh'
|
||||
|
||||
- if: steps.cron.outputs.failures > 0
|
||||
shell: bash
|
||||
# Must be inline, since context expressions are used.
|
||||
# Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions
|
||||
run: |
|
||||
set -eo pipefail
|
||||
(
|
||||
echo "Detected one or more Cirrus-CI cron-triggered jobs have failed recently:"
|
||||
echo ""
|
||||
|
||||
while read -r NAME BID; do
|
||||
echo "Cron build '$NAME' Failed: https://cirrus-ci.com/build/$BID"
|
||||
done < "$NAME_ID_FILEPATH"
|
||||
|
||||
echo ""
|
||||
echo "# Source: ${{ github.workflow }} workflow on ${{ github.repository }}."
|
||||
# Separate content from sendgrid.com automatic footer.
|
||||
echo ""
|
||||
) > ./artifacts/email_body.txt
|
||||
|
||||
- if: steps.cron.outputs.failures > 0
|
||||
id: mailto
|
||||
run: printf "::set-output name=csv::%s\n" $(cat "$FAILMAILCSV")
|
||||
|
||||
- if: steps.mailto.outputs.csv != ''
|
||||
name: Send failure notification e-mail
|
||||
# Ref: https://github.com/dawidd6/action-send-mail
|
||||
uses: dawidd6/action-send-mail@v2.2.2
|
||||
with:
|
||||
server_address: ${{secrets.ACTION_MAIL_SERVER}}
|
||||
server_port: 465
|
||||
username: ${{secrets.ACTION_MAIL_USERNAME}}
|
||||
password: ${{secrets.ACTION_MAIL_PASSWORD}}
|
||||
subject: Cirrus-CI cron build failures on ${{github.repository}}
|
||||
to: ${{steps.mailto.outputs.csv}}
|
||||
from: ${{secrets.ACTION_MAIL_SENDER}}
|
||||
body: file://./artifacts/email_body.txt
|
||||
|
||||
- if: always()
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ${{ github.job }}_artifacts
|
||||
path: artifacts/*
|
||||
|
|
@ -0,0 +1 @@
|
|||
rh.container.bot@gmail.com
|
||||
|
Loading…
Reference in New Issue