Add slack notifications to backend tests #481

.github/slack-notify.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+"""
+A script which runs as part of our GitHub Actions build to send notifications
+to Slack. Notifies when build status changes.
+Usage:
+    ./github-slack-notify.py $STATUS
+where $STATUS is the current build status.
+Requires GITHUB_TOKEN and SLACK_WEBHOOK_URL to be in the environment.
+"""
+import os
+import sys
+
+import requests
+
+
+def statusemoji(status):
+    return {"success": ":heavy_check_mark:", "failure": ":rotating_light:"}.get(
+        status, ":warning:"
+    )
+
+
+def get_message_title():
+    return os.getenv("SLACK_MESSAGE_TITLE", "GitHub Actions Tests")
+
+
+def get_build_url(job_name, run_id):
+    repo = os.environ["GITHUB_REPOSITORY"]
+    token = os.environ["GITHUB_TOKEN"]
+    response = requests.get(
+        f"https://api.github.com/repos/"+str(repo)+"/actions/runs/" + str(run_id) +"/jobs",
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/vnd.github.v3+json",
+        },
+    )
+    print("https://api.github.com/repos/"+str(repo)+"/actions/runs/" + str(run_id) +"/jobs", response.json())
+    jobs = response.json()["jobs"]
+    print(jobs)
+    for job in jobs:
+        if job_name in job["name"]:
+            return job["html_url"]
+
+
+    return "https://github.com/{GITHUB_REPOSITORY}/commit/{GITHUB_SHA}/checks".format(
+        **os.environ
+    )
+
+
+def is_pr_build():
+    return os.environ["GITHUB_REF"].startswith("refs/pull/")
+
+
+def get_branchname():
+    # split on slashes, strip 'refs/heads/*' , and rejoin
+    # this is also the tag name if a tag is used
+    return "/".join(os.environ["GITHUB_REF"].split("/")[2:])
+
+
+def _get_workflow_id_map():
+    repo = os.environ["GITHUB_REPOSITORY"]
+    token = os.environ["GITHUB_TOKEN"]
+    r = requests.get(
+        f"https://api.github.com/repos/{repo}/actions/workflows",
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/vnd.github.v3+json",
+        },
+    )
+
+    ret = {}
+    for w in r.json().get("workflows", []):
+        ret[w["name"]] = w["id"]
+    print(f">> workflow IDs: {ret}")
+    return ret
+
+
+def get_last_build_status():
+    branch = get_branchname()
+    repo = os.environ["GITHUB_REPOSITORY"]
+    token = os.environ["GITHUB_TOKEN"]
+    workflow_id = _get_workflow_id_map()[os.environ["GITHUB_WORKFLOW"]]
+
+    r = requests.get(
+        f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/runs",
+        params={"branch": branch, "status": "completed", "per_page": 1},
+        headers={
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/vnd.github.v3+json",
+        },
+    )
+    print(f">> get past workflow runs params: branch={branch},repo={repo}")
+    print(f">> get past workflow runs result: status={r.status_code}")
+    runs_docs = r.json().get("workflow_runs", [])
+    # no suitable status was found for a previous build, so the status is "None"
+    if not runs_docs:
+        print(">>> no previous run found for workflow")
+        return None
+    conclusion = runs_docs[0]["conclusion"]
+    print(f">>> previous run found with conclusion={conclusion}")
+    return conclusion
+
+
+def check_status_changed(status):
+    # NOTE: last_status==None is always considered a change. This is intentional
+    last_status = get_last_build_status()
+    res = last_status != status
+    if res:
+        print(f"status change detected (old={last_status}, new={status})")
+    else:
+        print(f"no status change detected (old={last_status}, new={status})")
+    return res
+
+
+def get_failure_message():
+    return os.getenv("SLACK_FAILURE_MESSAGE", "tests failed")
+
+
+def build_payload(status, job_name, run_id):
+    context = f"{statusemoji(status)} build for {get_branchname()}: {status}"
+    message = f"<{get_build_url(job_name, run_id)}|{get_message_title()}>"
+    if "fail" in status.lower():
+        message = f"{message}: {get_failure_message()}"
+    return {
+        "channel": os.getenv("SLACK_CHANNEL", "#servicex-github"),
+        "username": "Prajwal Kiran Kumar",
+        "blocks": [
+            {"type": "section", "text": {"type": "mrkdwn", "text": message}},
+            {"type": "context", "elements": [{"type": "mrkdwn", "text": context}]},
+        ],
+    }
+
+
+def on_main_repo():
+    """check if running from a fork"""
+    res = os.environ["GITHUB_REPOSITORY"].lower() == "ssl-hep/servicex-backend-tests"
+    print(f"Checking main repo: {res}")
+    return res
+
+
+def should_notify(status):
+    res = check_status_changed(status) and on_main_repo() and not is_pr_build()
+    print(f"Should notify: {res}")
+    return res
+
+
+
+def main():
+    status = sys.argv[1]
+    job_name = sys.argv[2]
+    run_id = sys.argv[3]
+
+    if should_notify(status):
+        r = requests.post(os.environ["SLACK_WEBHOOK_URL"], json=build_payload(status, job_name, run_id))
+        print(f">> webhook response: status={r.status_code}")
+
+if __name__ == "__main__":
+    main()

.github/workflows/daily_servicex_uproot_test.yml

@@ -34,3 +34,17 @@ jobs:
           TOKEN: ${{ secrets.SECRET_TESTING1_UPROOT_TOKEN }}
         run: |
           source test_uproot.sh $TOKEN https://servicex-release-int-uproot.servicex.ssl-hep.org
+    outputs:
+      job_name: ${{ github.job }}
+      run_id: ${{ github.run_id }}
+
+  call-slack-notify:
+    uses: ./.github/workflows/slack_notify.yml
+    if: always()
+    needs: [ run-tests ]
+    secrets: inherit
+    with:
+      result: ${{ needs.run-tests.result }}
+      job_name: ${{ needs.run-tests.outputs.job_name }}
+      run_id: ${{ needs.run-tests.outputs.run_id }}
+      slack_message_title: 'Daily Automated Tests - Uproot River Testing'

.github/workflows/daily_servicex_uproot_test_af.yml

@@ -32,3 +32,17 @@ jobs:
           TOKEN: ${{ secrets.SECRET_AF_UPROOT_TOKEN }}
         run: |
           source test_uproot.sh $TOKEN https://uproot-atlas.servicex.af.uchicago.edu/
+    outputs:
+      job_name: ${{ github.job }}
+      run_id: ${{ github.run_id }}
+
+  call-slack-notify:
+    uses: ./.github/workflows/slack_notify.yml
+    if: always()
+    needs: [ build ]
+    secrets: inherit
+    with:
+      result: ${{ needs.build.result }}
+      job_name: ${{ needs.build.outputs.job_name }}
+      run_id: ${{ needs.build.outputs.run_id }}
+      slack_message_title: 'Daily Automated Tests - Uproot AF Testing'

.github/workflows/daily_servicex_xaod_test.yml

@@ -34,3 +34,17 @@ jobs:
           TOKEN: ${{ secrets.SECRET_TESTING2_XAOD_TOKEN }}
         run: |
           source test_xaod.sh $TOKEN https://servicex-release-int-xaod.servicex.ssl-hep.org
+    outputs:
+      job_name: ${{ github.job }}
+      run_id: ${{ github.run_id }}
+
+  call-slack-notify:
+    uses: ./.github/workflows/slack_notify.yml
+    if: always()
+    needs: [ run-tests ]
+    secrets: inherit
+    with:
+      result: ${{ needs.run-tests.result }}
+      job_name: ${{ needs.run-tests.outputs.job_name }}
+      run_id: ${{ needs.run-tests.outputs.run_id }}
+      slack_message_title: 'Daily Automated Tests - xAOD River Testing'

.github/workflows/daily_servicex_xaod_test_af.yml

@@ -31,3 +31,18 @@ jobs:
           TOKEN: ${{ secrets.SECRET_AF_XAOD_TOKEN }}
         run: |
           source test_xaod.sh $TOKEN https://xaod.servicex.af.uchicago.edu/
+    outputs:
+      job_name: ${{ github.job }}
+      run_id: ${{ github.run_id }}
+
+  call-slack-notify:
+    uses: ./.github/workflows/slack_notify.yml
+    if: always()
+    needs: [ build ]
+    secrets: inherit
+    with:
+      result: ${{ needs.build.result }}
+      job_name: ${{ needs.build.outputs.job_name }}
+      run_id: ${{ needs.build.outputs.run_id }}
+      slack_message_title: 'Daily Automated Tests - xAOD AF Testing'
+

.github/workflows/slack_notify.yml

@@ -0,0 +1,37 @@
+
+name: Slack Notify
+
+on:
+  workflow_call:
+    inputs:
+      result:
+        required: true
+        type: string
+      job_name:
+        required: true
+        type: string
+      run_id:
+        required: true
+        type: string
+      slack_message_title:
+        required: true
+        type: string
+
+jobs:
+  slack-notify:
+    if: always()
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: python -m pip install -U requests
+      - name: notify slack
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_CHANNEL: '#servicex-github'
+          SLACK_MESSAGE_TITLE: ${{ inputs.slack_message_title }}
+          SLACK_FAILURE_MESSAGE: 'Daily run failed'
+        run: python ./.github/slack-notify.py ${{ inputs.result }} ${{ inputs.job_name }} ${{ inputs.run_id }}
+