Add nightly test dashboard to Nightly issue (#778)

* add workflow for init

* update install pre-request

* some clean up

* test in gha

* tweak the condition for the run

* get run logs

* update test workflow to add previous build run

* fix the error in test workflow

* tweak test workflow output

* test output

* only do short now

* remove extended

* test workflow

* fix the left from the refactor

* one more

* clean up

* clean up for PR

* Fix fetching of package run ID from test run ID.

* Fix logfile download.

* Fix app_check testapp name.

* Add the whole build report.

* Update build report to have the full report.

* Remove extraneous print.

* Some output cleanup.

* Modify header row.

* Fix log error.

---------

Co-authored-by: Cynthia Jiang <cynthiajiang@google.com>
Co-authored-by: Jon Simantov <jsimantov@google.com>

Author: 3 people

.github/workflows/build-report.yml

@@ -29,34 +29,39 @@ jobs:
         path: firebase-cpp-sdk
         submodules: true
         fetch-depth: 0
-    - name: Install Desktop SDK prerequisites
-      uses: nick-invision/retry@v2
+    - name: Setup python
+      uses: actions/setup-python@v4
       with:
-        timeout_minutes: 15
-        max_attempts: 3
-        command: |
-          python3 firebase-cpp-sdk/scripts/gha/install_prereqs_desktop.py --gha_build
-          python3 -m pip install requests python-dateutil progress attrs
+        python-version: '3.7'
+
+    - name: Install prerequisites
+      shell: bash
+      run: |
+        echo "FIREBASE_CPP_SDK_DIR=${{ github.workspace }}/firebase-cpp-sdk" >> $GITHUB_ENV
+        cd firebase-cpp-sdk
+        python3 scripts/gha/install_prereqs_desktop.py --gha_build
+        python3 -m pip install requests python-dateutil progress attrs
+        cd ..
     - name: Fetch GitHub jobs (with retry)
       uses: nick-invision/retry@v2
       with:
         timeout_minutes: 20
         max_attempts: 3
         shell: bash
-        command: python3 firebase-cpp-sdk/scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache
+        command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --write_cache build_status_short.cache
     - name: Fetch extended GitHub jobs (with retry)
       uses: nick-invision/retry@v2
       with:
         timeout_minutes: 80
         max_attempts: 3
         shell: bash
-        command: python3 firebase-cpp-sdk/scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache
+        command: python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --write_cache build_status.cache
     - name: Generate report files
       run: |
-        python3 firebase-cpp-sdk/scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md
-        python3 firebase-cpp-sdk/scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --output_markdown --read_cache build_status.cache > report_extended.md
-        python3 firebase-cpp-sdk/scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --output_markdown --read_cache build_status.cache --report=test_summary --summary_count=${{ env.numTestsSummary }} > test_summary.md
-        python3 firebase-cpp-sdk/scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --nooutput_header --read_cache build_status_short.cache > report.txt
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --output_markdown --read_cache build_status_short.cache > report_short.md
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --output_markdown --read_cache build_status.cache > report_extended.md
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDaysExtended }} --output_markdown --read_cache build_status.cache --report=test_summary --summary_count=${{ env.numTestsSummary }} > test_summary.md
+        python3 scripts/gha/report_build_status.py --token ${{ github.token }} --days ${{ env.numDays }} --nooutput_header --read_cache build_status_short.cache > report.txt
     - name: Generate comment string
       run: |
         echo -n > comment.md
@@ -95,4 +100,4 @@ jobs:
         timeout_minutes: 5
         max_attempts: 3
         shell: bash
-        command: python3 firebase-cpp-sdk/scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title '[Unity] Nightly Integration Testing Report' --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md
+        command: python3 scripts/gha/update_issue_comment.py --token ${{ github.token }} --issue_title '[Unity] Nightly Integration Testing Report' --start_tag build-dashboard-comment-start --end_tag build-dashboard-comment-end < comment.md

.github/workflows/build_android.yml

@@ -50,6 +50,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ inputs.unity_branch }}
+          fetch-depth: 0
 
       - name: Checkout CPP Repo
         uses: actions/checkout@v3
@@ -58,6 +59,7 @@ jobs:
           path: firebase-cpp-sdk
           ref: ${{ inputs.firebase_cpp_sdk_version }}
           submodules: true
+          fetch-depth: 0
 
       - name: Setup python
         uses: actions/setup-python@v4

.github/workflows/build_ios.yml

@@ -51,6 +51,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ inputs.unity_branch }}
+          fetch-depth: 0
 
       - name: Checkout CPP Repo
         uses: actions/checkout@v3
@@ -59,6 +60,7 @@ jobs:
           path: firebase-cpp-sdk
           ref: ${{ inputs.firebase_cpp_sdk_version }}
           submodules: true
+          fetch-depth: 0
 
       - name: Setup python
         uses: actions/setup-python@v4

.github/workflows/build_linux.yml

@@ -47,13 +47,15 @@ jobs:
         with:
           submodules: true
           ref: ${{ inputs.unity_branch }}
+          fetch-depth: 0
 
       - uses: actions/checkout@v3
         with:
           repository: firebase/firebase-cpp-sdk
           path: firebase-cpp-sdk
           submodules: true
           ref: ${{ inputs.firebase_cpp_sdk_version }}
+          fetch-depth: 0
 
       - name: Setup python
         uses: actions/setup-python@v4

.github/workflows/build_macos.yml

@@ -47,13 +47,15 @@ jobs:
         with:
           submodules: true
           ref: ${{ inputs.unity_branch }}
+          fetch-depth: 0
 
       - uses: actions/checkout@v3
         with:
           repository: firebase/firebase-cpp-sdk
           path: firebase-cpp-sdk
           submodules: true
           ref: ${{ inputs.firebase_cpp_sdk_version }}
+          fetch-depth: 0
 
       # Remove other python installs before installing the version we want
       - name: Remove other pythons

.github/workflows/build_tvos.yml

@@ -72,6 +72,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ inputs.unity_branch }}
+          fetch-depth: 0
 
       - name: Checkout CPP Repo
         uses: actions/checkout@v3
@@ -80,6 +81,7 @@ jobs:
           path: firebase-cpp-sdk
           ref: ${{ inputs.firebase_cpp_sdk_version }}
           submodules: true
+          fetch-depth: 0
 
       - name: Setup python
         uses: actions/setup-python@v4

.github/workflows/build_windows.yml

@@ -47,13 +47,15 @@ jobs:
         with:
           submodules: true
           ref: ${{ inputs.unity_branch }}
+          fetch-depth: 0
 
       - uses: actions/checkout@v3
         with:
           repository: firebase/firebase-cpp-sdk
           path: firebase-cpp-sdk
           submodules: true
           ref: ${{ inputs.firebase_cpp_sdk_version }}
+          fetch-depth: 0
 
       - name: Support longpaths
         run: git config --system core.longpaths true

.github/workflows/integration_tests.yml

@@ -61,6 +61,9 @@ jobs:
     ### trigger value: manual_trigger, scheduled_trigger, label_trigger, postsubmit_trigger
     - id: set_outputs
       run: |
+        if [[ "${{ github.event.inputs.packaged_sdk_run_id }}" != "" ]]; then
+          echo "::warning ::Downloading SDK package from previous run: https://github.com/${{github.repository}}/actions/runs/${{ github.event.inputs.packaged_sdk_run_id }}"
+        fi
         if [[ "${{ github.event.inputs.test_pull_request }}" == *"label-quick-packaging" ]]; then
           # Triggered by build_starter SDK workflow.
           echo "trigger=label_trigger" >> $GITHUB_OUTPUT

scripts/gha/firebase_github.py

@@ -31,6 +31,7 @@
 BACKOFF = 5
 RETRY_STATUS = (403, 500, 502, 504)
 TIMEOUT = 5
+TIMEOUT_LONG = 20
 
 OWNER = 'firebase'
 REPO = 'firebase-unity-sdk'
@@ -40,7 +41,7 @@
 logging.set_verbosity(logging.INFO)
 
 
-def set_repo_url(repo):  
+def set_repo_url(repo):
   match = re.match(r'https://github\.com/([^/]+)/([^/.]+)', repo)
   if not match:
     logging.info('Error, only pattern https://github.com/\{repo_owner\}/\{repo_name\} are allowed.')
@@ -178,14 +179,18 @@ def list_artifacts(token, run_id):
     return response.json()["artifacts"]
 
 
-def download_artifact(token, artifact_id, output_path):
+def download_artifact(token, artifact_id, output_path=None):
   """https://docs.github.com/en/rest/reference/actions#download-an-artifact"""
   url = f'{GITHUB_API_URL}/actions/artifacts/{artifact_id}/zip'
   headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
-  with requests.get(url, headers=headers, stream=True, timeout=TIMEOUT) as response:
+  with requests_retry_session().get(url, headers=headers, stream=True, timeout=TIMEOUT_LONG) as response:
     logging.info("download_artifact: %s response: %s", url, response)
-    with open(output_path, 'wb') as file:
-        shutil.copyfileobj(response.raw, file)
+    if output_path:
+      with open(output_path, 'wb') as file:
+          shutil.copyfileobj(response.raw, file)
+    elif response.status_code == 200:
+      return response.content
+  return None
 
 
 def dismiss_review(token, pull_number, review_id, message):
@@ -238,7 +243,7 @@ def list_workflows(token, workflow_id, branch):
   headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
   data = {'event': 'workflow_dispatch', 'branch': branch}
   with requests.get(url, headers=headers, data=json.dumps(data),
-                    stream=True, timeout=TIMEOUT) as response:
+                    stream=True, timeout=TIMEOUT_LONG) as response:
     logging.info("list_workflows: %s response: %s", url, response)
     return response.json()
 
@@ -272,9 +277,36 @@ def list_pull_requests(token, state, head, base):
     keep_going = False
     with requests_retry_session().get(url, headers=headers, params=params,
                       stream=True, timeout=TIMEOUT) as response:
-      logging.info("get_reviews: %s response: %s", url, response)
+      logging.info("list_pull_requests: %s response: %s", url, response)
       results = results + response.json()
       # If exactly per_page results were retrieved, read the next page.
       keep_going = (len(response.json()) == per_page)
   return results
 
+def list_workflow_runs(token, workflow_id, branch=None, event=None, limit=200):
+  """https://docs.github.com/en/rest/actions/workflow-runs?list-workflow-runs-for-a-required-workflow"""
+  url = f'{GITHUB_API_URL}/actions/workflows/{workflow_id}/runs'
+  headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
+  page = 1
+  per_page = 100
+  results = []
+  keep_going = True
+  while keep_going:
+    params = {'per_page': per_page, 'page': page}
+    if branch: params.update({'branch': branch})
+    if event: params.update({'event': event})
+    page = page + 1
+    keep_going = False
+    with requests_retry_session().get(url, headers=headers, params=params,
+                      stream=True, timeout=TIMEOUT) as response:
+      logging.info("list_workflow_runs: %s page %d, response: %s", url, params['page'], response)
+      if 'workflow_runs' not in response.json():
+        break
+      run_list_results = response.json()['workflow_runs']
+      results = results + run_list_results
+      # If exactly per_page results were retrieved, read the next page.
+      keep_going = (len(run_list_results) == per_page)
+      if limit > 0 and len(results) >= limit:
+        keep_going = False
+        results = results[:limit]
+  return results

scripts/gha/it_workflow.py

@@ -44,6 +44,7 @@
 
 from absl import app
 from absl import flags
+from absl import logging
 
 import firebase_github
 import summarize_test_results as summarize
@@ -65,7 +66,11 @@
 _COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n"
 
 _COMMENT_IDENTIFIER = "integration-test-status-comment"
-_COMMENT_SUFFIX = f'\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>'
+_COMMENT_HIDDEN_DIVIDER = f'\r\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>\r\n'
+
+_COMMENT_IDENTIFIER_DASHBOARD = "build-dashboard-comment"
+_COMMENT_DASHBOARD_START = f'\r\n<hidden value="{_COMMENT_IDENTIFIER_DASHBOARD}-start"></hidden>\r\n'
+_COMMENT_DASHBOARD_END = f'\r\n<hidden value="{_COMMENT_IDENTIFIER_DASHBOARD}-end"></hidden>\r\n'
 
 _LOG_ARTIFACT_NAME = "log-artifact"
 _LOG_OUTPUT_DIR = "test_results"
@@ -116,7 +121,7 @@ def test_start(token, issue_number, actor, commit, run_id):
 
   comment = (_COMMENT_TITLE_PROGESS +
              _get_description(actor, commit, run_id) +
-             _COMMENT_SUFFIX)
+             _COMMENT_HIDDEN_DIVIDER)
   _update_comment(token, issue_number, comment)
 
 
@@ -138,7 +143,7 @@ def test_progress(token, issue_number, actor, commit, run_id):
     comment = (title +
                _get_description(actor, commit, run_id) +
                log_summary +
-               _COMMENT_SUFFIX)
+               _COMMENT_HIDDEN_DIVIDER)
     _update_comment(token, issue_number, comment)
 
 
@@ -152,7 +157,7 @@ def test_end(token, issue_number, actor, commit, run_id, new_token):
     firebase_github.add_label(token, issue_number, _LABEL_SUCCEED)
     comment = (_COMMENT_TITLE_SUCCEED +
                _get_description(actor, commit, run_id) +
-               _COMMENT_SUFFIX)
+               _COMMENT_HIDDEN_DIVIDER)
     _update_comment(token, issue_number, comment)
   else:
     if success_or_only_flakiness:
@@ -166,7 +171,7 @@ def test_end(token, issue_number, actor, commit, run_id, new_token):
     comment = (title +
                _get_description(actor, commit, run_id) +
                log_summary +
-               _COMMENT_SUFFIX)
+               _COMMENT_HIDDEN_DIVIDER)
     _update_comment(token, issue_number, comment)
 
   firebase_github.delete_label(new_token, issue_number, _LABEL_PROGRESS)
@@ -178,18 +183,34 @@ def test_report(token, actor, commit, run_id):
   https://github.com/firebase/firebase-unity-sdk/issues?q=is%3Aissue+label%3Anightly-testing
   """
   issue_number = _get_issue_number(token, _REPORT_TITLE, _REPORT_LABEL)
+  previous_comment = firebase_github.get_issue_body(token, issue_number)
+  [previous_prefix, previous_comment_test_result] = previous_comment.split(_COMMENT_HIDDEN_DIVIDER) # TODO add more content
+  logging.info("Previous prefix: %s", previous_prefix)
+  # If there is a build dashboard, preserve it.
+  if (_COMMENT_DASHBOARD_START in previous_prefix and
+      _COMMENT_DASHBOARD_END in previous_prefix):
+    logging.info("Found dashboard comment, preserving.")
+    [_, previous_dashboard_plus_the_rest] = previous_prefix.split(_COMMENT_DASHBOARD_START)
+    [previous_dashboard, _] = previous_dashboard_plus_the_rest.split(_COMMENT_DASHBOARD_END)
+    prefix = prefix + _COMMENT_DASHBOARD_START + previous_dashboard + _COMMENT_DASHBOARD_END
+    logging.info("New prefix: %s", prefix)
+  else:
+    logging.info("No dashboard comment '%s' or '%s'", _COMMENT_DASHBOARD_START, _COMMENT_DASHBOARD_END)
+
   success_or_only_flakiness, log_summary = _get_summary_table(token, run_id)
   if success_or_only_flakiness:
     if not log_summary:
       # succeeded (without flakiness)
       title = _COMMENT_TITLE_SUCCEED
-      comment = title + _get_description(actor, commit, run_id)
+      test_result = title + _get_description(actor, commit, run_id)
     else:
       title = _COMMENT_TITLE_FLAKY
-      comment = title + _get_description(actor, commit, run_id) + log_summary
+      test_result = title + _get_description(actor, commit, run_id) + log_summary
   else:
     title = _COMMENT_TITLE_FAIL
-    comment = title + _get_description(actor, commit, run_id) + log_summary
+    test_result = title + _get_description(actor, commit, run_id) + log_summary
+
+  comment = prefix + _COMMENT_HIDDEN_DIVIDER + test_result
 
   if title == _COMMENT_TITLE_SUCCEED:
     firebase_github.close_issue(token, issue_number)