From 8557bb84b76a19e8fd4b7b81c2fb644460dde4c6 Mon Sep 17 00:00:00 2001 From: Amol Mane <22643905+cx-amol-mane@users.noreply.github.com> Date: Wed, 14 Jan 2026 22:07:30 +0530 Subject: [PATCH 1/6] Add Teams notification for failed integration tests and update test for notification validation --- .github/workflows/ci-tests.yml | 87 ++++++++++++++++++++ internal/commands/.scripts/integration_up.sh | 27 +++++- test/integration/auth_test.go | 1 + 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 217b51eb4..4cc4275da 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -50,6 +50,8 @@ jobs: pip install pre-commit pre-commit install - name: Go Integration test + id: integration_test + continue-on-error: true shell: bash env: CX_BASE_URI: ${{ secrets.CX_BASE_URI }} @@ -99,6 +101,91 @@ jobs: ./internal/commands/.scripts/integration_up.sh ./internal/commands/.scripts/integration_down.sh + - name: Extract failed tests + id: failed_tests + if: always() + shell: bash + run: | + if [ -f "failed_tests_summary.txt" ] && [ -s "failed_tests_summary.txt" ]; then + echo "has_failures=true" >> $GITHUB_OUTPUT + FAIL_COUNT=$(wc -l < failed_tests_summary.txt | tr -d ' ') + echo "fail_count=$FAIL_COUNT" >> $GITHUB_OUTPUT + # Format failed tests as bullet points for Teams + FAILED_LIST=$(awk '{print "• " $0}' failed_tests_summary.txt | tr '\n' ',' | sed 's/,$//' | sed 's/,/\\n/g') + echo "failed_list=$FAILED_LIST" >> $GITHUB_OUTPUT + else + echo "has_failures=false" >> $GITHUB_OUTPUT + echo "fail_count=0" >> $GITHUB_OUTPUT + echo "failed_list=" >> $GITHUB_OUTPUT + fi + + - name: Send failure notification to Teams + if: always() && steps.failed_tests.outputs.has_failures == 'true' + uses: Skitionek/notify-microsoft-teams@v1.0.8 + with: + webhook_url: ${{ secrets.MS_TEAMS_WEBHOOK_URL_INTEGRATION_TESTS }} + raw: > + { + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "content": { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.4", + "msteams": { + "width": "Full" + }, + "body": [ + { + "type": "TextBlock", + "text": "Integration Tests Failed", + "weight": "Bolder", + "size": "Large", + "color": "Attention" + }, + { + "type": "FactSet", + "facts": [ + { "title": "Repository:", "value": "${{ github.repository }}" }, + { "title": "Author:", "value": "${{ github.actor }}" }, + { "title": "Branch:", "value": "${{ github.head_ref }}" }, + { "title": "PR:", "value": "#${{ github.event.pull_request.number }}" }, + { "title": "Failed Tests:", "value": "${{ steps.failed_tests.outputs.fail_count }}" } + ] + }, + { + "type": "TextBlock", + "text": "**Failed Test Cases:**", + "weight": "Bolder", + "spacing": "Medium" + }, + { + "type": "TextBlock", + "text": "${{ steps.failed_tests.outputs.failed_list }}", + "wrap": true, + "fontType": "Monospace", + "spacing": "Small" + } + ], + "actions": [ + { + "type": "Action.OpenUrl", + "title": "View Workflow", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + }, + { + "type": "Action.OpenUrl", + "title": "View PR", + "url": "${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}" + } + ] + } + } + ] + } + - name: Coverage report uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 with: diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 62708467b..1d2b1ec6c 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -25,8 +25,9 @@ go test \ -tags integration \ -v \ -timeout 210m \ - -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ - -coverprofile cover.out \ + -run "^TestAuthValidate$" \ + # -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + # -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output.log # Generate the initial HTML coverage report @@ -83,7 +84,27 @@ fi echo "Running cleandata to clean up projects..." go test -v github.com/checkmarx/ast-cli/test/cleandata -# Step 8: Final cleanup and exit +# Step 8: Generate failed tests summary for CI/CD consumption +FINAL_FAILED_FILE="failed_tests_summary.txt" +if [ -s "$FAILED_TESTS_FILE" ] && [ $rerun_status -eq 1 ]; then + cp "$FAILED_TESTS_FILE" "$FINAL_FAILED_FILE" + + echo "" + echo "========================================================" + echo " FAILED INTEGRATION TESTS SUMMARY" + echo "========================================================" + echo "The following tests failed after retry:" + echo "--------------------------------------------------------" + while IFS= read -r testName; do + echo " - $testName" + done < "$FINAL_FAILED_FILE" + echo "--------------------------------------------------------" + echo "Total failed tests: $(wc -l < "$FINAL_FAILED_FILE" | tr -d ' ')" + echo "========================================================" + echo "" +fi + +# Step 9: Final cleanup and exit rm -f "$FAILED_TESTS_FILE" test_output.log if [ $status -ne 0 ] || [ $rerun_status -eq 1 ]; then diff --git a/test/integration/auth_test.go b/test/integration/auth_test.go index 3d2366a84..64d8ca3eb 100644 --- a/test/integration/auth_test.go +++ b/test/integration/auth_test.go @@ -26,6 +26,7 @@ const ( // Test validate with credentials used in test env func TestAuthValidate(t *testing.T) { + t.Fatal("Intentional failure to test Teams notification") err, buffer := executeCommand(t, "auth", "validate") assertSuccessAuthentication(t, err, buffer, defaultSuccessValidationMessage) } From ea49b74355f94a3ef462904b0ae5e5d73e945547 Mon Sep 17 00:00:00 2001 From: Amol Mane <22643905+cx-amol-mane@users.noreply.github.com> Date: Wed, 14 Jan 2026 22:13:21 +0530 Subject: [PATCH 2/6] Refactor integration test script to run only TestAuthValidate for quicker feedback and adjust timeout settings --- internal/commands/.scripts/integration_up.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 1d2b1ec6c..318f2862b 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -14,24 +14,24 @@ rm -rf ScaResolver-linux64.tar.gz # Step 1: Check if the failedTests file exists FAILED_TESTS_FILE="failedTests" +rerun_status=0 # Step 2: Create the failedTests file echo "Creating $FAILED_TESTS_FILE..." touch "$FAILED_TESTS_FILE" -# Step 3: Run all tests and write failed test names to failedTests file -echo "Running all tests..." +# Step 3: Run only TestAuthValidate for quick testing +echo "Running TestAuthValidate test..." go test \ -tags integration \ -v \ - -timeout 210m \ - -run "^TestAuthValidate$" \ - # -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ - # -coverprofile cover.out \ + -timeout 5m \ + -run "^TestAuthValidate$" \ github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output.log -# Generate the initial HTML coverage report -go tool cover -html=cover.out -o coverage.html +# Skip coverage for quick test +touch cover.out +echo "" > coverage.html # Extract names of failed tests and save them in the failedTests file grep -E "^--- FAIL: " test_output.log | awk '{print $3}' > "$FAILED_TESTS_FILE" From b77c389ecb9696bb3cc04e180b49c86cd7884211 Mon Sep 17 00:00:00 2001 From: Amol Mane <22643905+cx-amol-mane@users.noreply.github.com> Date: Wed, 14 Jan 2026 22:37:13 +0530 Subject: [PATCH 3/6] Update integration test script to run all tests and adjust timeout settings; remove intentional failure in TestAuthValidate --- .github/workflows/ci-tests.yml | 1 - internal/commands/.scripts/integration_up.sh | 14 +++++++------- test/integration/auth_test.go | 1 - 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4cc4275da..bff7a0e92 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -51,7 +51,6 @@ jobs: pre-commit install - name: Go Integration test id: integration_test - continue-on-error: true shell: bash env: CX_BASE_URI: ${{ secrets.CX_BASE_URI }} diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 318f2862b..d5389bdd8 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -20,18 +20,18 @@ rerun_status=0 echo "Creating $FAILED_TESTS_FILE..." touch "$FAILED_TESTS_FILE" -# Step 3: Run only TestAuthValidate for quick testing -echo "Running TestAuthValidate test..." +# Step 3: Run all tests and write failed test names to failedTests file +echo "Running all tests..." go test \ -tags integration \ -v \ - -timeout 5m \ - -run "^TestAuthValidate$" \ + -timeout 210m \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover.out \ github.com/checkmarx/ast-cli/test/integration 2>&1 | tee test_output.log -# Skip coverage for quick test -touch cover.out -echo "" > coverage.html +# Generate the initial HTML coverage report +go tool cover -html=cover.out -o coverage.html # Extract names of failed tests and save them in the failedTests file grep -E "^--- FAIL: " test_output.log | awk '{print $3}' > "$FAILED_TESTS_FILE" diff --git a/test/integration/auth_test.go b/test/integration/auth_test.go index 64d8ca3eb..3d2366a84 100644 --- a/test/integration/auth_test.go +++ b/test/integration/auth_test.go @@ -26,7 +26,6 @@ const ( // Test validate with credentials used in test env func TestAuthValidate(t *testing.T) { - t.Fatal("Intentional failure to test Teams notification") err, buffer := executeCommand(t, "auth", "validate") assertSuccessAuthentication(t, err, buffer, defaultSuccessValidationMessage) } From 820c2d7f58e2e4dd0c4786d869216830c412a1e7 Mon Sep 17 00:00:00 2001 From: Amol Mane <22643905+cx-amol-mane@users.noreply.github.com> Date: Thu, 15 Jan 2026 21:54:49 +0530 Subject: [PATCH 4/6] Update integration test script to use 'integration-check' tag for improved test categorization --- .github/workflows/ci-tests.yml | 2 ++ internal/commands/.scripts/integration_up.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index bff7a0e92..a5e140265 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -31,6 +31,7 @@ jobs: echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" exit 0 fi + integration-tests: runs-on: ubuntu-latest steps: @@ -204,6 +205,7 @@ jobs: echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" exit 0 fi + lint: name: lint runs-on: ubuntu-latest diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index d5389bdd8..438defe45 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -23,7 +23,7 @@ touch "$FAILED_TESTS_FILE" # Step 3: Run all tests and write failed test names to failedTests file echo "Running all tests..." go test \ - -tags integration \ + -tags integration-check \ -v \ -timeout 210m \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ From fa9cb52f9c336c25508532334893c988512ddc2f Mon Sep 17 00:00:00 2001 From: Amol Mane <22643905+cx-amol-mane@users.noreply.github.com> Date: Fri, 16 Jan 2026 09:45:08 +0530 Subject: [PATCH 5/6] Update integration test script to use 'integration-check' tag for consistency in test execution --- internal/commands/.scripts/integration_up.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index 438defe45..e733e4154 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -54,7 +54,7 @@ else rerun_status=0 while IFS= read -r testName; do go test \ - -tags integration \ + -tags integration-check \ -v \ -timeout 30m \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ From a60e9895ae6fe6c64b511615a8c07b6a69e3d6f6 Mon Sep 17 00:00:00 2001 From: Amol Mane <22643905+cx-amol-mane@users.noreply.github.com> Date: Fri, 16 Jan 2026 09:50:24 +0530 Subject: [PATCH 6/6] Update integration test script to use 'integration' tag for consistency in test execution --- internal/commands/.scripts/integration_up.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/commands/.scripts/integration_up.sh b/internal/commands/.scripts/integration_up.sh index e733e4154..d5389bdd8 100755 --- a/internal/commands/.scripts/integration_up.sh +++ b/internal/commands/.scripts/integration_up.sh @@ -23,7 +23,7 @@ touch "$FAILED_TESTS_FILE" # Step 3: Run all tests and write failed test names to failedTests file echo "Running all tests..." go test \ - -tags integration-check \ + -tags integration \ -v \ -timeout 210m \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ @@ -54,7 +54,7 @@ else rerun_status=0 while IFS= read -r testName; do go test \ - -tags integration-check \ + -tags integration \ -v \ -timeout 30m \ -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \