From eebfe02561036fc7a8382e0b0573c051b7a22e0c Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Thu, 4 Dec 2025 16:33:40 -0500 Subject: [PATCH 1/7] fix conda env --- .github/workflows/tritonbench.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tritonbench.yml b/.github/workflows/tritonbench.yml index 20d0b20..6087613 100644 --- a/.github/workflows/tritonbench.yml +++ b/.github/workflows/tritonbench.yml @@ -144,9 +144,10 @@ jobs: exit 1 fi + CONDA_ENV="triton-main" DOCKER_IMAGE="meta-pytorch/tritonbench:${IMAGE_SUFFIX}" echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> "$GITHUB_ENV" - echo "CONDA_ENV=triton-main" >> "$GITHUB_ENV" + echo "CONDA_ENV=$CONDA_ENV" >> "$GITHUB_ENV" echo "Using docker image: $DOCKER_IMAGE " echo "Using conda env: $CONDA_ENV " From 2a994cd6fb146931006176160c0a0adba75c1a34 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Thu, 4 Dec 2025 16:36:20 -0500 Subject: [PATCH 2/7] fix test --- .github/workflows/tritonbench.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tritonbench.yml b/.github/workflows/tritonbench.yml index 6087613..5f03daa 100644 --- a/.github/workflows/tritonbench.yml +++ b/.github/workflows/tritonbench.yml @@ -145,7 +145,7 @@ jobs: fi CONDA_ENV="triton-main" - DOCKER_IMAGE="meta-pytorch/tritonbench:${IMAGE_SUFFIX}" + DOCKER_IMAGE="ghcr.io/meta-pytorch/tritonbench:${IMAGE_SUFFIX}" echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> "$GITHUB_ENV" echo "CONDA_ENV=$CONDA_ENV" >> "$GITHUB_ENV" echo "Using docker image: $DOCKER_IMAGE " @@ -171,12 +171,9 @@ jobs: ) docker exec -t -w /tmp/workspace "${container_name}" bash -c " \ - set -eux && cd /workspace/tritonbench && - bash .ci/tritonbench/run-benchmark.sh ${{ matrix.BENCHMARKS }} --conda-env ${{ env.CONDA_ENV }} " - - docker exec -t -w /tmp/workspace "${container_name}" bash -c " \ - set -eux && cd /workspace/tritonbench && mv .benchmarks /tmp/workspace/triton-benchmarks/tritonbench/results - " + set -eux && cd /workspace/tritonbench && \ + bash .ci/tritonbench/run-benchmark.sh ${{ matrix.BENCHMARKS }} --conda-env ${{ env.CONDA_ENV }} && \ + mv /workspace/tritonbench/.benchmarks /tmp/workspace/triton-benchmarks/tritonbench/results " - name: Authenticate with AWS # AWS CUDA runners already have access to the bucket via its runner IAM role @@ -199,15 +196,15 @@ jobs: - name: Upload result to Scribe working-directory: triton-benchmarks/tritonbench run: | - latest_result_json=$(find ./results/${TRITONBENCH_SIDE_A_ENV} -name "result.json" | sort -r | head -n 1) + latest_result_json=$(find ./results/${CONDA_ENV} -name "result.json" | sort -r | head -n 1) python3 ./.ci/upload/scribe.py --json ${latest_result_json} - name: Rewrite Tritonbench result json to ClickHouse style working-directory: triton-benchmarks/tritonbench run: | - latest_result_json=$(find ./results/${TRITONBENCH_SIDE_A_ENV} -name "result.json" | sort -r | head -n 1) + latest_result_json=$(find ./results/${CONDA_ENV} -name "result.json" | sort -r | head -n 1) python3 ./.ci/test_infra/oss_ci_benchmark_v3.py --json ${latest_result_json} \ - --output clickhouse-results/result-${TRITONBENCH_SIDE_A_ENV}.json + --output clickhouse-results/result-${CONDA_ENV}.json - name: Upload result to ClickHouse uses: pytorch/test-infra/.github/actions/upload-benchmark-results@main From 2ef16fd8c72fc1f3984669b49c5538d5e3d600f7 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Fri, 5 Dec 2025 21:57:42 -0500 Subject: [PATCH 3/7] move result as root --- .github/workflows/tritonbench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tritonbench.yml b/.github/workflows/tritonbench.yml index 5f03daa..f005ad0 100644 --- a/.github/workflows/tritonbench.yml +++ b/.github/workflows/tritonbench.yml @@ -173,7 +173,7 @@ jobs: docker exec -t -w /tmp/workspace "${container_name}" bash -c " \ set -eux && cd /workspace/tritonbench && \ bash .ci/tritonbench/run-benchmark.sh ${{ matrix.BENCHMARKS }} --conda-env ${{ env.CONDA_ENV }} && \ - mv /workspace/tritonbench/.benchmarks /tmp/workspace/triton-benchmarks/tritonbench/results " + sudo mv /workspace/tritonbench/.benchmarks /tmp/workspace/triton-benchmarks/tritonbench/results " - name: Authenticate with AWS # AWS CUDA runners already have access to the bucket via its runner IAM role From 2f7dded087cc13f2b98545006d0124916bdfb76c Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Fri, 5 Dec 2025 22:01:21 -0500 Subject: [PATCH 4/7] move directory with sudo --- .github/workflows/tritonbench.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tritonbench.yml b/.github/workflows/tritonbench.yml index f005ad0..b2880e1 100644 --- a/.github/workflows/tritonbench.yml +++ b/.github/workflows/tritonbench.yml @@ -63,6 +63,8 @@ jobs: matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_matrix) }} fail-fast: false runs-on: ${{ matrix.runner }} + env: + TRITONBENCH_SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.TRITONBENCH_SCRIBE_GRAPHQL_ACCESS_TOKEN }} environment: pytorch-x-vllm permissions: id-token: write @@ -194,6 +196,7 @@ jobs: retention-days: 30 - name: Upload result to Scribe + if: ${{ env.TRITONBENCH_SCRIBE_GRAPHQL_ACCESS_TOKEN != '' }} working-directory: triton-benchmarks/tritonbench run: | latest_result_json=$(find ./results/${CONDA_ENV} -name "result.json" | sort -r | head -n 1) From d3df6b9c75f69f5e87be8faea586658c12c66dfa Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Sat, 6 Dec 2025 09:35:39 -0500 Subject: [PATCH 5/7] install requirements to upload --- .github/workflows/tritonbench.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tritonbench.yml b/.github/workflows/tritonbench.yml index b2880e1..89904e3 100644 --- a/.github/workflows/tritonbench.yml +++ b/.github/workflows/tritonbench.yml @@ -127,6 +127,14 @@ jobs: fi echo "DEVICE_TYPE=$DEVICE_TYPE" >> $GITHUB_ENV + - name: Install dependencies + shell: bash + working-directory: triton-benchmarks/tritonbench + run: | + set -eux + + pip install -r .ci/upload/requirements.txt + - name: Setup CUDA GPU_FLAG for docker run if: env.DEVICE_NAME == 'cuda' run: | From 8c7f8b81eb939fbc6562a023018dd3d660abe70f Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Sat, 6 Dec 2025 16:55:46 -0500 Subject: [PATCH 6/7] fix benchmark --- .github/workflows/tritonbench.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tritonbench.yml b/.github/workflows/tritonbench.yml index 89904e3..5abcdbb 100644 --- a/.github/workflows/tritonbench.yml +++ b/.github/workflows/tritonbench.yml @@ -183,7 +183,7 @@ jobs: docker exec -t -w /tmp/workspace "${container_name}" bash -c " \ set -eux && cd /workspace/tritonbench && \ bash .ci/tritonbench/run-benchmark.sh ${{ matrix.BENCHMARKS }} --conda-env ${{ env.CONDA_ENV }} && \ - sudo mv /workspace/tritonbench/.benchmarks /tmp/workspace/triton-benchmarks/tritonbench/results " + sudo mv /workspace/tritonbench/.benchmarks /tmp/workspace/triton-benchmarks/tritonbench/results-${{ env.CONDA_ENV }} " - name: Authenticate with AWS # AWS CUDA runners already have access to the bucket via its runner IAM role @@ -199,21 +199,21 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: tritonbench-results - path: triton-benchmarks/tritonbench/results + name: tritonbench-${{ matrix.runner }}-benchmark-single + path: triton-benchmarks/tritonbench/results-${{ env.CONDA_ENV }} retention-days: 30 - name: Upload result to Scribe if: ${{ env.TRITONBENCH_SCRIBE_GRAPHQL_ACCESS_TOKEN != '' }} working-directory: triton-benchmarks/tritonbench run: | - latest_result_json=$(find ./results/${CONDA_ENV} -name "result.json" | sort -r | head -n 1) + latest_result_json=$(find ./results-${CONDA_ENV} -name "result.json" | sort -r | head -n 1) python3 ./.ci/upload/scribe.py --json ${latest_result_json} - name: Rewrite Tritonbench result json to ClickHouse style working-directory: triton-benchmarks/tritonbench run: | - latest_result_json=$(find ./results/${CONDA_ENV} -name "result.json" | sort -r | head -n 1) + latest_result_json=$(find ./results-${CONDA_ENV} -name "result.json" | sort -r | head -n 1) python3 ./.ci/test_infra/oss_ci_benchmark_v3.py --json ${latest_result_json} \ --output clickhouse-results/result-${CONDA_ENV}.json From 0399a495322dd7fc041c401c5d871e003aa72702 Mon Sep 17 00:00:00 2001 From: Xu Zhao Date: Sat, 6 Dec 2025 19:30:19 -0500 Subject: [PATCH 7/7] add github env --- .github/workflows/tritonbench.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tritonbench.yml b/.github/workflows/tritonbench.yml index 5abcdbb..6fa4c6d 100644 --- a/.github/workflows/tritonbench.yml +++ b/.github/workflows/tritonbench.yml @@ -162,6 +162,7 @@ jobs: echo "Using conda env: $CONDA_ENV " - name: Run TritonBench benchmark + working-directory: triton-benchmarks/tritonbench run: | set -eux @@ -185,6 +186,11 @@ jobs: bash .ci/tritonbench/run-benchmark.sh ${{ matrix.BENCHMARKS }} --conda-env ${{ env.CONDA_ENV }} && \ sudo mv /workspace/tritonbench/.benchmarks /tmp/workspace/triton-benchmarks/tritonbench/results-${{ env.CONDA_ENV }} " + # post-process result.json + latest_result_json=$(find ./results-${CONDA_ENV} -name "result.json" | sort -r | head -n 1) + python3 ./.ci/test_infra/oss_ci_benchmark_v3.py --json ${latest_result_json} \ + --add-github-env --output ${latest_result_json} + - name: Authenticate with AWS # AWS CUDA runners already have access to the bucket via its runner IAM role if: env.DEVICE_NAME == 'rocm' || contains(env.DEVICE_TYPE, 'B200')