From a8179eeb07f815d1cb0408eb48c2a7b290956d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Mon, 8 Dec 2025 12:10:16 +0100 Subject: [PATCH 1/4] wip --- ci/ci-lib.sh | 14 +++++++++++++- ci/upload-files-to-s3.sh | 3 ++- ci/upload-report.sh | 8 ++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/ci/ci-lib.sh b/ci/ci-lib.sh index e23c867609..20b5f17ac7 100644 --- a/ci/ci-lib.sh +++ b/ci/ci-lib.sh @@ -26,10 +26,22 @@ function tag { return 0 } -function upload { +function upload_to_ror_data_bucket { BUCKET="readonlyrest-data" LOCAL_FILE="$1" S3_PATH="$2" # shellcheck disable=SC2154 "$CI_DIR"/s3-uploader.sh "$aws_access_key_id" "$aws_secret_access_key" "$BUCKET@eu-west-1" "$LOCAL_FILE" "$S3_PATH" } + +function upload_to_ror_data_xdelta_bucket { + BUCKET="readonlyrest-data-xdelta" + LOCAL_FILE="$1" + S3_PATH="$2" + FILE_NAME=$(basename $LOCAL_FILE) + + docker run --rm -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key \ + -v $LOCAL_FILE:/tmp/$FILE_NAME:ro \ + beshultd/deltaglider \ + cp /tmp/$FILE_NAME s3://$BUCKET/$S3_PATH +} \ No newline at end of file diff --git a/ci/upload-files-to-s3.sh b/ci/upload-files-to-s3.sh index 39fb7d7e5d..9b37870023 100755 --- a/ci/upload-files-to-s3.sh +++ b/ci/upload-files-to-s3.sh @@ -13,7 +13,8 @@ for ((i = 1; i < $#; i++)); do FILE="${!i}" echo "Uploading $FILE to $DEST ..." - upload "$FILE" "$DEST" + upload_to_ror_data_bucket "$FILE" "$DEST" + upload_to_ror_data_xdelta_bucket "$FILE" "$DEST" done echo "DONE" \ No newline at end of file diff --git a/ci/upload-report.sh b/ci/upload-report.sh index fb95550326..c826a14556 100755 --- a/ci/upload-report.sh +++ b/ci/upload-report.sh @@ -7,12 +7,16 @@ INT_TEST_REPORT=$(find . | grep report | grep integration | grep index.html) if [ ! -z "$UNIT_TEST_REPORT" ]; then echo "found unit test report UNIT_TEST_REPORT" - upload $UNIT_TEST_REPORT "test_reports/$(date +%Y-%m-%d_%H.%M)$ROR_TASK-unit.html" + S3_PATH="test_reports/$(date +%Y-%m-%d_%H.%M)$ROR_TASK-unit.html" + upload_to_ror_data_bucket $UNIT_TEST_REPORT $S3_PATH + upload_to_ror_data_xdelta_bucket $UNIT_TEST_REPORT $S3_PATH fi if [ ! -z "$INT_TEST_REPORT" ]; then echo "found integration test report $INT_TEST_REPORT" - upload $INT_TEST_REPORT "test_reports/$(date +%Y-%m-%d_%H.%M)_$ROR_TASK-integration.html" + S3_PATH="test_reports/$(date +%Y-%m-%d_%H.%M)_$ROR_TASK-integration.html" + upload_to_ror_data_bucket $INT_TEST_REPORT $S3_PATH + upload_to_ror_data_xdelta_bucket $INT_TEST_REPORT $S3_PATH fi From ea97b290e4405898290b614e056d3bd211204459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Mon, 8 Dec 2025 12:13:08 +0100 Subject: [PATCH 2/4] wip --- ci/upload-files-to-s3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/upload-files-to-s3.sh b/ci/upload-files-to-s3.sh index 9b37870023..49ef8ed130 100755 --- a/ci/upload-files-to-s3.sh +++ b/ci/upload-files-to-s3.sh @@ -13,7 +13,7 @@ for ((i = 1; i < $#; i++)); do FILE="${!i}" echo "Uploading $FILE to $DEST ..." - upload_to_ror_data_bucket "$FILE" "$DEST" + #upload_to_ror_data_bucket "$FILE" "$DEST" upload_to_ror_data_xdelta_bucket "$FILE" "$DEST" done From 7fed6dd989f0d0534ccfc58e14e5a185f5b80c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Mon, 8 Dec 2025 12:42:40 +0100 Subject: [PATCH 3/4] wip --- ci/ci-lib.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/ci-lib.sh b/ci/ci-lib.sh index 20b5f17ac7..23c88188af 100644 --- a/ci/ci-lib.sh +++ b/ci/ci-lib.sh @@ -35,13 +35,14 @@ function upload_to_ror_data_bucket { } function upload_to_ror_data_xdelta_bucket { + set -x BUCKET="readonlyrest-data-xdelta" LOCAL_FILE="$1" - S3_PATH="$2" + S3_PATH=$(echo "$2" | sed 's:/*$::') FILE_NAME=$(basename $LOCAL_FILE) docker run --rm -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key \ -v $LOCAL_FILE:/tmp/$FILE_NAME:ro \ - beshultd/deltaglider \ - cp /tmp/$FILE_NAME s3://$BUCKET/$S3_PATH + beshultd/deltaglider:main-012662c \ + cp /tmp/$FILE_NAME s3://$BUCKET/${S3_PATH}/${FILE_NAME} } \ No newline at end of file From 63fa5728da813e4c7eb60ebedc9ab6483e2edc9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Mon, 8 Dec 2025 12:46:07 +0100 Subject: [PATCH 4/4] wip --- ci/ci-lib.sh | 10 ++++++---- ci/upload-files-to-s3.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ci/ci-lib.sh b/ci/ci-lib.sh index 23c88188af..fab73d35ff 100644 --- a/ci/ci-lib.sh +++ b/ci/ci-lib.sh @@ -27,22 +27,24 @@ function tag { } function upload_to_ror_data_bucket { - BUCKET="readonlyrest-data" LOCAL_FILE="$1" S3_PATH="$2" + + BUCKET="readonlyrest-data" # shellcheck disable=SC2154 "$CI_DIR"/s3-uploader.sh "$aws_access_key_id" "$aws_secret_access_key" "$BUCKET@eu-west-1" "$LOCAL_FILE" "$S3_PATH" } function upload_to_ror_data_xdelta_bucket { - set -x - BUCKET="readonlyrest-data-xdelta" LOCAL_FILE="$1" S3_PATH=$(echo "$2" | sed 's:/*$::') FILE_NAME=$(basename $LOCAL_FILE) + BUCKET="readonlyrest-data-xdelta" + DELTA_GLIDER_VERSION="main-012662c" + docker run --rm -e AWS_ACCESS_KEY_ID=$aws_access_key_id -e AWS_SECRET_ACCESS_KEY=$aws_secret_access_key \ -v $LOCAL_FILE:/tmp/$FILE_NAME:ro \ - beshultd/deltaglider:main-012662c \ + beshultd/deltaglider:$DELTA_GLIDER_VERSION \ cp /tmp/$FILE_NAME s3://$BUCKET/${S3_PATH}/${FILE_NAME} } \ No newline at end of file diff --git a/ci/upload-files-to-s3.sh b/ci/upload-files-to-s3.sh index 49ef8ed130..9b37870023 100755 --- a/ci/upload-files-to-s3.sh +++ b/ci/upload-files-to-s3.sh @@ -13,7 +13,7 @@ for ((i = 1; i < $#; i++)); do FILE="${!i}" echo "Uploading $FILE to $DEST ..." - #upload_to_ror_data_bucket "$FILE" "$DEST" + upload_to_ror_data_bucket "$FILE" "$DEST" upload_to_ror_data_xdelta_bucket "$FILE" "$DEST" done