From 5e968463a387f8cc6cb9b9ff46bc7221c5657b93 Mon Sep 17 00:00:00 2001 From: bwplotka Date: Thu, 26 Jun 2025 13:28:08 +0100 Subject: [PATCH 1/3] feat: add release scripts See go/gmp:toil-automation for motivation. * listing Go vulns + severity * create a security vuln fix commit * create a new fork release that syncs with certain upstream tag (see go/gmp:fork-toil) * cut RC Signed-off-by: bwplotka chore: update chore: add go version check Signed-off-by: bwplotka --- .bingo/.gitignore | 1 + .bingo/README.md | 11 +- .bingo/Variables.mk | 24 +- .bingo/variables.env | 2 +- .github/workflows/release-bot.yml | 5 +- hack/bump-go.sh | 29 -- hack/go.mod | 14 + hack/go.sum | 12 + hack/lib.sh | 674 +++++++++++++++++++++++++++++ hack/lib_test.sh | 150 +++++++ hack/prepare_rc/cmd.go | 80 ---- hack/prepare_rc/main.go | 197 --------- hack/release-forksync.sh | 327 ++++++++++++++ hack/release-rc.sh | 175 ++++++++ hack/release-vulnfix.sh | 172 ++++++++ hack/test-lib.sh | 74 ++++ hack/vulnupdatelist/.gitignore | 1 + hack/vulnupdatelist/main.go | 140 ++++++ hack/vulnupdatelist/nvdapi.go | 107 +++++ hack/vulnupdatelist/nvdapi_test.go | 18 + hack/vulnupdatelist/vuln.go | 159 +++++++ 21 files changed, 2048 insertions(+), 324 deletions(-) delete mode 100755 hack/bump-go.sh create mode 100644 hack/go.mod create mode 100644 hack/go.sum create mode 100644 hack/lib.sh create mode 100644 hack/lib_test.sh delete mode 100644 hack/prepare_rc/cmd.go delete mode 100644 hack/prepare_rc/main.go create mode 100755 hack/release-forksync.sh create mode 100644 hack/release-rc.sh create mode 100755 hack/release-vulnfix.sh create mode 100644 hack/test-lib.sh create mode 100644 hack/vulnupdatelist/.gitignore create mode 100644 hack/vulnupdatelist/main.go create mode 100644 hack/vulnupdatelist/nvdapi.go create mode 100644 hack/vulnupdatelist/nvdapi_test.go create mode 100644 hack/vulnupdatelist/vuln.go diff --git a/.bingo/.gitignore b/.bingo/.gitignore index 9efccf683c..4428593c37 100644 --- a/.bingo/.gitignore +++ b/.bingo/.gitignore @@ -11,3 +11,4 @@ !variables.env *tmp.mod +*tmp.sum diff --git a/.bingo/README.md b/.bingo/README.md index 7a5c2d4f6d..c812e3a6c2 100644 --- a/.bingo/README.md +++ b/.bingo/README.md @@ -1,14 +1,13 @@ # Project Development Dependencies. -This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by https://github.com/bwplotka/bingo. +This is directory which stores Go modules with pinned buildable package that is used within this repository, managed by . * Run `bingo get` to install all tools having each own module file in this directory. -* Run `bingo get ` to install that have own module file in this directory. -* For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use $() variable where is the .bingo/.mod. +* Run `bingo get ` to install `` that have own module file in this directory. +* For Makefile: Make sure to put `include .bingo/Variables.mk` in your Makefile, then use `\$()` variable where `` is the .bingo/`.mod`. * For shell: Run `source .bingo/variables.env` to source all environment variable for each tool. -* For go: Import `.bingo/variables.go` to for variable names. -* See https://github.com/bwplotka/bingo or -h on how to add, remove or change binaries dependencies. +* See or -h on how to add, remove or change binaries dependencies. ## Requirements -* Go 1.14+ +* Go 1.24.x or 1.25.x diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk index 465f809ee4..50c1370c32 100644 --- a/.bingo/Variables.mk +++ b/.bingo/Variables.mk @@ -1,43 +1,49 @@ -# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT. +# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.10. DO NOT EDIT. # All tools are designed to be build inside $GOBIN. BINGO_DIR := $(dir $(lastword $(MAKEFILE_LIST))) GOPATH ?= $(shell go env GOPATH) GOBIN ?= $(firstword $(subst :, ,${GOPATH}))/bin GO ?= $(shell which go) +# Ensure bingo-managed tools are always built for the host platform, +# even when GOOS/GOARCH are set for cross-compilation of other targets. +GOHOSTOS ?= $(shell $(GO) env GOHOSTOS) +GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH) +GOHOSTARM ?= $(shell $(GO) env GOHOSTARM) + # Below generated variables ensure that every time a tool under each variable is invoked, the correct version # will be used; reinstalling only if needed. -# For example for addlicense variable: +# For example for controller-gen variable: # # In your main Makefile (for non array binaries): # #include .bingo/Variables.mk # Assuming -dir was set to .bingo . # -#command: $(ADDLICENSE) -# @echo "Running addlicense" -# @$(ADDLICENSE) +#command: $(CONTROLLER_GEN) +# @echo "Running controller-gen" +# @$(CONTROLLER_GEN) # CONTROLLER_GEN := $(GOBIN)/controller-gen-v0.17.1-0.20250103184936-50893dee96da $(CONTROLLER_GEN): $(BINGO_DIR)/controller-gen.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/controller-gen-v0.17.1-0.20250103184936-50893dee96da" - @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=controller-gen.mod -o=$(GOBIN)/controller-gen-v0.17.1-0.20250103184936-50893dee96da "sigs.k8s.io/controller-tools/cmd/controller-gen" + @cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=controller-gen.mod -o=$(GOBIN)/controller-gen-v0.17.1-0.20250103184936-50893dee96da "sigs.k8s.io/controller-tools/cmd/controller-gen" GEN_CRD_API_REFERENCE_DOCS := $(GOBIN)/gen-crd-api-reference-docs-v0.3.0 $(GEN_CRD_API_REFERENCE_DOCS): $(BINGO_DIR)/gen-crd-api-reference-docs.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/gen-crd-api-reference-docs-v0.3.0" - @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=gen-crd-api-reference-docs.mod -o=$(GOBIN)/gen-crd-api-reference-docs-v0.3.0 "github.com/ahmetb/gen-crd-api-reference-docs" + @cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=gen-crd-api-reference-docs.mod -o=$(GOBIN)/gen-crd-api-reference-docs-v0.3.0 "github.com/ahmetb/gen-crd-api-reference-docs" HELM := $(GOBIN)/helm-v3.14.0 $(HELM): $(BINGO_DIR)/helm.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/helm-v3.14.0" - @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=helm.mod -o=$(GOBIN)/helm-v3.14.0 "helm.sh/helm/v3/cmd/helm" + @cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=helm.mod -o=$(GOBIN)/helm-v3.14.0 "helm.sh/helm/v3/cmd/helm" MDOX := $(GOBIN)/mdox-v0.9.0 $(MDOX): $(BINGO_DIR)/mdox.mod @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. @echo "(re)installing $(GOBIN)/mdox-v0.9.0" - @cd $(BINGO_DIR) && GOWORK=off $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.9.0 "github.com/bwplotka/mdox" + @cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.9.0 "github.com/bwplotka/mdox" diff --git a/.bingo/variables.env b/.bingo/variables.env index 8aecf3f3e1..1a87f61d1b 100644 --- a/.bingo/variables.env +++ b/.bingo/variables.env @@ -1,4 +1,4 @@ -# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.9. DO NOT EDIT. +# Auto generated binary variables helper managed by https://github.com/bwplotka/bingo v0.10. DO NOT EDIT. # All tools are designed to be build inside $GOBIN. # Those variables will work only until 'bingo get' was invoked, or if tools were installed via Makefile's Variables.mk. GOBIN=${GOBIN:=$(go env GOBIN)} diff --git a/.github/workflows/release-bot.yml b/.github/workflows/release-bot.yml index f1bf3e1806..42f9c24b2d 100644 --- a/.github/workflows/release-bot.yml +++ b/.github/workflows/release-bot.yml @@ -135,7 +135,8 @@ jobs: id: prepare working-directory: ./main_branch run: | - go run ./hack/prepare_rc ${{ env.BRANCH_NAME }} ../release_branch + alias yq=go tool yq + go run ./hack/prepare_rc -branch ${{ env.BRANCH_NAME }} -dir ../release_branch - name: Regen files working-directory: ./release_branch # Workflows can't edit workflows. Better to create PR and let tests fail. @@ -188,4 +189,4 @@ jobs: --head $BOT_BRANCH \ --title "chore: prepare for $RC release" \ --body "Beep boop. Merging activates deployment. A fresh PR appears on merge. Boop beep." - fi \ No newline at end of file + fi diff --git a/hack/bump-go.sh b/hack/bump-go.sh deleted file mode 100755 index fec0f8133d..0000000000 --- a/hack/bump-go.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Bump Go images -GOLANG_REPO=google-go.pkg.dev/golang -GOLANG_TAG=$(go tool gcrane ls ${GOLANG_REPO} --json | jq --raw-output '.tags[]' | sort -V | tail -n1) -GOLANG_DIGEST=$(crane digest "${GOLANG_REPO}:${GOLANG_TAG}") -GOLANG_REF="${GOLANG_REPO}:${GOLANG_TAG}@${GOLANG_DIGEST}" -echo "${GOLANG_REF}" -find ./cmd ./examples ./hack -name Dockerfile -exec \ - sed -E "s#google-go\.pkg\.dev/golang:([0-9]+\.[0-9]+\.[0-9+][^@ ]*)?(@sha256:[0-9a-f]+)?#${GOLANG_REF}#g" -i {} \; - -# Bump golangci-lint -go get -tool github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest -LINTER_TAG=$(go list -mod=readonly -m github.com/golangci/golangci-lint/v2 | awk '{print $2}') -echo "golangci-lint@${LINTER_TAG}" -go tool yq -i ".jobs[\"golangci-lint\"].steps[2].with.version = \"${LINTER_TAG}\"" .github/workflows/presubmit.yml diff --git a/hack/go.mod b/hack/go.mod new file mode 100644 index 0000000000..e2622b7ab3 --- /dev/null +++ b/hack/go.mod @@ -0,0 +1,14 @@ +module github.com/GoogleCloudPlatform/promethue-engine/hack + +go 1.24.0 + +require ( + github.com/Masterminds/semver/v3 v3.3.1 + github.com/stretchr/testify v1.10.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/hack/go.sum b/hack/go.sum new file mode 100644 index 0000000000..3a1efbd155 --- /dev/null +++ b/hack/go.sum @@ -0,0 +1,12 @@ +github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= +github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hack/lib.sh b/hack/lib.sh new file mode 100644 index 0000000000..5ed1a59e39 --- /dev/null +++ b/hack/lib.sh @@ -0,0 +1,674 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE for contributors: Bash is funky, but still more readable than Go/easier to iterate. +# Eventually if we depend on those scripts and ppl want we could rewrite them to Go, but some +# parts likely is more readable and easier to bash. However, some rules to writing bash: +# * Use https://github.com/mvdan/sh?tab=readme-ov-file#shfmt on your IDE for formatting (TODO: ensure on CI). +# * lib.sh has ONLY functions. Make sure: +# * Function names have release-lib:: prefix to figure out where they come from. +# * Function check their required arguments +# * Especially for functions that return strings via stdout: +# * Ensure all error messages are redirected to stderr, use log_err func for this. +# * Be careful with pushd/popd which log to stdout, you can redirect those to stderr too. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +SCRIPT_DIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" + +# Extended regular expressions (ERE) regex matching paths to exclude from the fork commit. +# NOTE: # ^\..+ means all hidden files (e.g. changes to .golangci.yaml .gitignore or CI). +# TODO(bwplotka): Consider moving to globs with dotglob and extglob settings.. or Go (: +export RELEASE_LIB_EXCLUDE_RE="^\..+ +^README\.md +^CHANGELOG\.md +^MAINTAINERS\.md +^CONTRIBUTING\.md +^RELEASE\.md +^Dockerfile +^docs/.* +^documentation/.* +^google/.* +^.*go\..* +^.*\.gitignore +^.*package.json +^.*package-lock.json +^Makefile.* +^.*vendor/.* +^VERSION +^.*node_modules/.*" + +# Extended regular expressions (ERE) regex matching paths from EXCLUDE_RE that should be included. +# This is needed as it's simpler than implementing RE negative matchers. +# NOTE: For Prometheus the two specific documentation files are imported in Google Managed Prometheus docs, so keep those. +export RELEASE_LIB_DOCUMENTATION_INCLUDE_RE="^documentation/examples/prometheus-agent\.y.?ml +^documentation/examples/prometheus\.y.?ml" + +log_err() { + echo "❌ ${1}" >&2 +} + +release-lib::confirm() { + local prompt_message="${1:-Are you sure?}" + + # -p: Display the prompt string. + # -r: Prevents backslash interpretation. + # -n 1: Read only one character. + read -p "$prompt_message [y/n/CTR+C]: " -r -n 1 response + echo # Ensures the cursor moves to the next line after input. + case "$response" in + [yY]) + return 0 + ;; + [nN]) + log_err "The action has been cancelled as requested." + return 1 + ;; + *) + log_err "Invalid input. Exiting script." + exit 1 + ;; + esac +} + +# clone clones the $REMOTE_URL to $clone_dir at $source_branch version, then +# creates $target_branch from it, if set. +# +# Idempotence: If the $clone_dir exists, skip cloning and check if +# the $clone_dir is a git repo on the desired branch. +# TODO: Cloning takes time, consider resetting repo if present. +release-lib::idemp::clone() { + local clone_dir="${1}" + if [[ -z "${clone_dir}" ]]; then + log_err "clone_dir arg is not set." + return 1 + fi + local source_branch="${2}" # Branch to fetch when cloning, base for $target_branch + if [[ -z "${source_branch}" ]]; then + log_err "source_branch arg is not set." + return 1 + fi + local target_branch="${3}" # Branch to create from source_branch, + if [[ -z "${target_branch}" ]]; then + target_branch="${source_branch}" + fi + + if [[ -z "${REMOTE_URL}" ]]; then + log_err "REMOTE_URL environment variable is not set." + return 1 + fi + + if [[ ! -d "${clone_dir}" ]]; then + git clone -b "${source_branch}" "${REMOTE_URL}" "${clone_dir}" + if [[ "${source_branch}" != "${target_branch}" ]]; then + pushd "${clone_dir}" + git checkout -b "${target_branch}" + popd + fi + else + if ! release-lib::confirm "The repository clone on ${clone_dir} exists. Do you want to reuse this directory without resetting? 'n' will attempt a hard reset on the repo (quicker then re-clone)."; then + pushd "${clone_dir}" + git fetch origin + git checkout "${source_branch}" + git reset --hard "origin/${source_branch}" + # TODO: Remove tags? + popd + fi + fi + + pushd "${clone_dir}" + if [[ "$(git symbolic-ref --short HEAD)" != "${target_branch}" ]]; then + log_err "Malformed ${DIR}; expected ${target_branch} got $(git symbolic-ref --short HEAD); remove or fix manually the ${clone_dir} and rerun." + return 1 + fi + popd +} + +release-lib::remote_url_from_branch() { + local branch=$1 + # Check if the BRANCH environment variable is set. + if [[ -z "${branch}" ]]; then + log_err "branch is required." + return 1 + fi + + if [[ "${branch}" =~ release-(2|3)\.[0-9]+\.[0-9]+-gmp$ ]]; then + echo "git@github.com:GoogleCloudPlatform/prometheus.git" + elif [[ "${branch}" =~ release-0\.[0-9]+\.[0-9]+-gmp$ ]]; then + echo "git@github.com:GoogleCloudPlatform/alertmanager.git" + elif [[ "${branch}" =~ release/0\.[0-9]+$ ]]; then + echo "git@github.com:GoogleCloudPlatform/prometheus-engine.git" + else + log_err "No matching remote URL found for branch=${branch}" + return 1 + fi +} + +release-lib::upstream_remote_url() { + local project=$1 + if [[ -z "${project}" ]]; then + log_err "project is required." + return 1 + fi + + if [[ "${project}" == "prometheus" ]]; then + echo "git@github.com:prometheus/prometheus.git" + elif [[ "${project}" == "alertmanager" ]]; then + echo "git@github.com:prometheus/alertmanager.git" + else + log_err "No matching remote URL found for project='${project}'" + return 1 + fi +} + +release-lib::idemp::vulnlist() { + local dir="${1}" + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." + return 1 + fi + local vuln_file="${2}" + if [[ -z "${vuln_file}" ]]; then + log_err "vuln_file arg is required." + return 1 + fi + if [[ "${vuln_file}" != /* ]]; then + log_err "vuln_file arg must point to an absolute file path." + return 1 + fi + + if [[ -f "${vuln_file}" && ! -z $(cat "${vuln_file}") ]]; then + if ! release-lib::confirm "Found previous "${vuln_file}". Do you want to reuse this file? 'n' will re-run Go vulnlist check."; then + release-lib::vulnlist "${dir}" "${vuln_file}" + else + echo "⚠ī¸ Using existing ${vuln_file}" + fi + else + release-lib::vulnlist "${dir}" "${vuln_file}" + fi +} + +release-lib::dockerfiles() { + local dir="${1}" + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." + return 1 + fi + find "${dir}" -name "Dockerfile*" | grep -v "/third_party/" | grep -v "/examples/" | grep -v "/hack/" | grep -v "/ui/" +} + +release-lib::vulnlist() { + local dir="${1}" + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." + return 1 + fi + local vuln_file="${2}" + if [[ -z "${vuln_file}" ]]; then + log_err "vuln_file arg is required." + return 1 + fi + if [[ "${vuln_file}" != /* ]]; then + log_err "vuln_file arg must point to an absolute file path." + return 1 + fi + + readarray -t DOCKERFILES < <(release-lib::dockerfiles "${DIR}") + local go_version=$(release-lib::dockerfile_go_version "${DOCKERFILES[0]}") + if [[ -z "${go_version}" ]]; then + log_err "can't find any golang image in ${DOCKERFILES[0]}" + return 1 + fi + + echo "🔄 Detecting Go ${go_version} vulnerabilities to fix..." + pushd "${SCRIPT_DIR}/vulnupdatelist/" + if [[ ! -f "./api.text" ]]; then + log_err "$(pwd)/api.text file not found in your filesystem. Please create it with your NVD API key. See https://nvd.nist.gov/developers/request-an-api-key" + return 1 + fi + + go run "./..." \ + -go-version=${go_version} \ + -only-fixed \ + -dir="${dir}" \ + -nvd-api-key="$(cat "./api.text")" | tee "${vuln_file}" + if [[ -z $(cat "${vuln_file}") ]]; then + # Print this, otherwise error on the above might keep this file mistakenly empty. + echo "no vulnerabilities" >"${vuln_file}" + fi + popd +} + +release-lib::gomod_vulnfix() { + local dir=${1} + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." + return 1 + fi + + local vuln_file="${2}" + if [[ -z "${vuln_file}" ]]; then + log_err "vuln_file arg is required." + return 1 + fi + if [[ ! -f "${vuln_file}" ]]; then + log_err "no ${vuln_file} file found" + return 1 + fi + + if [[ "no vulnerabilities" == $(cat "${vuln_file}") ]]; then + log_err "${vuln_file} shows no vulnerabilities" + return 1 + fi + + # Read the vulnerability file line by line. + # The `|| [[ -n "$line" ]]` part handles the case where the last line doesn't have a newline. + while IFS= read -r line || [[ -n "$line" ]]; do + # Skip any empty lines in the input file. + if [ -z "$line" ]; then + continue + fi + + mod=$(echo "$line" | awk '{print $2}') + mod_path=$(echo "${mod}" | cut -d'@' -f1) + desired_version=$(echo "${mod}" | cut -d'@' -f2) + + if [[ -z "${mod_path}" ]] || [[ -z "${desired_version}" ]]; then + echo "⚠ī¸ Skipping malformed line: $line" + continue + fi + + echo "🔄 Updating module '${mod_path}' to version '${desired_version}'..." + gsed -i "s|\( ${mod_path} \).*|\1${desired_version}|" "${dir}/go.mod" + done <"${vuln_file}" + echo "🔄 Resolving ${dir}/go.mod..." + pushd "${dir}" + go mod tidy + popd +} + +release-lib::idemp::git_commit_amend_match() { + # Anything staged? + if ! git diff-index --quiet --cached HEAD; then + release-lib::git_commit_amend_match "${1}" + fi +} + +release-lib::git_commit_amend_match() { + local message="${1}" + if [[ -z "${message}" ]]; then + log_err "message is required." + return 1 + fi + if [[ "$(git log -1 --pretty=%s)" == "${message}" ]]; then + git commit -s --amend -m "${message}" + else + git commit -sm "${message}" + fi +} + +release-lib::needs_push() { + local branch_to_push="${1}" + if [[ -z "${branch_to_push}" ]]; then + log_err "branch_to_push environment variable is not set." + return 1 + fi + local base_branch_to_diff="${2}" + if [[ -z "${base_branch_to_diff}" ]]; then + log_err "base_branch_to_diff environment variable is not set." + return 1 + fi + git checkout "${branch_to_push}" + + # TODO: Fix edge case - deleting branch remotely does not delete it locally (origin ref). + if upstream_head=$(git fetch && git rev-parse "origin/${branch_to_push}"); then + if [[ "$(git rev-parse HEAD)" == "${upstream_head}" ]]; then + echo "⚠ī¸ Nothing to push; origin/${branch_to_push} is all up to date" + return 1 + fi + git --no-pager log --oneline "${upstream_head}"...HEAD + return 0 + fi + # Likely "origin/${branch_to_push}" does not exists yet, so definitely something to + # push (full $branch_to_push). Assuming the $branch_to_push will be proposed to be merged to + # $base_branch_to_diff, so showing a full diff vs $base_branch_to_diff. + if upstream_base_head=$(git fetch && git rev-parse "origin/${base_branch_to_diff}"); then + if [[ "$(git rev-parse HEAD)" == "${upstream_base_head}" ]]; then + echo "⚠ī¸ Nothing to push, even the base origin/${base_branch_to_diff} is up to date; did you expect that?" + return 1 + fi + git --no-pager log --oneline "${upstream_base_head}"...HEAD + return 0 + fi +} + +release-lib::exclude_changes_from_last_commit() { + local exclude_regexes=$1 + if [[ -z "${exclude_regexes}" ]]; then + log_err "exclude_regexes is required." + return 1 + fi + local include_regexes=$2 + if [[ -z "${include_regexes}" ]]; then + log_err "include_regexes is required." + return 1 + fi + local commit_title=$3 + if [[ -z "${commit_title}" ]]; then + log_err "commit_title is required." + return 1 + fi + + # Get all files touched by a git commit, delimited by space. + changed_files=$(git show --pretty="" --name-only "$(git rev-parse --verify HEAD)") + if [ -z "${changed_files}" ]; then + log_err "suspicious HEAD commit, no files changed." + return 1 + fi + + # Change to \n delimit (needed for grep to work) and exclude/include lines. + tmp_to_exclude=$(echo "${changed_files}" | tr ' ' '\n' | grep -E "${exclude_regexes}" | grep -v -E "${include_regexes}") + + # Group node_module and vendor changes, we know we want to get rid of full directories here -- too many of those files slowing things down and obscuring the summary in git commit -- git restore supports globs. + to_exclude=$(echo "${tmp_to_exclude}" | gsed -e 's|vendor/.*$|vendor/*|' -e 's|node_modules/.*$|node_modules/*|' | sort -u | tr ' ' '\n') + if [ -z "${to_exclude}" ]; then + # Nothing to exclude. + return 0 + fi + + echo "🔄 Excluding the following files from the fork squash commit: ${to_exclude}; appending this information to the git commit message" + curr_msg=$(git log --format=%B -n1) + + # Get all changes to be in stage area. + git reset --soft HEAD~1 + while IFS= read -r exclude_path; do + git restore -S "${exclude_path}" + done <<<"${to_exclude}" + # Commit after unstaging exclusions. + # TODO(bwplotka): Handle nothing to commit after exclusion case. + git commit -m "${commit_title}" -m "${curr_msg}" -m "Excluded files: +${to_exclude} +" + git restore . + git clean -fd +} + +# Return all images used in a Dockerfile, delimited by new-line. +# Use readarray if you need bash array: +# readarray -t IMAGES < <(release-lib::dockerfile_images_used "${dockerfile}") +release-lib::dockerfile_images_used() { + local dockerfile=${1} + if [[ -z "${dockerfile}" ]]; then + log_err "dir arg is required." + return 1 + fi + + if [ ! -f "${dockerfile}" ]; then + log_err "File not found: $DOCKERFILE" + return 1 + fi + + # FROM based image references e.g.: FROM --platform=$BUILDPLATFORM google-go.pkg.dev/golang:1.25.1@sha256:2a5741ae38c60d188ae9144d1b63730b8059d016216e785b9b35acf1ace69bc1 AS buildbase + grep '^FROM ' "${dockerfile}" | + gsed -e 's/^FROM //i' \ + -e 's/--platform=[^ ]* //' \ + -e 's/ AS .*$//i' | + while read -r full_image_string; do + # NOTE: We miss images like launcher.gcr.io/google/nodejs but that's fine, let's fix it by + # adding tag. + echo "${full_image_string}" | grep -E "^.*(:|@).*$" || true + done + + # Arg based image references e.g.: ARG IMAGE_BASE_DEBUG=gcr.io/distroless/base-nossl-debian12:debug + grep '^ARG ' "${dockerfile}" | + gsed -e 's/^ARG [^ ]*=//i' | + while read -r full_image_string; do + # NOTE: We miss images like launcher.gcr.io/google/nodejs but that's fine, let's fix it by + # adding tag. + echo "${full_image_string}" | grep -E "^.*(:|@).*$" || true + done + + return 0 +} + +release-lib::dockerfile_go_version() { + local dockerfile=${1} + if [[ -z "${dockerfile}" ]]; then + log_err "dir arg is required." + return 1 + fi + + if [ ! -f "${dockerfile}" ]; then + log_err "File not found: $DOCKERFILE" + return 1 + fi + + for image in $(release-lib::dockerfile_images_used "${dockerfile}"); do + # Initialize variables for each line + image_name="" + tag="" + sha="" + + # --- 1. Extract SHA --- + # Check if the string contains a SHA digest (delimited by '@') + if [[ "$image" == *@* ]]; then + # Use cut to split the string at the '@' + image_and_tag=$(echo "$image" | cut -d'@' -f1) + sha=$(echo "$image" | cut -d'@' -f2) + else + # No SHA found + image_and_tag="$image" + sha="" + fi + + # --- 2. Extract Tag --- + # A tag is the part after the *last* colon. + # We must check that this part doesn't contain a '/', + # which would mean it's part of a port number (e.g., localhost:5000/my-image) + + # Get the part after the last colon + last_part="${image_and_tag##*:}" + + if [[ "$last_part" == "$image_and_tag" || "$last_part" == */* ]]; then + # Case 1: No colon found (e.g., "alpine") + # Here, last_part == image_and_tag + # Case 2: Colon is part of a port/path (e.g., "my.registry:5000/image") + # Here, last_part == "5000/image", which matches */* + image_name="$image_and_tag" + tag="" + else + # Case 3: A valid tag was found (e.g., "alpine:latest") + # Here, last_part == "latest" + image_name="${image_and_tag%:*}" + tag="$last_part" + fi + + if [[ "${image_name}" == "google-go.pkg.dev/golang" ]]; then + go_tag="${tag}" + echo "${go_tag}" + return 0 + fi + done + + log_err "could not find golang image in Dockerfile: ${dockerfile}" + return 1 +} + +release-lib::dockerfile_update_image() { + local dockerfile=${1} + if [[ -z "${dockerfile}" ]]; then + log_err "dockerfile arg is required." + return 1 + fi + + if [ ! -f "${dockerfile}" ]; then + log_err "File not found: $DOCKERFILE" + return 1 + fi + + local image=${2} + if [[ -z "${image}" ]]; then + log_err "image arg is required." + return 1 + fi + + local tag_prefix=${3} + if [[ -z "${tag_prefix}" ]]; then + log_err "tag_prefix arg is required." + return 1 + fi + + # Use gcrane vs crane for --json. + local all_tags=$(go tool gcrane ls "${image}" --json | jq --raw-output '.tags[]' | sort -V) + # Exclude RC images. + all_tags=$(echo "${all_tags}" | grep -v "rc.*") + # Prefix allows sticking to e.g. latest minor. + all_tags=$(echo "${all_tags}" | grep "${tag_prefix}") + local latest_tag=$(echo "${all_tags}" | tail -n1) + + local latest_digest=$(crane digest "${image}:${latest_tag}") + local latest_image="${image}:${latest_tag}@${latest_digest}" + + echo "🔄 Ensuring ${latest_image} on ${dockerfile}..." + for img in $(release-lib::dockerfile_images_used "${dockerfile}"); do + if ! [[ "${img}" =~ ${image}* ]]; then + continue + fi + + # Found Go image, perform sed. + gsed -i "s#${img}#${latest_image}#g" "${dockerfile}" + done + + # TODO: Validate if swap was done? ::idemp? + return 0 +} + +release-lib::idemp::manifests_bash_image_bump() { + local dir=${1} + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." + return 1 + fi + + local values_file="${dir}/charts/values.global.yaml" + # TODO: Not enough, this has to check actual manifests. + local bash_tag=$(go tool yq '.images.bash.tag' "${values_file}") + + # Use gcrane vs crane for --json. + local latest_bash_tag=$(go tool gcrane ls "gke.gcr.io/gke-distroless/bash" --json | jq --raw-output '.tags[]' | grep "gke_distroless_" | sort -V | tail -n1) + if [[ "${bash_tag}" == "${latest_bash_tag}" ]]; then + echo "✅ Nothing to do; ${values_file} already uses ${latest_bash_tag}" + return 0 + fi + + # Upgrade. + echo "🔄 Ensuring ${latest_bash_tag} on ${values_file}..." + if ! gsed -i -E "s#tag: ${bash_tag}#tag: ${latest_bash_tag}#g" "${values_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + log_err "sed didn't replace?" + return 1 + fi + + # Regen only what's needed. + release-lib::manifests_regen "${dir}" + echo "✅ Done!" + return 0 +} + +release-lib::manifests_regen() { + local dir=${1} + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." + return 1 + fi + + source "${dir}/.bingo/variables.env" + YQ="${YQ:-}" HELM="${HELM}" ADDLICENSE="${ADDLICENSE:-}" bash "${dir}/hack/presubmit.sh" manifests + echo "✅ Manifests regenerated" + return 0 +} + +# Accepts "FORCE_NEW_PATCH_VERSION" +release-lib::next_release_tag() { + local dir=${1} + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." + return 1 + fi + + pushd "${dir}" >&2 + + # Get the latest tag from the current branch's history + # `git describe --tags --abbrev=0` finds the closest tag in the ancestry. + local LATEST_TAG="" + if ! LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null); then + log_err "No reachable tags found on this branch's history." + echo "Please ensure you have tags and have fetched them." >&2 + return 1 + fi + + # Apply bumping logic. + NEW_TAG="" + if [[ "${LATEST_TAG}" == *"-rc."* && -z "${FORCE_NEW_PATCH_VERSION:-}" ]]; then + # Get the part before "-rc." (e.g., "v1.2.3") + BASE_VERSION="${LATEST_TAG%-rc.*}" + # Get the part after "-rc." (e.g., "4") + RC_NUMBER="${LATEST_TAG##*-rc.}" + # Increment the RC number + NEW_RC_NUMBER=$((RC_NUMBER + 1)) + + NEW_TAG="${BASE_VERSION}-rc.${NEW_RC_NUMBER}" + else + # Preserve the 'v' prefix if it exists + PREFIX="" + if [[ "$LATEST_TAG" == v* ]]; then + PREFIX="v" + fi + # Remove 'v' prefix for parsing (e.g., "1.2.3") + VERSION_ONLY="${LATEST_TAG#v}" + # Remove rc suffix, if exists. + VERSION_ONLY="${VERSION_ONLY%-rc.*}" + + # Read major, minor, and patch into variables + # We use a default of 0 for missing components + IFS='.' read -r major minor patch <<<"$VERSION_ONLY" + major=${major:-0} + minor=${minor:-0} + patch=${patch:-0} + + # Check that the patch version is a valid number + if ! [[ "$patch" =~ ^[0-9]+$ ]]; then + log_err "Latest tag '$LATEST_TAG' does not have a numeric patch version (x.y.Z)." + return 1 + fi + # Increment the patch number + NEW_PATCH=$((patch + 1)) + NEW_TAG="${PREFIX}${major}.${minor}.${NEW_PATCH}-rc.0" + fi + + popd >&2 + + echo "${NEW_TAG}" + return 0 +} diff --git a/hack/lib_test.sh b/hack/lib_test.sh new file mode 100644 index 0000000000..11209d0a2b --- /dev/null +++ b/hack/lib_test.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +SCRIPT_DIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" +source "${SCRIPT_DIR}/test-lib.sh" +source "${SCRIPT_DIR}/lib.sh" + +TEST_FAILED=0 +cleanup() { + if ((TEST_FAILED)); then + echo "${line_break}" + echo "$test_name: FAILED" + echo "${line_break}" + # Print out the directories. + echo "CHECKOUT_DIR=${CHECKOUT_DIR}" + else + rm -rf \ + "${CHECKOUT_DIR:-}" + fi +} + +exit_on_error() { + if [ $? -ne 0 ]; then + echo "Error!" >&2 + exit 1 + fi +} + +setup() { + CHECKOUT_DIR=$(mktemp --tmpdir -d tmpcheckout.XXXXX) + export CHECKOUT_DIR +} + +create_file() { + mkdir -p "$(dirname $1)" + echo "change" >$1 +} +test_exclude_changes_from_last_commit() { + setup + + # Add a fake state and branches to fake repo. + git init "${CHECKOUT_DIR}" + pushd "${CHECKOUT_DIR}" + + git commit --allow-empty -m "COMMIT 1: initial commit" + + create_file cmd/prometheus/main.go + create_file config/config.go + create_file documentation/examples/prometheus-agent.yml + create_file documentation/examples/prometheus.yml + create_file tsdb/some.go + + # Files expected to be excluded. + create_file .github/workflows/some-ci.yaml + create_file README.md + create_file VERSION.md + create_file docs/command-line/prometheus.md + create_file documentation/examples/remote_storage/vendor/abc/d.go + create_file documentation/examples/remote_storage/vendor/abc2/a.ini + create_file go.mod + create_file go.sum + create_file vendor/abc2/a.ini + create_file web/ui/node_modules/webpack/whatever.js + + git add --all + git commit -m "some commit" + + release-lib::exclude_changes_from_last_commit "${RELEASE_LIB_EXCLUDE_RE}" "${RELEASE_LIB_DOCUMENTATION_INCLUDE_RE}" "COMMIT 2: my commit" + exit_on_error + + got="$(git -C "${CHECKOUT_DIR}" log --format=%B --no-decorate)" + expected=$( + cat <" - } - return fmt.Sprintf("%s %s", c.command, strings.Join(c.args, " ")) -} - -func (c *Command) run() (string, error) { - cmd := exec.Command(c.command, c.args...) - - var stdoutBuf bytes.Buffer - cmd.Stdout = io.MultiWriter(os.Stderr, &stdoutBuf) - cmd.Stderr = os.Stdout - - err := cmd.Start() - if err != nil { - return "", fmt.Errorf("failed to start command '%s': %w", - c, err) - } - waitErr := cmd.Wait() - result := strings.TrimSpace(stdoutBuf.String()) - - if waitErr != nil { - if exitErr, ok := waitErr.(*exec.ExitError); ok { - return result, fmt.Errorf("command '%s' exited with code %d: %w", - c, exitErr.ExitCode(), exitErr) - } - return result, fmt.Errorf("command '%s' failed after starting: %w", - c, waitErr) - } - - return result, nil -} - -func (c *Command) Run() (string, error) { - fmt.Printf("::group::Running: %s\n", c) - res, err := c.run() - fmt.Println("::endgroup::") - if err != nil { - fmt.Printf("::info::Command '%s' failed: %v\n", c, err) - } - return res, err -} diff --git a/hack/prepare_rc/main.go b/hack/prepare_rc/main.go deleted file mode 100644 index cf1bc185c7..0000000000 --- a/hack/prepare_rc/main.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" -) - -const ValuesFile = "charts/values.global.yaml" - -type Branch struct { - major int - minor int -} - -func getBranch(branch string) (*Branch, error) { - re := regexp.MustCompile(`^release/(\d+)\.(\d+)$`) - matches := re.FindStringSubmatch(branch) - if matches == nil { - return nil, errors.New("branch name must match regex release/([0-9]+).([0-9]+)") - } - majorRaw, minorRaw := matches[1], matches[2] - major, err := strconv.Atoi(majorRaw) - if err != nil { - return nil, fmt.Errorf("malformatted branch %q", branch) - } - minor, err := strconv.Atoi(minorRaw) - if err != nil { - return nil, fmt.Errorf("malformatted branch %q", branch) - } - - return &Branch{ - major: major, - minor: minor, - }, nil -} - -type ReleaseCandidate struct { - branch *Branch - patch int - rc int -} - -func (b *Branch) lastTag() (string, error) { - tagRegex := fmt.Sprintf("v%d.%d.*", b.major, b.minor) - tags, err := Cmd("git", "tag", "--list", tagRegex, "--sort=-v:refname").Run() - return strings.SplitN(tags, "\n", 2)[0], err -} - -func (b *Branch) lastReleaseCandidate() (*ReleaseCandidate, error) { - tag, err := b.lastTag() - if err != nil { - return nil, err - } - if tag == "" { - return nil, nil - } - re := regexp.MustCompile(`^v\d+.\d+.(\d+)-rc.(\d+)$`) - matches := re.FindStringSubmatch(tag) - if matches == nil { - return nil, fmt.Errorf("malformatted tag name %q", tag) - } - patchRaw, rcRaw := matches[1], matches[2] - patch, err := strconv.Atoi(patchRaw) - if err != nil { - return nil, fmt.Errorf("malformatted tag name %q", tag) - } - rc, err := strconv.Atoi(rcRaw) - if err != nil { - return nil, fmt.Errorf("malformatted tag name %q", tag) - } - return &ReleaseCandidate{ - branch: b, - patch: patch, - rc: rc, - }, nil -} - -func (b *Branch) nextReleaseCandidate() (*ReleaseCandidate, error) { - last, err := b.lastReleaseCandidate() - if err != nil { - return nil, err - } - if last == nil { - return &ReleaseCandidate{ - branch: b, - patch: 0, - rc: 0, - }, nil - } - hasRelease, err := last.hasRelease() - if err != nil { - return nil, err - } - if hasRelease { - return &ReleaseCandidate{ - branch: last.branch, - patch: last.patch + 1, - rc: 0, - }, nil - } - return &ReleaseCandidate{ - branch: last.branch, - patch: last.patch, - rc: last.rc + 1, - }, nil -} - -func (rc *ReleaseCandidate) version() string { - return fmt.Sprintf("v%d.%d.%d", rc.branch.major, rc.branch.minor, rc.patch) -} - -func (rc *ReleaseCandidate) hasRelease() (bool, error) { - out, err := Cmd("git", "tag", "--list", rc.version()).Run() - return out != "", err -} - -func (rc *ReleaseCandidate) updateValuesFile(repoPath string) error { - path := filepath.Join(repoPath, ValuesFile) - _, err := Cmd("go", "tool", "yq", "e", - fmt.Sprintf(`.version = "%d.%d.%d"`, rc.branch.major, rc.branch.minor, rc.patch), - "-i", path, - ).Run() - if err != nil { - return err - } - for _, image := range []string{"configReloader", "operator", "ruleEvaluator", "datasourceSyncer"} { - _, err := Cmd("go", "tool", "yq", "e", - fmt.Sprintf(`.images.%s.tag = "%s-gke.%d"`, image, rc.version(), rc.rc), - "-i", path, - ).Run() - if err != nil { - return err - } - } - return nil -} - -func main() { - if len(os.Args) < 3 { - fmt.Printf("::error::Usage: %s \n", os.Args[0]) - fmt.Println("::error::Error: Missing arguments") - os.Exit(1) - } - if _, err := Cmd("git", "fetch", "--tags", "-f").Run(); err != nil { - fmt.Printf("::error::Failed to fetch tags: %v\n", err) - } - branch, err := getBranch(os.Args[1]) - if err != nil { - fmt.Printf("::error::Failed to parse branch: %v\n", err) - } - nextRc, err := branch.nextReleaseCandidate() - if err != nil { - fmt.Printf("::error::Failed to get next release candidate: %v\n", err) - } - - repoPath := os.Args[2] - if err := nextRc.updateValuesFile(repoPath); err != nil { - fmt.Printf("::error::Failed to update value file: %v\n", err) - } - // For GH actions - outputPath := os.Getenv("GITHUB_OUTPUT") - if outputPath == "" { - fmt.Println("::error::GITHUB_OUTPUT environment variable not set.") - os.Exit(1) - } - outputFile, err := os.OpenFile(outputPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - fmt.Printf("::error::Failed to open GITHUB_OUTPUT: %v\n", err) - return - } - defer outputFile.Close() - - s := fmt.Sprintf("full_rc_version=%s-rc.%d\n", nextRc.version(), nextRc.rc) - _, err = outputFile.WriteString(s) - if err != nil { - fmt.Printf("::error::Failed to write to GITHUB_OUTPUT: %v\n", err) - } -} diff --git a/hack/release-forksync.sh b/hack/release-forksync.sh new file mode 100755 index 0000000000..d191211a9b --- /dev/null +++ b/hack/release-forksync.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +FORK_COMMIT_FILE=".git/fork-commit-sha.txt" + +# TODO(bwplotka): Clean err on missing deps e.g. gsed. + +SCRIPT_DIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" +source "${SCRIPT_DIR}/lib.sh" + +usage() { + local me + me="${BASH_SOURCE[0]}" + cat <<_EOM +usage: ${me} + +Create a new release of fork against an upstream version (tag) while carrying the patches from the +previous fork version. The process involves an "semantic fork git rebase -i" flow, similar to the normal git rebase -i (see semantic_fork_rebase for details). + +NOTE: The script is idempotent; to force it to recreate local artifacts (e.g. local clones, remote branches it created), remove the artifact you want to recreate. + +Example use: +* SOURCE_BRANCH=release-2.45.3-gmp UPSTREAM_TAG=v2.53.5 CHECKOUT_DIR=~/Repos/tmp-release bash ${me} + +Variables: +* CHECKOUT_DIR (required) - Local working directory e.g. for local clones. +* SOURCE_BRANCH (required) - Fork branch considered as a source for the forked changes. This branch will be semantically rebased on UPSTREAM_TAG. +* UPSTREAM_TAG (required) - Upstream tag to synchronize (rebase) to; this also controls the name of the eventual branch to use for the sync (release-$UPSTREAM_TAG-gmp). +* PR_BRANCH (default: $USER/cut-release-$UPSTREAM_TAG-gmp) - Upstream branch to push to. +_EOM +} + +if (($# > 0)); then + case $1 in + help) + usage + exit 1 + ;; + esac +fi + +if [[ -z "${CHECKOUT_DIR}" ]]; then + echo "❌ CHECKOUT_DIR environment variable is not set." >&2 + usage + exit 1 +fi +if [[ -z "${SOURCE_BRANCH}" ]]; then + echo "❌ SOURCE_BRANCH environment variable is not set." >&2 + usage + exit 1 +fi +if [[ -z "${UPSTREAM_TAG}" ]]; then + echo "❌ UPSTREAM_TAG environment variable is not set." >&2 + usage + exit 1 +fi + +idemp::create_fork_commit() { + if [[ ! -f "${FORK_COMMIT_FILE}" || -z $(cat "${FORK_COMMIT_FILE}") ]]; then + create_fork_commit + else + echo "⚠ī¸ Using existing ${FORK_COMMIT_FILE}" + fi +} + +# create_fork_commit prepare a squashed commit with the core fork changes to cherry-pick. +# It writes the SHA of that commit into the ${FORK_COMMIT_FILE} +# +# This commit excludes all files that are: +# * no longer needed (documentation) +# * can be easily recreated (e.g. by checking out the latest state or the automation like go mod vendor). +# +# Exclusion significantly simplifies the fork sync procedure. +create_fork_commit() { + # Create ephemeral branch from the source of the forked code. + git checkout "${SOURCE_BRANCH}" + git checkout --detach + + # Get the exact upstream tag we base our fork on. + source_tag="${SOURCE_BRANCH#release-}" + source_tag="v${source_tag%-gmp}" + + # Reset it to clean vanilla version. + git reset --hard "${source_tag}" + + # Squash all forked changes into a single commit. + # HEAD@{1} is where the branch was just before the previous command. + # TODO(bwplotka): Handle no change here. + git merge --squash "HEAD@{1}" && git commit --no-edit + + # We could take it as-is but there are trivial changes we know we have to recreate, exclude them. + release-lib::exclude_changes_from_last_commit "${RELEASE_LIB_EXCLUDE_RE}" "${RELEASE_LIB_DOCUMENTATION_INCLUDE_RE}" "google-patch[logic]: required upstream code modifications" + git rev-parse --verify HEAD >"${FORK_COMMIT_FILE}" + git checkout "${PR_BRANCH}" +} + +force_clean_git_local_changes() { + git restore -S . + git restore . + git clean -fd +} + +idemp::recreate_fork_base_files() { + pushd "${DIR}" + if [[ "$(git rev-parse --verify HEAD)" != "$(git rev-list -n 1 "${UPSTREAM_TAG}")" ]]; then + # There are some commits already, check if this step was done ~correctly. + # Get 2 first commits from the upstream tag to HEAD. + commits_state=$(git log --oneline --no-decorate "${UPSTREAM_TAG}"..HEAD | cut -d' ' -f2- | tail -2) + expected="google-patch[libs]: add google fork packages (export and secret) +google-patch[setup]: initial setup" + if [[ "${PROJECT}" != "prometheus" ]]; then + expected="google-patch[setup]: initial setup" + commits_state=$(echo "${commits_state}" | tail -1) + fi + if ! state_diff=$(diff <(echo "${commits_state}") <(echo "${expected}")); then + echo "❌ Unexpected commit state on $(pwd); expected first commits to match +${expected} + +diff: +${state_diff} + +Remove those commits or remove the dir to recreate and rerun." + exit 1 + fi + echo "⚠ī¸ Using existing commit state; looks clean: +${commits_state}" + return 0 + fi + if [ -n "$(git status --porcelain)" ]; then + echo "❌ Unclean git state on $(pwd); clean it (e.g. git clean -fd) or remove the dir to recreate; and rerun." + exit 1 + fi + recreate_fork_base_files +} + +# recreate_fork_base_files creates a few base commits: +# * setup (e.g. removal unnecessary files, documentation, gitignore, CI files, Dockerfile) +# * libs (e.g. any libs with non-conflicting code that we maintain on forks e.g. export pkg). +# +# NOTE: This is interconnected with $RELEASE_LIB_EXCLUDE_RE variable that excludes some of those files +# from cherry-pick to recreate it here. +recreate_fork_base_files() { + echo "🔄 Creating 'google-patch[setup]' on top of the ${UPSTREAM_TAG} state..." + + # Remove unnecessary files, especially documentation - we want ppl to use GCP or upstream Prometheus docs. + rm "MAINTAINERS.md" + rm "CHANGELOG.md" + rm "RELEASE.md" + rm -r ".circleci/" + rm -r ".github/" + rm -r "docs/" + # Remove all but important linked files. + find "documentation" -type f | grep -v -E "${RELEASE_LIB_DOCUMENTATION_INCLUDE_RE}" | xargs rm -- + find "documentation" -type d -empty -delete + + git checkout "${SOURCE_BRANCH}" -- "README.md" + git checkout "${SOURCE_BRANCH}" -- "CONTRIBUTING.md" + git checkout "${SOURCE_BRANCH}" -- ".gcloudignore" + + # Apply our simplified CI. + git checkout "${SOURCE_BRANCH}" -- ".github/" + + # Add our own build pipeline. + # TODO(bwplotka): We could consider Dockerfile.google and removal of vanilla Dockerfile + # This needs fix on Louhi side, so has to be carefully done. + git checkout "${SOURCE_BRANCH}" -- "Dockerfile" + + git add --all + git commit -s -m "google-patch[setup]: initial setup + +Changes: +* Removal of unnecessary files (e.g. documentation) to avoid confusion. +* Replacing README and CONTRIBUTING doc files with our wording. +* Replacing CI scripts with our own. +* Replacing Dockerfile with our own for Google assured build. +* Adding .gcloudignore +" + if [ -n "$(git status --porcelain)" ]; then + echo "❌ Unclean git state on $(pwd); clean it (e.g. git clean -fd) or remove the dir to recreate; and rerun." + exit 1 + fi + if [[ "${PROJECT}" == "prometheus" ]]; then + # After unfork this will change, but currently we have a separate package to maintain on Prometheus fork for: + # * secrets + # * lease + # * GCM export + # Recreate that instead of cherry-picking little commits. + echo "🔄 Creating 'google-patch[libs]' on top of the 'google-patch[setup]' commit..." + if ! git checkout "${SOURCE_BRANCH}" -- "google/"; then + echo "❌ Expected google lib is missing on ${SOURCE_BRANCH}; Perhaps source branch is missing that (too old) or we are in progress of the unforking; double check and update script or fetch it manuall and commit with the 'google-patch[libs]:' prefix" + exit 1 + fi + + git add --all + git commit -s -m "google-patch[libs]: add google fork packages (export and secret) + + Changes: + * Add google directory with export and secret code. + " + fi + # All done, nothing to do. +} + +REMOTE_URL=$(release-lib::remote_url_from_branch "${SOURCE_BRANCH}") +PROJECT=$( + tmp=${REMOTE_URL##*/} + echo "${tmp%.git}" +) +UPSTREAM_REMOTE_URL=$(release-lib::upstream_remote_url "${PROJECT}") +RELEASE_BRANCH="release-${UPSTREAM_TAG#v}-gmp" +PR_BRANCH=${PR_BRANCH:-"${USER}/cut-${RELEASE_BRANCH}"} + +echo "🔄 Assuming ${PROJECT} with remotes {internal: ${REMOTE_URL}, upstream: ${UPSTREAM_REMOTE_URL}}; syncing ${UPSTREAM_TAG} into ${RELEASE_BRANCH} with the internal patches synced to ${PR_BRANCH} from ${SOURCE_BRANCH}" + +DIR="${CHECKOUT_DIR}/${PROJECT}" +release-lib::idemp::clone "${DIR}" "${SOURCE_BRANCH}" "${PR_BRANCH}" + +pushd "${DIR}" + +if ! url=$(git remote get-url upstream 2>/dev/null) || [[ "${url}" != "${UPSTREAM_REMOTE_URL}" ]]; then + git remote add upstream "${UPSTREAM_REMOTE_URL}" +fi + +# Is cherry pick in progress? +if git rev-parse -q --verify CHERRY_PICK_HEAD; then + echo "❌ Cherry pick is in progress on ${DIR}; cd there and fix conflicts manually, then --continue and rerun the script" >&2 + exit 1 +fi + +# Step 1: Prepare ${RELEASE_BRANCH} and ${PR_BRANCH}, both targeting vanilla ${UPSTREAM_TAG} for now. This will be used as a base for a PR with rebased forked functionality. +if ! git fetch upstream "refs/tags/${UPSTREAM_TAG}:refs/tags/${UPSTREAM_TAG}"; then + echo "❌ Failed to fetch ${UPSTREAM_TAG} from the upstream ${UPSTREAM_REMOTE_URL}" >&2 + exit 1 +fi + +if git fetch origin && git rev-parse -q "origin/${RELEASE_BRANCH}" >/dev/null; then + echo "❌ The internal 'origin/${RELEASE_BRANCH}' branch already exists. This means that this script was already run sucessfully or the release branch is live; aborting. + + Remove that remote branch and ${DIR} if you wish to recreate this branch." >&2 + exit 1 +fi + +if git rev-parse -q "${RELEASE_BRANCH}" >/dev/null; then + release_branch_head=$(git rev-parse -q "${RELEASE_BRANCH}") + if [[ "${release_branch_head}" != "$(git rev-list -n 1 "${UPSTREAM_TAG}")" ]]; then + echo "❌ Internal '${RELEASE_BRANCH}' branch already exists and it's not pointing to the upstream ${UPSTREAM_TAG}; aborting. Remove that branch if you wish to resume this script." >&2 + exit 1 + fi +else + git checkout "${UPSTREAM_TAG}" --detach + git checkout -b "${RELEASE_BRANCH}" + + # Switch back to PR_BRANCH and reset it to RELEASE_BRANCH. + git checkout "${PR_BRANCH}" + git reset --hard "${RELEASE_BRANCH}" +fi + +# Step 2: Prepare a squashed commit for core fork changes to cherry-pick. Exclude +# all files that are either no longer needed (documentation) or can be easily recreated +# (e.g. by checking out the latest state or the automation). This significantly simplifies +# the fork sync procedure. +if ! idemp::create_fork_commit "${DIR}"; then + force_clean_git_local_changes + git checkout "${PR_BRANCH}" # Ensure clean state. + exit 1 +fi + +# Step 3: Recreate base fork commits like setup, build and vendor. +git checkout "${PR_BRANCH}" +idemp::recreate_fork_base_files + +# Step 4: Cherry-pick the squashed fork commit -- this will likely conflict and manual intervention and review is needed. + +if [[ "$(git log --oneline --no-decorate -n1 | cut -d' ' -f2- | cut -d':' -f1)" != "google-patch[logic]" ]]; then + echo "🔄 Cherry picking prepared commit from the ${FORK_COMMIT_FILE}..." + if ! git cherry-pick "$(cat "${FORK_COMMIT_FILE}")"; then + echo "❌ Cherry pick found some conflicts (generally expected for this commit). Go to the directory, fix the issues and run 'git cherry-pick --continue', don't change the commit message (!); then rerun the script. Dir: + cd ${DIR} + + NOTE: You can run 'make test' to run all tests. After cherry-pick --continue, you can also do git rebase -i ${UPSTREAM_TAG} and arrange/change commits as you need." >&2 + exit 1 + fi +else + echo "⚠ī¸ google-patch[logic]* is present; assuming cherry-pick was done successfully; proceeding." +fi + +if release-lib::needs_push "${PR_BRANCH}" "${RELEASE_BRANCH}"; then + git --no-pager log --oneline "${RELEASE_BRANCH}"...HEAD + + if release-lib::confirm "About to git push state from ${DIR} to origin/${PR_BRANCH}; also pushing ${RELEASE_BRANCH} with the upstream state; are you sure?"; then + git push origin "${PR_BRANCH}" + git push origin "${RELEASE_BRANCH}" + + # TODO(bwplotka): Use gh to creat this + echo "✅ Sync changes has been pushed on origin/${PR_BRANCH}; the vanilla upstream was pushed to ${RELEASE_BRANCH}; create a PR with origin/${PR_BRANCH} -> ${RELEASE_BRANCH}, ensure CI is passing and get the changes reviewed (especially the cherry-picked 'google-patch[logic]:' commit!)." + echo "ℹī¸ After PR is merged, consider running: +* Vulnerability check/fix: +BRANCH=${RELEASE_BRANCH} CHECKOUT_DIR=${CHECKOUT_DIR} bash ${SCRIPT_DIR}/release-vulnfix.sh +* RC release creation: +BRANCH=${RELEASE_BRANCH} TAG=${UPSTREAM_TAG}-gmp.0-rc.0 CHECKOUT_DIR=${CHECKOUT_DIR} bash ${SCRIPT_DIR}/release-rc.sh" + fi +else + exit 1 +fi diff --git a/hack/release-rc.sh b/hack/release-rc.sh new file mode 100644 index 0000000000..5731a4d5df --- /dev/null +++ b/hack/release-rc.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +# TODO(bwplotka): Clean err on missing deps e.g. gsed. + +SCRIPT_DIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" +source "${SCRIPT_DIR}/lib.sh" + +usage() { + local me + me="${BASH_SOURCE[0]}" + cat <<_EOM +usage: ${me} + +Release the RC. + +NOTE: The script is idempotent; to force it to recreate local artifacts (e.g. local clones, remote branches it created), remove the artifact you want to recreate. + +Example use: + * BRANCH=release/0.15 TAG=v0.15.4-rc.0 CHECKOUT_DIR=~/Repos/tmp-release ${me} + * BRANCH=release-2.45.3-gmp TAG=v2.45.3-gmp.13-rc.0 CHECKOUT_DIR=~/Repos/tmp-release ${me} + * BRANCH=release-0.27.0-gmp TAG=v0.27.0-gmp.4-rc.0 CHECKOUT_DIR=~/Repos/tmp-release ${me} + +Variables: +* BRANCH (required) - Release branch to work on; Project is auto-detected from this. +* CHECKOUT_DIR or DIR (required) - Local working directory e.g. for local clones. DIR is a working dir, CHECKOUT_DIR sets DIR to CHECKOUT_DIR/ from remote URL. +* TAG (optional) - Tag to release. If empty next tag version will be detected (double check this!) +* FORCE_NEW_PATCH_VERSION (optional) - If not empty, forces a new patch version as a new TAG (if TAG is empty). +_EOM +} + +if (($# > 0)); then + case $1 in + help) + usage + exit 0 + ;; + esac +fi + +# Check if the BRANCH environment variable is set. +if [[ -z "${BRANCH}" ]]; then + echo "❌ BRANCH environment variable is not set." + usage + exit 1 +fi + +REMOTE_URL=$(release-lib::remote_url_from_branch "${BRANCH}") +PROJECT=$( + tmp=${REMOTE_URL##*/} + echo ${tmp%.git} +) +PR_BRANCH=${BRANCH} # Same as branch because we push directly, without PR as per our process. + +echo "🔄 Assuming ${PROJECT} with remote ${REMOTE_URL}; changes will be pushed directly to ${PR_BRANCH}" + +if [[ -z "${CHECKOUT_DIR:-}" && -z "${DIR:-}" ]]; then + echo "❌ CHECKOUT_DIR or DIR environment variable has to be set." + usage + exit 1 +fi +DIR=${DIR:-"${CHECKOUT_DIR}/${PROJECT}"} + +release-lib::idemp::clone "${DIR}" "${BRANCH}" "${PR_BRANCH}" + +pushd "${DIR}" + +if [[ -z "${TAG:-}" ]]; then + TAG=$(release-lib::next_release_tag "${DIR}") + echo "✅ Detected next release tag: ${TAG}" +fi + +if [[ "${PROJECT}" == "prometheus-engine" ]]; then + CLEAN_TAG="${TAG%-rc.*}" + CLEAN_TAG="${CLEAN_TAG#v}" + if [[ "${BRANCH}" == "release/0.12" ]]; then + # A bit different flow. + chart_file="${DIR}/charts/operator/Chart.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." + if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + + chart_file="${DIR}/charts/rule-evaluator/Chart.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." + if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + else + # 0.12+ + values_file="${DIR}/charts/values.global.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${values_file}..." + if ! gsed -i -E "s#version:.*#version: ${CLEAN_TAG}#g" "${values_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + fi + # For versions with export embedded. + if [[ -f "${DIR}/pkg/export/export.go" ]]; then + echo "🔄 Ensuring ${TAG} in ${DIR}/pkg/export/export.go mainModuleVersion..." + if ! gsed -i -E "s#mainModuleVersion = .*#mainModuleVersion = \"${TAG}\"#g" "${DIR}/pkg/export/export.go"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + fi + + release-lib::manifests_regen "${DIR}" + git add --all +else + # Prometheus and Alertmanager fork needs just a correct version in the VERSION file, + # so the binary build (go_build_info) metrics and flags are correct. + temp=${TAG#v} # Remove v and then -rc.* suffix. + echo "${temp%-rc.*}" >VERSION + git add VERSION +fi + +if ! release-lib::confirm "About to create a commit and a local git tag for ${TAG} in ${DIR} on ${PR_BRANCH}; should I continue?"; then + exit 1 +fi + +# Commit if anything is staged. +release-lib::idemp::git_commit_amend_match "chore: prepare for ${TAG} release" + +# Check if tag exists. +if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then + # Tag exists, but is it tagged for the current HEAD? + if [[ "$(git rev-parse HEAD)" != "$(git rev-list -n 1 "${TAG}")" ]]; then + echo "❌ Tag ${TAG} exists already locally, not pointing to the HEAD; consider 'git tag -d' to remove it and rerun." + exit 1 + fi +else + echo "🔄 Creating a signed tag ${TAG}..." + # explicit TTY is often needed on Macs. + # TODO(bwplotka): Consider adding v0.x second tag for Prometheus fork (similar to how v0.300 Prometheus releases are structured). + # This is to have a little bit cleaner prometheus-engine go.mod version against the fork. + GPG_TTY=$(tty) git tag -s "${TAG}" -m "${TAG}" +fi + +if release-lib::needs_push "${PR_BRANCH}" "${BRANCH}" || ! git ls-remote --tags --exit-code origin "refs/tags/${TAG}" >/dev/null; then + if release-lib::confirm "About to git push state from ${DIR} to origin/${PR_BRANCH}; then ${TAG}; are you sure?"; then + git push origin "${PR_BRANCH}" + git push origin "${TAG}" + fi +else + exit 1 +fi diff --git a/hack/release-vulnfix.sh b/hack/release-vulnfix.sh new file mode 100755 index 0000000000..d02a1bbe0c --- /dev/null +++ b/hack/release-vulnfix.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +# TODO(bwplotka): Clean err on missing deps e.g. gsed. +# TODO(bwplotka): Consider automation for npm and docker images (Go, debian, similar to bump-go.sh) + +SCRIPT_DIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" +source "${SCRIPT_DIR}/lib.sh" + +usage() { + local me + me="${BASH_SOURCE[0]}" + cat <<_EOM +usage: ${me} + +Attempt a minimal dependency upgrade to solve fixable vulnerabilities. + +* Docker images: + * Distros use latest tag so rebuilding takes latest, nothing to do. + * google-go.pkg.dev/golang images are updated to the latest minor version using docker-bump-images.sh +* Manifests + * distroless bumped to latest (although our component tooling is capable of bumpting this too) +* Go deps: Upgrade to minimal required version per a known fixable vulnerability. +* Npm deps: Not implemented. + +NOTE: The script is idempotent; to force it to recreate local artifacts (e.g. local clones, remote branches it created), remove the artifact you want to recreate. + +Example use: + * BRANCH=release/0.15 CHECKOUT_DIR=~/Repos/tmp-release ${me} + * BRANCH=release-2.45.3-gmp CHECKOUT_DIR=~/Repos/tmp-release ${me} + * BRANCH=release-0.27.0-gmp CHECKOUT_DIR=~/Repos/tmp-release ${me} + +Variables: +* BRANCH (required) - Release branch to work on; Project is auto-detected from this. +* CHECKOUT_DIR or DIR (required) - Local working directory e.g. for local clones. DIR is a working dir, CHECKOUT_DIR sets DIR to CHECKOUT_DIR/ from remote URL. +* PR_BRANCH (default: USER/BRANCH-vulnfix) - Upstream branch to push to (user-confirmed first). +* SYNC_DOCKERFILES_FROM - optional branch name to sync manifests for each dockerfile. +_EOM +} + +if (($# > 0)); then + case $1 in + help) + usage + exit 0 + ;; + esac +fi + +# Check if the BRANCH environment variable is set. +if [[ -z "${BRANCH}" ]]; then + echo "❌ BRANCH environment variable is not set." + usage + exit 1 +fi + +REMOTE_URL=$(release-lib::remote_url_from_branch "${BRANCH}") +PROJECT=$( + tmp=${REMOTE_URL##*/} + echo ${tmp%.git} +) +PR_BRANCH=${PR_BRANCH:-"${USER}/${BRANCH}-vulnfix"} + +echo "🔄 Assuming ${PROJECT} with remote ${REMOTE_URL}; changes will be pushed to ${PR_BRANCH}" + +if [[ -z "${CHECKOUT_DIR:-}" && -z "${DIR:-}" ]]; then + echo "❌ CHECKOUT_DIR or DIR environment variable has to be set." + usage + exit 1 +fi +DIR=${DIR:-"${CHECKOUT_DIR}/${PROJECT}"} + +release-lib::idemp::clone "${DIR}" "${BRANCH}" "${PR_BRANCH}" + +readarray -t DOCKERFILES < <(release-lib::dockerfiles "${DIR}") + +# Sync dockerfiles if needed. +if [[ -n "${SYNC_DOCKERFILES_FROM:-}" ]]; then + pushd "${DIR}" + for dockerfile in "${DOCKERFILES[@]}"; do + # TODO: Should we ensure SYNC_DOCKERFILES_FROM if it's a branch is up to data with origin? + echo "🔄 Syncing ${dockerfile} from ${SYNC_DOCKERFILES_FROM}" + git checkout "${SYNC_DOCKERFILES_FROM}" -- "${dockerfile}" + done + popd +fi + +# Docker images bumps. + +# Get first dockerfile Go version. We will use this version to find minor version to stick to. +go_version=$(release-lib::dockerfile_go_version "${DOCKERFILES[0]}") +if [[ -z "${go_version}" ]]; then + echo "❌ can't find any golang image in ${DOCKERFILES[0]}" + exit 1 +fi + +# TODO: git add charts & vendor for old projects. + +# Update our images. +for dockerfile in "${DOCKERFILES[@]}"; do + release-lib::dockerfile_update_image "${dockerfile}" "google-go.pkg.dev/golang" $(echo "${go_version}" | cut -d '.' -f 1-2) + release-lib::dockerfile_update_image "${dockerfile}" "gke.gcr.io/gke-distroless/libc" "gke_distroless_" + pushd "${DIR}" + git add "${dockerfile}" + popd +done + +# bash manifest bump. +# Exclude 0.12 as values were inlined with each part, easy to manually sed for old versions. +if [[ "${PROJECT}" == "prometheus-engine" && "${BRANCH}" != "release/0.12" ]]; then + release-lib::idemp::manifests_bash_image_bump "${DIR}" +fi + +# Go vulnerabilities. +vuln_file="${DIR}/.git/vulnlist.txt" +pushd "${DIR}" + +release-lib::idemp::vulnlist "${DIR}" "${vuln_file}" + +if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then + # Attempt to update + go mod tidy. + release-lib::gomod_vulnfix "${DIR}" "${vuln_file}" + git add go.mod go.sum + + # Check if that helped. + echo "⚠ī¸ This will fail on older branches with vendoring; in this case, simply go to ${DIR}, run 'go mod vendor' and rerun." + release-lib::vulnlist "${DIR}" "${vuln_file}" + if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then + echo "❌ After go mod update some vulnerabilities are still found; go to ${DIR} and resolve it manually and remove the ./vulnlist.txt file and rerun." + exit 1 + fi +fi + +# TODO: Warn of unstaged files at this point. + +# Commit if anything is staged. +msg="google patch[deps]: fix ${BRANCH} vulnerabilities" +if [[ "${PROJECT}" == "prometheus-engine" ]]; then + msg="fix: fix ${BRANCH} vulnerabilities" +fi +release-lib::idemp::git_commit_amend_match "${msg}" + +if release-lib::needs_push "${PR_BRANCH}" "${BRANCH}"; then + if release-lib::confirm "About to FORCE git push from ${DIR} to origin/${PR_BRANCH}; are you sure?"; then + git push --force origin "${PR_BRANCH}" + fi +else + exit 1 +fi diff --git a/hack/test-lib.sh b/hack/test-lib.sh new file mode 100644 index 0000000000..8c8e0f77c0 --- /dev/null +++ b/hack/test-lib.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +_assert_equal() { + got="${1}" + expected="${2}" + + if [[ "${got}" == "${expected}" ]]; then + return + fi + + echo "assertion failure:" + cat <>>>>>> EXPECTED +EOF + + return 1 +} + +_assert_pass() { + if eval "$1"; then + return + fi + + return 1 +} + +_assert_fail() { + if ! eval "$1"; then + return + fi + + return 1 +} + +# Sanity check on pass/fail helpers. +_assert_pass true || exit 1 +_assert_fail false || exit 1 + +_assert_fail "_assert_pass false" || exit 1 +_assert_fail "_assert_fail true" || exit 1 + +_assert_pass "_assert_equal 1 1" || exit 1 +echo "NOTE: Expected failure output below on _assert_fail (exit code 0 though)" +_assert_fail "_assert_equal 1 2" || exit 1 + +# Stronger assertion helpers that exit the program on assertion failure. +assert_pass() { _assert_pass "$@" || exit 1; } +assert_fail() { _assert_fail "$@" || exit 1; } +assert_equal() { _assert_equal "$@" || exit 1; } diff --git a/hack/vulnupdatelist/.gitignore b/hack/vulnupdatelist/.gitignore new file mode 100644 index 0000000000..c50d3a14c5 --- /dev/null +++ b/hack/vulnupdatelist/.gitignore @@ -0,0 +1 @@ +api.text diff --git a/hack/vulnupdatelist/main.go b/hack/vulnupdatelist/main.go new file mode 100644 index 0000000000..a683953ade --- /dev/null +++ b/hack/vulnupdatelist/main.go @@ -0,0 +1,140 @@ +// package main implements vulnupdatelist script. +// +// Run this script to list all the vulnerable Go modules to upgrade. +// Compared to govulncheck binary, it also checks severity and groups the results +// into clear table per module to upgrade. +// +// Example use: +// +// go run ./... \ +// -go-version=1.23.0 \ +// -only-fixed \ +// -dir=../../../prometheus \ +// -nvd-api-key="$(cat ./api.text)" | tee vuln.txt +package main + +import ( + "bytes" + "flag" + "fmt" + "log" + "log/slog" + "os" + "os/exec" + "path/filepath" + + "github.com/Masterminds/semver/v3" +) + +var ( + goVersion = flag.String("go-version", "", "Go version to test vulnerabilities in (stdlib). Otherwise the `go env GOVERSION` is used") + dir = flag.String("dir", ".", "Where to run the script from") + nvdAPIKey = flag.String("nvd-api-key", "", "API Key for avoiding rate-limiting on severity checks; see https://nvd.nist.gov/developers/request-an-api-key") + onlyFixed = flag.Bool("only-fixed", false, "Don't print vulnerable modules without fixed version; note: fixed version often means sometimes that a new major version contains a fix.") +) + +// UpdateList presents the minimum version to upgrade to solve all CVEs with +// a fixed version. The CVE refers to the important CVE. +// For example critical CVE 1 is fixed in v0.5.1 and low is fixed in v0.10.1. +// TODO(bwplotka): Logically, there might be cases where low contains heavy breaking changes that we can't fix easily; add option to print those too. +type UpdateList struct { + CVE CVE // If CVE has + suffix, it means the top CVE. + AdditionalCVEs int // Lower priority CVEs included in the "fixed" version. + Module string + FixedVersion *semver.Version + Version string +} + +func (u UpdateList) String() string { + fixedVersion := "???" + if u.FixedVersion != nil { + fixedVersion = "v" + u.FixedVersion.String() + } + if u.AdditionalCVEs > 0 { + return fmt.Sprintf("%s %s@%s %s(+%d more) now@%s", u.CVE.Severity, u.Module, fixedVersion, u.CVE.ID, u.AdditionalCVEs, u.Version) + } + return fmt.Sprintf("%s %s@%s %s now@%s", u.CVE.Severity, u.Module, fixedVersion, u.CVE.ID, u.Version) +} + +func main() { + flag.Parse() + + workDir, err := filepath.Abs(*dir) + if err != nil { + log.Fatalf("Failed to resolve work dir: %v", err) + } + slog.Info("Running vulnupdatelist", "dir", workDir) + + if err := ensureGovulncheck(workDir); err != nil { + log.Fatalf("Failed to ensure govulncheck is installed: %v", err) + } + + slog.Info("Running govulncheck... ") + vulnJSON, err := runGovulncheck(workDir, *goVersion) + if err != nil { + log.Fatalf("Error running govulncheck: %v", err) + } + + if len(vulnJSON) == 0 { + slog.Info("govulncheck produced no output; no vulnerabilities found.") + os.Exit(0) + } + + slog.Info("Parsing vulnerabilities and finding updates...") + updates, err := compileUpdateList(bytes.NewReader(vulnJSON), *onlyFixed) + if err != nil { + log.Fatalf("Error parsing govulncheck output: %v", err) + } + if len(updates) == 0 { + slog.Info("No actionable vulnerabilities with fixed versions found.") + os.Exit(0) + } + for _, up := range updates { + fmt.Println(up.String()) + } +} + +// ensureGovulncheck checks if govulncheck is in the PATH, and installs it if not. +func ensureGovulncheck(dir string) error { + _, err := exec.LookPath("govulncheck") + if err == nil { + slog.Info("govulncheck is already installed") + return nil + } + + slog.Info("govulncheck not found. Installing...") + cmd := exec.Command("go", "install", "golang.org/x/vuln/cmd/govulncheck@latest") + cmd.Dir = dir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to run 'go install': %w", err) + } + slog.Info("govulncheck installed successfully.") + return nil +} + +// runGovulncheck executes `govulncheck -json ./...` and returns the output. +func runGovulncheck(dir string, goVersion string) ([]byte, error) { + cmd := exec.Command("govulncheck", "--format=json", "./...") + if goVersion != "" { + cmd.Env = append(os.Environ(), "GOVERSION="+goVersion) + } + + cmd.Dir = dir + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + // govulncheck exits with a non-zero status code if vulns are found. + // We ignore the exit code and check stderr instead. If stderr is empty, + // it's a successful run (even with vulnerabilities). + _ = cmd.Run() + + if stderr.Len() > 0 { + // Only return an error if stderr is not empty, as this indicates a real execution problem. + return nil, fmt.Errorf("govulncheck execution error: %s", stderr.String()) + } + return out.Bytes(), nil +} diff --git a/hack/vulnupdatelist/nvdapi.go b/hack/vulnupdatelist/nvdapi.go new file mode 100644 index 0000000000..7d2a8afba6 --- /dev/null +++ b/hack/vulnupdatelist/nvdapi.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "time" +) + +// NVDResponse is the top-level object for the NVD CVE API. +type NVDResponse struct { + Vulnerabilities []struct { + CVE struct { + ID string `json:"id"` + Metrics struct { + CVSSMetricV31 []struct { + CVSSData struct { + BaseSeverity string `json:"baseSeverity"` + } `json:"cvssData"` + } `json:"cvssMetricV31"` + } `json:"metrics"` + } `json:"cve"` + } `json:"vulnerabilities"` +} + +// getCVSSSeverity fetches vulnerability details from the NVD API and returns the CVSS V3 severity. +func getCVSSSeverity(apiKey string, cveID string) (string, error) { + // https://nvd.nist.gov/developers/vulnerabilities + apiURL := fmt.Sprintf("https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=%s", cveID) + + client := &http.Client{Timeout: 15 * time.Second} + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + if apiKey != "" { + req.Header.Set("apiKey", apiKey) + } + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to make request to NVD API: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("NVD API returned non-200 status: %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %w", err) + } + + var nvdResponse NVDResponse + if err := json.Unmarshal(body, &nvdResponse); err != nil { + return "", fmt.Errorf("failed to parse JSON from NVD API: %w", err) + } + + if len(nvdResponse.Vulnerabilities) > 0 { + metrics := nvdResponse.Vulnerabilities[0].CVE.Metrics + if len(metrics.CVSSMetricV31) > 0 { + return metrics.CVSSMetricV31[0].CVSSData.BaseSeverity, nil + } + } + + return "UNKNOWN", nil +} + +type CVE struct { + ID string + Severity string +} + +func (a CVE) LessThan(b CVE) bool { + order := map[string]int{ + "CRITICAL": 0, + "HIGH": 1, + "MEDIUM": 2, + "UNKNOWN": 3, + "": 3, + } + return order[a.Severity] < order[b.Severity] +} + +func getCVEDetails(apiKey string, osv OSV) CVE { + cveID := CVE{Severity: "UNKNOWN"} + // Assume ID is GO-... ID and use it as a fallback. + for _, a := range osv.Aliases { + if strings.HasPrefix(a, "CVE") { + cveID.ID = a + break + } + } + if cveID.ID == "" { + return CVE{ID: osv.ID, Severity: "UNKNOWN"} // Fallback to GO ID. + } + sev, err := getCVSSSeverity(apiKey, cveID.ID) + if err != nil { + slog.Error("failed to find severity", "cve", cveID, "err", err) + } else { + cveID.Severity = sev + } + return cveID +} diff --git a/hack/vulnupdatelist/nvdapi_test.go b/hack/vulnupdatelist/nvdapi_test.go new file mode 100644 index 0000000000..a350f4d1b5 --- /dev/null +++ b/hack/vulnupdatelist/nvdapi_test.go @@ -0,0 +1,18 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetCVEDetails(t *testing.T) { + t.Skip("depends on NVDE API") + + c := getCVEDetails("", OSV{ + ID: "GO-2021-0065", + Aliases: []string{"GHSA-jmrx-5g74-6v2f", "CVE-2019-11250"}, + }) + require.Equal(t, "CVE-2019-11250", c.ID) + require.Equal(t, "MEDIUM", c.Severity) +} diff --git a/hack/vulnupdatelist/vuln.go b/hack/vulnupdatelist/vuln.go new file mode 100644 index 0000000000..05ed1155c7 --- /dev/null +++ b/hack/vulnupdatelist/vuln.go @@ -0,0 +1,159 @@ +package main + +import ( + "encoding/json" + "io" + "log/slog" + "maps" + "slices" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" +) + +// Vuln represents the top-level structure of a single vulnerability report item. +type Vuln struct { + OSV *OSV `json:"osv"` + Finding *Finding `json:"finding"` +} + +// OSV contains the details of the Open Source Vulnerability. +type OSV struct { + ID string `json:"id"` + Aliases []string `json:"aliases"` + Summary string `json:"summary"` + Details string `json:"details"` + Affected []Affected `json:"affected"` +} + +func (o OSV) CVEs() string { + return strings.Join(append([]string{o.ID}, o.Aliases...), ", ") +} + +// Affected describes a package that is affected by the vulnerability. +type Affected struct { + Package Package `json:"package"` + Ranges []Range `json:"ranges"` +} + +// Package holds the name and ecosystem of the vulnerable package. +type Package struct { + Name string `json:"name"` + Ecosystem string `json:"ecosystem"` +} + +// Range specifies the version ranges affected by the vulnerability. +type Range struct { + Type string `json:"type"` + Events []Event `json:"events"` +} + +// Event marks the introduction or fixing of a vulnerability in the version history. +type Event struct { + Introduced string `json:"introduced,omitempty"` + Fixed string `json:"fixed,omitempty"` +} + +type Finding struct { + OSVID string `json:"osv"` + FixedVersion string `json:"fixed_version"` + Trace []FindingTrace +} + +type FindingTrace struct { + Module string `json:"module"` + Version string `json:"version"` +} + +// compileUpdateList decodes the JSON stream from govulncheck and extracts +// a list of modules that need to be updated to a fixed version. +func compileUpdateList(jsonData io.Reader, onlyFixed bool) ([]UpdateList, error) { + updates := make(map[string]UpdateList) + osvs := make(map[string]*OSV) + decoder := json.NewDecoder(jsonData) + + for { + var v Vuln + if err := decoder.Decode(&v); err == io.EOF { + break // End of JSON stream + } else if err != nil { + // It might not be a JSON error, could be other text in the stream. + // We continue to try and find valid JSON objects. + continue + } + + if v.OSV == nil && v.Finding == nil { + continue + } + if v.OSV != nil { + osvs[v.OSV.ID] = v.OSV + continue + } + + // Parse finding. + // We assume OSVs are printed first. + osv := osvs[v.Finding.OSVID] + cve := CVE{} + allCVEs := v.Finding.OSVID + if osv != nil { + cve = getCVEDetails(*nvdAPIKey, *osv) + allCVEs = osv.CVEs() + } else { + slog.Error("Malformed govulncheck input; a finding without a OSV entry; assuming unkown severity.", "finding.osv", v.Finding.OSVID) + cve.ID = v.Finding.OSVID // Fallback to GO ID + cve.Severity = "UNKNOWN" + } + if len(v.Finding.Trace) == 0 { + slog.Error("Malformed govulncheck input; a finding with empty trace; ignoring.", "finding.osv", v.Finding.OSVID) + continue + } + + module := v.Finding.Trace[0].Module + + var fixVersion *semver.Version + if v.Finding.FixedVersion != "" { + var err error + fixVersion, err = semver.NewVersion(v.Finding.FixedVersion) + if err != nil { + slog.Warn("Found Go vulnerability with a fix that is not a correct semver version; assuming no fix version", "mod", module, "osv", cve, "fixedVersion", v.Finding.FixedVersion, "err", err) + } + } + + if onlyFixed && fixVersion == nil { + slog.Warn("IMPORTANT: Found Go vulnerability without a fixed version. Ignoring this module, given the -only-fixed flag...", "mod", module, "osv", cve) + continue + } + + up, ok := updates[module] + if !ok { + slog.Info("Found Go vulnerability with a fix; queuing...", "mod", module, "CVEs", allCVEs) + updates[module] = UpdateList{ + CVE: cve, + Module: module, + FixedVersion: fixVersion, + Version: v.Finding.Trace[0].Version, + } + continue + } + + // Check if there are more CVE IDs corresponding to the vulnerability, which can give more context. + slog.Debug("Found Go vulnerability with a fix, the module was already queued; resolving version...", "mod", module, "CVEs", allCVEs) + up.AdditionalCVEs++ + if fixVersion != nil { + if up.FixedVersion == nil || fixVersion.GreaterThan(up.FixedVersion) { + up.FixedVersion = fixVersion + } + } + if !cve.LessThan(up.CVE) { + up.CVE = cve + } + updates[module] = up + } + + updateList := slices.Collect(maps.Values(updates)) + sort.Slice(updateList, func(i, j int) bool { + return updateList[i].CVE.LessThan(updateList[j].CVE) + }) + return updateList, nil +} From 5cc808d9fdfcd747143e53297e7f58c67e4893bc Mon Sep 17 00:00:00 2001 From: bwplotka Date: Wed, 3 Dec 2025 22:11:12 +0000 Subject: [PATCH 2/3] feat: move to gmpctl; adding shmft Signed-off-by: bwplotka --- .bingo/Variables.mk | 6 + .bingo/shfmt.mod | 5 + .bingo/shfmt.sum | 12 + .bingo/variables.env | 2 + .github/workflows/release-bot.yml | 192 ---------- hack/gmpctl.sh | 34 ++ hack/gmpctl/.gitignore | 1 + hack/gmpctl/.gmpctl.default.yaml | 15 + hack/gmpctl/README.md | 107 ++++++ hack/gmpctl/cmd_release.go | 102 +++++ hack/gmpctl/cmd_vulnfix.go | 117 ++++++ hack/gmpctl/dialog.go | 44 +++ hack/gmpctl/git.go | 108 ++++++ hack/gmpctl/gmp.go | 128 +++++++ hack/{ => gmpctl}/lib.sh | 393 ++++++++++---------- hack/gmpctl/main.go | 219 +++++++++++ hack/{ => gmpctl}/vulnupdatelist/.gitignore | 0 hack/{ => gmpctl}/vulnupdatelist/main.go | 16 +- hack/{ => gmpctl}/vulnupdatelist/nvdapi.go | 16 +- hack/gmpctl/vulnupdatelist/nvdapi_test.go | 32 ++ hack/{ => gmpctl}/vulnupdatelist/vuln.go | 14 + hack/go.mod | 36 +- hack/go.sum | 91 ++++- hack/lib_test.sh | 150 -------- hack/presubmit.sh | 11 +- hack/release-forksync.sh | 327 ---------------- hack/release-rc.sh | 175 --------- hack/release-vulnfix.sh | 172 --------- hack/test-lib.sh | 74 ---- hack/vulnupdatelist/nvdapi_test.go | 18 - 30 files changed, 1301 insertions(+), 1316 deletions(-) create mode 100644 .bingo/shfmt.mod create mode 100644 .bingo/shfmt.sum delete mode 100644 .github/workflows/release-bot.yml create mode 100755 hack/gmpctl.sh create mode 100644 hack/gmpctl/.gitignore create mode 100644 hack/gmpctl/.gmpctl.default.yaml create mode 100644 hack/gmpctl/README.md create mode 100644 hack/gmpctl/cmd_release.go create mode 100644 hack/gmpctl/cmd_vulnfix.go create mode 100644 hack/gmpctl/dialog.go create mode 100644 hack/gmpctl/git.go create mode 100644 hack/gmpctl/gmp.go rename hack/{ => gmpctl}/lib.sh (66%) mode change 100644 => 100755 create mode 100644 hack/gmpctl/main.go rename hack/{ => gmpctl}/vulnupdatelist/.gitignore (100%) rename hack/{ => gmpctl}/vulnupdatelist/main.go (87%) rename hack/{ => gmpctl}/vulnupdatelist/nvdapi.go (79%) create mode 100644 hack/gmpctl/vulnupdatelist/nvdapi_test.go rename hack/{ => gmpctl}/vulnupdatelist/vuln.go (88%) delete mode 100644 hack/lib_test.sh delete mode 100755 hack/release-forksync.sh delete mode 100644 hack/release-rc.sh delete mode 100755 hack/release-vulnfix.sh delete mode 100644 hack/test-lib.sh delete mode 100644 hack/vulnupdatelist/nvdapi_test.go diff --git a/.bingo/Variables.mk b/.bingo/Variables.mk index 50c1370c32..fe36118517 100644 --- a/.bingo/Variables.mk +++ b/.bingo/Variables.mk @@ -47,3 +47,9 @@ $(MDOX): $(BINGO_DIR)/mdox.mod @echo "(re)installing $(GOBIN)/mdox-v0.9.0" @cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=mdox.mod -o=$(GOBIN)/mdox-v0.9.0 "github.com/bwplotka/mdox" +SHFMT := $(GOBIN)/shfmt-v3.12.0 +$(SHFMT): $(BINGO_DIR)/shfmt.mod + @# Install binary/ries using Go 1.14+ build command. This is using bwplotka/bingo-controlled, separate go module with pinned dependencies. + @echo "(re)installing $(GOBIN)/shfmt-v3.12.0" + @cd $(BINGO_DIR) && GOWORK=off GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) GOARM=$(GOHOSTARM) $(GO) build -mod=mod -modfile=shfmt.mod -o=$(GOBIN)/shfmt-v3.12.0 "mvdan.cc/sh/v3/cmd/shfmt" + diff --git a/.bingo/shfmt.mod b/.bingo/shfmt.mod new file mode 100644 index 0000000000..da9d2c3927 --- /dev/null +++ b/.bingo/shfmt.mod @@ -0,0 +1,5 @@ +module _ // Auto generated by https://github.com/bwplotka/bingo. DO NOT EDIT + +go 1.25.0 + +require mvdan.cc/sh/v3 v3.12.0 // cmd/shfmt diff --git a/.bingo/shfmt.sum b/.bingo/shfmt.sum new file mode 100644 index 0000000000..cd6a3b51f7 --- /dev/null +++ b/.bingo/shfmt.sum @@ -0,0 +1,12 @@ +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +mvdan.cc/editorconfig v0.3.0 h1:D1D2wLYEYGpawWT5SpM5pRivgEgXjtEXwC9MWhEY0gQ= +mvdan.cc/editorconfig v0.3.0/go.mod h1:NcJHuDtNOTEJ6251indKiWuzK6+VcrMuLzGMLKBFupQ= +mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI= +mvdan.cc/sh/v3 v3.12.0/go.mod h1:Se6Cj17eYSn+sNooLZiEUnNNmNxg0imoYlTu4CyaGyg= diff --git a/.bingo/variables.env b/.bingo/variables.env index 1a87f61d1b..5297d7d808 100644 --- a/.bingo/variables.env +++ b/.bingo/variables.env @@ -16,3 +16,5 @@ HELM="${GOBIN}/helm-v3.14.0" MDOX="${GOBIN}/mdox-v0.9.0" +SHFMT="${GOBIN}/shfmt-v3.12.0" + diff --git a/.github/workflows/release-bot.yml b/.github/workflows/release-bot.yml deleted file mode 100644 index 42f9c24b2d..0000000000 --- a/.github/workflows/release-bot.yml +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Release bot - -on: - workflow_call: - inputs: - branch_name: - required: true - type: string - commit_sha: - required: true - type: string -env: - REGISTRY: ghcr.io - IMAGE_NAME: googlecloudplatform/gmp/hermetic-build - BRANCH_NAME: ${{ inputs.branch_name }} -concurrency: - group: ${{inputs.branch_name}} - cancel-in-progress: true -jobs: - auto_tag: - runs-on: ubuntu-latest - permissions: - contents: write - concurrency: - group: ${{inputs.commit_sha}} - cancel-in-progress: false - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ env.BRANCH_NAME }} - - name: Setup git user - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - name: Get the other commit from the merge if there is one - id: merge - run: | - parents=$(git log --pretty=%P -n 1 ${{ inputs.commit_sha }}) - IFS=' ' read -r parent1 parent2 <<< "$parents" - echo "sha=$parent2" >> $GITHUB_OUTPUT - - name: Check if commit is tagged - if: ${{ steps.merge.outputs.sha != '' }} - id: check - run: | - TAG=$(git tag --points-at ${{ steps.merge.outputs.sha }} | grep releasebot || true) - echo "tag=$TAG" >> $GITHUB_OUTPUT - - name: Push tag - if: ${{ steps.check.outputs.tag != '' }} - run: | - set -ex - NEW_TAG=$(echo "${{ steps.check.outputs.tag }}" | awk '{gsub(/releasebot\//, ""); print}') - git tag "$NEW_TAG" - git push origin "$NEW_TAG" - echo "::notice::Successfully created and pushed new tag: '$NEW_TAG'." - - build_and_push_image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - outputs: - image_tag: ${{ steps.create_tag.outputs.image_tag }} - steps: - - name: Get docker tag - id: create_tag - run: | - TAG=$(echo "${{ env.BRANCH_NAME }}" | awk '{gsub(/\//, "-"); print}') - echo "image_tag=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:$TAG" >> $GITHUB_OUTPUT - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ env.BRANCH_NAME }} - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build and push Docker image - uses: docker/build-push-action@v6 - with: - file: ./hack/Dockerfile - target: hermetic - push: true - tags: ${{ steps.create_tag.outputs.image_tag }} - cache-from: type=registry,ref=${{ steps.create_tag.outputs.image_tag }}-cache - cache-to: type=registry,ref=${{ steps.create_tag.outputs.image_tag }}-cache,mode=max - prepare_rc: - runs-on: ubuntu-latest - permissions: - contents: write - packages: read - needs: - - build_and_push_image - - auto_tag - outputs: - full_rc_version: ${{ steps.prepare.outputs.full_rc_version }} - bot_branch: ${{ steps.push.outputs.bot_branch }} - steps: - - name: Checkout release - uses: actions/checkout@v4 - with: - ref: ${{ env.BRANCH_NAME }} - path: release_branch - - name: Checkout main - uses: actions/checkout@v4 - with: - ref: main - path: main_branch - - name: Set up Git Identity - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - working-directory: ./release_branch - - name: Create RC Version and Update Files - id: prepare - working-directory: ./main_branch - run: | - alias yq=go tool yq - go run ./hack/prepare_rc -branch ${{ env.BRANCH_NAME }} -dir ../release_branch - - name: Regen files - working-directory: ./release_branch - # Workflows can't edit workflows. Better to create PR and let tests fail. - run: | - mv .github .. - make regen CACHE_IMAGE_FROM=${{ needs.build_and_push_image.outputs.image_tag }}-cache - mv ../.github . - - name: Commit and Tag Release Candidate - if: steps.prepare.outputs.full_rc_version != '' - id: push - working-directory: ./release_branch - run: | - set -e - BOT_BRANCH=$(echo "${{ env.BRANCH_NAME }}" | awk '{gsub(/release/, "releasebot"); print}') - echo "bot_branch=$BOT_BRANCH" >> $GITHUB_OUTPUT - git checkout -b "$BOT_BRANCH" - git add . - RC="${{ steps.prepare.outputs.full_rc_version }}" - git commit -as -m"chore: prepare for $RC release" - git tag -a "releasebot/$RC" -m "releasebot release candidate $RC" - git push -f origin "$BOT_BRANCH" "releasebot/$RC" - manage_pr: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - needs: - prepare_rc - steps: - - name: Checkout main - uses: actions/checkout@v4 - with: - ref: ${{needs.prepare_rc.outputs.bot_branch}} - - name: Manage Release PR - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - BOT_BRANCH=${{needs.prepare_rc.outputs.bot_branch}} - RC=${{ needs.prepare_rc.outputs.full_rc_version }} - EXISTING_PR_NUMBER=$(gh pr list \ - --base ${{ env.BRANCH_NAME }} \ - --head $BOT_BRANCH \ - --state open \ - --json number \ - -q '.[0].number // empty') - - if [[ -z "$EXISTING_PR_NUMBER" ]]; then - gh pr create \ - --base ${{ env.BRANCH_NAME }} \ - --head $BOT_BRANCH \ - --title "chore: prepare for $RC release" \ - --body "Beep boop. Merging activates deployment. A fresh PR appears on merge. Boop beep." - fi diff --git a/hack/gmpctl.sh b/hack/gmpctl.sh new file mode 100755 index 0000000000..607a483126 --- /dev/null +++ b/hack/gmpctl.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +SCRIPT_DIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" + +pushd "${SCRIPT_DIR}/gmpctl" >/dev/null +# NOTE gmpctl expects the whole gmpctl directory to be present. +# We could consider embedding bash scripts, config into binary, but it's good +# for now. +go run ./ "$@" +popd >/dev/null diff --git a/hack/gmpctl/.gitignore b/hack/gmpctl/.gitignore new file mode 100644 index 0000000000..1269488f7f --- /dev/null +++ b/hack/gmpctl/.gitignore @@ -0,0 +1 @@ +data diff --git a/hack/gmpctl/.gmpctl.default.yaml b/hack/gmpctl/.gmpctl.default.yaml new file mode 100644 index 0000000000..b44eb2fef8 --- /dev/null +++ b/hack/gmpctl/.gmpctl.default.yaml @@ -0,0 +1,15 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dir: "./data" diff --git a/hack/gmpctl/README.md b/hack/gmpctl/README.md new file mode 100644 index 0000000000..c4a588fb69 --- /dev/null +++ b/hack/gmpctl/README.md @@ -0,0 +1,107 @@ +# gmpctl + +`gmpctl` is an interactive CLI for common operations on GMP projects. + +It's a starting point for smaller or bigger automation on OSS side (e.g. releasing, syncing or even debugging). + +> NOTE: This script is far from perfect, but it's better than doing things manually. Feel free to contribute, +> fix bugs and add more automation for common tasks! + +## Setup + +1. To start using `gmpctl` you need to have a clone of `prometheus-engine` on your machine (you probably have already one! + to fetch the latest `main` for the best experience (latest scripts). + +2. The next this is to obtain NVD API key to avoid rate-limits when querying CVE DB. See https://nvd.nist.gov/developers/request-an-api-key and save this key to `hack/vulnupdatelist/api.text` + +3. You can configure different work directory for gmpctl via `-c` flag. By default, `gmpctl` does the work in `hack/gmpctl/.data`) + +Enjoy! + +## Usage + +Generally `gmpctl` does not need flags for general usage. It interactively asks you for +key information and confirmations e.g. + +```bash +./hack/gmpctl.sh release +┃ What do you want to release? + ┃ > release/0.17 + ┃ release/0.15 + ┃ release/0.14 + ┃ release/0.12 + ┃ release-2.45.3-gmp + ┃ release-2.53.5-gmp +↑ up â€ĸ ↓ down â€ĸ / filter â€ĸ enter submit +``` + +`gmpctl` maintains 1 git clone for each project and uses `git worktree` for each command and branch. + +`gmpctl` commands are aimed to be **idempotent**, meaning you should be able to run it multiple times with the +same parameters, and it will continue the previous work or at least yield same results. This is crucial when iterating +on breaking go mod updates for vulnerabilities or fork sync conflicts. + +```text mdox-exec="bash hack/gmpctl.sh --help" +Usage: gmpctl [COMMAND] [FLAGS] + -c string + Path to the configuration file. See config.go#Config for the structure. (default ".gmpctl.default.yaml") + -v Enabled verbose, debug output (e.g. logging os.Exec commands) + +--- Commands --- +[release] Usage of release: + -b string + Release branch to work on; Project is auto-detected from this + -patch + If true, and --tag is empty, forces a new patch version as a new TAG. + -t string + Tag to release. If empty, next TAG version will be auto-detected (double check this!) + +[vulnfix] Usage of vulnfix: + -b string + Release branch to work on; Project is auto-detected from this + -pr-branch string + (default: $USER/BRANCH-vulnfix) Upstream branch to push to (user-confirmed first). + -sync-dockerfiles-from + Optional branch name to sync Dockerfiles from. Useful when things changed. +``` + +## `gmpctl` development + +Some rules to follow: + +* Downstream functions should literally use `panicf` for error handling. This improves readability and enormously help + with debugging errors. The obvious exception is when code needs to handle this error. Then swap panic with a proper `err error` pattern. +* Offer choice, be interactive! See `dialog.go` and https://github.com/charmbracelet/huh on what's possible. + +## Bash development + +While the `gmpctl` is written in Go, you might notice some functionalities are in Bash. + +Bash is funky, but sometimes more readable than Go/easier to iterate. +Eventually, we could rewrite more critical pieces to Go, but you're welcome to add some quick +pieces in bash to automate some stuff. + +It's trivial to call bash function from `gmpctl` e.g.: + +```go +if err := runLibFunction(dir, opts, "release-lib:vulnfix"); err != nil { + return err +} +``` + +Some rules to follow: +* CI checks bash formatting via https://github.com/mvdan/sh?tab=readme-ov-file#shfmt. You can install this on your IDE for formatting. +* Write only libraries (functions). The starting point for scripts should be always Go gmpctl CLI. +* Function names have `release-lib::` prefix to figure out where they come from. +* Function check their required arguments/envvars; always. +* Especially for functions that return strings via stdout: + * Ensure all error messages are redirected to stderr, use log_err func for this. + * Be careful with pushd/popd which log to stdout, you can redirect those to stderr too. + +## TODO + +* Ability to configure NVD API key in gmpctl config. +* Port fork-sync script from the old PR. +* Generate some on-demand query of vulnerabilities for all releases (aka dashboard.) +* Fix NPM vulns (although it's rate). +* Ability to schedule multiple scripts at once and managing that? (lot's of work vs multiple terminals) diff --git a/hack/gmpctl/cmd_release.go b/hack/gmpctl/cmd_release.go new file mode 100644 index 0000000000..cf851040fd --- /dev/null +++ b/hack/gmpctl/cmd_release.go @@ -0,0 +1,102 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "flag" + "fmt" +) + +var ( + releaseFlags = flag.NewFlagSet("release", flag.ExitOnError) + releaseBranch = releaseFlags.String("b", "", "Release branch to work on; Project is auto-detected from this") + releaseTag = releaseFlags.String("t", "", "Tag to release. If empty, next TAG version will be auto-detected (double check this!)") + releasePatch = releaseFlags.Bool("patch", false, "If true, and --tag is empty, forces a new patch version as a new TAG.") +) + +func release() error { + _ = releaseFlags.Parse(flag.Args()[1:]) + + cfg, err := loadConfig() + if err != nil { + return err + } + + var ( + proj Project + branch = *releaseBranch + tag = *releaseTag + ok bool + ) + if branch == "" { + branch = selectBranch("What do you want to release?") + } + proj, ok = projectFromBranch(branch) + if !ok { + return fmt.Errorf("couldn't find project from branch %s", branch) + } + + logf("Assuming %q with remote %q; branch to release: %q", proj.Name, proj.RemoteURL, branch) + dir := proj.WorkDir(cfg.Directory, branch, "release") + + mustFetchAll(dir) + + if tag == "" { + var envs []string + if *releasePatch { + envs = append(envs, "FORCE_NEW_PATCH_VERSION=true") + } + tag, err = getFromLibFunction(dir, envs, "release-lib::next_release_tag", dir) + if err != nil { + return err + } + } + logf("Selected %v tag", tag) + + if err := runLibFunction(dir, []string{ + fmt.Sprintf("DIR=%v", dir), + fmt.Sprintf("BRANCH=%v", branch), + fmt.Sprintf("PROJECT=%v", proj.Name), + fmt.Sprintf("TAG=%v", tag), + }, "release-lib::pre-release-rc"); err != nil { + return err + } + + msg := fmt.Sprintf("chore: prepare for %v release", tag) + // TODO(bwplotka): Port to Go, make it more reliable. + // TODO(bwplotka): Quote otherwise it's split into separate args... port it so it works better (: + if err := runLibFunction(dir, nil, "release-lib::idemp::git_commit_amend_match", "\""+msg+"\""); err != nil { + return err + } + + // TODO(bwplotka): Check if tag exists. + mustCreateSignedTag(dir, tag) + + // TODO check if anything is needed to push? + // TODO(bwplotka): Add option to print more debug/open terminal with the workdir? + if confirmf("About to git push state from %q to \"origin/%v\"; then push %q tag; are you sure?", dir, branch, tag) { + // We are in detached state + mustPush(dir, fmt.Sprintf("HEAD:%v", branch)) + mustPush(dir, tag) + } else { + return errors.New("aborting") + } + + if confirmf("Do you want to remove the %v worktree (recommended)?", dir) { + proj.RemoveWorkDir(cfg.Directory, dir) + } + return nil +} diff --git a/hack/gmpctl/cmd_vulnfix.go b/hack/gmpctl/cmd_vulnfix.go new file mode 100644 index 0000000000..244b0bb9fd --- /dev/null +++ b/hack/gmpctl/cmd_vulnfix.go @@ -0,0 +1,117 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "flag" + "fmt" + "os" +) + +var ( + vulnfixFlags = flag.NewFlagSet("vulnfix", flag.ExitOnError) + vulnfixBranch = vulnfixFlags.String("b", "", "Release branch to work on; Project is auto-detected from this") + vulnfixPRBranch = vulnfixFlags.String("pr-branch", "", "(default: $USER/BRANCH-vulnfix) Upstream branch to push to (user-confirmed first).") + vulnfixSyncDockerfilesFrom = vulnfixFlags.Bool("sync-dockerfiles-from", false, "Optional branch name to sync Dockerfiles from. Useful when things changed.") +) + +// Attempt a minimal dependency upgrade to solve fixable vulnerabilities. +// +// * Docker images: +// - Distros use latest tag so rebuilding takes latest, nothing to do. +// - google-go.pkg.dev/golang images are updated to the latest minor version using docker-bump-images.sh +// +// * Manifests +// - distroless bumped to latest (although our component tooling is capable of bumping this too) +// +// * Go deps: Upgrade to minimal required version per a known fixable vulnerability. +// * Npm deps: Not implemented. +// +// NOTE: The script is idempotent; to force it to recreate local artifacts (e.g. local clones, remote branches it created), remove the artifact you want to recreate. +func vulnfix() error { + _ = vulnfixFlags.Parse(flag.Args()[1:]) + + cfg, err := loadConfig() + if err != nil { + return err + } + var ( + proj Project + branch = *vulnfixBranch + prBranch = *vulnfixPRBranch + ok bool + ) + if branch == "" { + branch = selectBranch("What do you want to release?") + } + proj, ok = projectFromBranch(branch) + if !ok { + return fmt.Errorf("couldn't find project from branch %s", branch) + } + + // TODO(bwplotka): We are force pushing prBranch. Shall we add safety check it's not a release branch? + // Perhaps validated in mustForcePush? + if prBranch == "" { + prBranch = fmt.Sprintf("%v/%v-gmpctl-vulnfix", os.Getenv("USER"), branch) + } + + logf("Assuming %q with remote %q; on %q; changes will be pushed to %q", proj.Name, proj.RemoteURL, branch, prBranch) + dir := proj.WorkDir(cfg.Directory, branch, "vulnfix") + + // Refresh. + mustFetchAll(dir) + + opts := []string{ + fmt.Sprintf("DIR=%v", dir), + fmt.Sprintf("BRANCH=%v", branch), + fmt.Sprintf("PROJECT=%v", proj.Name), + } + if *vulnfixSyncDockerfilesFrom { + opts = append(opts, "SYNC_DOCKERFILES_FROM=true") + } + + // TODO(bwplotka): Add NPM vulnfix. + if err := runLibFunction(dir, opts, "release-lib::vulnfix"); err != nil { + return err + } + + // TODO: Warn of unstaged files at this point. + + // Commit if anything is staged. + msg := fmt.Sprintf("google patch[deps]: fix %v vulnerabilities", branch) + if proj.Name == "prometheus-engine" { + msg = fmt.Sprintf("fix: fix %v vulnerabilities", branch) + } + // TODO(bwplotka): Port to Go, make it more reliable. + // TODO(bwplotka): Quote otherwise it's split into separate args... port it so it works better (: + if err := runLibFunction(dir, nil, "release-lib::idemp::git_commit_amend_match", "\""+msg+"\""); err != nil { + return err + } + // TODO(bwplotka): Check if needs pushing? + // TODO(bwplotka): Add option to print more debug/open terminal with the workdir? + if confirmf("About to FORCE git push state from %q to \"origin/%v\"; are you sure?", dir, prBranch) { + // We are in detached state, so be explicit what to push and from where. + mustRecreateBranch(dir, prBranch) + mustForcePush(dir, prBranch) + } else { + return errors.New("aborting") + } + + if confirmf("Do you want to remove the %v worktree (recommended)?", dir) { + proj.RemoveWorkDir(cfg.Directory, dir) + } + return nil +} diff --git a/hack/gmpctl/dialog.go b/hack/gmpctl/dialog.go new file mode 100644 index 0000000000..3580b8996c --- /dev/null +++ b/hack/gmpctl/dialog.go @@ -0,0 +1,44 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/charmbracelet/huh" +) + +func selectBranch(question string) (branch string) { + var opts []huh.Option[string] + for _, b := range ReleaseBranches { + opts = append(opts, huh.NewOption[string](b, b)) + } + if err := huh.NewForm(huh.NewGroup( + huh.NewSelect[string]().Title(question).Options(opts...).Value(&branch), + )).Run(); err != nil { + os.Exit(1) // Abort. + } + return branch +} + +func confirmf(question string, args ...any) (confirmed bool) { + if err := huh.NewForm(huh.NewGroup( + huh.NewConfirm().Title(fmt.Sprintf(question, args...)).Inline(true).Value(&confirmed), + )).Run(); err != nil { + os.Exit(1) // Abort. + } + return confirmed +} diff --git a/hack/gmpctl/git.go b/hack/gmpctl/git.go new file mode 100644 index 0000000000..962bb387e3 --- /dev/null +++ b/hack/gmpctl/git.go @@ -0,0 +1,108 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" +) + +func mustCloneRepo(repoURL, destinationDir string) { + if _, err := runCommand(nil, "git", "clone", repoURL, destinationDir); err != nil { + panicf(err.Error()) + } +} + +func mustAddWorktree(dir, newWorktreeDir, branchName string) { + // TODO(bwplotka): Ideally we want fresh changes. + mustFetchAll(dir) + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "worktree", "add", newWorktreeDir, "origin/"+branchName); err != nil { + panicf(err.Error()) + } +} + +func mustRemoveWorktree(dir, worktreeDir string) { + // Without force and some local modifications, worktree fails with "contains + // modified or untracked files, use --force to delete it". + // TODO(bwplotka): Perhaps a good safety guard? + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "worktree", "remove", "-f", worktreeDir); err != nil { + panicf(err.Error()) + } +} + +func mustFetchAll(dir string) { + logf("Fetching origin...") + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "fetch", "--tags"); err != nil { + panicf("failed to fetch: %v", err) + } +} + +func mustCreateSignedTag(dir, tag string) { + logf("Creating a signed tag %v...", tag) + + // explicit TTY is often needed on Macs. + // TODO(bwplotka): Consider adding v0.x second tag for Prometheus fork (similar to how v0.300 Prometheus releases are structured). + // This is to have a little bit cleaner prometheus-engine go.mod version against the fork. + if _, err := runCommand( + &cmdOpts{Dir: dir}, + "bash", "-c", + fmt.Sprintf("GPG_TTY=$(tty) git tag -s %v -m %v", tag, tag), + ); err != nil { + panicf(err.Error()) + } +} + +func mustPush(dir, what string) { + logf("Pushing %v...", what) + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "push", "origin", what); err != nil { + panicf("failed to push: %v", err) + } +} + +func mustForcePush(dir, what string) { + logf("FORCE Pushing %v...", what) + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "push", "--force", "origin", what); err != nil { + panicf("failed to force push: %v", err) + } +} + +func mustRecreateBranch(dir, branch string) { + // TODO(bwplotka): Yolo, check error etc. + _, _ = runCommand(&cmdOpts{Dir: dir}, "git", "branch", "-D", branch) + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "branch", branch); err != nil { + panicf("failed to fopush: %v", err) + } +} + +func checkoutBranch(dir, branchName string) { + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "checkout", branchName); err != nil { + panicf(err.Error()) + } +} + +func resetBranch(dir, target string) { + if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "reset", "--hard", target); err != nil { + panicf(err.Error()) + } +} + +func getLatestTagForBranch(dir, branchName string) (string, error) { + // --abbrev=0 suppresses the suffix (e.g., returns "v1.0" instead of "v1.0-5-g3a1b2") + // --tags allows lightweight tags, not just annotated ones + tag, err := runCommand(&cmdOpts{Dir: dir}, "git", "describe", "--tags", "--abbrev=0", branchName) + if err != nil { + return "", fmt.Errorf("no tags found reachable from %s or error: %w", branchName, err) + } + return tag, nil +} diff --git a/hack/gmpctl/gmp.go b/hack/gmpctl/gmp.go new file mode 100644 index 0000000000..6e2d515a63 --- /dev/null +++ b/hack/gmpctl/gmp.go @@ -0,0 +1,128 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +var ( + Prometheus = Project{ + Name: "prometheus", + RemoteURL: "git@github.com:GoogleCloudPlatform/prometheus.git", + BranchRE: regexp.MustCompile(`^release-[23]\.[0-9]+\.[0-9]+-gmp$`), + } + Alertmanager = Project{ + Name: "alertmanager", + RemoteURL: "git@github.com:GoogleCloudPlatform/alertmanager.git", + BranchRE: regexp.MustCompile(`^release-0\.[0-9]+\.[0-9]+-gmp$`), + } + PrometheusEngine = Project{ + Name: "prometheus-engine", + RemoteURL: "git@github.com:GoogleCloudPlatform/prometheus-engine.git", + BranchRE: regexp.MustCompile(`^release/0\.[0-9]+$`), + } + + // ReleaseBranches contains hardcoded list of active branches. We could pull it out from somewhere. + ReleaseBranches = []string{ + "release/0.17", + "release/0.15", + "release/0.14", + "release/0.12", + "release-2.45.3-gmp", + "release-2.53.5-gmp", + "release-0.27.0-gmp", + } +) + +func projectFromBranch(branch string) (Project, bool) { + switch { + case Prometheus.BranchRE.MatchString(branch): + return Prometheus, true + case Alertmanager.BranchRE.MatchString(branch): + return Alertmanager, true + case PrometheusEngine.BranchRE.MatchString(branch): + return PrometheusEngine, true + } + return Project{}, false +} + +type Project struct { + Name string + RemoteURL string + BranchRE *regexp.Regexp +} + +func (p Project) cloneDir(dir string) (cloneDir string) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err := os.MkdirAll(dir, 0o755); err != nil { + panicf(err.Error()) + } + } + + cloneDir = filepath.Join(dir, p.Name) + _, err := os.Stat(cloneDir) + if err == nil { + return cloneDir + } + if !errors.Is(err, os.ErrNotExist) { + panicf("failed to stat %s: %v", cloneDir, err) + } + logf("Cloning %q into %q", p.RemoteURL, cloneDir) + mustCloneRepo(p.RemoteURL, cloneDir) + return cloneDir +} + +func (p Project) workDir(dir, branch, suffix string) string { + subDir := strings.ToLower(fmt.Sprintf("%v_%v", branch, suffix)) + subDir = strings.ReplaceAll(subDir, "/", "_") + return filepath.Join(dir, p.Name, subDir) +} + +// WorkDir returns a new working directory. +func (p Project) WorkDir(dir, branch, suffix string) (workDir string) { + cloneDir := p.cloneDir(dir) + + workDir = p.workDir(dir, branch, suffix) + if _, err := os.Stat(workDir); err == nil { + if confirmf("Found worktree %q; do you want to reuse it (without reset)?", workDir) { + logf("Reusing %q worktree", workDir) + return workDir + } + logf("Removing %q worktree", workDir) + mustRemoveWorktree(cloneDir, workDir) + } + + logf("Creating new worktree %q from %q", workDir, cloneDir) + mustAddWorktree(cloneDir, workDir, branch) + // Whenever we start work tree, we + return workDir +} + +func (p Project) RemoveWorkDir(dir, workDir string) { + _, err := os.Stat(workDir) + if err == nil { + mustRemoveWorktree(filepath.Join(dir, p.Name), workDir) + return + } + if !errors.Is(err, os.ErrNotExist) { + panicf("failed to stat %s: %v", workDir, err) + } +} diff --git a/hack/lib.sh b/hack/gmpctl/lib.sh old mode 100644 new mode 100755 similarity index 66% rename from hack/lib.sh rename to hack/gmpctl/lib.sh index 5ed1a59e39..dea00ab263 --- a/hack/lib.sh +++ b/hack/gmpctl/lib.sh @@ -13,16 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE for contributors: Bash is funky, but still more readable than Go/easier to iterate. -# Eventually if we depend on those scripts and ppl want we could rewrite them to Go, but some -# parts likely is more readable and easier to bash. However, some rules to writing bash: -# * Use https://github.com/mvdan/sh?tab=readme-ov-file#shfmt on your IDE for formatting (TODO: ensure on CI). -# * lib.sh has ONLY functions. Make sure: -# * Function names have release-lib:: prefix to figure out where they come from. -# * Function check their required arguments -# * Especially for functions that return strings via stdout: -# * Ensure all error messages are redirected to stderr, use log_err func for this. -# * Be careful with pushd/popd which log to stdout, you can redirect those to stderr too. +# NOTE for contributors: Bash is funky, but sometimes more readable than Go/easier to iterate. +# Eventually, we could rewrite more critical pieces to Go, but you're welcome to add some quick +# pieces in bash to automate some stuff. +# +# See README.md#bash for rules to writing bash. set -o errexit set -o pipefail @@ -32,43 +27,16 @@ if [[ -n "${DEBUG_MODE:-}" ]]; then set -o xtrace fi -SCRIPT_DIR="$( - cd -- "$(dirname "$0")" >/dev/null 2>&1 - pwd -P -)" - -# Extended regular expressions (ERE) regex matching paths to exclude from the fork commit. -# NOTE: # ^\..+ means all hidden files (e.g. changes to .golangci.yaml .gitignore or CI). -# TODO(bwplotka): Consider moving to globs with dotglob and extglob settings.. or Go (: -export RELEASE_LIB_EXCLUDE_RE="^\..+ -^README\.md -^CHANGELOG\.md -^MAINTAINERS\.md -^CONTRIBUTING\.md -^RELEASE\.md -^Dockerfile -^docs/.* -^documentation/.* -^google/.* -^.*go\..* -^.*\.gitignore -^.*package.json -^.*package-lock.json -^Makefile.* -^.*vendor/.* -^VERSION -^.*node_modules/.*" - -# Extended regular expressions (ERE) regex matching paths from EXCLUDE_RE that should be included. -# This is needed as it's simpler than implementing RE negative matchers. -# NOTE: For Prometheus the two specific documentation files are imported in Google Managed Prometheus docs, so keep those. -export RELEASE_LIB_DOCUMENTATION_INCLUDE_RE="^documentation/examples/prometheus-agent\.y.?ml -^documentation/examples/prometheus\.y.?ml" - log_err() { echo "❌ ${1}" >&2 } +# TODO(bwplotka): Finding correct script dir is not so trivial. +if [[ -z "${SCRIPT_DIR}" ]]; then + log_err "SCRIPT_DIR envvar is required." + return 1 +fi + release-lib::confirm() { local prompt_message="${1:-Are you sure?}" @@ -92,96 +60,6 @@ release-lib::confirm() { esac } -# clone clones the $REMOTE_URL to $clone_dir at $source_branch version, then -# creates $target_branch from it, if set. -# -# Idempotence: If the $clone_dir exists, skip cloning and check if -# the $clone_dir is a git repo on the desired branch. -# TODO: Cloning takes time, consider resetting repo if present. -release-lib::idemp::clone() { - local clone_dir="${1}" - if [[ -z "${clone_dir}" ]]; then - log_err "clone_dir arg is not set." - return 1 - fi - local source_branch="${2}" # Branch to fetch when cloning, base for $target_branch - if [[ -z "${source_branch}" ]]; then - log_err "source_branch arg is not set." - return 1 - fi - local target_branch="${3}" # Branch to create from source_branch, - if [[ -z "${target_branch}" ]]; then - target_branch="${source_branch}" - fi - - if [[ -z "${REMOTE_URL}" ]]; then - log_err "REMOTE_URL environment variable is not set." - return 1 - fi - - if [[ ! -d "${clone_dir}" ]]; then - git clone -b "${source_branch}" "${REMOTE_URL}" "${clone_dir}" - if [[ "${source_branch}" != "${target_branch}" ]]; then - pushd "${clone_dir}" - git checkout -b "${target_branch}" - popd - fi - else - if ! release-lib::confirm "The repository clone on ${clone_dir} exists. Do you want to reuse this directory without resetting? 'n' will attempt a hard reset on the repo (quicker then re-clone)."; then - pushd "${clone_dir}" - git fetch origin - git checkout "${source_branch}" - git reset --hard "origin/${source_branch}" - # TODO: Remove tags? - popd - fi - fi - - pushd "${clone_dir}" - if [[ "$(git symbolic-ref --short HEAD)" != "${target_branch}" ]]; then - log_err "Malformed ${DIR}; expected ${target_branch} got $(git symbolic-ref --short HEAD); remove or fix manually the ${clone_dir} and rerun." - return 1 - fi - popd -} - -release-lib::remote_url_from_branch() { - local branch=$1 - # Check if the BRANCH environment variable is set. - if [[ -z "${branch}" ]]; then - log_err "branch is required." - return 1 - fi - - if [[ "${branch}" =~ release-(2|3)\.[0-9]+\.[0-9]+-gmp$ ]]; then - echo "git@github.com:GoogleCloudPlatform/prometheus.git" - elif [[ "${branch}" =~ release-0\.[0-9]+\.[0-9]+-gmp$ ]]; then - echo "git@github.com:GoogleCloudPlatform/alertmanager.git" - elif [[ "${branch}" =~ release/0\.[0-9]+$ ]]; then - echo "git@github.com:GoogleCloudPlatform/prometheus-engine.git" - else - log_err "No matching remote URL found for branch=${branch}" - return 1 - fi -} - -release-lib::upstream_remote_url() { - local project=$1 - if [[ -z "${project}" ]]; then - log_err "project is required." - return 1 - fi - - if [[ "${project}" == "prometheus" ]]; then - echo "git@github.com:prometheus/prometheus.git" - elif [[ "${project}" == "alertmanager" ]]; then - echo "git@github.com:prometheus/alertmanager.git" - else - log_err "No matching remote URL found for project='${project}'" - return 1 - fi -} - release-lib::idemp::vulnlist() { local dir="${1}" if [[ -z "${dir}" ]]; then @@ -198,24 +76,8 @@ release-lib::idemp::vulnlist() { return 1 fi - if [[ -f "${vuln_file}" && ! -z $(cat "${vuln_file}") ]]; then - if ! release-lib::confirm "Found previous "${vuln_file}". Do you want to reuse this file? 'n' will re-run Go vulnlist check."; then - release-lib::vulnlist "${dir}" "${vuln_file}" - else - echo "⚠ī¸ Using existing ${vuln_file}" - fi - else - release-lib::vulnlist "${dir}" "${vuln_file}" - fi -} - -release-lib::dockerfiles() { - local dir="${1}" - if [[ -z "${dir}" ]]; then - log_err "dir arg is required." - return 1 - fi - find "${dir}" -name "Dockerfile*" | grep -v "/third_party/" | grep -v "/examples/" | grep -v "/hack/" | grep -v "/ui/" + # TODO(bwplotka): We could ask user if we should reuse existing vulnfile. + release-lib::vulnlist "${dir}" "${vuln_file}" } release-lib::vulnlist() { @@ -234,7 +96,7 @@ release-lib::vulnlist() { return 1 fi - readarray -t DOCKERFILES < <(release-lib::dockerfiles "${DIR}") + readarray -t DOCKERFILES < <(release-lib::dockerfiles "${dir}") local go_version=$(release-lib::dockerfile_go_version "${DOCKERFILES[0]}") if [[ -z "${go_version}" ]]; then log_err "can't find any golang image in ${DOCKERFILES[0]}" @@ -308,6 +170,94 @@ release-lib::gomod_vulnfix() { popd } +# Also accepts SYNC_DOCKERFILES_FROM. +function release-lib::vulnfix() { + if [[ -z "${DIR}" ]]; then + log_err "DIR envvar is required." + return 1 + fi + + if [[ -z "${BRANCH}" ]]; then + log_err "BRANCH envvar is required." + return 1 + fi + + if [[ -z "${PROJECT}" ]]; then + log_err "PROJECT envvar is required." + return 1 + fi + + echo "${DIR}" + echo "${SCRIPT_DIR}" + + readarray -t DOCKERFILES < <(release-lib::dockerfiles "${DIR}") + + # Sync dockerfiles if needed. + if [[ -n "${SYNC_DOCKERFILES_FROM:-}" ]]; then + pushd "${DIR}" + for dockerfile in "${DOCKERFILES[@]}"; do + # TODO: Should we ensure SYNC_DOCKERFILES_FROM if it's a branch is up to data with origin? + echo "🔄 Syncing ${dockerfile} from ${SYNC_DOCKERFILES_FROM}" + git checkout "${SYNC_DOCKERFILES_FROM}" -- "${dockerfile}" + done + popd + fi + + # Docker images bumps. + + # Get first dockerfile Go version. We will use this version to find minor version to stick to. + go_version=$(release-lib::dockerfile_go_version "${DOCKERFILES[0]}") + if [[ -z "${go_version}" ]]; then + echo "❌ can't find any golang image in ${DOCKERFILES[0]}" + return 1 + fi + + # TODO: git add charts & vendor for old projects? + + # Update our images. + for dockerfile in "${DOCKERFILES[@]}"; do + release-lib::dockerfile_update_image "${dockerfile}" "google-go.pkg.dev/golang" $(echo "${go_version}" | cut -d '.' -f 1-2) + release-lib::dockerfile_update_image "${dockerfile}" "gke.gcr.io/gke-distroless/libc" "gke_distroless_" + pushd "${DIR}" + git add "${dockerfile}" + popd + done + + # bash manifest bump. + # Exclude 0.12 as values were inlined with each part, easy to manually sed for old versions. + if [[ "${PROJECT}" == "prometheus-engine" && "${BRANCH}" != "release/0.12" ]]; then + release-lib::idemp::manifests_bash_image_bump "${DIR}" + fi + + # Go vulnerabilities. + # TODO(bwplotka): Find better place to put this? + mkdir -p "${DIR}/.gmpctl/" + echo "*" >>"${DIR}/.gmpctl/.gitignore" + vuln_file="${DIR}/.gmpctl/vulnlist.txt" + pushd "${DIR}" + + release-lib::idemp::vulnlist "${DIR}" "${vuln_file}" + + if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then + # Attempt to update + go mod tidy. + release-lib::gomod_vulnfix "${DIR}" "${vuln_file}" + git add go.mod go.sum + + if [ -d "${DIR}/vendor" ]; then + go mod vendor + git add --all # TODO: Can be flaky. + fi + + # Check if that helped. + echo "⚠ī¸ This will fail on older branches with vendoring; in this case, simply go to ${DIR}, run 'go mod vendor' and rerun." + release-lib::vulnlist "${DIR}" "${vuln_file}" + if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then + echo "❌ After go mod update some vulnerabilities are still found; go to ${DIR} and resolve it manually (select not reusing the ./vulnlist.txt file) and rerun." + exit 1 + fi + fi +} + release-lib::idemp::git_commit_amend_match() { # Anything staged? if ! git diff-index --quiet --cached HEAD; then @@ -328,6 +278,7 @@ release-lib::git_commit_amend_match() { fi } +# TODO(bwplotka): Move params to envvars for consistency. release-lib::needs_push() { local branch_to_push="${1}" if [[ -z "${branch_to_push}" ]]; then @@ -363,55 +314,13 @@ release-lib::needs_push() { fi } -release-lib::exclude_changes_from_last_commit() { - local exclude_regexes=$1 - if [[ -z "${exclude_regexes}" ]]; then - log_err "exclude_regexes is required." - return 1 - fi - local include_regexes=$2 - if [[ -z "${include_regexes}" ]]; then - log_err "include_regexes is required." - return 1 - fi - local commit_title=$3 - if [[ -z "${commit_title}" ]]; then - log_err "commit_title is required." - return 1 - fi - - # Get all files touched by a git commit, delimited by space. - changed_files=$(git show --pretty="" --name-only "$(git rev-parse --verify HEAD)") - if [ -z "${changed_files}" ]; then - log_err "suspicious HEAD commit, no files changed." +release-lib::dockerfiles() { + local dir="${1}" + if [[ -z "${dir}" ]]; then + log_err "dir arg is required." return 1 fi - - # Change to \n delimit (needed for grep to work) and exclude/include lines. - tmp_to_exclude=$(echo "${changed_files}" | tr ' ' '\n' | grep -E "${exclude_regexes}" | grep -v -E "${include_regexes}") - - # Group node_module and vendor changes, we know we want to get rid of full directories here -- too many of those files slowing things down and obscuring the summary in git commit -- git restore supports globs. - to_exclude=$(echo "${tmp_to_exclude}" | gsed -e 's|vendor/.*$|vendor/*|' -e 's|node_modules/.*$|node_modules/*|' | sort -u | tr ' ' '\n') - if [ -z "${to_exclude}" ]; then - # Nothing to exclude. - return 0 - fi - - echo "🔄 Excluding the following files from the fork squash commit: ${to_exclude}; appending this information to the git commit message" - curr_msg=$(git log --format=%B -n1) - - # Get all changes to be in stage area. - git reset --soft HEAD~1 - while IFS= read -r exclude_path; do - git restore -S "${exclude_path}" - done <<<"${to_exclude}" - # Commit after unstaging exclusions. - # TODO(bwplotka): Handle nothing to commit after exclusion case. - git commit -m "${commit_title}" -m "${curr_msg}" -m "Excluded files: -${to_exclude} -" - git restore . - git clean -fd + find "${dir}" -name "Dockerfile*" | grep -v "${dir}/third_party/" | grep -v "${dir}/examples/" | grep -v "${dir}/hack/" | grep -v "${dir}/ui/" } # Return all images used in a Dockerfile, delimited by new-line. @@ -452,6 +361,7 @@ release-lib::dockerfile_images_used() { return 0 } +# TODO(bwplotka): Move params to envvars for consistency. release-lib::dockerfile_go_version() { local dockerfile=${1} if [[ -z "${dockerfile}" ]]; then @@ -515,6 +425,7 @@ release-lib::dockerfile_go_version() { return 1 } +# TODO(bwplotka): Move params to envvars for consistency. release-lib::dockerfile_update_image() { local dockerfile=${1} if [[ -z "${dockerfile}" ]]; then @@ -539,8 +450,11 @@ release-lib::dockerfile_update_image() { return 1 fi + # Prerequisite tool. + go install github.com/google/go-containerregistry/cmd/gcrane@latest + # Use gcrane vs crane for --json. - local all_tags=$(go tool gcrane ls "${image}" --json | jq --raw-output '.tags[]' | sort -V) + local all_tags=$(gcrane ls "${image}" --json | jq --raw-output '.tags[]' | sort -V) # Exclude RC images. all_tags=$(echo "${all_tags}" | grep -v "rc.*") # Prefix allows sticking to e.g. latest minor. @@ -564,6 +478,7 @@ release-lib::dockerfile_update_image() { return 0 } +# TODO(bwplotka): Move params to envvars for consistency. release-lib::idemp::manifests_bash_image_bump() { local dir=${1} if [[ -z "${dir}" ]]; then @@ -596,6 +511,7 @@ release-lib::idemp::manifests_bash_image_bump() { return 0 } +# TODO(bwplotka): Move params to envvars for consistency. release-lib::manifests_regen() { local dir=${1} if [[ -z "${dir}" ]]; then @@ -603,13 +519,16 @@ release-lib::manifests_regen() { return 1 fi + # TODO(bwplotka): Manage deps better. It's getting confusing what bins we should use (worktree bingo? script bingo?). + # bingo get is sort of necessary here? source "${dir}/.bingo/variables.env" YQ="${YQ:-}" HELM="${HELM}" ADDLICENSE="${ADDLICENSE:-}" bash "${dir}/hack/presubmit.sh" manifests echo "✅ Manifests regenerated" return 0 } -# Accepts "FORCE_NEW_PATCH_VERSION" +# Also accepts "FORCE_NEW_PATCH_VERSION" envvar. +# TODO(bwplotka): Move params to envvars for consistency. release-lib::next_release_tag() { local dir=${1} if [[ -z "${dir}" ]]; then @@ -672,3 +591,75 @@ release-lib::next_release_tag() { echo "${NEW_TAG}" return 0 } + +function release-lib::pre-release-rc() { + if [[ -z "${DIR}" ]]; then + log_err "DIR envvar is required." + return 1 + fi + + if [[ -z "${BRANCH}" ]]; then + log_err "BRANCH envvar is required." + return 1 + fi + + if [[ -z "${TAG}" ]]; then + log_err "TAG envvar is required." + return 1 + fi + + if [[ -z "${PROJECT}" ]]; then + log_err "PROJECT envvar is required." + return 1 + fi + + if [[ "${PROJECT}" == "prometheus-engine" ]]; then + local CLEAN_TAG="${TAG%-rc.*}" + CLEAN_TAG="${CLEAN_TAG#v}" + if [[ "${BRANCH}" == "release/0.12" ]]; then + # A bit different flow. + local chart_file="${DIR}/charts/operator/Chart.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." + if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + return 1 + fi + + chart_file="${DIR}/charts/rule-evaluator/Chart.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." + if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + return 1 + fi + else + # 0.12+ + local values_file="${DIR}/charts/values.global.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${values_file}..." + if ! gsed -i -E "s#version:.*#version: ${CLEAN_TAG}#g" "${values_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + return 1 + fi + fi + # For versions with export embedded. + if [[ -f "${DIR}/pkg/export/export.go" ]]; then + echo "🔄 Ensuring ${TAG} in ${DIR}/pkg/export/export.go mainModuleVersion..." + if ! gsed -i -E "s#mainModuleVersion = .*#mainModuleVersion = \"${TAG}\"#g" "${DIR}/pkg/export/export.go"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + return 1 + fi + fi + + release-lib::manifests_regen "${DIR}" + git add --all + else + # Prometheus and Alertmanager fork needs just a correct version in the VERSION file, + # so the binary build (go_build_info) metrics and flags are correct. + local temp=${TAG#v} # Remove v and then -rc.* suffix. + echo "${temp%-rc.*}" >VERSION + git add VERSION + fi +} diff --git a/hack/gmpctl/main.go b/hack/gmpctl/main.go new file mode 100644 index 0000000000..42613146a6 --- /dev/null +++ b/hack/gmpctl/main.go @@ -0,0 +1,219 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +var ( + cfgPath = flag.String("c", ".gmpctl.default.yaml", "Path to the configuration file. See config.go#Config for the structure.") + verbose = flag.Bool("v", false, "Enabled verbose, debug output (e.g. logging os.Exec commands)") +) + +type Config struct { + // Directory for the gmpctl work, notably for project clones and git worktrees. + Directory string `yaml:"dir"` +} + +func loadConfig() (ret *Config, _ error) { + b, err := os.ReadFile(*cfgPath) + if err != nil { + return nil, err + } + if err := yaml.Unmarshal(b, &ret); err != nil { + return nil, err + } + ret.Directory, err = filepath.Abs(ret.Directory) + return ret, err +} + +func main() { + flag.Usage = func() { + fmt.Println("Usage: gmpctl [COMMAND] [FLAGS]") + flag.PrintDefaults() + fmt.Print("\n--- Commands ---\n") + fmt.Print("[release] ") + releaseFlags.Usage() + fmt.Println() + fmt.Print("[vulnfix] ") + vulnfixFlags.Usage() + } + + // Help. + for _, cmd := range os.Args[1:] { + if cmd == "-h" || cmd == "--help" { + flag.Usage() + os.Exit(0) + } + } + + flag.Parse() + if flag.NArg() == 0 { + errf("expected at least one argument, got none") + flag.Usage() + os.Exit(1) + } + + var ( + err error + cmd = flag.Arg(0) + ) + switch cmd { + + case "release": + err = release() + case "vulnfix": + err = vulnfix() + default: + errf("Unknown command: %q", cmd) + flag.Usage() + os.Exit(1) + } + + if err != nil { + errf("Command %q failed: %v", cmd, err) + os.Exit(1) + } + successf("Command %q succeded", cmd) + os.Exit(0) +} + +type cmdOpts struct { + // Dir configures the directory that command will be run from. + Dir string + + // Envs configures additional OS environments. + Envs []string + // HideOutputs disables stdout and stderr streaming. + // This is useful for ~porcelain like commands that are meant to pass state via + // stdout. + HideOutputs bool +} + +// libScriptFile is contains useful shell functions. +// Sometimes it's just easier to hack something in bash before porting to Go. +// TODO(bwplotka): go embed this for portability? +const libScriptFile = "lib.sh" + +// getFromLibFunction runs certain function from libScript that's expected to pass a return +// parameter via stdout. +func getFromLibFunction(dir string, envs []string, function string, args ...string) (string, error) { + curr, err := filepath.Abs("") // Hacky. TODO(bwplotka): Improve dir management. + if err != nil { + return "", err + } + libScript := filepath.Join(curr, libScriptFile) + + envs = append(envs, + fmt.Sprintf("SCRIPT_DIR=%v", curr), + ) + return runCommand( + &cmdOpts{Dir: dir, Envs: envs, HideOutputs: true}, + "bash", "-c", fmt.Sprintf(". %v && %v %v", libScript, function, strings.Join(args, " ")), + ) +} + +// runLibFunction runs certain function from libScript that is not expected +// to pass any return parameters. +func runLibFunction(dir string, envs []string, function string, args ...string) error { + curr, err := filepath.Abs("") // Hacky. TODO(bwplotka): Improve dir management. + if err != nil { + return err + } + libScript := filepath.Join(curr, libScriptFile) + + envs = append(envs, fmt.Sprintf("SCRIPT_DIR=%v", curr)) + _, err = runCommand( + &cmdOpts{Dir: dir, Envs: envs, HideOutputs: false}, + "bash", "-c", fmt.Sprintf(". %v && %v %v", libScript, function, strings.Join(args, " ")), + ) + return err +} + +// runCommand executes a command in a specific directory +func runCommand(opts *cmdOpts, args ...string) (string, error) { + if len(args) == 0 { + return "", fmt.Errorf("no command to execute") + } + + if opts == nil { + opts = &cmdOpts{} + } + + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = opts.Dir + if cmd.Dir == "" { + cmd.Dir, _ = filepath.Abs("") + } + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, opts.Envs...) + + if *verbose { + cmd.Env = append(cmd.Env, "DEBUG_MODE=yes") + logf("DEBUG: Executing %q from %q", cmd.String(), cmd.Dir) + } + + var ( + out bytes.Buffer + stderr bytes.Buffer + ) + cmd.Stdin = os.Stdin + cmd.Stdout = &out + cmd.Stderr = &stderr + + if !opts.HideOutputs { + cmd.Stdout = io.MultiWriter(os.Stdout, &out) + cmd.Stderr = io.MultiWriter(os.Stderr, &out) + } + if err := cmd.Run(); err != nil { + if opts.HideOutputs { + return "", fmt.Errorf("%v failed: %s; %s", args, err, stderr.String()) + } + return "", fmt.Errorf("%v failed: %s", args, err) + } + return strings.TrimSpace(out.String()), nil +} + +func logf(msg string, args ...any) { + _, _ = fmt.Fprintf(os.Stderr, "🔄 "+msg, args...) + _, _ = fmt.Fprintln(os.Stderr) +} + +func errf(msg string, args ...any) { + _, _ = fmt.Fprintf(os.Stderr, " ❌ "+msg, args...) + _, _ = fmt.Fprintln(os.Stderr) +} + +func panicf(msg string, args ...any) { + // TODO(bwplotka): Panics are much better for scripting. The alternative + // is a strict wrapping (with extra lib). I'd suggest we panic on things + // we know we never handle errors on. + panic(fmt.Sprintf(" ❌ "+msg, args...)) +} + +func successf(msg string, args ...any) { + _, _ = fmt.Fprintf(os.Stderr, " ✅ "+msg+"!", args...) + _, _ = fmt.Fprintln(os.Stderr) +} diff --git a/hack/vulnupdatelist/.gitignore b/hack/gmpctl/vulnupdatelist/.gitignore similarity index 100% rename from hack/vulnupdatelist/.gitignore rename to hack/gmpctl/vulnupdatelist/.gitignore diff --git a/hack/vulnupdatelist/main.go b/hack/gmpctl/vulnupdatelist/main.go similarity index 87% rename from hack/vulnupdatelist/main.go rename to hack/gmpctl/vulnupdatelist/main.go index a683953ade..3a2e09fb7f 100644 --- a/hack/vulnupdatelist/main.go +++ b/hack/gmpctl/vulnupdatelist/main.go @@ -1,3 +1,17 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // package main implements vulnupdatelist script. // // Run this script to list all the vulnerable Go modules to upgrade. @@ -115,7 +129,7 @@ func ensureGovulncheck(dir string) error { } // runGovulncheck executes `govulncheck -json ./...` and returns the output. -func runGovulncheck(dir string, goVersion string) ([]byte, error) { +func runGovulncheck(dir, goVersion string) ([]byte, error) { cmd := exec.Command("govulncheck", "--format=json", "./...") if goVersion != "" { cmd.Env = append(os.Environ(), "GOVERSION="+goVersion) diff --git a/hack/vulnupdatelist/nvdapi.go b/hack/gmpctl/vulnupdatelist/nvdapi.go similarity index 79% rename from hack/vulnupdatelist/nvdapi.go rename to hack/gmpctl/vulnupdatelist/nvdapi.go index 7d2a8afba6..3a59085f7c 100644 --- a/hack/vulnupdatelist/nvdapi.go +++ b/hack/gmpctl/vulnupdatelist/nvdapi.go @@ -1,3 +1,17 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( @@ -27,7 +41,7 @@ type NVDResponse struct { } // getCVSSSeverity fetches vulnerability details from the NVD API and returns the CVSS V3 severity. -func getCVSSSeverity(apiKey string, cveID string) (string, error) { +func getCVSSSeverity(apiKey, cveID string) (string, error) { // https://nvd.nist.gov/developers/vulnerabilities apiURL := fmt.Sprintf("https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=%s", cveID) diff --git a/hack/gmpctl/vulnupdatelist/nvdapi_test.go b/hack/gmpctl/vulnupdatelist/nvdapi_test.go new file mode 100644 index 0000000000..01b00b25a1 --- /dev/null +++ b/hack/gmpctl/vulnupdatelist/nvdapi_test.go @@ -0,0 +1,32 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetCVEDetails(t *testing.T) { + t.Skip("depends on NVDE API") + + c := getCVEDetails("", OSV{ + ID: "GO-2021-0065", + Aliases: []string{"GHSA-jmrx-5g74-6v2f", "CVE-2019-11250"}, + }) + require.Equal(t, "CVE-2019-11250", c.ID) + require.Equal(t, "MEDIUM", c.Severity) +} diff --git a/hack/vulnupdatelist/vuln.go b/hack/gmpctl/vulnupdatelist/vuln.go similarity index 88% rename from hack/vulnupdatelist/vuln.go rename to hack/gmpctl/vulnupdatelist/vuln.go index 05ed1155c7..35548da9da 100644 --- a/hack/vulnupdatelist/vuln.go +++ b/hack/gmpctl/vulnupdatelist/vuln.go @@ -1,3 +1,17 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( diff --git a/hack/go.mod b/hack/go.mod index e2622b7ab3..decc056ade 100644 --- a/hack/go.mod +++ b/hack/go.mod @@ -1,14 +1,46 @@ module github.com/GoogleCloudPlatform/promethue-engine/hack -go 1.24.0 +go 1.25.0 require ( github.com/Masterminds/semver/v3 v3.3.1 + github.com/charmbracelet/huh v0.8.0 github.com/stretchr/testify v1.10.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/catppuccin/go v0.3.0 // indirect + github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect + github.com/charmbracelet/bubbletea v1.3.6 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.10.1 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/hack/go.sum b/hack/go.sum index 3a1efbd155..253d2e2145 100644 --- a/hack/go.sum +++ b/hack/go.sum @@ -1,12 +1,101 @@ +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= +github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= +github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= +github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= +github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 h1:JFgG/xnwFfbezlUnFMJy0nusZvytYysV4SCS2cYbvws= +github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7/go.mod h1:ISC1gtLcVilLOf23wvTfoQuYbW2q0JevFxPfUzZ9Ybw= +github.com/charmbracelet/bubbletea v1.3.6 h1:VkHIxPJQeDt0aFJIsVxw8BQdh/F/L2KKZGsK6et5taU= +github.com/charmbracelet/bubbletea v1.3.6/go.mod h1:oQD9VCRQFF8KplacJLo28/jofOI2ToOfGYeFgBBxHOc= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/huh v0.8.0 h1:Xz/Pm2h64cXQZn/Jvele4J3r7DDiqFCNIVteYukxDvY= +github.com/charmbracelet/huh v0.8.0/go.mod h1:5YVc+SlZ1IhQALxRPpkGwwEKftN/+OlJlnJYlDRFqN4= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ= +github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= +github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= +github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= +github.com/charmbracelet/x/conpty v0.1.0/go.mod h1:rMFsDJoDwVmiYM10aD4bH2XiRgwI7NYJtQgl5yskjEQ= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 h1:qko3AQ4gK1MTS/de7F5hPGx6/k1u0w4TeYmBFwzYVP4= +github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= +github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= +github.com/charmbracelet/x/xpty v0.1.2 h1:Pqmu4TEJ8KeA9uSkISKMU3f+C1F6OGBn8ABuGlqCbtI= +github.com/charmbracelet/x/xpty v0.1.2/go.mod h1:XK2Z0id5rtLWcpeNiMYBccNNBrP2IJnzHI0Lq13Xzq4= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hack/lib_test.sh b/hack/lib_test.sh deleted file mode 100644 index 11209d0a2b..0000000000 --- a/hack/lib_test.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail -set -o nounset - -if [[ -n "${DEBUG_MODE:-}" ]]; then - set -o xtrace -fi - -SCRIPT_DIR="$( - cd -- "$(dirname "$0")" >/dev/null 2>&1 - pwd -P -)" -source "${SCRIPT_DIR}/test-lib.sh" -source "${SCRIPT_DIR}/lib.sh" - -TEST_FAILED=0 -cleanup() { - if ((TEST_FAILED)); then - echo "${line_break}" - echo "$test_name: FAILED" - echo "${line_break}" - # Print out the directories. - echo "CHECKOUT_DIR=${CHECKOUT_DIR}" - else - rm -rf \ - "${CHECKOUT_DIR:-}" - fi -} - -exit_on_error() { - if [ $? -ne 0 ]; then - echo "Error!" >&2 - exit 1 - fi -} - -setup() { - CHECKOUT_DIR=$(mktemp --tmpdir -d tmpcheckout.XXXXX) - export CHECKOUT_DIR -} - -create_file() { - mkdir -p "$(dirname $1)" - echo "change" >$1 -} -test_exclude_changes_from_last_commit() { - setup - - # Add a fake state and branches to fake repo. - git init "${CHECKOUT_DIR}" - pushd "${CHECKOUT_DIR}" - - git commit --allow-empty -m "COMMIT 1: initial commit" - - create_file cmd/prometheus/main.go - create_file config/config.go - create_file documentation/examples/prometheus-agent.yml - create_file documentation/examples/prometheus.yml - create_file tsdb/some.go - - # Files expected to be excluded. - create_file .github/workflows/some-ci.yaml - create_file README.md - create_file VERSION.md - create_file docs/command-line/prometheus.md - create_file documentation/examples/remote_storage/vendor/abc/d.go - create_file documentation/examples/remote_storage/vendor/abc2/a.ini - create_file go.mod - create_file go.sum - create_file vendor/abc2/a.ini - create_file web/ui/node_modules/webpack/whatever.js - - git add --all - git commit -m "some commit" - - release-lib::exclude_changes_from_last_commit "${RELEASE_LIB_EXCLUDE_RE}" "${RELEASE_LIB_DOCUMENTATION_INCLUDE_RE}" "COMMIT 2: my commit" - exit_on_error - - got="$(git -C "${CHECKOUT_DIR}" log --format=%B --no-decorate)" - expected=$( - cat <>> running unit tests" - go test $(go list ${REPO_ROOT}/... | grep -v e2e | grep -v export/bench | grep -v export/gcm) + go test $(go list ${REPO_ROOT}/... | grep -v e2e | grep -v gmpctl/data | grep -v export/bench | grep -v export/gcm) } reformat() { @@ -128,7 +129,13 @@ reformat() { pushd "${REPO_ROOT}" go fmt ./... popd - ${MDOX} fmt --soft-wraps "${REPO_ROOT}"/*.md "${REPO_ROOT}"/cmd/**/*.md + + pushd "${REPO_ROOT}/hack/" + go mod download # get all deps to avoid garbage output on --help when auto-generating docs. + popd + ${MDOX} fmt --soft-wraps "${REPO_ROOT}"/*.md "${REPO_ROOT}"/cmd/**/*.md "${REPO_ROOT}"/hack/gmpctl/*.md + # TODO: Fix and apply this to all .sh scripts we host. + ${SHFMT} -l -w "${REPO_ROOT}/hack/gmpctl/lib.sh" "${REPO_ROOT}/hack/presubmit.sh" } exit_msg() { diff --git a/hack/release-forksync.sh b/hack/release-forksync.sh deleted file mode 100755 index d191211a9b..0000000000 --- a/hack/release-forksync.sh +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail -set -o nounset - -if [[ -n "${DEBUG_MODE:-}" ]]; then - set -o xtrace -fi - -FORK_COMMIT_FILE=".git/fork-commit-sha.txt" - -# TODO(bwplotka): Clean err on missing deps e.g. gsed. - -SCRIPT_DIR="$( - cd -- "$(dirname "$0")" >/dev/null 2>&1 - pwd -P -)" -source "${SCRIPT_DIR}/lib.sh" - -usage() { - local me - me="${BASH_SOURCE[0]}" - cat <<_EOM -usage: ${me} - -Create a new release of fork against an upstream version (tag) while carrying the patches from the -previous fork version. The process involves an "semantic fork git rebase -i" flow, similar to the normal git rebase -i (see semantic_fork_rebase for details). - -NOTE: The script is idempotent; to force it to recreate local artifacts (e.g. local clones, remote branches it created), remove the artifact you want to recreate. - -Example use: -* SOURCE_BRANCH=release-2.45.3-gmp UPSTREAM_TAG=v2.53.5 CHECKOUT_DIR=~/Repos/tmp-release bash ${me} - -Variables: -* CHECKOUT_DIR (required) - Local working directory e.g. for local clones. -* SOURCE_BRANCH (required) - Fork branch considered as a source for the forked changes. This branch will be semantically rebased on UPSTREAM_TAG. -* UPSTREAM_TAG (required) - Upstream tag to synchronize (rebase) to; this also controls the name of the eventual branch to use for the sync (release-$UPSTREAM_TAG-gmp). -* PR_BRANCH (default: $USER/cut-release-$UPSTREAM_TAG-gmp) - Upstream branch to push to. -_EOM -} - -if (($# > 0)); then - case $1 in - help) - usage - exit 1 - ;; - esac -fi - -if [[ -z "${CHECKOUT_DIR}" ]]; then - echo "❌ CHECKOUT_DIR environment variable is not set." >&2 - usage - exit 1 -fi -if [[ -z "${SOURCE_BRANCH}" ]]; then - echo "❌ SOURCE_BRANCH environment variable is not set." >&2 - usage - exit 1 -fi -if [[ -z "${UPSTREAM_TAG}" ]]; then - echo "❌ UPSTREAM_TAG environment variable is not set." >&2 - usage - exit 1 -fi - -idemp::create_fork_commit() { - if [[ ! -f "${FORK_COMMIT_FILE}" || -z $(cat "${FORK_COMMIT_FILE}") ]]; then - create_fork_commit - else - echo "⚠ī¸ Using existing ${FORK_COMMIT_FILE}" - fi -} - -# create_fork_commit prepare a squashed commit with the core fork changes to cherry-pick. -# It writes the SHA of that commit into the ${FORK_COMMIT_FILE} -# -# This commit excludes all files that are: -# * no longer needed (documentation) -# * can be easily recreated (e.g. by checking out the latest state or the automation like go mod vendor). -# -# Exclusion significantly simplifies the fork sync procedure. -create_fork_commit() { - # Create ephemeral branch from the source of the forked code. - git checkout "${SOURCE_BRANCH}" - git checkout --detach - - # Get the exact upstream tag we base our fork on. - source_tag="${SOURCE_BRANCH#release-}" - source_tag="v${source_tag%-gmp}" - - # Reset it to clean vanilla version. - git reset --hard "${source_tag}" - - # Squash all forked changes into a single commit. - # HEAD@{1} is where the branch was just before the previous command. - # TODO(bwplotka): Handle no change here. - git merge --squash "HEAD@{1}" && git commit --no-edit - - # We could take it as-is but there are trivial changes we know we have to recreate, exclude them. - release-lib::exclude_changes_from_last_commit "${RELEASE_LIB_EXCLUDE_RE}" "${RELEASE_LIB_DOCUMENTATION_INCLUDE_RE}" "google-patch[logic]: required upstream code modifications" - git rev-parse --verify HEAD >"${FORK_COMMIT_FILE}" - git checkout "${PR_BRANCH}" -} - -force_clean_git_local_changes() { - git restore -S . - git restore . - git clean -fd -} - -idemp::recreate_fork_base_files() { - pushd "${DIR}" - if [[ "$(git rev-parse --verify HEAD)" != "$(git rev-list -n 1 "${UPSTREAM_TAG}")" ]]; then - # There are some commits already, check if this step was done ~correctly. - # Get 2 first commits from the upstream tag to HEAD. - commits_state=$(git log --oneline --no-decorate "${UPSTREAM_TAG}"..HEAD | cut -d' ' -f2- | tail -2) - expected="google-patch[libs]: add google fork packages (export and secret) -google-patch[setup]: initial setup" - if [[ "${PROJECT}" != "prometheus" ]]; then - expected="google-patch[setup]: initial setup" - commits_state=$(echo "${commits_state}" | tail -1) - fi - if ! state_diff=$(diff <(echo "${commits_state}") <(echo "${expected}")); then - echo "❌ Unexpected commit state on $(pwd); expected first commits to match -${expected} - -diff: -${state_diff} - -Remove those commits or remove the dir to recreate and rerun." - exit 1 - fi - echo "⚠ī¸ Using existing commit state; looks clean: -${commits_state}" - return 0 - fi - if [ -n "$(git status --porcelain)" ]; then - echo "❌ Unclean git state on $(pwd); clean it (e.g. git clean -fd) or remove the dir to recreate; and rerun." - exit 1 - fi - recreate_fork_base_files -} - -# recreate_fork_base_files creates a few base commits: -# * setup (e.g. removal unnecessary files, documentation, gitignore, CI files, Dockerfile) -# * libs (e.g. any libs with non-conflicting code that we maintain on forks e.g. export pkg). -# -# NOTE: This is interconnected with $RELEASE_LIB_EXCLUDE_RE variable that excludes some of those files -# from cherry-pick to recreate it here. -recreate_fork_base_files() { - echo "🔄 Creating 'google-patch[setup]' on top of the ${UPSTREAM_TAG} state..." - - # Remove unnecessary files, especially documentation - we want ppl to use GCP or upstream Prometheus docs. - rm "MAINTAINERS.md" - rm "CHANGELOG.md" - rm "RELEASE.md" - rm -r ".circleci/" - rm -r ".github/" - rm -r "docs/" - # Remove all but important linked files. - find "documentation" -type f | grep -v -E "${RELEASE_LIB_DOCUMENTATION_INCLUDE_RE}" | xargs rm -- - find "documentation" -type d -empty -delete - - git checkout "${SOURCE_BRANCH}" -- "README.md" - git checkout "${SOURCE_BRANCH}" -- "CONTRIBUTING.md" - git checkout "${SOURCE_BRANCH}" -- ".gcloudignore" - - # Apply our simplified CI. - git checkout "${SOURCE_BRANCH}" -- ".github/" - - # Add our own build pipeline. - # TODO(bwplotka): We could consider Dockerfile.google and removal of vanilla Dockerfile - # This needs fix on Louhi side, so has to be carefully done. - git checkout "${SOURCE_BRANCH}" -- "Dockerfile" - - git add --all - git commit -s -m "google-patch[setup]: initial setup - -Changes: -* Removal of unnecessary files (e.g. documentation) to avoid confusion. -* Replacing README and CONTRIBUTING doc files with our wording. -* Replacing CI scripts with our own. -* Replacing Dockerfile with our own for Google assured build. -* Adding .gcloudignore -" - if [ -n "$(git status --porcelain)" ]; then - echo "❌ Unclean git state on $(pwd); clean it (e.g. git clean -fd) or remove the dir to recreate; and rerun." - exit 1 - fi - if [[ "${PROJECT}" == "prometheus" ]]; then - # After unfork this will change, but currently we have a separate package to maintain on Prometheus fork for: - # * secrets - # * lease - # * GCM export - # Recreate that instead of cherry-picking little commits. - echo "🔄 Creating 'google-patch[libs]' on top of the 'google-patch[setup]' commit..." - if ! git checkout "${SOURCE_BRANCH}" -- "google/"; then - echo "❌ Expected google lib is missing on ${SOURCE_BRANCH}; Perhaps source branch is missing that (too old) or we are in progress of the unforking; double check and update script or fetch it manuall and commit with the 'google-patch[libs]:' prefix" - exit 1 - fi - - git add --all - git commit -s -m "google-patch[libs]: add google fork packages (export and secret) - - Changes: - * Add google directory with export and secret code. - " - fi - # All done, nothing to do. -} - -REMOTE_URL=$(release-lib::remote_url_from_branch "${SOURCE_BRANCH}") -PROJECT=$( - tmp=${REMOTE_URL##*/} - echo "${tmp%.git}" -) -UPSTREAM_REMOTE_URL=$(release-lib::upstream_remote_url "${PROJECT}") -RELEASE_BRANCH="release-${UPSTREAM_TAG#v}-gmp" -PR_BRANCH=${PR_BRANCH:-"${USER}/cut-${RELEASE_BRANCH}"} - -echo "🔄 Assuming ${PROJECT} with remotes {internal: ${REMOTE_URL}, upstream: ${UPSTREAM_REMOTE_URL}}; syncing ${UPSTREAM_TAG} into ${RELEASE_BRANCH} with the internal patches synced to ${PR_BRANCH} from ${SOURCE_BRANCH}" - -DIR="${CHECKOUT_DIR}/${PROJECT}" -release-lib::idemp::clone "${DIR}" "${SOURCE_BRANCH}" "${PR_BRANCH}" - -pushd "${DIR}" - -if ! url=$(git remote get-url upstream 2>/dev/null) || [[ "${url}" != "${UPSTREAM_REMOTE_URL}" ]]; then - git remote add upstream "${UPSTREAM_REMOTE_URL}" -fi - -# Is cherry pick in progress? -if git rev-parse -q --verify CHERRY_PICK_HEAD; then - echo "❌ Cherry pick is in progress on ${DIR}; cd there and fix conflicts manually, then --continue and rerun the script" >&2 - exit 1 -fi - -# Step 1: Prepare ${RELEASE_BRANCH} and ${PR_BRANCH}, both targeting vanilla ${UPSTREAM_TAG} for now. This will be used as a base for a PR with rebased forked functionality. -if ! git fetch upstream "refs/tags/${UPSTREAM_TAG}:refs/tags/${UPSTREAM_TAG}"; then - echo "❌ Failed to fetch ${UPSTREAM_TAG} from the upstream ${UPSTREAM_REMOTE_URL}" >&2 - exit 1 -fi - -if git fetch origin && git rev-parse -q "origin/${RELEASE_BRANCH}" >/dev/null; then - echo "❌ The internal 'origin/${RELEASE_BRANCH}' branch already exists. This means that this script was already run sucessfully or the release branch is live; aborting. - - Remove that remote branch and ${DIR} if you wish to recreate this branch." >&2 - exit 1 -fi - -if git rev-parse -q "${RELEASE_BRANCH}" >/dev/null; then - release_branch_head=$(git rev-parse -q "${RELEASE_BRANCH}") - if [[ "${release_branch_head}" != "$(git rev-list -n 1 "${UPSTREAM_TAG}")" ]]; then - echo "❌ Internal '${RELEASE_BRANCH}' branch already exists and it's not pointing to the upstream ${UPSTREAM_TAG}; aborting. Remove that branch if you wish to resume this script." >&2 - exit 1 - fi -else - git checkout "${UPSTREAM_TAG}" --detach - git checkout -b "${RELEASE_BRANCH}" - - # Switch back to PR_BRANCH and reset it to RELEASE_BRANCH. - git checkout "${PR_BRANCH}" - git reset --hard "${RELEASE_BRANCH}" -fi - -# Step 2: Prepare a squashed commit for core fork changes to cherry-pick. Exclude -# all files that are either no longer needed (documentation) or can be easily recreated -# (e.g. by checking out the latest state or the automation). This significantly simplifies -# the fork sync procedure. -if ! idemp::create_fork_commit "${DIR}"; then - force_clean_git_local_changes - git checkout "${PR_BRANCH}" # Ensure clean state. - exit 1 -fi - -# Step 3: Recreate base fork commits like setup, build and vendor. -git checkout "${PR_BRANCH}" -idemp::recreate_fork_base_files - -# Step 4: Cherry-pick the squashed fork commit -- this will likely conflict and manual intervention and review is needed. - -if [[ "$(git log --oneline --no-decorate -n1 | cut -d' ' -f2- | cut -d':' -f1)" != "google-patch[logic]" ]]; then - echo "🔄 Cherry picking prepared commit from the ${FORK_COMMIT_FILE}..." - if ! git cherry-pick "$(cat "${FORK_COMMIT_FILE}")"; then - echo "❌ Cherry pick found some conflicts (generally expected for this commit). Go to the directory, fix the issues and run 'git cherry-pick --continue', don't change the commit message (!); then rerun the script. Dir: - cd ${DIR} - - NOTE: You can run 'make test' to run all tests. After cherry-pick --continue, you can also do git rebase -i ${UPSTREAM_TAG} and arrange/change commits as you need." >&2 - exit 1 - fi -else - echo "⚠ī¸ google-patch[logic]* is present; assuming cherry-pick was done successfully; proceeding." -fi - -if release-lib::needs_push "${PR_BRANCH}" "${RELEASE_BRANCH}"; then - git --no-pager log --oneline "${RELEASE_BRANCH}"...HEAD - - if release-lib::confirm "About to git push state from ${DIR} to origin/${PR_BRANCH}; also pushing ${RELEASE_BRANCH} with the upstream state; are you sure?"; then - git push origin "${PR_BRANCH}" - git push origin "${RELEASE_BRANCH}" - - # TODO(bwplotka): Use gh to creat this - echo "✅ Sync changes has been pushed on origin/${PR_BRANCH}; the vanilla upstream was pushed to ${RELEASE_BRANCH}; create a PR with origin/${PR_BRANCH} -> ${RELEASE_BRANCH}, ensure CI is passing and get the changes reviewed (especially the cherry-picked 'google-patch[logic]:' commit!)." - echo "ℹī¸ After PR is merged, consider running: -* Vulnerability check/fix: -BRANCH=${RELEASE_BRANCH} CHECKOUT_DIR=${CHECKOUT_DIR} bash ${SCRIPT_DIR}/release-vulnfix.sh -* RC release creation: -BRANCH=${RELEASE_BRANCH} TAG=${UPSTREAM_TAG}-gmp.0-rc.0 CHECKOUT_DIR=${CHECKOUT_DIR} bash ${SCRIPT_DIR}/release-rc.sh" - fi -else - exit 1 -fi diff --git a/hack/release-rc.sh b/hack/release-rc.sh deleted file mode 100644 index 5731a4d5df..0000000000 --- a/hack/release-rc.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail -set -o nounset - -if [[ -n "${DEBUG_MODE:-}" ]]; then - set -o xtrace -fi - -# TODO(bwplotka): Clean err on missing deps e.g. gsed. - -SCRIPT_DIR="$( - cd -- "$(dirname "$0")" >/dev/null 2>&1 - pwd -P -)" -source "${SCRIPT_DIR}/lib.sh" - -usage() { - local me - me="${BASH_SOURCE[0]}" - cat <<_EOM -usage: ${me} - -Release the RC. - -NOTE: The script is idempotent; to force it to recreate local artifacts (e.g. local clones, remote branches it created), remove the artifact you want to recreate. - -Example use: - * BRANCH=release/0.15 TAG=v0.15.4-rc.0 CHECKOUT_DIR=~/Repos/tmp-release ${me} - * BRANCH=release-2.45.3-gmp TAG=v2.45.3-gmp.13-rc.0 CHECKOUT_DIR=~/Repos/tmp-release ${me} - * BRANCH=release-0.27.0-gmp TAG=v0.27.0-gmp.4-rc.0 CHECKOUT_DIR=~/Repos/tmp-release ${me} - -Variables: -* BRANCH (required) - Release branch to work on; Project is auto-detected from this. -* CHECKOUT_DIR or DIR (required) - Local working directory e.g. for local clones. DIR is a working dir, CHECKOUT_DIR sets DIR to CHECKOUT_DIR/ from remote URL. -* TAG (optional) - Tag to release. If empty next tag version will be detected (double check this!) -* FORCE_NEW_PATCH_VERSION (optional) - If not empty, forces a new patch version as a new TAG (if TAG is empty). -_EOM -} - -if (($# > 0)); then - case $1 in - help) - usage - exit 0 - ;; - esac -fi - -# Check if the BRANCH environment variable is set. -if [[ -z "${BRANCH}" ]]; then - echo "❌ BRANCH environment variable is not set." - usage - exit 1 -fi - -REMOTE_URL=$(release-lib::remote_url_from_branch "${BRANCH}") -PROJECT=$( - tmp=${REMOTE_URL##*/} - echo ${tmp%.git} -) -PR_BRANCH=${BRANCH} # Same as branch because we push directly, without PR as per our process. - -echo "🔄 Assuming ${PROJECT} with remote ${REMOTE_URL}; changes will be pushed directly to ${PR_BRANCH}" - -if [[ -z "${CHECKOUT_DIR:-}" && -z "${DIR:-}" ]]; then - echo "❌ CHECKOUT_DIR or DIR environment variable has to be set." - usage - exit 1 -fi -DIR=${DIR:-"${CHECKOUT_DIR}/${PROJECT}"} - -release-lib::idemp::clone "${DIR}" "${BRANCH}" "${PR_BRANCH}" - -pushd "${DIR}" - -if [[ -z "${TAG:-}" ]]; then - TAG=$(release-lib::next_release_tag "${DIR}") - echo "✅ Detected next release tag: ${TAG}" -fi - -if [[ "${PROJECT}" == "prometheus-engine" ]]; then - CLEAN_TAG="${TAG%-rc.*}" - CLEAN_TAG="${CLEAN_TAG#v}" - if [[ "${BRANCH}" == "release/0.12" ]]; then - # A bit different flow. - chart_file="${DIR}/charts/operator/Chart.yaml" - echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." - if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - exit 1 - fi - - chart_file="${DIR}/charts/rule-evaluator/Chart.yaml" - echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." - if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - exit 1 - fi - else - # 0.12+ - values_file="${DIR}/charts/values.global.yaml" - echo "🔄 Ensuring ${CLEAN_TAG} on ${values_file}..." - if ! gsed -i -E "s#version:.*#version: ${CLEAN_TAG}#g" "${values_file}"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - exit 1 - fi - fi - # For versions with export embedded. - if [[ -f "${DIR}/pkg/export/export.go" ]]; then - echo "🔄 Ensuring ${TAG} in ${DIR}/pkg/export/export.go mainModuleVersion..." - if ! gsed -i -E "s#mainModuleVersion = .*#mainModuleVersion = \"${TAG}\"#g" "${DIR}/pkg/export/export.go"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - exit 1 - fi - fi - - release-lib::manifests_regen "${DIR}" - git add --all -else - # Prometheus and Alertmanager fork needs just a correct version in the VERSION file, - # so the binary build (go_build_info) metrics and flags are correct. - temp=${TAG#v} # Remove v and then -rc.* suffix. - echo "${temp%-rc.*}" >VERSION - git add VERSION -fi - -if ! release-lib::confirm "About to create a commit and a local git tag for ${TAG} in ${DIR} on ${PR_BRANCH}; should I continue?"; then - exit 1 -fi - -# Commit if anything is staged. -release-lib::idemp::git_commit_amend_match "chore: prepare for ${TAG} release" - -# Check if tag exists. -if git rev-parse -q --verify "refs/tags/${TAG}" >/dev/null; then - # Tag exists, but is it tagged for the current HEAD? - if [[ "$(git rev-parse HEAD)" != "$(git rev-list -n 1 "${TAG}")" ]]; then - echo "❌ Tag ${TAG} exists already locally, not pointing to the HEAD; consider 'git tag -d' to remove it and rerun." - exit 1 - fi -else - echo "🔄 Creating a signed tag ${TAG}..." - # explicit TTY is often needed on Macs. - # TODO(bwplotka): Consider adding v0.x second tag for Prometheus fork (similar to how v0.300 Prometheus releases are structured). - # This is to have a little bit cleaner prometheus-engine go.mod version against the fork. - GPG_TTY=$(tty) git tag -s "${TAG}" -m "${TAG}" -fi - -if release-lib::needs_push "${PR_BRANCH}" "${BRANCH}" || ! git ls-remote --tags --exit-code origin "refs/tags/${TAG}" >/dev/null; then - if release-lib::confirm "About to git push state from ${DIR} to origin/${PR_BRANCH}; then ${TAG}; are you sure?"; then - git push origin "${PR_BRANCH}" - git push origin "${TAG}" - fi -else - exit 1 -fi diff --git a/hack/release-vulnfix.sh b/hack/release-vulnfix.sh deleted file mode 100755 index d02a1bbe0c..0000000000 --- a/hack/release-vulnfix.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail -set -o nounset - -if [[ -n "${DEBUG_MODE:-}" ]]; then - set -o xtrace -fi - -# TODO(bwplotka): Clean err on missing deps e.g. gsed. -# TODO(bwplotka): Consider automation for npm and docker images (Go, debian, similar to bump-go.sh) - -SCRIPT_DIR="$( - cd -- "$(dirname "$0")" >/dev/null 2>&1 - pwd -P -)" -source "${SCRIPT_DIR}/lib.sh" - -usage() { - local me - me="${BASH_SOURCE[0]}" - cat <<_EOM -usage: ${me} - -Attempt a minimal dependency upgrade to solve fixable vulnerabilities. - -* Docker images: - * Distros use latest tag so rebuilding takes latest, nothing to do. - * google-go.pkg.dev/golang images are updated to the latest minor version using docker-bump-images.sh -* Manifests - * distroless bumped to latest (although our component tooling is capable of bumpting this too) -* Go deps: Upgrade to minimal required version per a known fixable vulnerability. -* Npm deps: Not implemented. - -NOTE: The script is idempotent; to force it to recreate local artifacts (e.g. local clones, remote branches it created), remove the artifact you want to recreate. - -Example use: - * BRANCH=release/0.15 CHECKOUT_DIR=~/Repos/tmp-release ${me} - * BRANCH=release-2.45.3-gmp CHECKOUT_DIR=~/Repos/tmp-release ${me} - * BRANCH=release-0.27.0-gmp CHECKOUT_DIR=~/Repos/tmp-release ${me} - -Variables: -* BRANCH (required) - Release branch to work on; Project is auto-detected from this. -* CHECKOUT_DIR or DIR (required) - Local working directory e.g. for local clones. DIR is a working dir, CHECKOUT_DIR sets DIR to CHECKOUT_DIR/ from remote URL. -* PR_BRANCH (default: USER/BRANCH-vulnfix) - Upstream branch to push to (user-confirmed first). -* SYNC_DOCKERFILES_FROM - optional branch name to sync manifests for each dockerfile. -_EOM -} - -if (($# > 0)); then - case $1 in - help) - usage - exit 0 - ;; - esac -fi - -# Check if the BRANCH environment variable is set. -if [[ -z "${BRANCH}" ]]; then - echo "❌ BRANCH environment variable is not set." - usage - exit 1 -fi - -REMOTE_URL=$(release-lib::remote_url_from_branch "${BRANCH}") -PROJECT=$( - tmp=${REMOTE_URL##*/} - echo ${tmp%.git} -) -PR_BRANCH=${PR_BRANCH:-"${USER}/${BRANCH}-vulnfix"} - -echo "🔄 Assuming ${PROJECT} with remote ${REMOTE_URL}; changes will be pushed to ${PR_BRANCH}" - -if [[ -z "${CHECKOUT_DIR:-}" && -z "${DIR:-}" ]]; then - echo "❌ CHECKOUT_DIR or DIR environment variable has to be set." - usage - exit 1 -fi -DIR=${DIR:-"${CHECKOUT_DIR}/${PROJECT}"} - -release-lib::idemp::clone "${DIR}" "${BRANCH}" "${PR_BRANCH}" - -readarray -t DOCKERFILES < <(release-lib::dockerfiles "${DIR}") - -# Sync dockerfiles if needed. -if [[ -n "${SYNC_DOCKERFILES_FROM:-}" ]]; then - pushd "${DIR}" - for dockerfile in "${DOCKERFILES[@]}"; do - # TODO: Should we ensure SYNC_DOCKERFILES_FROM if it's a branch is up to data with origin? - echo "🔄 Syncing ${dockerfile} from ${SYNC_DOCKERFILES_FROM}" - git checkout "${SYNC_DOCKERFILES_FROM}" -- "${dockerfile}" - done - popd -fi - -# Docker images bumps. - -# Get first dockerfile Go version. We will use this version to find minor version to stick to. -go_version=$(release-lib::dockerfile_go_version "${DOCKERFILES[0]}") -if [[ -z "${go_version}" ]]; then - echo "❌ can't find any golang image in ${DOCKERFILES[0]}" - exit 1 -fi - -# TODO: git add charts & vendor for old projects. - -# Update our images. -for dockerfile in "${DOCKERFILES[@]}"; do - release-lib::dockerfile_update_image "${dockerfile}" "google-go.pkg.dev/golang" $(echo "${go_version}" | cut -d '.' -f 1-2) - release-lib::dockerfile_update_image "${dockerfile}" "gke.gcr.io/gke-distroless/libc" "gke_distroless_" - pushd "${DIR}" - git add "${dockerfile}" - popd -done - -# bash manifest bump. -# Exclude 0.12 as values were inlined with each part, easy to manually sed for old versions. -if [[ "${PROJECT}" == "prometheus-engine" && "${BRANCH}" != "release/0.12" ]]; then - release-lib::idemp::manifests_bash_image_bump "${DIR}" -fi - -# Go vulnerabilities. -vuln_file="${DIR}/.git/vulnlist.txt" -pushd "${DIR}" - -release-lib::idemp::vulnlist "${DIR}" "${vuln_file}" - -if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then - # Attempt to update + go mod tidy. - release-lib::gomod_vulnfix "${DIR}" "${vuln_file}" - git add go.mod go.sum - - # Check if that helped. - echo "⚠ī¸ This will fail on older branches with vendoring; in this case, simply go to ${DIR}, run 'go mod vendor' and rerun." - release-lib::vulnlist "${DIR}" "${vuln_file}" - if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then - echo "❌ After go mod update some vulnerabilities are still found; go to ${DIR} and resolve it manually and remove the ./vulnlist.txt file and rerun." - exit 1 - fi -fi - -# TODO: Warn of unstaged files at this point. - -# Commit if anything is staged. -msg="google patch[deps]: fix ${BRANCH} vulnerabilities" -if [[ "${PROJECT}" == "prometheus-engine" ]]; then - msg="fix: fix ${BRANCH} vulnerabilities" -fi -release-lib::idemp::git_commit_amend_match "${msg}" - -if release-lib::needs_push "${PR_BRANCH}" "${BRANCH}"; then - if release-lib::confirm "About to FORCE git push from ${DIR} to origin/${PR_BRANCH}; are you sure?"; then - git push --force origin "${PR_BRANCH}" - fi -else - exit 1 -fi diff --git a/hack/test-lib.sh b/hack/test-lib.sh deleted file mode 100644 index 8c8e0f77c0..0000000000 --- a/hack/test-lib.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o pipefail -set -o nounset - -if [[ -n "${DEBUG_MODE:-}" ]]; then - set -o xtrace -fi - -_assert_equal() { - got="${1}" - expected="${2}" - - if [[ "${got}" == "${expected}" ]]; then - return - fi - - echo "assertion failure:" - cat <>>>>>> EXPECTED -EOF - - return 1 -} - -_assert_pass() { - if eval "$1"; then - return - fi - - return 1 -} - -_assert_fail() { - if ! eval "$1"; then - return - fi - - return 1 -} - -# Sanity check on pass/fail helpers. -_assert_pass true || exit 1 -_assert_fail false || exit 1 - -_assert_fail "_assert_pass false" || exit 1 -_assert_fail "_assert_fail true" || exit 1 - -_assert_pass "_assert_equal 1 1" || exit 1 -echo "NOTE: Expected failure output below on _assert_fail (exit code 0 though)" -_assert_fail "_assert_equal 1 2" || exit 1 - -# Stronger assertion helpers that exit the program on assertion failure. -assert_pass() { _assert_pass "$@" || exit 1; } -assert_fail() { _assert_fail "$@" || exit 1; } -assert_equal() { _assert_equal "$@" || exit 1; } diff --git a/hack/vulnupdatelist/nvdapi_test.go b/hack/vulnupdatelist/nvdapi_test.go deleted file mode 100644 index a350f4d1b5..0000000000 --- a/hack/vulnupdatelist/nvdapi_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestGetCVEDetails(t *testing.T) { - t.Skip("depends on NVDE API") - - c := getCVEDetails("", OSV{ - ID: "GO-2021-0065", - Aliases: []string{"GHSA-jmrx-5g74-6v2f", "CVE-2019-11250"}, - }) - require.Equal(t, "CVE-2019-11250", c.ID) - require.Equal(t, "MEDIUM", c.Severity) -} From d61c0bbe63cf404aaeba92b30456aa3541151bcd Mon Sep 17 00:00:00 2001 From: bwplotka Date: Thu, 11 Dec 2025 15:42:24 +0000 Subject: [PATCH 3/3] chore: move gmpctl to ops/ --- .dockerignore | 1 + hack/Dockerfile | 1 + hack/gmpctl.sh | 7 +- hack/presubmit.sh | 24 ++- ops/README.md | 7 + ops/gmpctl.sh | 37 ++++ {hack => ops}/gmpctl/.gitignore | 0 {hack => ops}/gmpctl/.gmpctl.default.yaml | 0 {hack => ops}/gmpctl/README.md | 23 ++- {hack => ops}/gmpctl/cmd_release.go | 24 ++- {hack => ops}/gmpctl/cmd_vulnfix.go | 14 +- {hack => ops}/gmpctl/dialog.go | 0 {hack => ops}/gmpctl/git.go | 21 ++ {hack => ops}/gmpctl/gmp.go | 22 +- {hack => ops/gmpctl}/go.mod | 25 ++- {hack => ops/gmpctl}/go.sum | 58 +++++- {hack => ops}/gmpctl/lib.sh | 193 ++---------------- {hack => ops}/gmpctl/main.go | 18 +- ops/gmpctl/prep-rc.sh | 104 ++++++++++ ops/gmpctl/vulnfix.sh | 124 +++++++++++ .../gmpctl/vulnupdatelist/.gitignore | 0 {hack => ops}/gmpctl/vulnupdatelist/main.go | 0 {hack => ops}/gmpctl/vulnupdatelist/nvdapi.go | 0 .../gmpctl/vulnupdatelist/nvdapi_test.go | 0 {hack => ops}/gmpctl/vulnupdatelist/vuln.go | 0 25 files changed, 481 insertions(+), 222 deletions(-) create mode 100644 ops/README.md create mode 100755 ops/gmpctl.sh rename {hack => ops}/gmpctl/.gitignore (100%) rename {hack => ops}/gmpctl/.gmpctl.default.yaml (100%) rename {hack => ops}/gmpctl/README.md (83%) rename {hack => ops}/gmpctl/cmd_release.go (82%) rename {hack => ops}/gmpctl/cmd_vulnfix.go (91%) rename {hack => ops}/gmpctl/dialog.go (100%) rename {hack => ops}/gmpctl/git.go (84%) rename {hack => ops}/gmpctl/gmp.go (86%) rename {hack => ops/gmpctl}/go.mod (63%) rename {hack => ops/gmpctl}/go.sum (65%) rename {hack => ops}/gmpctl/lib.sh (70%) rename {hack => ops}/gmpctl/main.go (87%) create mode 100644 ops/gmpctl/prep-rc.sh create mode 100644 ops/gmpctl/vulnfix.sh rename {hack => ops}/gmpctl/vulnupdatelist/.gitignore (100%) rename {hack => ops}/gmpctl/vulnupdatelist/main.go (100%) rename {hack => ops}/gmpctl/vulnupdatelist/nvdapi.go (100%) rename {hack => ops}/gmpctl/vulnupdatelist/nvdapi_test.go (100%) rename {hack => ops}/gmpctl/vulnupdatelist/vuln.go (100%) diff --git a/.dockerignore b/.dockerignore index faa0312e4c..647755ec3a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ !examples !manifests !hack +!ops !pkg !e2e !third_party diff --git a/hack/Dockerfile b/hack/Dockerfile index d2ddeff8c9..bb6e0fab19 100644 --- a/hack/Dockerfile +++ b/hack/Dockerfile @@ -35,6 +35,7 @@ WORKDIR /workspace COPY *.md *.md COPY vendor* vendor COPY hack hack +COPY ops ops COPY .bingo .bingo COPY go.mod go.mod COPY go.sum go.sum diff --git a/hack/gmpctl.sh b/hack/gmpctl.sh index 607a483126..fcc937b992 100755 --- a/hack/gmpctl.sh +++ b/hack/gmpctl.sh @@ -26,9 +26,4 @@ SCRIPT_DIR="$( pwd -P )" -pushd "${SCRIPT_DIR}/gmpctl" >/dev/null -# NOTE gmpctl expects the whole gmpctl directory to be present. -# We could consider embedding bash scripts, config into binary, but it's good -# for now. -go run ./ "$@" -popd >/dev/null +bash "${SCRIPT_DIR}/../ops/gmpctl.sh" "$@" diff --git a/hack/presubmit.sh b/hack/presubmit.sh index ef6699f636..d2bfbdd872 100755 --- a/hack/presubmit.sh +++ b/hack/presubmit.sh @@ -38,7 +38,7 @@ warn() { echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: Warning: $*" >&2 } -REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +REPO_ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd) CRD_DIR=${REPO_ROOT}/charts/operator/crds SED=$(which gsed 2>/dev/null || which sed) @@ -77,7 +77,7 @@ combine() { update_crdgen() { echo ">>> regenerating CRD yamls" API_DIR=${REPO_ROOT}/pkg/operator/apis/... - ${CONTROLLER_GEN} crd paths=./${API_DIR} output:crd:dir=${CRD_DIR} + ${CONTROLLER_GEN} crd paths=${API_DIR} output:crd:dir=${CRD_DIR} CRD_YAMLS=$(find ${CRD_DIR} -iname '*.yaml' | sort) for i in $CRD_YAMLS; do @@ -124,18 +124,28 @@ run_tests() { } reformat() { - echo ">>> reformatting" - go mod tidy + find . -name "go.mod" | grep -v gmpctl/data | grep -v ".bingo" | while read -r file; do + dir=$(dirname "$file") + pushd "${dir}" + echo ">>> go mod tidy $dir" + go mod tidy + popd + done + + echo ">>> formatting Go files" pushd "${REPO_ROOT}" go fmt ./... popd - pushd "${REPO_ROOT}/hack/" + echo ">>> formatting docs" + pushd "${REPO_ROOT}/ops/gmpctl" go mod download # get all deps to avoid garbage output on --help when auto-generating docs. popd - ${MDOX} fmt --soft-wraps "${REPO_ROOT}"/*.md "${REPO_ROOT}"/cmd/**/*.md "${REPO_ROOT}"/hack/gmpctl/*.md + ${MDOX} fmt --soft-wraps "${REPO_ROOT}"/*.md "${REPO_ROOT}"/cmd/**/*.md "${REPO_ROOT}"/ops/gmpctl/*.md + + echo ">>> formatting bash scripts" # TODO: Fix and apply this to all .sh scripts we host. - ${SHFMT} -l -w "${REPO_ROOT}/hack/gmpctl/lib.sh" "${REPO_ROOT}/hack/presubmit.sh" + ${SHFMT} -l -w "${REPO_ROOT}/ops/gmpctl/lib.sh" "${REPO_ROOT}/hack/presubmit.sh" } exit_msg() { diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 0000000000..7954a007d3 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,7 @@ +### ops + +This directory contains scripts and automation for common GMP dev operations. + +Resources in `/ops` are meant to be used in the latest form (the `main` branch). Notably, +when using those scripts in CI, use the latest version in main branch. + diff --git a/ops/gmpctl.sh b/ops/gmpctl.sh new file mode 100755 index 0000000000..794a7a0828 --- /dev/null +++ b/ops/gmpctl.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +SCRIPT_DIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" + +# NOTE gmpctl expects the gmpctl directory to be present on execution for +# local bash scripts and configuration. +# +# We could consider embedding bash scripts, config into binary for portability, +# but it's good enough for now. + +pushd "${SCRIPT_DIR}/gmpctl" >/dev/null +go run ./ "$@" +popd >/dev/null diff --git a/hack/gmpctl/.gitignore b/ops/gmpctl/.gitignore similarity index 100% rename from hack/gmpctl/.gitignore rename to ops/gmpctl/.gitignore diff --git a/hack/gmpctl/.gmpctl.default.yaml b/ops/gmpctl/.gmpctl.default.yaml similarity index 100% rename from hack/gmpctl/.gmpctl.default.yaml rename to ops/gmpctl/.gmpctl.default.yaml diff --git a/hack/gmpctl/README.md b/ops/gmpctl/README.md similarity index 83% rename from hack/gmpctl/README.md rename to ops/gmpctl/README.md index c4a588fb69..b005a2191a 100644 --- a/hack/gmpctl/README.md +++ b/ops/gmpctl/README.md @@ -14,7 +14,13 @@ It's a starting point for smaller or bigger automation on OSS side (e.g. releasi 2. The next this is to obtain NVD API key to avoid rate-limits when querying CVE DB. See https://nvd.nist.gov/developers/request-an-api-key and save this key to `hack/vulnupdatelist/api.text` -3. You can configure different work directory for gmpctl via `-c` flag. By default, `gmpctl` does the work in `hack/gmpctl/.data`) +3. Ensure you have installed: + * new-ish `bash` (MacOS: `brew install bash`) + * `gsed` (MacOS: `brew install gsed`) + * `gcloud` (https://docs.cloud.google.com/sdk/docs/install-sdk) (and `gcloud auth login`) + * `gpg` (MacOS: `brew install gpg`) + +4. You can configure different work directory for gmpctl via `-c` flag. By default, `gmpctl` does the work in `hack/gmpctl/.data`) Enjoy! @@ -45,6 +51,8 @@ on breaking go mod updates for vulnerabilities or fork sync conflicts. Usage: gmpctl [COMMAND] [FLAGS] -c string Path to the configuration file. See config.go#Config for the structure. (default ".gmpctl.default.yaml") + -git.prefer-https + If true, uses HTTPS protocol instead of git for remote URLs. -v Enabled verbose, debug output (e.g. logging os.Exec commands) --- Commands --- @@ -98,10 +106,11 @@ Some rules to follow: * Ensure all error messages are redirected to stderr, use log_err func for this. * Be careful with pushd/popd which log to stdout, you can redirect those to stderr too. -## TODO +## TODO / Known issues. -* Ability to configure NVD API key in gmpctl config. -* Port fork-sync script from the old PR. -* Generate some on-demand query of vulnerabilities for all releases (aka dashboard.) -* Fix NPM vulns (although it's rate). -* Ability to schedule multiple scripts at once and managing that? (lot's of work vs multiple terminals) +* [ ] Port bash to Go for stable commands. +* [ ] Ability to configure NVD API key in gmpctl config. +* [ ] Port fork-sync script from the old PR. +* [ ] Generate some on-demand query of vulnerabilities for all releases (aka dashboard.) +* [ ] Fix NPM vulns (although it's rate). +* [ ] Ability to schedule multiple scripts at once and managing that? (lot's of work vs multiple terminals) diff --git a/hack/gmpctl/cmd_release.go b/ops/gmpctl/cmd_release.go similarity index 82% rename from hack/gmpctl/cmd_release.go rename to ops/gmpctl/cmd_release.go index cf851040fd..1f3f6b62a7 100644 --- a/hack/gmpctl/cmd_release.go +++ b/ops/gmpctl/cmd_release.go @@ -49,7 +49,7 @@ func release() error { return fmt.Errorf("couldn't find project from branch %s", branch) } - logf("Assuming %q with remote %q; branch to release: %q", proj.Name, proj.RemoteURL, branch) + logf("Assuming %q with remote %q; branch to release: %q", proj.Name, proj.RemoteURL(), branch) dir := proj.WorkDir(cfg.Directory, branch, "release") mustFetchAll(dir) @@ -66,35 +66,39 @@ func release() error { } logf("Selected %v tag", tag) - if err := runLibFunction(dir, []string{ + if err := runLocalBash(dir, []string{ fmt.Sprintf("DIR=%v", dir), fmt.Sprintf("BRANCH=%v", branch), fmt.Sprintf("PROJECT=%v", proj.Name), fmt.Sprintf("TAG=%v", tag), - }, "release-lib::pre-release-rc"); err != nil { + }, "prep-rc.sh"); err != nil { return err } msg := fmt.Sprintf("chore: prepare for %v release", tag) // TODO(bwplotka): Port to Go, make it more reliable. // TODO(bwplotka): Quote otherwise it's split into separate args... port it so it works better (: + // TODO(bwplotka): Add message about a script command. if err := runLibFunction(dir, nil, "release-lib::idemp::git_commit_amend_match", "\""+msg+"\""); err != nil { return err } + if !mustIsRemoteUpToDate(dir, branch) { + if confirmf("About to git push state from %q to \"origin/%v\" for %q tag; are you sure?", dir, branch, tag) { + // We are in detached state, so use the HEAD reference. + mustPush(dir, fmt.Sprintf("HEAD:%v", branch)) + } else { + return errors.New("aborting") + } + } + // TODO(bwplotka): Check if tag exists. mustCreateSignedTag(dir, tag) - - // TODO check if anything is needed to push? - // TODO(bwplotka): Add option to print more debug/open terminal with the workdir? - if confirmf("About to git push state from %q to \"origin/%v\"; then push %q tag; are you sure?", dir, branch, tag) { - // We are in detached state - mustPush(dir, fmt.Sprintf("HEAD:%v", branch)) + if confirmf("About to git push %q tag from %q to \"origin/%v\"; are you sure?", tag, dir, branch) { mustPush(dir, tag) } else { return errors.New("aborting") } - if confirmf("Do you want to remove the %v worktree (recommended)?", dir) { proj.RemoveWorkDir(cfg.Directory, dir) } diff --git a/hack/gmpctl/cmd_vulnfix.go b/ops/gmpctl/cmd_vulnfix.go similarity index 91% rename from hack/gmpctl/cmd_vulnfix.go rename to ops/gmpctl/cmd_vulnfix.go index 244b0bb9fd..ae10c633c7 100644 --- a/hack/gmpctl/cmd_vulnfix.go +++ b/ops/gmpctl/cmd_vulnfix.go @@ -68,7 +68,7 @@ func vulnfix() error { prBranch = fmt.Sprintf("%v/%v-gmpctl-vulnfix", os.Getenv("USER"), branch) } - logf("Assuming %q with remote %q; on %q; changes will be pushed to %q", proj.Name, proj.RemoteURL, branch, prBranch) + logf("Assuming %q with remote %q; on %q; changes will be pushed to %q", proj.Name, proj.RemoteURL(), branch, prBranch) dir := proj.WorkDir(cfg.Directory, branch, "vulnfix") // Refresh. @@ -84,7 +84,7 @@ func vulnfix() error { } // TODO(bwplotka): Add NPM vulnfix. - if err := runLibFunction(dir, opts, "release-lib::vulnfix"); err != nil { + if err := runLocalBash(dir, opts, "vulnfix.sh"); err != nil { return err } @@ -100,17 +100,21 @@ func vulnfix() error { if err := runLibFunction(dir, nil, "release-lib::idemp::git_commit_amend_match", "\""+msg+"\""); err != nil { return err } - // TODO(bwplotka): Check if needs pushing? + + if mustIsRemoteUpToDate(dir, branch) { + return fmt.Errorf("nothing to push from %q to \"origin/%v\"; aborting", dir, prBranch) + } + // TODO(bwplotka): Add option to print more debug/open terminal with the workdir? if confirmf("About to FORCE git push state from %q to \"origin/%v\"; are you sure?", dir, prBranch) { - // We are in detached state, so be explicit what to push and from where. + // We are in detached state, so be explicit what to push and from where, by recreating the local prBranch. mustRecreateBranch(dir, prBranch) mustForcePush(dir, prBranch) } else { return errors.New("aborting") } - if confirmf("Do you want to remove the %v worktree (recommended)?", dir) { + if confirmf("Do you want to remove the %v worktree?", dir) { proj.RemoveWorkDir(cfg.Directory, dir) } return nil diff --git a/hack/gmpctl/dialog.go b/ops/gmpctl/dialog.go similarity index 100% rename from hack/gmpctl/dialog.go rename to ops/gmpctl/dialog.go diff --git a/hack/gmpctl/git.go b/ops/gmpctl/git.go similarity index 84% rename from hack/gmpctl/git.go rename to ops/gmpctl/git.go index 962bb387e3..4de71c9dbe 100644 --- a/hack/gmpctl/git.go +++ b/ops/gmpctl/git.go @@ -16,6 +16,7 @@ package main import ( "fmt" + "strings" ) func mustCloneRepo(repoURL, destinationDir string) { @@ -63,6 +64,26 @@ func mustCreateSignedTag(dir, tag string) { } } +// mustIsRemoteUpToDate returns true if HEAD points to the same commit as +// the origin branch +func mustIsRemoteUpToDate(dir, branch string) bool { + // Fetch to ensure we have the latest remote state. + mustFetchAll(dir) + + // Get the commit hash of the local HEAD. + localHead, err := runCommand(&cmdOpts{Dir: dir, HideOutputs: true}, "git", "rev-parse", "HEAD") + if err != nil { + panicf(err.Error()) + } + + // Get the commit hash of the remote branch. + remoteHead, err := runCommand(&cmdOpts{Dir: dir, HideOutputs: true}, "git", "rev-parse", "origin/"+branch) + if err != nil { + panicf(err.Error()) + } + return strings.TrimSpace(localHead) == strings.TrimSpace(remoteHead) +} + func mustPush(dir, what string) { logf("Pushing %v...", what) if _, err := runCommand(&cmdOpts{Dir: dir}, "git", "push", "origin", what); err != nil { diff --git a/hack/gmpctl/gmp.go b/ops/gmpctl/gmp.go similarity index 86% rename from hack/gmpctl/gmp.go rename to ops/gmpctl/gmp.go index 6e2d515a63..f70c1ed32c 100644 --- a/hack/gmpctl/gmp.go +++ b/ops/gmpctl/gmp.go @@ -26,17 +26,17 @@ import ( var ( Prometheus = Project{ Name: "prometheus", - RemoteURL: "git@github.com:GoogleCloudPlatform/prometheus.git", + remoteURL: "git@github.com:GoogleCloudPlatform/prometheus.git", BranchRE: regexp.MustCompile(`^release-[23]\.[0-9]+\.[0-9]+-gmp$`), } Alertmanager = Project{ Name: "alertmanager", - RemoteURL: "git@github.com:GoogleCloudPlatform/alertmanager.git", + remoteURL: "git@github.com:GoogleCloudPlatform/alertmanager.git", BranchRE: regexp.MustCompile(`^release-0\.[0-9]+\.[0-9]+-gmp$`), } PrometheusEngine = Project{ Name: "prometheus-engine", - RemoteURL: "git@github.com:GoogleCloudPlatform/prometheus-engine.git", + remoteURL: "git@github.com:GoogleCloudPlatform/prometheus-engine.git", BranchRE: regexp.MustCompile(`^release/0\.[0-9]+$`), } @@ -66,7 +66,7 @@ func projectFromBranch(branch string) (Project, bool) { type Project struct { Name string - RemoteURL string + remoteURL string BranchRE *regexp.Regexp } @@ -85,8 +85,8 @@ func (p Project) cloneDir(dir string) (cloneDir string) { if !errors.Is(err, os.ErrNotExist) { panicf("failed to stat %s: %v", cloneDir, err) } - logf("Cloning %q into %q", p.RemoteURL, cloneDir) - mustCloneRepo(p.RemoteURL, cloneDir) + logf("Cloning %q into %q", p.RemoteURL(), cloneDir) + mustCloneRepo(p.RemoteURL(), cloneDir) return cloneDir } @@ -96,6 +96,16 @@ func (p Project) workDir(dir, branch, suffix string) string { return filepath.Join(dir, p.Name, subDir) } +func (p Project) RemoteURL() string { + if *gitPreferHTTPS { + return "https://" + + strings.TrimSuffix( + strings.TrimPrefix(strings.ReplaceAll(p.remoteURL, ":", "/"), "git@"), + ".git") + } + return p.remoteURL +} + // WorkDir returns a new working directory. func (p Project) WorkDir(dir, branch, suffix string) (workDir string) { cloneDir := p.cloneDir(dir) diff --git a/hack/go.mod b/ops/gmpctl/go.mod similarity index 63% rename from hack/go.mod rename to ops/gmpctl/go.mod index decc056ade..4a03352c99 100644 --- a/hack/go.mod +++ b/ops/gmpctl/go.mod @@ -1,4 +1,4 @@ -module github.com/GoogleCloudPlatform/promethue-engine/hack +module github.com/GoogleCloudPlatform/promethue-engine/ops/gmpctl go 1.25.0 @@ -10,6 +10,7 @@ require ( ) require ( + cloud.google.com/go/compute/metadata v0.7.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/catppuccin/go v0.3.0 // indirect @@ -22,25 +23,43 @@ require ( github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.18.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/cli v29.0.3+incompatible // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/go-containerregistry v0.20.7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.10.1 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/vbatts/tar-split v0.12.2 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.36.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.29.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) + +tool github.com/google/go-containerregistry/cmd/gcrane diff --git a/hack/go.sum b/ops/gmpctl/go.sum similarity index 65% rename from hack/go.sum rename to ops/gmpctl/go.sum index 253d2e2145..acf4862dfd 100644 --- a/hack/go.sum +++ b/ops/gmpctl/go.sum @@ -1,3 +1,5 @@ +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= @@ -40,15 +42,33 @@ github.com/charmbracelet/x/xpty v0.1.2 h1:Pqmu4TEJ8KeA9uSkISKMU3f+C1F6OGBn8ABuGl github.com/charmbracelet/x/xpty v0.1.2/go.mod h1:XK2Z0id5rtLWcpeNiMYBccNNBrP2IJnzHI0Lq13Xzq4= github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/containerd/stargz-snapshotter/estargz v0.18.1 h1:cy2/lpgBXDA3cDKSyEfNOFMA/c10O1axL69EU7iirO8= +github.com/containerd/stargz-snapshotter/estargz v0.18.1/go.mod h1:ALIEqa7B6oVDsrF37GkGN20SuvG/pIMm7FwP7ZmRb0Q= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v29.0.3+incompatible h1:8J+PZIcF2xLd6h5sHPsp5pvvJA+Sr2wGQxHkRl53a1E= +github.com/docker/cli v29.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.20.7 h1:24VGNpS0IwrOZ2ms2P1QE3Xa5X9p4phx0aUgzYzHW6I= +github.com/google/go-containerregistry v0.20.7/go.mod h1:Lx5LCZQjLH1QBaMPeGwsME9biPeo1lPx6lbGj/UmzgM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -64,6 +84,8 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -72,7 +94,13 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -80,22 +108,44 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= +github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/hack/gmpctl/lib.sh b/ops/gmpctl/lib.sh similarity index 70% rename from hack/gmpctl/lib.sh rename to ops/gmpctl/lib.sh index dea00ab263..351193c2b7 100755 --- a/hack/gmpctl/lib.sh +++ b/ops/gmpctl/lib.sh @@ -122,6 +122,8 @@ release-lib::vulnlist() { popd } +# TODO(bwplotka): There's so much more we can do to guide us here. +# e.g. not updating too much, updating k8s and prom deps in bulk etc. release-lib::gomod_vulnfix() { local dir=${1} if [[ -z "${dir}" ]]; then @@ -170,94 +172,6 @@ release-lib::gomod_vulnfix() { popd } -# Also accepts SYNC_DOCKERFILES_FROM. -function release-lib::vulnfix() { - if [[ -z "${DIR}" ]]; then - log_err "DIR envvar is required." - return 1 - fi - - if [[ -z "${BRANCH}" ]]; then - log_err "BRANCH envvar is required." - return 1 - fi - - if [[ -z "${PROJECT}" ]]; then - log_err "PROJECT envvar is required." - return 1 - fi - - echo "${DIR}" - echo "${SCRIPT_DIR}" - - readarray -t DOCKERFILES < <(release-lib::dockerfiles "${DIR}") - - # Sync dockerfiles if needed. - if [[ -n "${SYNC_DOCKERFILES_FROM:-}" ]]; then - pushd "${DIR}" - for dockerfile in "${DOCKERFILES[@]}"; do - # TODO: Should we ensure SYNC_DOCKERFILES_FROM if it's a branch is up to data with origin? - echo "🔄 Syncing ${dockerfile} from ${SYNC_DOCKERFILES_FROM}" - git checkout "${SYNC_DOCKERFILES_FROM}" -- "${dockerfile}" - done - popd - fi - - # Docker images bumps. - - # Get first dockerfile Go version. We will use this version to find minor version to stick to. - go_version=$(release-lib::dockerfile_go_version "${DOCKERFILES[0]}") - if [[ -z "${go_version}" ]]; then - echo "❌ can't find any golang image in ${DOCKERFILES[0]}" - return 1 - fi - - # TODO: git add charts & vendor for old projects? - - # Update our images. - for dockerfile in "${DOCKERFILES[@]}"; do - release-lib::dockerfile_update_image "${dockerfile}" "google-go.pkg.dev/golang" $(echo "${go_version}" | cut -d '.' -f 1-2) - release-lib::dockerfile_update_image "${dockerfile}" "gke.gcr.io/gke-distroless/libc" "gke_distroless_" - pushd "${DIR}" - git add "${dockerfile}" - popd - done - - # bash manifest bump. - # Exclude 0.12 as values were inlined with each part, easy to manually sed for old versions. - if [[ "${PROJECT}" == "prometheus-engine" && "${BRANCH}" != "release/0.12" ]]; then - release-lib::idemp::manifests_bash_image_bump "${DIR}" - fi - - # Go vulnerabilities. - # TODO(bwplotka): Find better place to put this? - mkdir -p "${DIR}/.gmpctl/" - echo "*" >>"${DIR}/.gmpctl/.gitignore" - vuln_file="${DIR}/.gmpctl/vulnlist.txt" - pushd "${DIR}" - - release-lib::idemp::vulnlist "${DIR}" "${vuln_file}" - - if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then - # Attempt to update + go mod tidy. - release-lib::gomod_vulnfix "${DIR}" "${vuln_file}" - git add go.mod go.sum - - if [ -d "${DIR}/vendor" ]; then - go mod vendor - git add --all # TODO: Can be flaky. - fi - - # Check if that helped. - echo "⚠ī¸ This will fail on older branches with vendoring; in this case, simply go to ${DIR}, run 'go mod vendor' and rerun." - release-lib::vulnlist "${DIR}" "${vuln_file}" - if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then - echo "❌ After go mod update some vulnerabilities are still found; go to ${DIR} and resolve it manually (select not reusing the ./vulnlist.txt file) and rerun." - exit 1 - fi - fi -} - release-lib::idemp::git_commit_amend_match() { # Anything staged? if ! git diff-index --quiet --cached HEAD; then @@ -320,7 +234,7 @@ release-lib::dockerfiles() { log_err "dir arg is required." return 1 fi - find "${dir}" -name "Dockerfile*" | grep -v "${dir}/third_party/" | grep -v "${dir}/examples/" | grep -v "${dir}/hack/" | grep -v "${dir}/ui/" + find "${dir}" -name "Dockerfile*" | grep -v "${dir}/third_party/" | grep -v "${dir}/hack/" | grep -v "${dir}/ui/" | grep -v "vendor/" } # Return all images used in a Dockerfile, delimited by new-line. @@ -450,10 +364,7 @@ release-lib::dockerfile_update_image() { return 1 fi - # Prerequisite tool. - go install github.com/google/go-containerregistry/cmd/gcrane@latest - - # Use gcrane vs crane for --json. + # Use gcrane (over crane) for --json. local all_tags=$(gcrane ls "${image}" --json | jq --raw-output '.tags[]' | sort -V) # Exclude RC images. all_tags=$(echo "${all_tags}" | grep -v "rc.*") @@ -461,7 +372,7 @@ release-lib::dockerfile_update_image() { all_tags=$(echo "${all_tags}" | grep "${tag_prefix}") local latest_tag=$(echo "${all_tags}" | tail -n1) - local latest_digest=$(crane digest "${image}:${latest_tag}") + local latest_digest=$(gcrane digest "${image}:${latest_tag}") local latest_image="${image}:${latest_tag}@${latest_digest}" echo "🔄 Ensuring ${latest_image} on ${dockerfile}..." @@ -486,12 +397,14 @@ release-lib::idemp::manifests_bash_image_bump() { return 1 fi + go install github.com/mikefarah/yq/v4@latest + local values_file="${dir}/charts/values.global.yaml" # TODO: Not enough, this has to check actual manifests. - local bash_tag=$(go tool yq '.images.bash.tag' "${values_file}") + local bash_tag=$(yq '.images.bash.tag' "${values_file}") - # Use gcrane vs crane for --json. - local latest_bash_tag=$(go tool gcrane ls "gke.gcr.io/gke-distroless/bash" --json | jq --raw-output '.tags[]' | grep "gke_distroless_" | sort -V | tail -n1) + # Use gcrane (over crane) for --json. + local latest_bash_tag=$(gcrane ls "gke.gcr.io/gke-distroless/bash" --json | jq --raw-output '.tags[]' | grep "gke_distroless_" | sort -V | tail -n1) if [[ "${bash_tag}" == "${latest_bash_tag}" ]]; then echo "✅ Nothing to do; ${values_file} already uses ${latest_bash_tag}" return 0 @@ -520,9 +433,17 @@ release-lib::manifests_regen() { fi # TODO(bwplotka): Manage deps better. It's getting confusing what bins we should use (worktree bingo? script bingo?). - # bingo get is sort of necessary here? - source "${dir}/.bingo/variables.env" - YQ="${YQ:-}" HELM="${HELM}" ADDLICENSE="${ADDLICENSE:-}" bash "${dir}/hack/presubmit.sh" manifests + go install helm.sh/helm/v3/cmd/helm@latest + go install github.com/google/addlicense@latest + go install github.com/mikefarah/yq/v4@latest + + # Hack: Do the bingo variable swap. This allows injecting our own. + # This is faster than running requiring bingo and running bingo get. + cp "${dir}/.bingo/variables.env" "${dir}/.bingo/variables.env.bak" + echo "#!/bin/bash" >"${dir}/.bingo/variables.env" # Clean the file. + YQ="$(which yq)" HELM="$(which helm)" ADDLICENSE="$(which addlicense)" bash "${dir}/hack/presubmit.sh" manifests + cp "${dir}/.bingo/variables.env.bak" "${dir}/.bingo/variables.env" + echo "✅ Manifests regenerated" return 0 } @@ -591,75 +512,3 @@ release-lib::next_release_tag() { echo "${NEW_TAG}" return 0 } - -function release-lib::pre-release-rc() { - if [[ -z "${DIR}" ]]; then - log_err "DIR envvar is required." - return 1 - fi - - if [[ -z "${BRANCH}" ]]; then - log_err "BRANCH envvar is required." - return 1 - fi - - if [[ -z "${TAG}" ]]; then - log_err "TAG envvar is required." - return 1 - fi - - if [[ -z "${PROJECT}" ]]; then - log_err "PROJECT envvar is required." - return 1 - fi - - if [[ "${PROJECT}" == "prometheus-engine" ]]; then - local CLEAN_TAG="${TAG%-rc.*}" - CLEAN_TAG="${CLEAN_TAG#v}" - if [[ "${BRANCH}" == "release/0.12" ]]; then - # A bit different flow. - local chart_file="${DIR}/charts/operator/Chart.yaml" - echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." - if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - return 1 - fi - - chart_file="${DIR}/charts/rule-evaluator/Chart.yaml" - echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." - if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - return 1 - fi - else - # 0.12+ - local values_file="${DIR}/charts/values.global.yaml" - echo "🔄 Ensuring ${CLEAN_TAG} on ${values_file}..." - if ! gsed -i -E "s#version:.*#version: ${CLEAN_TAG}#g" "${values_file}"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - return 1 - fi - fi - # For versions with export embedded. - if [[ -f "${DIR}/pkg/export/export.go" ]]; then - echo "🔄 Ensuring ${TAG} in ${DIR}/pkg/export/export.go mainModuleVersion..." - if ! gsed -i -E "s#mainModuleVersion = .*#mainModuleVersion = \"${TAG}\"#g" "${DIR}/pkg/export/export.go"; then - # TODO: This is flaky, no failing actually on no match. Common bug is - echo "❌ sed didn't replace?" - return 1 - fi - fi - - release-lib::manifests_regen "${DIR}" - git add --all - else - # Prometheus and Alertmanager fork needs just a correct version in the VERSION file, - # so the binary build (go_build_info) metrics and flags are correct. - local temp=${TAG#v} # Remove v and then -rc.* suffix. - echo "${temp%-rc.*}" >VERSION - git add VERSION - fi -} diff --git a/hack/gmpctl/main.go b/ops/gmpctl/main.go similarity index 87% rename from hack/gmpctl/main.go rename to ops/gmpctl/main.go index 42613146a6..602e66823f 100644 --- a/hack/gmpctl/main.go +++ b/ops/gmpctl/main.go @@ -28,8 +28,9 @@ import ( ) var ( - cfgPath = flag.String("c", ".gmpctl.default.yaml", "Path to the configuration file. See config.go#Config for the structure.") - verbose = flag.Bool("v", false, "Enabled verbose, debug output (e.g. logging os.Exec commands)") + cfgPath = flag.String("c", ".gmpctl.default.yaml", "Path to the configuration file. See config.go#Config for the structure.") + verbose = flag.Bool("v", false, "Enabled verbose, debug output (e.g. logging os.Exec commands)") + gitPreferHTTPS = flag.Bool("git.prefer-https", false, "If true, uses HTTPS protocol instead of git for remote URLs. ") ) type Config struct { @@ -152,6 +153,19 @@ func runLibFunction(dir string, envs []string, function string, args ...string) return err } +func runLocalBash(dir string, envs []string, file string, args ...string) error { + curr, err := filepath.Abs("") // Hacky. TODO(bwplotka): Improve dir management. + if err != nil { + return err + } + + cmdArgs := []string{"bash", filepath.Join(curr, file)} + cmdArgs = append(cmdArgs, args...) + envs = append(envs, fmt.Sprintf("SCRIPT_DIR=%v", curr)) + _, err = runCommand(&cmdOpts{Dir: dir, Envs: envs, HideOutputs: false}, cmdArgs...) + return err +} + // runCommand executes a command in a specific directory func runCommand(opts *cmdOpts, args ...string) (string, error) { if len(args) == 0 { diff --git a/ops/gmpctl/prep-rc.sh b/ops/gmpctl/prep-rc.sh new file mode 100644 index 0000000000..1f23ae9803 --- /dev/null +++ b/ops/gmpctl/prep-rc.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE for contributors: Bash is funky, but sometimes more readable than Go/easier to iterate. +# Eventually, we could rewrite more critical pieces to Go, but you're welcome to add some quick +# pieces in bash to automate some stuff. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +# TODO(bwplotka): Finding correct script dir is not so trivial. +if [[ -z "${SCRIPT_DIR}" ]]; then + log_err "SCRIPT_DIR envvar is required." + exit 1 +fi + +source "${SCRIPT_DIR}/lib.sh" + +if [[ -z "${DIR}" ]]; then + log_err "DIR envvar is required." + exit 1 +fi + +if [[ -z "${BRANCH}" ]]; then + log_err "BRANCH envvar is required." + exit 1 +fi + +if [[ -z "${TAG}" ]]; then + log_err "TAG envvar is required." + exit 1 +fi + +if [[ -z "${PROJECT}" ]]; then + log_err "PROJECT envvar is required." + exit 1 +fi + +if [[ "${PROJECT}" == "prometheus-engine" ]]; then + CLEAN_TAG="${TAG%-rc.*}" + CLEAN_TAG="${CLEAN_TAG#v}" + if [[ "${BRANCH}" == "release/0.12" ]]; then + # A bit different flow. + chart_file="${DIR}/charts/operator/Chart.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." + if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + + chart_file="${DIR}/charts/rule-evaluator/Chart.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${chart_file}..." + if ! gsed -i -E "s#appVersion:.*#appVersion: ${CLEAN_TAG}#g" "${chart_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + else + # 0.12+ + values_file="${DIR}/charts/values.global.yaml" + echo "🔄 Ensuring ${CLEAN_TAG} on ${values_file}..." + if ! gsed -i -E "s#version:.*#version: ${CLEAN_TAG}#g" "${values_file}"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + fi + # For versions with export embedded. + if [[ -f "${DIR}/pkg/export/export.go" ]]; then + echo "🔄 Ensuring ${TAG} in ${DIR}/pkg/export/export.go mainModuleVersion..." + if ! gsed -i -E "s#mainModuleVersion = .*#mainModuleVersion = \"${TAG}\"#g" "${DIR}/pkg/export/export.go"; then + # TODO: This is flaky, no failing actually on no match. Common bug is + echo "❌ sed didn't replace?" + exit 1 + fi + fi + + release-lib::manifests_regen "${DIR}" + git add --all +else + # Prometheus and Alertmanager fork needs just a correct version in the VERSION file, + # so the binary build (go_build_info) metrics and flags are correct. + temp=${TAG#v} # Remove v and then -rc.* suffix. + echo "${temp%-rc.*}" >VERSION + git add VERSION +fi diff --git a/ops/gmpctl/vulnfix.sh b/ops/gmpctl/vulnfix.sh new file mode 100644 index 0000000000..eb4f67f3c9 --- /dev/null +++ b/ops/gmpctl/vulnfix.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE for contributors: Bash is funky, but sometimes more readable than Go/easier to iterate. +# Eventually, we could rewrite more critical pieces to Go, but you're welcome to add some quick +# pieces in bash to automate some stuff. + +set -o errexit +set -o pipefail +set -o nounset + +if [[ -n "${DEBUG_MODE:-}" ]]; then + set -o xtrace +fi + +# TODO(bwplotka): Finding correct script dir is not so trivial. +if [[ -z "${SCRIPT_DIR}" ]]; then + log_err "SCRIPT_DIR envvar is required." + return 1 +fi + +source "${SCRIPT_DIR}/lib.sh" + +# TODO: Find better way. Go tool grane is tricky as we run in different directory. +go install github.com/google/go-containerregistry/cmd/gcrane@latest + +# Also accepts SYNC_DOCKERFILES_FROM. + +if [[ -z "${DIR}" ]]; then + log_err "DIR envvar is required." + exit 1 +fi + +if [[ -z "${BRANCH}" ]]; then + log_err "BRANCH envvar is required." + exit 1 +fi + +if [[ -z "${PROJECT}" ]]; then + log_err "PROJECT envvar is required." + exit 1 +fi + +echo "${DIR}" +echo "${SCRIPT_DIR}" + +readarray -t DOCKERFILES < <(release-lib::dockerfiles "${DIR}") + +# Sync dockerfiles if needed. +if [[ -n "${SYNC_DOCKERFILES_FROM:-}" ]]; then + pushd "${DIR}" + for dockerfile in "${DOCKERFILES[@]}"; do + # TODO: Should we ensure SYNC_DOCKERFILES_FROM if it's a branch is up to data with origin? + echo "🔄 Syncing ${dockerfile} from ${SYNC_DOCKERFILES_FROM}" + git checkout "${SYNC_DOCKERFILES_FROM}" -- "${dockerfile}" + done + popd +fi + +# Docker images bumps. + +# Get first dockerfile Go version. We will use this version to find minor version to stick to. +go_version=$(release-lib::dockerfile_go_version "${DOCKERFILES[0]}") +if [[ -z "${go_version}" ]]; then + echo "❌ can't find any golang image in ${DOCKERFILES[0]}" + exit 1 +fi + +# TODO: git add charts & vendor for old projects? + +# Update our images. +for dockerfile in "${DOCKERFILES[@]}"; do + release-lib::dockerfile_update_image "${dockerfile}" "google-go.pkg.dev/golang" $(echo "${go_version}" | cut -d '.' -f 1-2) + release-lib::dockerfile_update_image "${dockerfile}" "gke.gcr.io/gke-distroless/libc" "gke_distroless_" + pushd "${DIR}" + git add "${dockerfile}" + popd +done + +# bash manifest bump. +# Exclude 0.12 as values were inlined with each part, easy to manually sed for old versions. +if [[ "${PROJECT}" == "prometheus-engine" && "${BRANCH}" != "release/0.12" ]]; then + release-lib::idemp::manifests_bash_image_bump "${DIR}" +fi + +# Go vulnerabilities. +# TODO(bwplotka): Find better place to put this? +mkdir -p "${DIR}/.gmpctl/" +echo "*" >>"${DIR}/.gmpctl/.gitignore" +vuln_file="${DIR}/.gmpctl/vulnlist.txt" +pushd "${DIR}" + +release-lib::idemp::vulnlist "${DIR}" "${vuln_file}" + +if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then + # Attempt to update + go mod tidy. + release-lib::gomod_vulnfix "${DIR}" "${vuln_file}" + git add go.mod go.sum + + if [ -d "${DIR}/vendor" ]; then + go mod vendor + git add --all # TODO: Can be flaky. + fi + + # Check if that helped. + echo "⚠ī¸ This will fail on older branches with vendoring; in this case, simply go to ${DIR}, run 'go mod vendor' and rerun." + release-lib::vulnlist "${DIR}" "${vuln_file}" + if [[ "no vulnerabilities" != $(cat "${vuln_file}") ]]; then + echo "❌ After go mod update some vulnerabilities are still found; go to ${DIR} and resolve it manually (select not reusing the ./vulnlist.txt file) and rerun." + exit 1 + fi +fi diff --git a/hack/gmpctl/vulnupdatelist/.gitignore b/ops/gmpctl/vulnupdatelist/.gitignore similarity index 100% rename from hack/gmpctl/vulnupdatelist/.gitignore rename to ops/gmpctl/vulnupdatelist/.gitignore diff --git a/hack/gmpctl/vulnupdatelist/main.go b/ops/gmpctl/vulnupdatelist/main.go similarity index 100% rename from hack/gmpctl/vulnupdatelist/main.go rename to ops/gmpctl/vulnupdatelist/main.go diff --git a/hack/gmpctl/vulnupdatelist/nvdapi.go b/ops/gmpctl/vulnupdatelist/nvdapi.go similarity index 100% rename from hack/gmpctl/vulnupdatelist/nvdapi.go rename to ops/gmpctl/vulnupdatelist/nvdapi.go diff --git a/hack/gmpctl/vulnupdatelist/nvdapi_test.go b/ops/gmpctl/vulnupdatelist/nvdapi_test.go similarity index 100% rename from hack/gmpctl/vulnupdatelist/nvdapi_test.go rename to ops/gmpctl/vulnupdatelist/nvdapi_test.go diff --git a/hack/gmpctl/vulnupdatelist/vuln.go b/ops/gmpctl/vulnupdatelist/vuln.go similarity index 100% rename from hack/gmpctl/vulnupdatelist/vuln.go rename to ops/gmpctl/vulnupdatelist/vuln.go