wolfi: Rebuild wolfi base images when env WOLFI_BASE_REBUILD=true (#57541)

* Rebuild wolfi base images when env WOLFI_BASE_REBUILD=true

* Return operations from wolfiRebuildAllBaseImages

* Move wolfiRebuildAllBaseImages

* Also refactor addWolfiOps

* Update docs

Author: willdollman

dev/ci/internal/ci/config.go

@@ -65,9 +65,10 @@ func NewConfig(now time.Time) Config {
 		tag    = os.Getenv("BUILDKITE_TAG")
 		// evaluates what type of pipeline run this is
 		runType = runtype.Compute(tag, branch, map[string]string{
-			"BEXT_NIGHTLY":    os.Getenv("BEXT_NIGHTLY"),
-			"RELEASE_NIGHTLY": os.Getenv("RELEASE_NIGHTLY"),
-			"VSCE_NIGHTLY":    os.Getenv("VSCE_NIGHTLY"),
+			"BEXT_NIGHTLY":       os.Getenv("BEXT_NIGHTLY"),
+			"RELEASE_NIGHTLY":    os.Getenv("RELEASE_NIGHTLY"),
+			"VSCE_NIGHTLY":       os.Getenv("VSCE_NIGHTLY"),
+			"WOLFI_BASE_REBUILD": os.Getenv("WOLFI_BASE_REBUILD"),
 		})
 		// defaults to 0
 		buildNumber, _ = strconv.Atoi(os.Getenv("BUILDKITE_BUILD_NUMBER"))

dev/ci/internal/ci/operations.go

@@ -88,9 +88,10 @@ func addSgLints(targets []string) func(pipeline *bk.Pipeline) {
 		tag    = os.Getenv("BUILDKITE_TAG")
 		// evaluates what type of pipeline run this is
 		runType = runtype.Compute(tag, branch, map[string]string{
-			"BEXT_NIGHTLY":    os.Getenv("BEXT_NIGHTLY"),
-			"RELEASE_NIGHTLY": os.Getenv("RELEASE_NIGHTLY"),
-			"VSCE_NIGHTLY":    os.Getenv("VSCE_NIGHTLY"),
+			"BEXT_NIGHTLY":       os.Getenv("BEXT_NIGHTLY"),
+			"RELEASE_NIGHTLY":    os.Getenv("RELEASE_NIGHTLY"),
+			"VSCE_NIGHTLY":       os.Getenv("VSCE_NIGHTLY"),
+			"WOLFI_BASE_REBUILD": os.Getenv("WOLFI_BASE_REBUILD"),
 		})
 	)
 

dev/ci/internal/ci/pipeline.go

@@ -143,7 +143,13 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) {
 		ops.Merge(securityOps)
 
 		// Wolfi package and base images
-		addWolfiOps(c, ops)
+		packageOps, baseImageOps := addWolfiOps(c)
+		if packageOps != nil {
+			ops.Merge(packageOps)
+		}
+		if baseImageOps != nil {
+			ops.Merge(baseImageOps)
+		}
 
 		// Now we set up conditional operations that only apply to pull requests.
 		if c.Diff.Has(changed.Client) {
@@ -189,6 +195,13 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) {
 			addVsceTests,
 		)
 
+	case runtype.WolfiBaseRebuild:
+		// If this is a Wolfi base image rebuild, rebuild all Wolfi base images and push to registry
+		baseImageOps := wolfiRebuildAllBaseImages(c)
+		if baseImageOps != nil {
+			ops.Merge(baseImageOps)
+		}
+
 	case runtype.CandidatesNoTest:
 		imageBuildOps := operations.NewNamedSet("Image builds")
 		imageBuildOps.Append(bazelBuildCandidateDockerImages(legacyDockerImages, c.Version, c.candidateImageTag(), c.RunType))
@@ -302,7 +315,13 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) {
 		))
 
 		// Wolfi package and base images
-		addWolfiOps(c, ops)
+		packageOps, baseImageOps := addWolfiOps(c)
+		if packageOps != nil {
+			ops.Merge(packageOps)
+		}
+		if baseImageOps != nil {
+			ops.Merge(baseImageOps)
+		}
 
 		// All operations before this point are required
 		ops.Append(wait)
@@ -403,33 +422,3 @@ func withAgentLostRetries(s *bk.Step) {
 		ExitStatus: -1,
 	})
 }
-
-// addWolfiOps adds operations to rebuild modified Wolfi packages and base images.
-func addWolfiOps(c Config, ops *operations.Set) {
-	// Rebuild Wolfi packages that have config changes
-	var updatedPackages []string
-	if c.Diff.Has(changed.WolfiPackages) {
-		var packageOps *operations.Set
-		packageOps, updatedPackages = WolfiPackagesOperations(c.ChangedFiles[changed.WolfiPackages])
-		ops.Merge(packageOps)
-	}
-
-	// Rebuild Wolfi base images
-	// Inspect package dependencies, and rebuild base images with updated packages
-	_, imagesWithChangedPackages, err := GetDependenciesOfPackages(updatedPackages, "sourcegraph")
-	if err != nil {
-		panic(err)
-	}
-	// Rebuild base images with package changes AND with config changes
-	imagesToRebuild := append(imagesWithChangedPackages, c.ChangedFiles[changed.WolfiBaseImages]...)
-	imagesToRebuild = sortUniq(imagesToRebuild)
-
-	if len(imagesToRebuild) > 0 {
-		baseImageOps, _ := WolfiBaseImagesOperations(
-			imagesToRebuild,
-			c.Version,
-			(len(updatedPackages) > 0),
-		)
-		ops.Merge(baseImageOps)
-	}
-}

dev/ci/internal/ci/wolfi_operations.go

@@ -11,6 +11,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	bk "github.com/sourcegraph/sourcegraph/dev/ci/internal/buildkite"
+	"github.com/sourcegraph/sourcegraph/dev/ci/internal/ci/changed"
 	"github.com/sourcegraph/sourcegraph/dev/ci/internal/ci/operations"
 	"github.com/sourcegraph/sourcegraph/dev/sg/root"
 	"github.com/sourcegraph/sourcegraph/internal/lazyregexp"
@@ -265,3 +266,62 @@ func getPackagesFromBaseImageConfig(configFile string) ([]string, error) {
 
 	return config.Contents.Packages, nil
 }
+
+// addWolfiOps adds operations to rebuild modified Wolfi packages and base images.
+func addWolfiOps(c Config) (packageOps, baseImageOps *operations.Set) {
+	// Rebuild Wolfi packages that have config changes
+	var updatedPackages []string
+	if c.Diff.Has(changed.WolfiPackages) {
+		packageOps, updatedPackages = WolfiPackagesOperations(c.ChangedFiles[changed.WolfiPackages])
+	}
+
+	// Rebuild Wolfi base images
+	// Inspect package dependencies, and rebuild base images with updated packages
+	_, imagesWithChangedPackages, err := GetDependenciesOfPackages(updatedPackages, "sourcegraph")
+	if err != nil {
+		panic(err)
+	}
+	// Rebuild base images with package changes AND with config changes
+	imagesToRebuild := append(imagesWithChangedPackages, c.ChangedFiles[changed.WolfiBaseImages]...)
+	imagesToRebuild = sortUniq(imagesToRebuild)
+
+	if len(imagesToRebuild) > 0 {
+		baseImageOps, _ = WolfiBaseImagesOperations(
+			imagesToRebuild,
+			c.Version,
+			(len(updatedPackages) > 0),
+		)
+	}
+
+	return packageOps, baseImageOps
+}
+
+// wolfiRebuildAllBaseImages adds operations to rebuild all Wolfi base images and push to registry
+func wolfiRebuildAllBaseImages(c Config) *operations.Set {
+	// List all YAML files in wolfi-images/
+	dir := "wolfi-images"
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		panic(err)
+	}
+
+	var wolfiBaseImages []string
+	for _, f := range files {
+		if filepath.Ext(f.Name()) == ".yaml" {
+			fullPath := filepath.Join(dir, f.Name())
+			wolfiBaseImages = append(wolfiBaseImages, fullPath)
+		}
+	}
+
+	// Rebuild all images
+	var baseImageOps *operations.Set
+	if len(wolfiBaseImages) > 0 {
+		baseImageOps, _ = WolfiBaseImagesOperations(
+			wolfiBaseImages,
+			c.Version,
+			false,
+		)
+	}
+
+	return baseImageOps
+}

dev/ci/runtype/runtype.go

@@ -18,11 +18,12 @@ const (
 
 	// Nightly builds - must be first because they take precedence
 
-	ReleaseNightly // release branch nightly healthcheck builds
-	BextNightly    // browser extension nightly build
-	VsceNightly    // vs code extension nightly build
-	AppRelease     // app release build
-	AppInsiders    // app insiders build
+	ReleaseNightly   // release branch nightly healthcheck builds
+	BextNightly      // browser extension nightly build
+	VsceNightly      // vs code extension nightly build
+	AppRelease       // app release build
+	AppInsiders      // app insiders build
+	WolfiBaseRebuild // wolfi base image build
 
 	// Release branches
 
@@ -108,6 +109,12 @@ func (t RunType) Matcher() *RunTypeMatcher {
 			Branch:      "vsce/release",
 			BranchExact: true,
 		}
+	case WolfiBaseRebuild:
+		return &RunTypeMatcher{
+			EnvIncludes: map[string]string{
+				"WOLFI_BASE_REBUILD": "true",
+			},
+		}
 
 	case AppRelease:
 		return &RunTypeMatcher{
@@ -192,6 +199,8 @@ func (t RunType) String() string {
 		return "Browser extension nightly release build"
 	case VsceNightly:
 		return "VS Code extension nightly release build"
+	case WolfiBaseRebuild:
+		return "Wolfi base images rebuild"
 	case AppRelease:
 		return "App release build"
 	case AppInsiders:

dev/ci/runtype/runtype_test.go

@@ -60,6 +60,15 @@ func TestComputeRunType(t *testing.T) {
 			},
 		},
 		want: VsceNightly,
+	}, {
+		name: "wolfi base image rebuild",
+		args: args{
+			branch: "main",
+			env: map[string]string{
+				"WOLFI_BASE_REBUILD": "true",
+			},
+		},
+		want: WolfiBaseRebuild,
 	}, {
 		name: "vsce release",
 		args: args{

doc/dev/how-to/wolfi/add_update_images.md

@@ -16,14 +16,16 @@ These configuration files can be processed with apko, which will generate a base
 
 Before each release, we should update the base images to ensure we include any updated packages and vulnerability fixes.
 
-This is currently a two-step process, which will be further automated in the future:
-
-- Run [`wolfi-images/rebuild-images.sh`](https://sourcegraph.com/github.com/sourcegraph/sourcegraph@588463afbb0904c125cdcf78c7b182f43328504e/-/blob/wolfi-images/rebuild-images.sh) script, commit the updated YAML files, and merge to main.
-- Wait for the `main` branch's Buildkite run to complete.
-  - Buildkite will rebuild the base images and publish them to Dockerhub.
 - Run `sg wolfi update-hashes` locally to update the base image hashes in `dev/oci_deps.bzl`. Commit these changes and merge to `main`.
-  - This fetches the updated base image hashes from the images that were pushed to Dockerhub in the previous step.
-- Backport the PR that updated `dev/oci_deps.bzl` to the release branch.
+- Backport the PR to the release branch.
+
+#### Automation
+
+This process is partially automated by Buildkite. A scheduled build runs daily to rebuild Wolfi base images - pulling in any updated dependencies - then push them to Docker Hub. When `sg wolfi update-hashes` is run, it pulls these latest image hashes from Docker Hub to update the references in `dev/oci_deps.bzl`.
+
+To rebuild the images (perhaps to pick up a just-released package version), [find a scheduled build](https://buildkite.com/sourcegraph/sourcegraph/builds?branch=main) in Buildkite named "Nightly Rebuild of Wolfi Base Images", and hit "Rebuild".
+
+It is also possible to manually rebuild individual images by running `wolfi-images/rebuild-images.sh` locally, then pushing and merging.
 
 ### Modify an existing base image