From f7fb41eddda30eef503d4131c1fc0021b41ab22a Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Thu, 25 Dec 2025 21:38:31 +0300 Subject: [PATCH 01/11] fix: splitn bug Signed-off-by: Timur Tuktamyshev --- pkg/libmirror/images/digests.go | 18 ++++++++---------- pkg/libmirror/layouts/layouts.go | 18 ++++++++---------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/pkg/libmirror/images/digests.go b/pkg/libmirror/images/digests.go index 94129292..9748850a 100644 --- a/pkg/libmirror/images/digests.go +++ b/pkg/libmirror/images/digests.go @@ -195,23 +195,21 @@ func FindVexImage( return "", fmt.Errorf("parse reference: %w", err) } - split := strings.SplitN(vexImageName, ":", 2) - imagePath := split[0] - tag := split[1] - - imageSegmentsRaw := strings.TrimPrefix(imagePath, client.GetRegistry()) - imageSegments := strings.Split(imageSegmentsRaw, "/") - - for i, segment := range imageSegments { - client = client.WithSegment(segment) - logger.Debugf("Segment %d: %s", i, segment) + // Use LastIndex to correctly handle URLs with port (e.g., localhost:443/repo:tag) + splitIndex := strings.LastIndex(vexImageName, ":") + if splitIndex == -1 { + return "", fmt.Errorf("invalid vex image name format: %s", vexImageName) } + tag := vexImageName[splitIndex+1:] err = client.CheckImageExists(context.TODO(), tag) if errors.Is(err, regclient.ErrImageNotFound) { // Image not found, which is expected for non-vulnerable images return "", nil } + if err != nil { + return "", fmt.Errorf("check VEX image exists: %w", err) + } return vexImageName, nil } diff --git a/pkg/libmirror/layouts/layouts.go b/pkg/libmirror/layouts/layouts.go index afd0d8ef..2e4cc61f 100644 --- a/pkg/libmirror/layouts/layouts.go +++ b/pkg/libmirror/layouts/layouts.go @@ -493,23 +493,21 @@ func FindVexImage( return "", fmt.Errorf("parse reference: %w", err) } - split := strings.SplitN(vexImageName, ":", 2) - imagePath := split[0] - tag := split[1] - - imageSegmentsRaw := strings.TrimPrefix(imagePath, client.GetRegistry()) - imageSegments := strings.Split(imageSegmentsRaw, "/") - - for i, segment := range imageSegments { - client = client.WithSegment(segment) - logger.Debugf("Segment %d: %s", i, segment) + // Use LastIndex to correctly handle URLs with port (e.g., localhost:443/repo:tag) + splitIndex := strings.LastIndex(vexImageName, ":") + if splitIndex == -1 { + return "", fmt.Errorf("invalid vex image name format: %s", vexImageName) } + tag := vexImageName[splitIndex+1:] err = client.CheckImageExists(context.TODO(), tag) if errors.Is(err, regclient.ErrImageNotFound) { // Image not found, which is expected for non-vulnerable images return "", nil } + if err != nil { + return "", fmt.Errorf("check VEX image exists: %w", err) + } return vexImageName, nil } From e0e18fbf29391dc248a094bce2b8464eebb9f1fb Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Thu, 25 Dec 2025 21:46:49 +0300 Subject: [PATCH 02/11] fix: tests Signed-off-by: Timur Tuktamyshev --- pkg/libmirror/images/digests_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/libmirror/images/digests_test.go b/pkg/libmirror/images/digests_test.go index 772bd414..7359667a 100644 --- a/pkg/libmirror/images/digests_test.go +++ b/pkg/libmirror/images/digests_test.go @@ -47,8 +47,6 @@ func TestExtractImageDigestsFromDeckhouseInstaller(t *testing.T) { installersLayout := createOCILayoutWithInstallerImage(t, "nonexistent.registry.com/deckhouse", installerTag, expectedImages) client := mock.NewRegistryClientMock(t) - client.GetRegistryMock.Return("nonexistent.registry.com") - client.WithSegmentMock.Return(client) client.CheckImageExistsMock.Return(nil) images, err := ExtractImageDigestsFromDeckhouseInstaller( ¶ms.PullParams{BaseParams: params.BaseParams{ From 60c540011e425e7e053eb9dd79aaf25e57b1bcce Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Thu, 25 Dec 2025 21:09:11 +0300 Subject: [PATCH 03/11] test Signed-off-by: Timur Tuktamyshev --- testing/e2e/mirror/README.md | 178 ++++++++++ testing/e2e/mirror/config.go | 143 ++++++++ testing/e2e/mirror/digest_collector.go | 241 ++++++++++++++ testing/e2e/mirror/digest_comparator.go | 233 +++++++++++++ testing/e2e/mirror/mirror_e2e_test.go | 388 ++++++++++++---------- testing/e2e/mirror/structure_validator.go | 250 ++++++++++++++ testing/util/mirror/registry.go | 58 +++- 7 files changed, 1301 insertions(+), 190 deletions(-) create mode 100644 testing/e2e/mirror/README.md create mode 100644 testing/e2e/mirror/config.go create mode 100644 testing/e2e/mirror/digest_collector.go create mode 100644 testing/e2e/mirror/digest_comparator.go create mode 100644 testing/e2e/mirror/structure_validator.go diff --git a/testing/e2e/mirror/README.md b/testing/e2e/mirror/README.md new file mode 100644 index 00000000..fe60c0d6 --- /dev/null +++ b/testing/e2e/mirror/README.md @@ -0,0 +1,178 @@ +# E2E Tests for d8 mirror + +End-to-end tests for the `d8 mirror pull` and `d8 mirror push` commands. + +## Overview + +These tests perform a complete mirror cycle: +1. Collect reference digests from source registry +2. Pull images to a local bundle +3. Push bundle to a target registry +4. Validate the target registry structure +5. Compare all digests between source and target + +## Requirements + +- Built `d8` binary (run `task build` from project root) +- Valid license token for the source registry +- Network access to the source registry + +## Running Tests + +### Basic Usage + +```bash +# Run with license token +go test -v ./testing/e2e/mirror/... \ + -license-token=YOUR_LICENSE_TOKEN + +# Or use environment variables (recommended) +E2E_LICENSE_TOKEN=YOUR_LICENSE_TOKEN \ +go test -v ./testing/e2e/mirror/... +``` + +### Full Configuration + +```bash +# Using license token +go test -v ./testing/e2e/mirror/... \ + -source-registry=registry.deckhouse.ru/deckhouse/fe \ + -license-token=YOUR_LICENSE_TOKEN \ + -target-registry=my-registry.local:5000/deckhouse \ + -target-user=admin \ + -target-password=secret \ + -tls-skip-verify \ + -keep-bundle + +# Using explicit source credentials (with self-signed cert) +go test -v ./testing/e2e/mirror/... \ + -source-registry=my-source-registry.local/deckhouse \ + -source-user=admin \ + -source-password=secret \ + -target-registry=my-target-registry.local:5000/deckhouse \ + -tls-skip-verify + +# Using environment variables (recommended) +E2E_SOURCE_REGISTRY=localhost:443/deckhouse-etalon \ +E2E_SOURCE_USER=admin \ +E2E_SOURCE_PASSWORD=secret \ +E2E_TLS_SKIP_VERIFY=true \ +go test -v ./testing/e2e/mirror/... +``` + +### Environment Variables + +All flags can be set via environment variables: + +| Flag | Environment Variable | Default | Description | +|------|---------------------|---------|-------------| +| `-source-registry` | `E2E_SOURCE_REGISTRY` | `registry.deckhouse.ru/deckhouse/fe` | Source registry to pull from | +| `-source-user` | `E2E_SOURCE_USER` | | Source registry username (alternative to license-token) | +| `-source-password` | `E2E_SOURCE_PASSWORD` | | Source registry password | +| `-license-token` | `E2E_LICENSE_TOKEN` | | License token for Deckhouse registry (shortcut for source-user=license-token) | +| `-target-registry` | `E2E_TARGET_REGISTRY` | (empty = in-memory) | Target registry to push to | +| `-target-user` | `E2E_TARGET_USER` | | Target registry username | +| `-target-password` | `E2E_TARGET_PASSWORD` | | Target registry password | +| `-tls-skip-verify` | `E2E_TLS_SKIP_VERIFY` | `false` | Skip TLS certificate verification (for self-signed certs) | +| `-keep-bundle` | `E2E_KEEP_BUNDLE` | `false` | Keep bundle directory after test | +| `-d8-binary` | `E2E_D8_BINARY` | `bin/d8` | Path to d8 binary | + +**Note:** Either `-license-token` OR `-source-user`/`-source-password` must be provided for authentication. + +## Test Scenarios + +### TestMirrorE2E_FullCycle + +Complete end-to-end test that: +1. Collects all image digests from source registry +2. Runs `d8 mirror pull` to create a bundle +3. Runs `d8 mirror push` to push to target +4. Validates that all segments exist in target +5. Compares every digest between source and target + +### TestMirrorE2E_PullOnly + +Tests only the pull operation: +1. Runs `d8 mirror pull` to create a bundle +2. Verifies the bundle directory contains expected files + +## Target Registry Options + +### In-Memory Registry (Default) + +When `-target-registry` is not specified, tests use an in-memory registry. +This is useful for CI/CD and quick local testing. + +### External Registry + +Specify `-target-registry` to push to a real registry: + +```bash +# Docker registry with self-signed cert +go test -v ./testing/e2e/mirror/... \ + -license-token=TOKEN \ + -target-registry=localhost:5000/deckhouse \ + -tls-skip-verify + +# Registry with auth +go test -v ./testing/e2e/mirror/... \ + -license-token=TOKEN \ + -target-registry=registry.example.com/deckhouse \ + -target-user=admin \ + -target-password=secret +``` + +## What Gets Validated + +### Structure Validation + +- Root Deckhouse images exist +- `/install` segment exists with release channel tags +- `/install-standalone` segment exists +- `/release-channel` segment exists +- `/security/*` databases exist (if present in source) +- `/modules/*` exist (if present in source) + +### Digest Comparison + +Every image tag in the source is compared with the target: +- All images must be present in target +- All digests must match exactly (SHA256) + +## Timeouts + +The full cycle test has a 60-minute timeout. This can be adjusted in the test code if needed. + +## Troubleshooting + +### "License token not provided" + +Set the license token via flag or environment variable: +```bash +E2E_LICENSE_TOKEN=your_token go test -v ./testing/e2e/mirror/... +``` + +### "Pull failed" + +Check that: +1. The `d8` binary exists at `bin/d8` (run `task build` first) or specify path with `-d8-binary` +2. You have network access to the source registry +3. The license token is valid + +### "Push failed" + +Check that: +1. The target registry is accessible +2. Credentials are correct (if using authenticated registry) +3. Use `-tls-skip-verify` for self-signed certificates + +### Viewing Bundle Contents + +Use `-keep-bundle` to preserve the bundle directory: +```bash +go test -v ./testing/e2e/mirror/... \ + -license-token=TOKEN \ + -keep-bundle + +# Bundle will be at /tmp/d8-mirror-e2e-TIMESTAMP/ +``` diff --git a/testing/e2e/mirror/config.go b/testing/e2e/mirror/config.go new file mode 100644 index 00000000..6bec647e --- /dev/null +++ b/testing/e2e/mirror/config.go @@ -0,0 +1,143 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "flag" + "os" + + "github.com/google/go-containerregistry/pkg/authn" +) + +// Test configuration flags +var ( + // Source registry configuration + sourceRegistry = flag.String("source-registry", + getEnvOrDefault("E2E_SOURCE_REGISTRY", "registry.deckhouse.ru/deckhouse/fe"), + "Reference registry to pull from") + sourceUser = flag.String("source-user", + getEnvOrDefault("E2E_SOURCE_USER", ""), + "Source registry username (alternative to license-token)") + sourcePassword = flag.String("source-password", + getEnvOrDefault("E2E_SOURCE_PASSWORD", ""), + "Source registry password (alternative to license-token)") + licenseToken = flag.String("license-token", + getEnvOrDefault("E2E_LICENSE_TOKEN", ""), + "License token for source registry authentication (shortcut for source-user=license-token)") + + // Target registry configuration + targetRegistry = flag.String("target-registry", + getEnvOrDefault("E2E_TARGET_REGISTRY", ""), + "Target registry to push to (empty = use in-memory registry)") + targetUser = flag.String("target-user", + getEnvOrDefault("E2E_TARGET_USER", ""), + "Target registry username") + targetPassword = flag.String("target-password", + getEnvOrDefault("E2E_TARGET_PASSWORD", ""), + "Target registry password") + + // Test options + tlsSkipVerify = flag.Bool("tls-skip-verify", + getEnvOrDefault("E2E_TLS_SKIP_VERIFY", "") == "true", + "Skip TLS certificate verification (for self-signed certs)") + keepBundle = flag.Bool("keep-bundle", + getEnvOrDefault("E2E_KEEP_BUNDLE", "") == "true", + "Keep bundle directory after test") + d8Binary = flag.String("d8-binary", + getEnvOrDefault("E2E_D8_BINARY", "../../../bin/d8"), + "Path to d8 binary") +) + +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +// Config holds the parsed test configuration +type Config struct { + SourceRegistry string + SourceUser string + SourcePassword string + LicenseToken string + + TargetRegistry string + TargetUser string + TargetPassword string + + TLSSkipVerify bool + KeepBundle bool + D8Binary string +} + +// GetConfig returns the current test configuration from flags +func GetConfig() *Config { + // flag.Parse() is called automatically by go test + return &Config{ + SourceRegistry: *sourceRegistry, + SourceUser: *sourceUser, + SourcePassword: *sourcePassword, + LicenseToken: *licenseToken, + TargetRegistry: *targetRegistry, + TargetUser: *targetUser, + TargetPassword: *targetPassword, + TLSSkipVerify: *tlsSkipVerify, + KeepBundle: *keepBundle, + D8Binary: *d8Binary, + } +} + +// GetSourceAuth returns authenticator for source registry +func (c *Config) GetSourceAuth() authn.Authenticator { + // Explicit credentials take precedence + if c.SourceUser != "" { + return authn.FromConfig(authn.AuthConfig{ + Username: c.SourceUser, + Password: c.SourcePassword, + }) + } + // License token is a shortcut for source-user=license-token + if c.LicenseToken != "" { + return authn.FromConfig(authn.AuthConfig{ + Username: "license-token", + Password: c.LicenseToken, + }) + } + return authn.Anonymous +} + +// HasSourceAuth returns true if any source authentication is configured +func (c *Config) HasSourceAuth() bool { + return c.SourceUser != "" || c.LicenseToken != "" +} + +// GetTargetAuth returns authenticator for target registry +func (c *Config) GetTargetAuth() authn.Authenticator { + if c.TargetUser != "" { + return authn.FromConfig(authn.AuthConfig{ + Username: c.TargetUser, + Password: c.TargetPassword, + }) + } + return authn.Anonymous +} + +// UseInMemoryRegistry returns true if we should use in-memory registry +func (c *Config) UseInMemoryRegistry() bool { + return c.TargetRegistry == "" +} diff --git a/testing/e2e/mirror/digest_collector.go b/testing/e2e/mirror/digest_collector.go new file mode 100644 index 00000000..fb22803e --- /dev/null +++ b/testing/e2e/mirror/digest_collector.go @@ -0,0 +1,241 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "path" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/deckhouse/deckhouse-cli/internal" +) + +// DigestMap maps image reference (repo:tag) to its digest +type DigestMap map[string]string + +// DigestCollector collects digests from a container registry +type DigestCollector struct { + registry string + auth authn.Authenticator + + nameOpts []name.Option + remoteOpts []remote.Option +} + +// NewDigestCollector creates a new digest collector +// tlsSkipVerify: skip TLS certificate verification (for self-signed certs) +func NewDigestCollector(registry string, authenticator authn.Authenticator, tlsSkipVerify bool) *DigestCollector { + nameOpts := []name.Option{} + remoteOpts := []remote.Option{} + + if tlsSkipVerify { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + remoteOpts = append(remoteOpts, remote.WithTransport(transport)) + } + + if authenticator != nil && authenticator != authn.Anonymous { + remoteOpts = append(remoteOpts, remote.WithAuth(authenticator)) + } + + return &DigestCollector{ + registry: registry, + auth: authenticator, + nameOpts: nameOpts, + remoteOpts: remoteOpts, + } +} + +// CollectAll collects digests for all images in the registry +func (c *DigestCollector) CollectAll(ctx context.Context) (DigestMap, error) { + digests := make(DigestMap) + + // Collect Deckhouse root images + if err := c.collectDeckhouseRoot(ctx, digests); err != nil { + return nil, fmt.Errorf("collect deckhouse root: %w", err) + } + + // Collect install images + if err := c.collectSegment(ctx, digests, internal.InstallSegment); err != nil { + return nil, fmt.Errorf("collect install: %w", err) + } + + // Collect install-standalone images + if err := c.collectSegment(ctx, digests, internal.InstallStandaloneSegment); err != nil { + return nil, fmt.Errorf("collect install-standalone: %w", err) + } + + // Collect release-channel images + if err := c.collectSegment(ctx, digests, internal.ReleaseChannelSegment); err != nil { + return nil, fmt.Errorf("collect release-channel: %w", err) + } + + // Collect security databases + if err := c.collectSecurity(ctx, digests); err != nil { + return nil, fmt.Errorf("collect security: %w", err) + } + + // Collect modules + if err := c.collectModules(ctx, digests); err != nil { + return nil, fmt.Errorf("collect modules: %w", err) + } + + return digests, nil +} + +// collectDeckhouseRoot collects digests from the root registry path +func (c *DigestCollector) collectDeckhouseRoot(ctx context.Context, digests DigestMap) error { + return c.collectTagsFromRepo(ctx, digests, c.registry) +} + +// collectSegment collects digests from a specific segment (install, release-channel, etc) +func (c *DigestCollector) collectSegment(ctx context.Context, digests DigestMap, segment string) error { + repo := path.Join(c.registry, segment) + return c.collectTagsFromRepo(ctx, digests, repo) +} + +// collectSecurity collects security database digests +func (c *DigestCollector) collectSecurity(ctx context.Context, digests DigestMap) error { + securityImages := map[string]string{ + path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyDBSegment): "2", + path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyBDUSegment): "1", + path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyJavaDBSegment): "1", + path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyChecksSegment): "0", + } + + for repo, tag := range securityImages { + digest, err := c.getDigest(ctx, repo+":"+tag) + if err != nil { + // Security databases might not exist, skip with warning + continue + } + digests[repo+":"+tag] = digest + } + + return nil +} + +// collectModules collects digests for all modules +func (c *DigestCollector) collectModules(ctx context.Context, digests DigestMap) error { + modulesRepo := path.Join(c.registry, internal.ModulesSegment) + + // List all modules (they are tags in the modules repo) + modules, err := c.listTags(ctx, modulesRepo) + if err != nil { + // Modules might not be accessible + return nil + } + + for _, moduleName := range modules { + // Module root images + moduleRepo := path.Join(modulesRepo, moduleName) + if err := c.collectTagsFromRepo(ctx, digests, moduleRepo); err != nil { + continue + } + + // Module release channels + releaseRepo := path.Join(moduleRepo, "release") + if err := c.collectTagsFromRepo(ctx, digests, releaseRepo); err != nil { + // Release repo might not exist + continue + } + } + + return nil +} + +// collectTagsFromRepo lists all tags in a repo and collects their digests +func (c *DigestCollector) collectTagsFromRepo(ctx context.Context, digests DigestMap, repo string) error { + tags, err := c.listTags(ctx, repo) + if err != nil { + return err + } + + for _, tag := range tags { + ref := repo + ":" + tag + digest, err := c.getDigest(ctx, ref) + if err != nil { + continue + } + digests[ref] = digest + } + + return nil +} + +// listTags lists all tags in a repository +func (c *DigestCollector) listTags(ctx context.Context, repo string) ([]string, error) { + repoRef, err := name.NewRepository(repo, c.nameOpts...) + if err != nil { + return nil, fmt.Errorf("parse repo %s: %w", repo, err) + } + + tags, err := remote.List(repoRef, c.remoteOpts...) + if err != nil { + return nil, fmt.Errorf("list tags for %s: %w", repo, err) + } + + return tags, nil +} + +// getDigest gets the digest for a specific image reference +func (c *DigestCollector) getDigest(ctx context.Context, ref string) (string, error) { + imgRef, err := name.ParseReference(ref, c.nameOpts...) + if err != nil { + return "", fmt.Errorf("parse ref %s: %w", ref, err) + } + + desc, err := remote.Head(imgRef, c.remoteOpts...) + if err != nil { + return "", fmt.Errorf("get digest for %s: %w", ref, err) + } + + return desc.Digest.String(), nil +} + +// NormalizeRef normalizes a reference to allow comparison between registries +// Strips the registry host from the reference +func NormalizeRef(ref string) string { + // Remove scheme if present + ref = strings.TrimPrefix(ref, "https://") + ref = strings.TrimPrefix(ref, "http://") + + // Find the first slash after the host + slashIdx := strings.Index(ref, "/") + if slashIdx == -1 { + return ref + } + + // Return everything after the host + return ref[slashIdx+1:] +} + +// NormalizeDigests creates a new map with normalized references +func NormalizeDigests(digests DigestMap) DigestMap { + normalized := make(DigestMap, len(digests)) + for ref, digest := range digests { + normalized[NormalizeRef(ref)] = digest + } + return normalized +} diff --git a/testing/e2e/mirror/digest_comparator.go b/testing/e2e/mirror/digest_comparator.go new file mode 100644 index 00000000..529184c1 --- /dev/null +++ b/testing/e2e/mirror/digest_comparator.go @@ -0,0 +1,233 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "fmt" + "sort" + "strings" +) + +// Mismatch represents a digest mismatch between source and target +type Mismatch struct { + // Ref is the normalized image reference (path:tag) + Ref string + + // ExpectedDigest is the digest from the source registry + ExpectedDigest string + + // ActualDigest is the digest from the target registry (empty if not found) + ActualDigest string + + // Type indicates the type of mismatch + Type MismatchType +} + +// MismatchType categorizes the type of mismatch +type MismatchType int + +const ( + // MismatchTypeMissing means the image is missing in the target + MismatchTypeMissing MismatchType = iota + + // MismatchTypeDigestDifferent means the digests don't match + MismatchTypeDigestDifferent + + // MismatchTypeExtra means the image exists in target but not in source + MismatchTypeExtra +) + +func (t MismatchType) String() string { + switch t { + case MismatchTypeMissing: + return "MISSING" + case MismatchTypeDigestDifferent: + return "DIGEST_MISMATCH" + case MismatchTypeExtra: + return "EXTRA" + default: + return "UNKNOWN" + } +} + +// String returns a human-readable representation of the mismatch +func (m Mismatch) String() string { + switch m.Type { + case MismatchTypeMissing: + return fmt.Sprintf("[%s] %s: expected %s, not found in target", + m.Type, m.Ref, m.ExpectedDigest) + case MismatchTypeDigestDifferent: + return fmt.Sprintf("[%s] %s: expected %s, got %s", + m.Type, m.Ref, m.ExpectedDigest, m.ActualDigest) + case MismatchTypeExtra: + return fmt.Sprintf("[%s] %s: unexpected image with digest %s", + m.Type, m.Ref, m.ActualDigest) + default: + return fmt.Sprintf("[%s] %s", m.Type, m.Ref) + } +} + +// CompareResult contains the result of digest comparison +type CompareResult struct { + // Mismatches lists all found mismatches + Mismatches []Mismatch + + // MatchedCount is the number of images that matched + MatchedCount int + + // TotalExpected is the total number of expected images + TotalExpected int + + // TotalActual is the total number of actual images + TotalActual int +} + +// IsMatch returns true if all digests matched +func (r *CompareResult) IsMatch() bool { + return len(r.Mismatches) == 0 +} + +// MissingCount returns the number of missing images +func (r *CompareResult) MissingCount() int { + count := 0 + for _, m := range r.Mismatches { + if m.Type == MismatchTypeMissing { + count++ + } + } + return count +} + +// DigestMismatchCount returns the number of digest mismatches +func (r *CompareResult) DigestMismatchCount() int { + count := 0 + for _, m := range r.Mismatches { + if m.Type == MismatchTypeDigestDifferent { + count++ + } + } + return count +} + +// String returns a summary of the comparison +func (r *CompareResult) String() string { + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("Comparison Summary:\n")) + sb.WriteString(fmt.Sprintf(" Expected: %d images\n", r.TotalExpected)) + sb.WriteString(fmt.Sprintf(" Actual: %d images\n", r.TotalActual)) + sb.WriteString(fmt.Sprintf(" Matched: %d images\n", r.MatchedCount)) + sb.WriteString(fmt.Sprintf(" Missing: %d images\n", r.MissingCount())) + sb.WriteString(fmt.Sprintf(" Digest mismatches: %d\n", r.DigestMismatchCount())) + + if !r.IsMatch() { + sb.WriteString("\nMismatches:\n") + for _, m := range r.Mismatches { + sb.WriteString(" " + m.String() + "\n") + } + } + + return sb.String() +} + +// Compare compares two digest maps and returns mismatches +// Both maps should have normalized references (using NormalizeDigests) +func Compare(expected, actual DigestMap) *CompareResult { + result := &CompareResult{ + Mismatches: make([]Mismatch, 0), + TotalExpected: len(expected), + TotalActual: len(actual), + } + + // Check all expected images + for ref, expectedDigest := range expected { + actualDigest, exists := actual[ref] + + if !exists { + result.Mismatches = append(result.Mismatches, Mismatch{ + Ref: ref, + ExpectedDigest: expectedDigest, + Type: MismatchTypeMissing, + }) + continue + } + + if expectedDigest != actualDigest { + result.Mismatches = append(result.Mismatches, Mismatch{ + Ref: ref, + ExpectedDigest: expectedDigest, + ActualDigest: actualDigest, + Type: MismatchTypeDigestDifferent, + }) + continue + } + + result.MatchedCount++ + } + + // Sort mismatches by ref for consistent output + sort.Slice(result.Mismatches, func(i, j int) bool { + return result.Mismatches[i].Ref < result.Mismatches[j].Ref + }) + + return result +} + +// CompareStrict also reports extra images in the target +func CompareStrict(expected, actual DigestMap) *CompareResult { + result := Compare(expected, actual) + + // Find extra images in actual that don't exist in expected + for ref, actualDigest := range actual { + if _, exists := expected[ref]; !exists { + result.Mismatches = append(result.Mismatches, Mismatch{ + Ref: ref, + ActualDigest: actualDigest, + Type: MismatchTypeExtra, + }) + } + } + + // Re-sort after adding extras + sort.Slice(result.Mismatches, func(i, j int) bool { + return result.Mismatches[i].Ref < result.Mismatches[j].Ref + }) + + return result +} + +// FormatMismatches formats mismatches for test output +func FormatMismatches(mismatches []Mismatch) string { + if len(mismatches) == 0 { + return "no mismatches" + } + + var sb strings.Builder + for i, m := range mismatches { + if i > 0 { + sb.WriteString("\n") + } + sb.WriteString(m.String()) + + // Limit output for very long lists + if i >= 50 { + sb.WriteString(fmt.Sprintf("\n... and %d more mismatches", len(mismatches)-i-1)) + break + } + } + return sb.String() +} diff --git a/testing/e2e/mirror/mirror_e2e_test.go b/testing/e2e/mirror/mirror_e2e_test.go index 44f46e21..e7c3c000 100644 --- a/testing/e2e/mirror/mirror_e2e_test.go +++ b/testing/e2e/mirror/mirror_e2e_test.go @@ -17,235 +17,255 @@ limitations under the License. package mirror import ( - "encoding/json" + "context" "fmt" - "math/rand" "os" + "os/exec" "path/filepath" - "strings" "testing" + "time" - "github.com/Masterminds/semver/v3" - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/crane" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/random" - "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/stretchr/testify/require" - "sigs.k8s.io/yaml" - "github.com/deckhouse/deckhouse-cli/internal" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/operations/params" - "github.com/deckhouse/deckhouse-cli/pkg/libmirror/util/auth" + mirrorutil "github.com/deckhouse/deckhouse-cli/testing/util/mirror" ) -func TestMirrorE2E(t *testing.T) { - t.SkipNow() -} +// TestMirrorE2E_FullCycle performs a complete mirror cycle: +// 1. Collects reference digests from source registry +// 2. Pulls images to local bundle +// 3. Pushes bundle to target registry +// 4. Validates target registry structure +// 5. Compares all digests between source and target +// +// Run with: +// +// go test -v ./testing/e2e/mirror/... \ +// -license-token=YOUR_TOKEN \ +// -source-registry=registry.deckhouse.ru/deckhouse/fe +// +// Or using environment variables: +// +// E2E_LICENSE_TOKEN=YOUR_TOKEN \ +// E2E_SOURCE_REGISTRY=registry.deckhouse.ru/deckhouse/fe \ +// go test -v ./testing/e2e/mirror/... +func TestMirrorE2E_FullCycle(t *testing.T) { + cfg := GetConfig() + + // Debug: show config + t.Logf("Config: SourceRegistry=%s, SourceUser=%s, HasAuth=%v", + cfg.SourceRegistry, cfg.SourceUser, cfg.HasSourceAuth()) + + if !cfg.HasSourceAuth() { + t.Skip("Source authentication not provided (use -license-token or -source-user/-source-password)") + } -func createDeckhouseReleaseChannelsInRegistry(t *testing.T, repo string) { - t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + defer cancel() - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.AlphaChannel, "v1.56.5") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.BetaChannel, "v1.56.5") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.EarlyAccessChannel, "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.StableChannel, "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", internal.RockSolidChannel, "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", "v1.55.7", "v1.55.7") - createDeckhouseReleaseChannelImageInRegistry(t, repo+"/release-channel", "v1.56.5", "v1.56.5") -} + // Setup target registry + targetHost, targetPath, cleanup := setupTargetRegistry(t, cfg) + defer cleanup() -func createTrivyVulnerabilityDatabasesInRegistry(t *testing.T, repo string, insecure, useTLS bool) { - t.Helper() - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(authn.Anonymous, insecure, useTLS) + targetRegistry := targetHost + targetPath + t.Logf("Target registry: %s", targetRegistry) - images := []string{ - repo + "/security/trivy-db:2", - repo + "/security/trivy-bdu:1", - repo + "/security/trivy-java-db:1", - repo + "/security/trivy-checks:0", + // Create bundle directory + bundleDir := t.TempDir() + if cfg.KeepBundle { + bundleDir = filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-e2e-%d", time.Now().Unix())) + require.NoError(t, os.MkdirAll(bundleDir, 0755)) + t.Logf("Bundle directory (will be kept): %s", bundleDir) + } else { + t.Logf("Bundle directory: %s", bundleDir) } - for _, image := range images { - ref, err := name.ParseReference(image, nameOpts...) - require.NoError(t, err) - wantImage, err := random.Image(256, 1) - require.NoError(t, err) - require.NoError(t, remote.Write(ref, wantImage, remoteOpts...)) + // Step 1: Collect reference digests from source + t.Log("Step 1: Collecting reference digests from source registry...") + t.Logf("Source: %s, tlsSkipVerify: %v", cfg.SourceRegistry, cfg.TLSSkipVerify) + sourceCollector := NewDigestCollector(cfg.SourceRegistry, cfg.GetSourceAuth(), cfg.TLSSkipVerify) + referenceDigests, err := sourceCollector.CollectAll(ctx) + require.NoError(t, err, "Failed to collect reference digests") + t.Logf("Collected %d reference digests", len(referenceDigests)) + + // Step 2: Execute pull + t.Log("Step 2: Pulling images to bundle...") + pullCmd := buildPullCommand(cfg, bundleDir) + t.Logf("Running: %s", pullCmd.String()) + + pullOutput, err := pullCmd.CombinedOutput() + if err != nil { + t.Logf("Pull output:\n%s", string(pullOutput)) } -} + require.NoError(t, err, "Pull failed") + t.Log("Pull completed successfully") -func createDeckhouseControllersAndInstallersInRegistry(t *testing.T, repo string) { - t.Helper() - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(nil, true, false) + // Verify bundle was created + bundleFiles, err := os.ReadDir(bundleDir) + require.NoError(t, err) + t.Logf("Bundle contains %d files/dirs", len(bundleFiles)) + for _, f := range bundleFiles { + t.Logf(" - %s", f.Name()) + } - createRandomImageInRegistry(t, repo+":"+internal.AlphaChannel) - createRandomImageInRegistry(t, repo+":"+internal.BetaChannel) - createRandomImageInRegistry(t, repo+":"+internal.EarlyAccessChannel) - createRandomImageInRegistry(t, repo+":"+internal.StableChannel) - createRandomImageInRegistry(t, repo+":"+internal.RockSolidChannel) - createRandomImageInRegistry(t, repo+":v1.56.5") - createRandomImageInRegistry(t, repo+":v1.55.7") + // Step 3: Execute push + t.Log("Step 3: Pushing bundle to target registry...") + pushCmd := buildPushCommand(cfg, bundleDir, targetRegistry) + t.Logf("Running: %s", pushCmd.String()) - installers := map[string]v1.Image{ - "v1.56.5": createSyntheticInstallerImage(t, "v1.56.5", repo), - "v1.55.7": createSyntheticInstallerImage(t, "v1.55.7", repo), + pushOutput, err := pushCmd.CombinedOutput() + if err != nil { + t.Logf("Push output:\n%s", string(pushOutput)) } - installers[internal.AlphaChannel] = installers["v1.56.5"] - installers[internal.BetaChannel] = installers["v1.56.5"] - installers[internal.EarlyAccessChannel] = installers["v1.55.7"] - installers[internal.StableChannel] = installers["v1.55.7"] - installers[internal.RockSolidChannel] = installers["v1.55.7"] + require.NoError(t, err, "Push failed") + t.Log("Push completed successfully") - for shortTag, installer := range installers { - ref, err := name.ParseReference(repo+"/install:"+shortTag, nameOpts...) - require.NoError(t, err) + // Step 4: Validate structure + t.Log("Step 4: Validating target registry structure...") + validator := NewStructureValidator(targetRegistry, cfg.GetTargetAuth(), cfg.TLSSkipVerify) + validator.SetExpectedFromDigests(NormalizeDigests(referenceDigests)) - err = remote.Write(ref, installer, remoteOpts...) - require.NoError(t, err) + validationResult, err := validator.ValidateMinimal(ctx) + require.NoError(t, err, "Validation error") - ref, err = name.ParseReference(repo+"/install-standalone:"+shortTag, nameOpts...) - require.NoError(t, err) - - err = remote.Write(ref, installer, remoteOpts...) - require.NoError(t, err) + if !validationResult.IsValid() { + t.Logf("Validation issues:\n%s", validationResult.String()) + } + require.True(t, validationResult.IsValid(), "Structure validation failed") + t.Log("Structure validation passed") + + // Step 5: Compare digests + t.Log("Step 5: Comparing digests...") + targetCollector := NewDigestCollector(targetRegistry, cfg.GetTargetAuth(), cfg.TLSSkipVerify) + targetDigests, err := targetCollector.CollectAll(ctx) + require.NoError(t, err, "Failed to collect target digests") + t.Logf("Collected %d target digests", len(targetDigests)) + + // Normalize both maps for comparison + normalizedReference := NormalizeDigests(referenceDigests) + normalizedTarget := NormalizeDigests(targetDigests) + + compareResult := Compare(normalizedReference, normalizedTarget) + if !compareResult.IsMatch() { + t.Logf("Comparison failed:\n%s", compareResult.String()) } + require.True(t, compareResult.IsMatch(), + "Digest comparison failed: %d mismatches found\n%s", + len(compareResult.Mismatches), + FormatMismatches(compareResult.Mismatches)) + + t.Logf("SUCCESS: All %d digests match!", compareResult.MatchedCount) } -func createSyntheticInstallerImage(t *testing.T, version, repo string) v1.Image { - t.Helper() +// TestMirrorE2E_PullOnly tests only the pull operation +func TestMirrorE2E_PullOnly(t *testing.T) { + cfg := GetConfig() - // FROM scratch - base := empty.Image - layers := make([]v1.Layer, 0) - - // COPY ./version /deckhouse/version - // COPY ./images_digests.json /deckhouse/candi/images_digests.json - imagesDigests, err := json.Marshal( - map[string]map[string]string{ - "common": { - "alpine": createRandomImageInRegistry(t, repo+":alpine"+version), - }, - "nodeManager": { - "bashibleApiserver": createRandomImageInRegistry(t, repo+":bashibleApiserver"+version), - }, - }) - require.NoError(t, err) - l, err := crane.Layer(map[string][]byte{ - "deckhouse/version": []byte(version), - "deckhouse/candi/images_digests.json": imagesDigests, - }) - require.NoError(t, err) - layers = append(layers, l) + if !cfg.HasSourceAuth() { + t.Skip("Source authentication not provided") + } - img, err := mutate.AppendLayers(base, layers...) - require.NoError(t, err) + bundleDir := t.TempDir() + if cfg.KeepBundle { + bundleDir = filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-pull-%d", time.Now().Unix())) + require.NoError(t, os.MkdirAll(bundleDir, 0755)) + t.Logf("Bundle directory (will be kept): %s", bundleDir) + } - // ENTRYPOINT ["/bin/bash"] - img, err = mutate.Config(img, v1.Config{ - Entrypoint: []string{"/bin/bash"}, - }) - require.NoError(t, err) + pullCmd := buildPullCommand(cfg, bundleDir) + t.Logf("Running: %s", pullCmd.String()) + + output, err := pullCmd.CombinedOutput() + t.Logf("Output:\n%s", string(output)) + require.NoError(t, err, "Pull failed") - return img + // Verify bundle was created + bundleFiles, err := os.ReadDir(bundleDir) + require.NoError(t, err) + require.NotEmpty(t, bundleFiles, "Bundle directory is empty") + + t.Logf("Bundle created with %d files:", len(bundleFiles)) + for _, f := range bundleFiles { + info, _ := f.Info() + if info != nil { + t.Logf(" - %s (%d bytes)", f.Name(), info.Size()) + } else { + t.Logf(" - %s", f.Name()) + } + } } -func createRandomImageInRegistry(t *testing.T, tag string) (digest string) { +// setupTargetRegistry sets up the target registry for testing +// Returns host, path, and cleanup function +func setupTargetRegistry(t *testing.T, cfg *Config) (string, string, func()) { t.Helper() - img, err := random.Image(int64(rand.Intn(1024)+1), int64(rand.Intn(5)+1)) - require.NoError(t, err) - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(nil, true, false) - ref, err := name.ParseReference(tag, nameOpts...) - require.NoError(t, err) + if cfg.UseInMemoryRegistry() { + // Use in-memory registry + host, path, _ := mirrorutil.SetupEmptyRegistryRepo(false) + t.Logf("Started in-memory registry at %s%s", host, path) + return host, path, func() { + // In-memory registry is cleaned up when test ends + } + } - err = remote.Write(ref, img, remoteOpts...) - require.NoError(t, err) + // Use external registry + return cfg.TargetRegistry, "", func() { + // External registry cleanup is user's responsibility + } +} - digestHash, err := img.Digest() - require.NoError(t, err) +// buildPullCommand builds the d8 mirror pull command +func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { + args := []string{ + "mirror", "pull", + "--source", cfg.SourceRegistry, + "--force", // overwrite if exists + } - return digestHash.String() -} + // Add authentication flags + if cfg.SourceUser != "" { + args = append(args, "--source-login", cfg.SourceUser) + args = append(args, "--source-password", cfg.SourcePassword) + } else if cfg.LicenseToken != "" { + args = append(args, "--license", cfg.LicenseToken) + } -func createDeckhouseReleaseChannelImageInRegistry(t *testing.T, repo, tag, version string) (digest string) { - t.Helper() + // Add TLS skip verify flag (for self-signed certs) + if cfg.TLSSkipVerify { + args = append(args, "--tls-skip-verify") + } - // FROM scratch - base := empty.Image - layers := make([]v1.Layer, 0) + args = append(args, bundleDir) - // COPY ./version.json /version.json - changelog, err := yaml.JSONToYAML([]byte(`{"candi":{"fixes":[{"summary":"Fix deckhouse containerd start after installing new containerd-deckhouse package.","pull_request":"https://github.com/deckhouse/deckhouse/pull/6329"}]}}`)) - require.NoError(t, err) - versionInfo := fmt.Sprintf( - `{"disruptions":{"1.56":["ingressNginx"]},"requirements":{"containerdOnAllNodes":"true","ingressNginx":"1.1","k8s":"1.23.0","nodesMinimalOSVersionUbuntu":"18.04"},"version":%q}`, - "v"+version, + cmd := exec.Command(cfg.D8Binary, args...) + cmd.Env = append(os.Environ(), + "HOME="+os.Getenv("HOME"), ) - l, err := crane.Layer(map[string][]byte{ - "version.json": []byte(versionInfo), - "changelog.yaml": changelog, - }) - layers = append(layers, l) - img, err := mutate.AppendLayers(base, layers...) - require.NoError(t, err) - - nameOpts, remoteOpts := auth.MakeRemoteRegistryRequestOptions(nil, true, false) - ref, err := name.ParseReference(repo+":"+tag, nameOpts...) - require.NoError(t, err) + return cmd +} - err = remote.Write(ref, img, remoteOpts...) - require.NoError(t, err) +// buildPushCommand builds the d8 mirror push command +func buildPushCommand(cfg *Config, bundleDir, targetRegistry string) *exec.Cmd { + args := []string{ + "mirror", "push", + bundleDir, + targetRegistry, + } - digestHash, err := img.Digest() - require.NoError(t, err) + if cfg.TLSSkipVerify { + args = append(args, "--tls-skip-verify") + } - return digestHash.String() -} + if cfg.TargetUser != "" { + args = append(args, "--registry-login", cfg.TargetUser) + args = append(args, "--registry-password", cfg.TargetPassword) + } -func validateDeckhouseReleasesManifests(t *testing.T, pullCtx *params.PullParams, versions []semver.Version) { - t.Helper() - deckhouseReleasesManifestsFilepath := filepath.Join(pullCtx.BundleDir, "deckhousereleases.yaml") - actualManifests, err := os.ReadFile(deckhouseReleasesManifestsFilepath) - require.NoError(t, err) + cmd := exec.Command(cfg.D8Binary, args...) + cmd.Env = append(os.Environ(), + "HOME="+os.Getenv("HOME"), + ) - expectedManifests := strings.Builder{} - for _, version := range versions { - expectedManifests.WriteString(fmt.Sprintf(`--- -apiVersion: deckhouse.io/v1alpha1 -approved: false -kind: DeckhouseRelease -metadata: - creationTimestamp: null - name: v%[1]s -spec: - changelog: - candi: - fixes: - - summary: Fix deckhouse containerd start after installing new containerd-deckhouse package. - pull_request: https://github.com/deckhouse/deckhouse/pull/6329 - changelogLink: https://github.com/deckhouse/deckhouse/releases/tag/v%[1]s - disruptions: - - ingressNginx - requirements: - containerdOnAllNodes: 'true' - ingressNginx: '1.1' - k8s: 1.23.0 - nodesMinimalOSVersionUbuntu: '18.04' - version: v%[1]s -status: - approved: false - message: "" - transitionTime: "0001-01-01T00:00:00Z" -`, version.String())) - } - - require.FileExists(t, deckhouseReleasesManifestsFilepath, "deckhousereleases.yaml should be generated next tar bundle") - require.YAMLEq(t, expectedManifests.String(), string(actualManifests)) + return cmd } diff --git a/testing/e2e/mirror/structure_validator.go b/testing/e2e/mirror/structure_validator.go new file mode 100644 index 00000000..ae797558 --- /dev/null +++ b/testing/e2e/mirror/structure_validator.go @@ -0,0 +1,250 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "path" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/samber/lo" + + "github.com/deckhouse/deckhouse-cli/internal" +) + +// ValidationResult contains the results of structure validation +type ValidationResult struct { + // MissingRepos lists repositories that should exist but don't + MissingRepos []string + + // MissingTags maps repository to list of missing tags + MissingTags map[string][]string + + // ExtraRepos lists unexpected repositories found + ExtraRepos []string + + // Errors contains any errors encountered during validation + Errors []error +} + +// IsValid returns true if validation passed +func (r *ValidationResult) IsValid() bool { + return len(r.MissingRepos) == 0 && + len(r.MissingTags) == 0 && + len(r.Errors) == 0 +} + +// String returns a human-readable summary +func (r *ValidationResult) String() string { + var sb strings.Builder + + if len(r.MissingRepos) > 0 { + sb.WriteString("Missing repositories:\n") + for _, repo := range r.MissingRepos { + sb.WriteString(" - " + repo + "\n") + } + } + + if len(r.MissingTags) > 0 { + sb.WriteString("Missing tags:\n") + for repo, tags := range r.MissingTags { + sb.WriteString(" " + repo + ":\n") + for _, tag := range tags { + sb.WriteString(" - " + tag + "\n") + } + } + } + + if len(r.Errors) > 0 { + sb.WriteString("Errors:\n") + for _, err := range r.Errors { + sb.WriteString(" - " + err.Error() + "\n") + } + } + + if r.IsValid() { + sb.WriteString("Validation passed\n") + } + + return sb.String() +} + +// StructureValidator validates the structure of a mirrored registry +type StructureValidator struct { + registry string + auth authn.Authenticator + + nameOpts []name.Option + remoteOpts []remote.Option + + // Expected structure from reference registry + expectedRepos []string + expectedTags map[string][]string +} + +// NewStructureValidator creates a new structure validator +// tlsSkipVerify: skip TLS certificate verification (for self-signed certs) +func NewStructureValidator(registry string, authenticator authn.Authenticator, tlsSkipVerify bool) *StructureValidator { + nameOpts := []name.Option{} + remoteOpts := []remote.Option{} + + if tlsSkipVerify { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + remoteOpts = append(remoteOpts, remote.WithTransport(transport)) + } + + if authenticator != nil && authenticator != authn.Anonymous { + remoteOpts = append(remoteOpts, remote.WithAuth(authenticator)) + } + + return &StructureValidator{ + registry: registry, + auth: authenticator, + nameOpts: nameOpts, + remoteOpts: remoteOpts, + expectedRepos: make([]string, 0), + expectedTags: make(map[string][]string), + } +} + +// SetExpectedFromDigests sets expected structure from a DigestMap +func (v *StructureValidator) SetExpectedFromDigests(digests DigestMap) { + repoSet := make(map[string]struct{}) + tagsByRepo := make(map[string][]string) + + for ref := range digests { + // Parse ref to extract repo and tag + parts := strings.Split(ref, ":") + if len(parts) != 2 { + continue + } + repo := parts[0] + tag := parts[1] + + repoSet[repo] = struct{}{} + tagsByRepo[repo] = append(tagsByRepo[repo], tag) + } + + v.expectedRepos = lo.Keys(repoSet) + v.expectedTags = tagsByRepo +} + +// Validate checks the registry structure +func (v *StructureValidator) Validate(ctx context.Context) (*ValidationResult, error) { + result := &ValidationResult{ + MissingRepos: make([]string, 0), + MissingTags: make(map[string][]string), + ExtraRepos: make([]string, 0), + Errors: make([]error, 0), + } + + // Validate each expected repository + for _, expectedRepo := range v.expectedRepos { + actualRepo := v.translateRepo(expectedRepo) + + // Check if repo exists by trying to list tags + tags, err := v.listTags(ctx, actualRepo) + if err != nil { + result.MissingRepos = append(result.MissingRepos, actualRepo) + result.Errors = append(result.Errors, fmt.Errorf("repo %s: %w", actualRepo, err)) + continue + } + + // Check for missing tags + expectedTags := v.expectedTags[expectedRepo] + missingTags := lo.Filter(expectedTags, func(tag string, _ int) bool { + return !lo.Contains(tags, tag) + }) + + if len(missingTags) > 0 { + result.MissingTags[actualRepo] = missingTags + } + } + + return result, nil +} + +// ValidateMinimal performs minimal validation (just checks key segments exist) +func (v *StructureValidator) ValidateMinimal(ctx context.Context) (*ValidationResult, error) { + result := &ValidationResult{ + MissingRepos: make([]string, 0), + MissingTags: make(map[string][]string), + Errors: make([]error, 0), + } + + // Check required segments + segments := []string{ + "", // root + internal.InstallSegment, // install + internal.InstallStandaloneSegment, // install-standalone + internal.ReleaseChannelSegment, // release-channel + } + + for _, segment := range segments { + repo := v.registry + if segment != "" { + repo = path.Join(v.registry, segment) + } + + tags, err := v.listTags(ctx, repo) + if err != nil { + result.MissingRepos = append(result.MissingRepos, repo) + result.Errors = append(result.Errors, fmt.Errorf("segment %s: %w", segment, err)) + continue + } + + // Check that at least one release channel tag exists + hasReleaseChannel := lo.ContainsBy(tags, func(tag string) bool { + return lo.Contains(internal.GetAllDefaultReleaseChannels(), tag) + }) + if !hasReleaseChannel { + result.MissingTags[repo] = []string{""} + } + } + + return result, nil +} + +// translateRepo translates a source repo path to target repo path +func (v *StructureValidator) translateRepo(sourceRepo string) string { + // Normalize the source repo to get the path portion + normalizedPath := NormalizeRef(sourceRepo) + // Prepend the target registry + return path.Join(v.registry, normalizedPath) +} + +// listTags lists all tags in a repository +func (v *StructureValidator) listTags(ctx context.Context, repo string) ([]string, error) { + repoRef, err := name.NewRepository(repo, v.nameOpts...) + if err != nil { + return nil, fmt.Errorf("parse repo: %w", err) + } + + tags, err := remote.List(repoRef, v.remoteOpts...) + if err != nil { + return nil, fmt.Errorf("list tags: %w", err) + } + + return tags, nil +} diff --git a/testing/util/mirror/registry.go b/testing/util/mirror/registry.go index e7dbf56a..16190a83 100644 --- a/testing/util/mirror/registry.go +++ b/testing/util/mirror/registry.go @@ -28,6 +28,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" ) +// ListableBlobHandler wraps a BlobHandler to track ingested blobs type ListableBlobHandler struct { registry.BlobHandler registry.BlobPutHandler @@ -36,6 +37,7 @@ type ListableBlobHandler struct { ingestedBlobs []string } +// Get implements registry.BlobHandler and tracks accessed blobs func (h *ListableBlobHandler) Get(ctx context.Context, repo string, hash v1.Hash) (io.ReadCloser, error) { h.mu.Lock() defer h.mu.Unlock() @@ -44,19 +46,52 @@ func (h *ListableBlobHandler) Get(ctx context.Context, repo string, hash v1.Hash return h.BlobHandler.Get(ctx, repo, hash) } +// ListBlobs returns all blobs that have been accessed func (h *ListableBlobHandler) ListBlobs() []string { - return h.ingestedBlobs + h.mu.Lock() + defer h.mu.Unlock() + return append([]string{}, h.ingestedBlobs...) +} + +// TestRegistry holds the test registry server and its resources +type TestRegistry struct { + Server *httptest.Server + Host string + RepoPath string + BlobHandler *ListableBlobHandler +} + +// Close stops the test registry server +func (r *TestRegistry) Close() { + if r.Server != nil { + r.Server.Close() + } } +// FullPath returns the full registry path including host and repo +func (r *TestRegistry) FullPath() string { + return r.Host + r.RepoPath +} + +// SetupEmptyRegistryRepo creates an in-memory registry for testing +// Returns host, repoPath, and a ListableBlobHandler to track blob access func SetupEmptyRegistryRepo(useTLS bool) ( /*host*/ string /*repoPath*/, string, *ListableBlobHandler) { - var host, repoPath string + reg := SetupTestRegistry(useTLS) + return reg.Host, reg.RepoPath, reg.BlobHandler +} +// SetupTestRegistry creates an in-memory registry for testing and returns a TestRegistry +func SetupTestRegistry(useTLS bool) *TestRegistry { memBlobHandler := registry.NewInMemoryBlobHandler() bh := &ListableBlobHandler{ BlobHandler: memBlobHandler, BlobPutHandler: memBlobHandler.(registry.BlobPutHandler), } - registryHandler := registry.New(registry.WithBlobHandler(bh), registry.Logger(golog.New(io.Discard, "", 0))) + + registryHandler := registry.New( + registry.WithBlobHandler(bh), + registry.Logger(golog.New(io.Discard, "", 0)), + ) server := httptest.NewUnstartedServer(registryHandler) if useTLS { @@ -65,11 +100,22 @@ func SetupEmptyRegistryRepo(useTLS bool) ( /*host*/ string /*repoPath*/, string, server.Start() } - host = strings.TrimPrefix(server.URL, "http://") - repoPath = "/deckhouse/ee" + host := strings.TrimPrefix(server.URL, "http://") if useTLS { host = strings.TrimPrefix(server.URL, "https://") } - return host, repoPath, bh + return &TestRegistry{ + Server: server, + Host: host, + RepoPath: "/deckhouse/ee", + BlobHandler: bh, + } +} + +// SetupTestRegistryWithPath creates an in-memory registry with a custom repo path +func SetupTestRegistryWithPath(useTLS bool, repoPath string) *TestRegistry { + reg := SetupTestRegistry(useTLS) + reg.RepoPath = repoPath + return reg } From fa68cf6f88a7627671849436ec66a5c1661d0c31 Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Fri, 26 Dec 2025 10:55:02 +0300 Subject: [PATCH 04/11] feat: e2e tests Signed-off-by: Timur Tuktamyshev --- .gitignore | 5 +- go.mod | 5 + go.sum | 10 + pkg/libmirror/layouts/push_test.go | 24 +- testing/e2e/mirror/README.md | 240 +++--- testing/e2e/mirror/config.go | 9 + testing/e2e/mirror/digest_collector.go | 241 ------ testing/e2e/mirror/digest_comparator.go | 233 ------ testing/e2e/mirror/mirror_e2e_test.go | 648 ++++++++++++--- testing/e2e/mirror/registry_comparator.go | 936 ++++++++++++++++++++++ testing/e2e/mirror/structure_validator.go | 250 ------ testing/util/mirror/registry.go | 150 ++-- 12 files changed, 1737 insertions(+), 1014 deletions(-) delete mode 100644 testing/e2e/mirror/digest_collector.go delete mode 100644 testing/e2e/mirror/digest_comparator.go create mode 100644 testing/e2e/mirror/registry_comparator.go delete mode 100644 testing/e2e/mirror/structure_validator.go diff --git a/.gitignore b/.gitignore index 30a43a92..8a0a35b5 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,7 @@ dist/ build/ # Entrypoint for the application -!/cmd/d8 \ No newline at end of file +!/cmd/d8 + +# E2E test logs +testing/e2e/.logs/ \ No newline at end of file diff --git a/go.mod b/go.mod index 8d07317d..821dc8e2 100644 --- a/go.mod +++ b/go.mod @@ -134,6 +134,11 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/chanced/caps v1.0.2 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.8.0 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cilium/ebpf v0.12.3 // indirect github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible // indirect diff --git a/go.sum b/go.sum index 243393f9..9d86d774 100644 --- a/go.sum +++ b/go.sum @@ -300,6 +300,16 @@ github.com/chainguard-dev/git-urls v1.0.2 h1:pSpT7ifrpc5X55n4aTTm7FFUE+ZQHKiqpiw github.com/chainguard-dev/git-urls v1.0.2/go.mod h1:rbGgj10OS7UgZlbzdUQIQpT0k/D4+An04HJY7Ol+Y/o= github.com/chanced/caps v1.0.2 h1:RELvNN4lZajqSXJGzPaU7z8B4LK2+o2Oc/upeWdgMOA= github.com/chanced/caps v1.0.2/go.mod h1:SJhRzeYLKJ3OmzyQXhdZ7Etj7lqqWoPtQ1zcSJRtQjs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= +github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= diff --git a/pkg/libmirror/layouts/push_test.go b/pkg/libmirror/layouts/push_test.go index 3e2f3311..813a339f 100644 --- a/pkg/libmirror/layouts/push_test.go +++ b/pkg/libmirror/layouts/push_test.go @@ -38,7 +38,9 @@ func TestPushLayoutToRepoWithParallelism(t *testing.T) { const totalImages, layersPerImage = 10, 3 imagesLayout := createEmptyOCILayout(t) - host, repoPath, _ := mirrorTestUtils.SetupEmptyRegistryRepo(false) + reg := mirrorTestUtils.SetupTestRegistry(false) + defer reg.Close() + repoPath := reg.Host + "/deckhouse/ee" client := mock.NewRegistryClientMock(t) client.PushImageMock.Return(nil) @@ -50,7 +52,7 @@ func TestPushLayoutToRepoWithParallelism(t *testing.T) { digest, err := img.Digest() s.NoError(err) err = imagesLayout.AppendImage(img, platformOpt, layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": host + repoPath + "@" + digest.String(), + "org.opencontainers.image.ref.name": repoPath + "@" + digest.String(), "io.deckhouse.image.short_tag": digest.Hex, })) s.NoError(err) @@ -59,7 +61,7 @@ func TestPushLayoutToRepoWithParallelism(t *testing.T) { err := PushLayoutToRepo( client, imagesLayout, - host+repoPath, // Images repo + repoPath, authn.Anonymous, log.NewSLogger(slog.LevelDebug), params.ParallelismConfig{ @@ -81,7 +83,9 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { const totalImages, layersPerImage = 10, 3 imagesLayout := createEmptyOCILayout(t) - host, repoPath, _ := mirrorTestUtils.SetupEmptyRegistryRepo(false) + reg := mirrorTestUtils.SetupTestRegistry(false) + defer reg.Close() + repoPath := reg.Host + "/deckhouse/ee" client := mock.NewRegistryClientMock(t) client.PushImageMock.Return(nil) @@ -93,7 +97,7 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { digest, err := img.Digest() s.NoError(err) err = imagesLayout.AppendImage(img, platformOpt, layout.WithAnnotations(map[string]string{ - "org.opencontainers.image.ref.name": host + repoPath + "@" + digest.String(), + "org.opencontainers.image.ref.name": repoPath + "@" + digest.String(), "io.deckhouse.image.short_tag": digest.Hex, })) s.NoError(err) @@ -102,7 +106,7 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { err := PushLayoutToRepo( client, imagesLayout, - host+repoPath, // Images repo + repoPath, authn.Anonymous, log.NewSLogger(slog.LevelDebug), params.ParallelismConfig{ @@ -121,7 +125,9 @@ func TestPushLayoutToRepoWithoutParallelism(t *testing.T) { func TestPushEmptyLayoutToRepo(t *testing.T) { s := require.New(t) - host, repoPath, blobHandler := mirrorTestUtils.SetupEmptyRegistryRepo(false) + reg := mirrorTestUtils.SetupTestRegistry(false) + defer reg.Close() + repoPath := reg.Host + "/deckhouse/ee" client := mock.NewRegistryClientMock(t) @@ -129,7 +135,7 @@ func TestPushEmptyLayoutToRepo(t *testing.T) { err := PushLayoutToRepo( client, emptyLayout, - host+repoPath, + repoPath, authn.Anonymous, log.NewSLogger(slog.LevelDebug), params.DefaultParallelism, @@ -137,5 +143,5 @@ func TestPushEmptyLayoutToRepo(t *testing.T) { false, // TLS verification irrelevant to HTTP requests ) s.NoError(err, "Push should not fail") - s.Len(blobHandler.ListBlobs(), 0, "No blobs should be pushed to registry") + s.Len(reg.ListBlobs(), 0, "No blobs should be pushed to registry") } diff --git a/testing/e2e/mirror/README.md b/testing/e2e/mirror/README.md index fe60c0d6..47b58817 100644 --- a/testing/e2e/mirror/README.md +++ b/testing/e2e/mirror/README.md @@ -1,178 +1,206 @@ # E2E Tests for d8 mirror -End-to-end tests for the `d8 mirror pull` and `d8 mirror push` commands. +Heavy end-to-end tests for the `d8 mirror pull` and `d8 mirror push` commands. ## Overview -These tests perform a complete mirror cycle: -1. Collect reference digests from source registry -2. Pull images to a local bundle -3. Push bundle to a target registry -4. Validate the target registry structure -5. Compare all digests between source and target +These tests perform a **complete mirror cycle with deep comparison** to ensure source and target registries are **100% identical**. + +### Test Steps + +1. **Analyze source registry** - Discover all repositories and count all images +2. **Pull images** - Execute `d8 mirror pull` to create a bundle +3. **Push images** - Execute `d8 mirror push` to target registry +4. **Deep comparison** - Compare EVERY repository, tag, and digest between source and target + +### What Gets Compared (Deep Comparison) + +- **Repository level**: All repositories in source must exist in target +- **Tag level**: All tags in each repository must exist in target +- **Manifest digest**: Every image manifest digest must match (SHA256) +- **Config digest**: Image config blob digest is verified +- **Layer digests**: ALL layer digests of every image are compared +- **Layer count**: Number of layers must match + +This ensures **byte-for-byte identical** registries. ## Requirements - Built `d8` binary (run `task build` from project root) -- Valid license token for the source registry +- Valid credentials for the source registry - Network access to the source registry +- Sufficient disk space for the bundle (can be several GB) ## Running Tests ### Basic Usage ```bash -# Run with license token +# Run with license token (official Deckhouse registry) go test -v ./testing/e2e/mirror/... \ -license-token=YOUR_LICENSE_TOKEN -# Or use environment variables (recommended) -E2E_LICENSE_TOKEN=YOUR_LICENSE_TOKEN \ -go test -v ./testing/e2e/mirror/... +# Run with local registry (self-signed cert) +go test -v ./testing/e2e/mirror/... \ + -source-registry=localhost:443/deckhouse \ + -source-user=admin \ + -source-password=secret \ + -tls-skip-verify ``` ### Full Configuration ```bash -# Using license token go test -v ./testing/e2e/mirror/... \ - -source-registry=registry.deckhouse.ru/deckhouse/fe \ - -license-token=YOUR_LICENSE_TOKEN \ - -target-registry=my-registry.local:5000/deckhouse \ + -source-registry=my-registry.local/deckhouse \ + -source-user=admin \ + -source-password=secret \ + -target-registry=my-target.local:5000/deckhouse \ -target-user=admin \ -target-password=secret \ -tls-skip-verify \ -keep-bundle - -# Using explicit source credentials (with self-signed cert) -go test -v ./testing/e2e/mirror/... \ - -source-registry=my-source-registry.local/deckhouse \ - -source-user=admin \ - -source-password=secret \ - -target-registry=my-target-registry.local:5000/deckhouse \ - -tls-skip-verify - -# Using environment variables (recommended) -E2E_SOURCE_REGISTRY=localhost:443/deckhouse-etalon \ -E2E_SOURCE_USER=admin \ -E2E_SOURCE_PASSWORD=secret \ -E2E_TLS_SKIP_VERIFY=true \ -go test -v ./testing/e2e/mirror/... ``` ### Environment Variables -All flags can be set via environment variables: - | Flag | Environment Variable | Default | Description | |------|---------------------|---------|-------------| -| `-source-registry` | `E2E_SOURCE_REGISTRY` | `registry.deckhouse.ru/deckhouse/fe` | Source registry to pull from | -| `-source-user` | `E2E_SOURCE_USER` | | Source registry username (alternative to license-token) | +| `-source-registry` | `E2E_SOURCE_REGISTRY` | `registry.deckhouse.ru/deckhouse/fe` | Source registry | +| `-source-user` | `E2E_SOURCE_USER` | | Source registry username | | `-source-password` | `E2E_SOURCE_PASSWORD` | | Source registry password | -| `-license-token` | `E2E_LICENSE_TOKEN` | | License token for Deckhouse registry (shortcut for source-user=license-token) | -| `-target-registry` | `E2E_TARGET_REGISTRY` | (empty = in-memory) | Target registry to push to | +| `-license-token` | `E2E_LICENSE_TOKEN` | | License token | +| `-target-registry` | `E2E_TARGET_REGISTRY` | (local disk-based registry) | Target registry | | `-target-user` | `E2E_TARGET_USER` | | Target registry username | | `-target-password` | `E2E_TARGET_PASSWORD` | | Target registry password | -| `-tls-skip-verify` | `E2E_TLS_SKIP_VERIFY` | `false` | Skip TLS certificate verification (for self-signed certs) | -| `-keep-bundle` | `E2E_KEEP_BUNDLE` | `false` | Keep bundle directory after test | +| `-tls-skip-verify` | `E2E_TLS_SKIP_VERIFY` | `false` | Skip TLS verification | +| `-keep-bundle` | `E2E_KEEP_BUNDLE` | `false` | Keep bundle after test | | `-d8-binary` | `E2E_D8_BINARY` | `bin/d8` | Path to d8 binary | -**Note:** Either `-license-token` OR `-source-user`/`-source-password` must be provided for authentication. - -## Test Scenarios - -### TestMirrorE2E_FullCycle +## Test Output -Complete end-to-end test that: -1. Collects all image digests from source registry -2. Runs `d8 mirror pull` to create a bundle -3. Runs `d8 mirror push` to push to target -4. Validates that all segments exist in target -5. Compares every digest between source and target +### Log Directory Structure -### TestMirrorE2E_PullOnly +Test logs are stored in `testing/e2e/.logs/-/`: -Tests only the pull operation: -1. Runs `d8 mirror pull` to create a bundle -2. Verifies the bundle directory contains expected files - -## Target Registry Options - -### In-Memory Registry (Default) - -When `-target-registry` is not specified, tests use an in-memory registry. -This is useful for CI/CD and quick local testing. - -### External Registry - -Specify `-target-registry` to push to a real registry: - -```bash -# Docker registry with self-signed cert -go test -v ./testing/e2e/mirror/... \ - -license-token=TOKEN \ - -target-registry=localhost:5000/deckhouse \ - -tls-skip-verify - -# Registry with auth -go test -v ./testing/e2e/mirror/... \ - -license-token=TOKEN \ - -target-registry=registry.example.com/deckhouse \ - -target-user=admin \ - -target-password=secret +``` +testing/e2e/.logs/fullcycle-20251225-123456/ +├── test.log # Full command output (pull/push) +├── report.txt # Test summary report +└── comparison.txt # Detailed registry comparison ``` -## What Gets Validated +### Sample Report Output -### Structure Validation +``` +================================================================================ +E2E TEST REPORT: TestMirrorE2E_FullCycle +================================================================================ + +Duration: 25m30s + +REGISTRIES: + Source: localhost:443/deckhouse (15 repos, 1847 images) + Target: 127.0.0.1:54321/deckhouse/ee (15 repos, 1847 images) + +COMPARISON: + ✓ Matched: 1847 images (manifest digest) + ✓ Deep checked: 1847 images + ✓ Layers: 15234 verified + +STEPS: + ✓ Analyze source (15 repos, 1847 images) (45s) + ✓ Pull images (5.23 GB bundle) (12m15s) + ✓ Push to registry (8m30s) + ✓ Deep comparison (1847 images verified) (4m00s) + +-------------------------------------------------------------------------------- +RESULT: PASSED - REGISTRIES ARE IDENTICAL + 1847 images, 15234 layers - all hashes verified +================================================================================ +``` -- Root Deckhouse images exist -- `/install` segment exists with release channel tags -- `/install-standalone` segment exists -- `/release-channel` segment exists -- `/security/*` databases exist (if present in source) -- `/modules/*` exist (if present in source) +### Comparison Report (comparison.txt) -### Digest Comparison +Contains detailed breakdown per repository with layer-level verification: -Every image tag in the source is compared with the target: -- All images must be present in target -- All digests must match exactly (SHA256) +``` +REGISTRY COMPARISON SUMMARY +=========================== + +Source: localhost:443/deckhouse +Target: 127.0.0.1:54321/deckhouse/ee +Duration: 4m0s + +REPOSITORIES: + Source: 15 + Target: 15 + Missing in target: 0 + Extra in target: 0 + +IMAGES (manifest digest comparison): + Source: 1847 + Target: 1847 + Matched: 1847 + Missing: 0 + Mismatched: 0 + Extra: 0 + +DEEP COMPARISON (layers + config): + Images deep-checked: 1847 + Source layers: 15234 + Target layers: 15234 + Matched layers: 15234 + Missing layers: 0 + Config mismatches: 0 + +✓ REGISTRIES ARE IDENTICAL (all hashes match) + +REPOSITORY BREAKDOWN: +--------------------- +✓ (root): 523/523 tags, 4521 layers checked +✓ install: 6/6 tags, 48 layers checked +✓ install-standalone: 78/78 tags, 624 layers checked +✓ release-channel: 6/6 tags, 12 layers checked +✓ security/trivy-db: 1/1 tags, 2 layers checked +✓ modules/deckhouse-admin: 15/15 tags, 120 layers checked +... +``` ## Timeouts -The full cycle test has a 60-minute timeout. This can be adjusted in the test code if needed. +The test has a **120-minute timeout** to handle large registries. ## Troubleshooting -### "License token not provided" +### "Source authentication not provided" -Set the license token via flag or environment variable: +Set credentials: ```bash E2E_LICENSE_TOKEN=your_token go test -v ./testing/e2e/mirror/... +# or +E2E_SOURCE_USER=admin E2E_SOURCE_PASSWORD=secret go test -v ./testing/e2e/mirror/... ``` -### "Pull failed" +### "Pull failed" or "Push failed" -Check that: -1. The `d8` binary exists at `bin/d8` (run `task build` first) or specify path with `-d8-binary` -2. You have network access to the source registry -3. The license token is valid +1. Check `d8` binary exists (`task build`) +2. Check network access +3. Use `-tls-skip-verify` for self-signed certs +4. Check credentials -### "Push failed" +### "Registries differ" -Check that: -1. The target registry is accessible -2. Credentials are correct (if using authenticated registry) -3. Use `-tls-skip-verify` for self-signed certificates +Check `comparison.txt` for details: +- **Missing images**: Images in source but not in target +- **Mismatched digests**: Images exist but have different content ### Viewing Bundle Contents -Use `-keep-bundle` to preserve the bundle directory: ```bash go test -v ./testing/e2e/mirror/... \ -license-token=TOKEN \ -keep-bundle -# Bundle will be at /tmp/d8-mirror-e2e-TIMESTAMP/ +# Bundle location shown in output ``` diff --git a/testing/e2e/mirror/config.go b/testing/e2e/mirror/config.go index 6bec647e..c5a0443e 100644 --- a/testing/e2e/mirror/config.go +++ b/testing/e2e/mirror/config.go @@ -60,6 +60,11 @@ var ( d8Binary = flag.String("d8-binary", getEnvOrDefault("E2E_D8_BINARY", "../../../bin/d8"), "Path to d8 binary") + + // Debug/test options + noModules = flag.Bool("no-modules", + getEnvOrDefault("E2E_NO_MODULES", "") == "true", + "Skip modules during pull (for testing failure scenarios)") ) func getEnvOrDefault(key, defaultValue string) string { @@ -83,6 +88,9 @@ type Config struct { TLSSkipVerify bool KeepBundle bool D8Binary string + + // Debug/test options + NoModules bool // Skip modules during pull (for testing failure scenarios) } // GetConfig returns the current test configuration from flags @@ -99,6 +107,7 @@ func GetConfig() *Config { TLSSkipVerify: *tlsSkipVerify, KeepBundle: *keepBundle, D8Binary: *d8Binary, + NoModules: *noModules, } } diff --git a/testing/e2e/mirror/digest_collector.go b/testing/e2e/mirror/digest_collector.go deleted file mode 100644 index fb22803e..00000000 --- a/testing/e2e/mirror/digest_collector.go +++ /dev/null @@ -1,241 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "context" - "crypto/tls" - "fmt" - "net/http" - "path" - "strings" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - - "github.com/deckhouse/deckhouse-cli/internal" -) - -// DigestMap maps image reference (repo:tag) to its digest -type DigestMap map[string]string - -// DigestCollector collects digests from a container registry -type DigestCollector struct { - registry string - auth authn.Authenticator - - nameOpts []name.Option - remoteOpts []remote.Option -} - -// NewDigestCollector creates a new digest collector -// tlsSkipVerify: skip TLS certificate verification (for self-signed certs) -func NewDigestCollector(registry string, authenticator authn.Authenticator, tlsSkipVerify bool) *DigestCollector { - nameOpts := []name.Option{} - remoteOpts := []remote.Option{} - - if tlsSkipVerify { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - remoteOpts = append(remoteOpts, remote.WithTransport(transport)) - } - - if authenticator != nil && authenticator != authn.Anonymous { - remoteOpts = append(remoteOpts, remote.WithAuth(authenticator)) - } - - return &DigestCollector{ - registry: registry, - auth: authenticator, - nameOpts: nameOpts, - remoteOpts: remoteOpts, - } -} - -// CollectAll collects digests for all images in the registry -func (c *DigestCollector) CollectAll(ctx context.Context) (DigestMap, error) { - digests := make(DigestMap) - - // Collect Deckhouse root images - if err := c.collectDeckhouseRoot(ctx, digests); err != nil { - return nil, fmt.Errorf("collect deckhouse root: %w", err) - } - - // Collect install images - if err := c.collectSegment(ctx, digests, internal.InstallSegment); err != nil { - return nil, fmt.Errorf("collect install: %w", err) - } - - // Collect install-standalone images - if err := c.collectSegment(ctx, digests, internal.InstallStandaloneSegment); err != nil { - return nil, fmt.Errorf("collect install-standalone: %w", err) - } - - // Collect release-channel images - if err := c.collectSegment(ctx, digests, internal.ReleaseChannelSegment); err != nil { - return nil, fmt.Errorf("collect release-channel: %w", err) - } - - // Collect security databases - if err := c.collectSecurity(ctx, digests); err != nil { - return nil, fmt.Errorf("collect security: %w", err) - } - - // Collect modules - if err := c.collectModules(ctx, digests); err != nil { - return nil, fmt.Errorf("collect modules: %w", err) - } - - return digests, nil -} - -// collectDeckhouseRoot collects digests from the root registry path -func (c *DigestCollector) collectDeckhouseRoot(ctx context.Context, digests DigestMap) error { - return c.collectTagsFromRepo(ctx, digests, c.registry) -} - -// collectSegment collects digests from a specific segment (install, release-channel, etc) -func (c *DigestCollector) collectSegment(ctx context.Context, digests DigestMap, segment string) error { - repo := path.Join(c.registry, segment) - return c.collectTagsFromRepo(ctx, digests, repo) -} - -// collectSecurity collects security database digests -func (c *DigestCollector) collectSecurity(ctx context.Context, digests DigestMap) error { - securityImages := map[string]string{ - path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyDBSegment): "2", - path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyBDUSegment): "1", - path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyJavaDBSegment): "1", - path.Join(c.registry, internal.SecuritySegment, internal.SecurityTrivyChecksSegment): "0", - } - - for repo, tag := range securityImages { - digest, err := c.getDigest(ctx, repo+":"+tag) - if err != nil { - // Security databases might not exist, skip with warning - continue - } - digests[repo+":"+tag] = digest - } - - return nil -} - -// collectModules collects digests for all modules -func (c *DigestCollector) collectModules(ctx context.Context, digests DigestMap) error { - modulesRepo := path.Join(c.registry, internal.ModulesSegment) - - // List all modules (they are tags in the modules repo) - modules, err := c.listTags(ctx, modulesRepo) - if err != nil { - // Modules might not be accessible - return nil - } - - for _, moduleName := range modules { - // Module root images - moduleRepo := path.Join(modulesRepo, moduleName) - if err := c.collectTagsFromRepo(ctx, digests, moduleRepo); err != nil { - continue - } - - // Module release channels - releaseRepo := path.Join(moduleRepo, "release") - if err := c.collectTagsFromRepo(ctx, digests, releaseRepo); err != nil { - // Release repo might not exist - continue - } - } - - return nil -} - -// collectTagsFromRepo lists all tags in a repo and collects their digests -func (c *DigestCollector) collectTagsFromRepo(ctx context.Context, digests DigestMap, repo string) error { - tags, err := c.listTags(ctx, repo) - if err != nil { - return err - } - - for _, tag := range tags { - ref := repo + ":" + tag - digest, err := c.getDigest(ctx, ref) - if err != nil { - continue - } - digests[ref] = digest - } - - return nil -} - -// listTags lists all tags in a repository -func (c *DigestCollector) listTags(ctx context.Context, repo string) ([]string, error) { - repoRef, err := name.NewRepository(repo, c.nameOpts...) - if err != nil { - return nil, fmt.Errorf("parse repo %s: %w", repo, err) - } - - tags, err := remote.List(repoRef, c.remoteOpts...) - if err != nil { - return nil, fmt.Errorf("list tags for %s: %w", repo, err) - } - - return tags, nil -} - -// getDigest gets the digest for a specific image reference -func (c *DigestCollector) getDigest(ctx context.Context, ref string) (string, error) { - imgRef, err := name.ParseReference(ref, c.nameOpts...) - if err != nil { - return "", fmt.Errorf("parse ref %s: %w", ref, err) - } - - desc, err := remote.Head(imgRef, c.remoteOpts...) - if err != nil { - return "", fmt.Errorf("get digest for %s: %w", ref, err) - } - - return desc.Digest.String(), nil -} - -// NormalizeRef normalizes a reference to allow comparison between registries -// Strips the registry host from the reference -func NormalizeRef(ref string) string { - // Remove scheme if present - ref = strings.TrimPrefix(ref, "https://") - ref = strings.TrimPrefix(ref, "http://") - - // Find the first slash after the host - slashIdx := strings.Index(ref, "/") - if slashIdx == -1 { - return ref - } - - // Return everything after the host - return ref[slashIdx+1:] -} - -// NormalizeDigests creates a new map with normalized references -func NormalizeDigests(digests DigestMap) DigestMap { - normalized := make(DigestMap, len(digests)) - for ref, digest := range digests { - normalized[NormalizeRef(ref)] = digest - } - return normalized -} diff --git a/testing/e2e/mirror/digest_comparator.go b/testing/e2e/mirror/digest_comparator.go deleted file mode 100644 index 529184c1..00000000 --- a/testing/e2e/mirror/digest_comparator.go +++ /dev/null @@ -1,233 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "fmt" - "sort" - "strings" -) - -// Mismatch represents a digest mismatch between source and target -type Mismatch struct { - // Ref is the normalized image reference (path:tag) - Ref string - - // ExpectedDigest is the digest from the source registry - ExpectedDigest string - - // ActualDigest is the digest from the target registry (empty if not found) - ActualDigest string - - // Type indicates the type of mismatch - Type MismatchType -} - -// MismatchType categorizes the type of mismatch -type MismatchType int - -const ( - // MismatchTypeMissing means the image is missing in the target - MismatchTypeMissing MismatchType = iota - - // MismatchTypeDigestDifferent means the digests don't match - MismatchTypeDigestDifferent - - // MismatchTypeExtra means the image exists in target but not in source - MismatchTypeExtra -) - -func (t MismatchType) String() string { - switch t { - case MismatchTypeMissing: - return "MISSING" - case MismatchTypeDigestDifferent: - return "DIGEST_MISMATCH" - case MismatchTypeExtra: - return "EXTRA" - default: - return "UNKNOWN" - } -} - -// String returns a human-readable representation of the mismatch -func (m Mismatch) String() string { - switch m.Type { - case MismatchTypeMissing: - return fmt.Sprintf("[%s] %s: expected %s, not found in target", - m.Type, m.Ref, m.ExpectedDigest) - case MismatchTypeDigestDifferent: - return fmt.Sprintf("[%s] %s: expected %s, got %s", - m.Type, m.Ref, m.ExpectedDigest, m.ActualDigest) - case MismatchTypeExtra: - return fmt.Sprintf("[%s] %s: unexpected image with digest %s", - m.Type, m.Ref, m.ActualDigest) - default: - return fmt.Sprintf("[%s] %s", m.Type, m.Ref) - } -} - -// CompareResult contains the result of digest comparison -type CompareResult struct { - // Mismatches lists all found mismatches - Mismatches []Mismatch - - // MatchedCount is the number of images that matched - MatchedCount int - - // TotalExpected is the total number of expected images - TotalExpected int - - // TotalActual is the total number of actual images - TotalActual int -} - -// IsMatch returns true if all digests matched -func (r *CompareResult) IsMatch() bool { - return len(r.Mismatches) == 0 -} - -// MissingCount returns the number of missing images -func (r *CompareResult) MissingCount() int { - count := 0 - for _, m := range r.Mismatches { - if m.Type == MismatchTypeMissing { - count++ - } - } - return count -} - -// DigestMismatchCount returns the number of digest mismatches -func (r *CompareResult) DigestMismatchCount() int { - count := 0 - for _, m := range r.Mismatches { - if m.Type == MismatchTypeDigestDifferent { - count++ - } - } - return count -} - -// String returns a summary of the comparison -func (r *CompareResult) String() string { - var sb strings.Builder - - sb.WriteString(fmt.Sprintf("Comparison Summary:\n")) - sb.WriteString(fmt.Sprintf(" Expected: %d images\n", r.TotalExpected)) - sb.WriteString(fmt.Sprintf(" Actual: %d images\n", r.TotalActual)) - sb.WriteString(fmt.Sprintf(" Matched: %d images\n", r.MatchedCount)) - sb.WriteString(fmt.Sprintf(" Missing: %d images\n", r.MissingCount())) - sb.WriteString(fmt.Sprintf(" Digest mismatches: %d\n", r.DigestMismatchCount())) - - if !r.IsMatch() { - sb.WriteString("\nMismatches:\n") - for _, m := range r.Mismatches { - sb.WriteString(" " + m.String() + "\n") - } - } - - return sb.String() -} - -// Compare compares two digest maps and returns mismatches -// Both maps should have normalized references (using NormalizeDigests) -func Compare(expected, actual DigestMap) *CompareResult { - result := &CompareResult{ - Mismatches: make([]Mismatch, 0), - TotalExpected: len(expected), - TotalActual: len(actual), - } - - // Check all expected images - for ref, expectedDigest := range expected { - actualDigest, exists := actual[ref] - - if !exists { - result.Mismatches = append(result.Mismatches, Mismatch{ - Ref: ref, - ExpectedDigest: expectedDigest, - Type: MismatchTypeMissing, - }) - continue - } - - if expectedDigest != actualDigest { - result.Mismatches = append(result.Mismatches, Mismatch{ - Ref: ref, - ExpectedDigest: expectedDigest, - ActualDigest: actualDigest, - Type: MismatchTypeDigestDifferent, - }) - continue - } - - result.MatchedCount++ - } - - // Sort mismatches by ref for consistent output - sort.Slice(result.Mismatches, func(i, j int) bool { - return result.Mismatches[i].Ref < result.Mismatches[j].Ref - }) - - return result -} - -// CompareStrict also reports extra images in the target -func CompareStrict(expected, actual DigestMap) *CompareResult { - result := Compare(expected, actual) - - // Find extra images in actual that don't exist in expected - for ref, actualDigest := range actual { - if _, exists := expected[ref]; !exists { - result.Mismatches = append(result.Mismatches, Mismatch{ - Ref: ref, - ActualDigest: actualDigest, - Type: MismatchTypeExtra, - }) - } - } - - // Re-sort after adding extras - sort.Slice(result.Mismatches, func(i, j int) bool { - return result.Mismatches[i].Ref < result.Mismatches[j].Ref - }) - - return result -} - -// FormatMismatches formats mismatches for test output -func FormatMismatches(mismatches []Mismatch) string { - if len(mismatches) == 0 { - return "no mismatches" - } - - var sb strings.Builder - for i, m := range mismatches { - if i > 0 { - sb.WriteString("\n") - } - sb.WriteString(m.String()) - - // Limit output for very long lists - if i >= 50 { - sb.WriteString(fmt.Sprintf("\n... and %d more mismatches", len(mismatches)-i-1)) - break - } - } - return sb.String() -} diff --git a/testing/e2e/mirror/mirror_e2e_test.go b/testing/e2e/mirror/mirror_e2e_test.go index e7c3c000..52804ff7 100644 --- a/testing/e2e/mirror/mirror_e2e_test.go +++ b/testing/e2e/mirror/mirror_e2e_test.go @@ -17,56 +17,157 @@ limitations under the License. package mirror import ( + "bytes" "context" "fmt" + "io" "os" "os/exec" "path/filepath" + "strings" "testing" "time" + "github.com/charmbracelet/lipgloss" + "github.com/muesli/termenv" "github.com/stretchr/testify/require" + "golang.org/x/term" mirrorutil "github.com/deckhouse/deckhouse-cli/testing/util/mirror" ) -// TestMirrorE2E_FullCycle performs a complete mirror cycle: -// 1. Collects reference digests from source registry -// 2. Pulls images to local bundle -// 3. Pushes bundle to target registry -// 4. Validates target registry structure -// 5. Compares all digests between source and target +func init() { + // Force color output - go test buffers stdout which disables color detection + // Check stderr instead (usually unbuffered) or honor FORCE_COLOR env + if term.IsTerminal(int(os.Stderr.Fd())) || os.Getenv("FORCE_COLOR") != "" || os.Getenv("TERM") != "" { + lipgloss.DefaultRenderer().SetColorProfile(termenv.TrueColor) + } +} + +// output is a helper to write to stderr (which preserves colors in go test) +var output = os.Stderr + +func printLine(format string, args ...interface{}) { + fmt.Fprintf(output, format+"\n", args...) +} + +func print(format string, args ...interface{}) { + fmt.Fprintf(output, format, args...) +} + +// Lipgloss styles for beautiful terminal output +var ( + // Colors + cyan = lipgloss.Color("6") + green = lipgloss.Color("2") + red = lipgloss.Color("1") + yellow = lipgloss.Color("3") + blue = lipgloss.Color("4") + white = lipgloss.Color("15") + gray = lipgloss.Color("8") + + // Text styles + titleStyle = lipgloss.NewStyle().Bold(true).Foreground(white) + headerStyle = lipgloss.NewStyle().Bold(true).Foreground(cyan) + labelStyle = lipgloss.NewStyle().Foreground(gray) + valueStyle = lipgloss.NewStyle().Foreground(white) + dimStyle = lipgloss.NewStyle().Foreground(gray) + successStyle = lipgloss.NewStyle().Foreground(green) + errorStyle = lipgloss.NewStyle().Foreground(red) + + // Badge styles + okBadge = lipgloss.NewStyle().Bold(true).Foreground(green).Render("[OK]") + failBadge = lipgloss.NewStyle().Bold(true).Foreground(red).Render("[FAIL]") + skipBadge = lipgloss.NewStyle().Foreground(yellow).Render("[SKIP]") + + // Step styles + stepNumStyle = lipgloss.NewStyle().Bold(true).Foreground(blue) + stepTextStyle = lipgloss.NewStyle().Bold(true).Foreground(white) + + // Separator + separatorStyle = lipgloss.NewStyle().Foreground(cyan) +) + +// printStep prints a formatted step header +func printStep(num int, description string) { + badge := stepNumStyle.Render(fmt.Sprintf("[STEP %d]", num)) + text := stepTextStyle.Render(description) + printLine("\n%s %s", badge, text) +} + +// renderSeparator creates a separator line +func renderSeparator(char string, width int) string { + return separatorStyle.Render(strings.Repeat(char, width)) +} + +// TestMirrorE2E_FullCycle performs a complete mirror cycle and validates +// that source and target registries are identical. +// +// This is a heavy E2E test that: +// 1. Discovers all repositories in source registry +// 2. Pulls all images to local bundle using d8 mirror pull +// 3. Pushes bundle to target registry using d8 mirror push +// 4. Discovers all repositories in target registry +// 5. Compares EVERY tag and digest between source and target +// 6. Generates detailed comparison report // // Run with: // // go test -v ./testing/e2e/mirror/... \ -// -license-token=YOUR_TOKEN \ -// -source-registry=registry.deckhouse.ru/deckhouse/fe -// -// Or using environment variables: -// -// E2E_LICENSE_TOKEN=YOUR_TOKEN \ -// E2E_SOURCE_REGISTRY=registry.deckhouse.ru/deckhouse/fe \ -// go test -v ./testing/e2e/mirror/... +// -source-registry=localhost:443/deckhouse \ +// -source-user=admin -source-password=admin \ +// -tls-skip-verify func TestMirrorE2E_FullCycle(t *testing.T) { cfg := GetConfig() - // Debug: show config - t.Logf("Config: SourceRegistry=%s, SourceUser=%s, HasAuth=%v", - cfg.SourceRegistry, cfg.SourceUser, cfg.HasSourceAuth()) + // Create log directory in project (gitignored) + logDir := getLogDir("fullcycle") + require.NoError(t, os.MkdirAll(logDir, 0755)) + logFile := filepath.Join(logDir, "test.log") + reportFile := filepath.Join(logDir, "report.txt") + comparisonFile := filepath.Join(logDir, "comparison.txt") + + printLine("") + printLine(renderSeparator("═", 80)) + printLine(" %s", titleStyle.Render("E2E TEST: Mirror Full Cycle")) + printLine(renderSeparator("═", 80)) + printLine(" %s %s", labelStyle.Render("Source:"), valueStyle.Render(cfg.SourceRegistry)) + printLine(" %s %s", labelStyle.Render("Logs: "), dimStyle.Render(logDir)) + printLine(renderSeparator("═", 80)) + printLine("") if !cfg.HasSourceAuth() { t.Skip("Source authentication not provided (use -license-token or -source-user/-source-password)") } - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Minute) defer cancel() + // Initialize report + report := &TestReport{ + TestName: "TestMirrorE2E_FullCycle", + StartTime: time.Now(), + SourceRegistry: cfg.SourceRegistry, + LogDir: logDir, + } + + // Ensure report is written at the end + defer func() { + report.EndTime = time.Now() + report.Print() + if err := report.WriteToFile(reportFile); err != nil { + t.Logf("Warning: failed to write report: %v", err) + } else { + t.Logf("Report written to: %s", reportFile) + } + }() + // Setup target registry targetHost, targetPath, cleanup := setupTargetRegistry(t, cfg) defer cleanup() targetRegistry := targetHost + targetPath + report.TargetRegistry = targetRegistry t.Logf("Target registry: %s", targetRegistry) // Create bundle directory @@ -79,119 +180,178 @@ func TestMirrorE2E_FullCycle(t *testing.T) { t.Logf("Bundle directory: %s", bundleDir) } - // Step 1: Collect reference digests from source - t.Log("Step 1: Collecting reference digests from source registry...") - t.Logf("Source: %s, tlsSkipVerify: %v", cfg.SourceRegistry, cfg.TLSSkipVerify) - sourceCollector := NewDigestCollector(cfg.SourceRegistry, cfg.GetSourceAuth(), cfg.TLSSkipVerify) - referenceDigests, err := sourceCollector.CollectAll(ctx) - require.NoError(t, err, "Failed to collect reference digests") - t.Logf("Collected %d reference digests", len(referenceDigests)) + // ======================================================================== + // STEP 1: Analyze source registry + // ======================================================================== + stepStart := time.Now() + printStep(1, "Analyzing source registry") + + sourceComparator := NewRegistryComparator( + cfg.SourceRegistry, "", + cfg.GetSourceAuth(), nil, + cfg.TLSSkipVerify, + ) + sourceComparator.SetProgressCallback(func(msg string) { + t.Logf(" %s", msg) + }) + + sourceRepos, err := sourceComparator.discoverRepositories(ctx, cfg.SourceRegistry, sourceComparator.sourceRemoteOpts) + if err != nil { + report.AddStep("Analyze source registry", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Failed to analyze source registry") + } + + report.SourceRepoCount = len(sourceRepos) + report.AddStep(fmt.Sprintf("Analyze source (%d repos)", len(sourceRepos)), + "PASS", time.Since(stepStart), nil) + t.Logf("Source registry: %d repositories", len(sourceRepos)) + + // ======================================================================== + // STEP 2: Execute pull + // ======================================================================== + stepStart = time.Now() + printStep(2, "Pulling images to bundle") - // Step 2: Execute pull - t.Log("Step 2: Pulling images to bundle...") pullCmd := buildPullCommand(cfg, bundleDir) t.Logf("Running: %s", pullCmd.String()) - pullOutput, err := pullCmd.CombinedOutput() + _, err = runCommandWithLog(t, pullCmd, logFile) if err != nil { - t.Logf("Pull output:\n%s", string(pullOutput)) + report.AddStep("Pull images", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Pull failed") } - require.NoError(t, err, "Pull failed") - t.Log("Pull completed successfully") - // Verify bundle was created + // Log bundle contents bundleFiles, err := os.ReadDir(bundleDir) require.NoError(t, err) - t.Logf("Bundle contains %d files/dirs", len(bundleFiles)) + var totalSize int64 for _, f := range bundleFiles { - t.Logf(" - %s", f.Name()) + if info, err := f.Info(); err == nil { + totalSize += info.Size() + t.Logf(" %s (%.2f MB)", f.Name(), float64(info.Size())/(1024*1024)) + } } + report.BundleSize = totalSize + report.AddStep(fmt.Sprintf("Pull images (%.2f GB bundle)", float64(totalSize)/(1024*1024*1024)), + "PASS", time.Since(stepStart), nil) + t.Logf("Pull completed: %d files, %.2f GB total", len(bundleFiles), float64(totalSize)/(1024*1024*1024)) + + // ======================================================================== + // STEP 3: Execute push + // ======================================================================== + stepStart = time.Now() + printStep(3, "Pushing bundle to target registry") - // Step 3: Execute push - t.Log("Step 3: Pushing bundle to target registry...") pushCmd := buildPushCommand(cfg, bundleDir, targetRegistry) t.Logf("Running: %s", pushCmd.String()) - pushOutput, err := pushCmd.CombinedOutput() + _, err = runCommandWithLog(t, pushCmd, logFile) if err != nil { - t.Logf("Push output:\n%s", string(pushOutput)) + report.AddStep("Push to registry", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Push failed") } - require.NoError(t, err, "Push failed") + report.AddStep("Push to registry", "PASS", time.Since(stepStart), nil) t.Log("Push completed successfully") - // Step 4: Validate structure - t.Log("Step 4: Validating target registry structure...") - validator := NewStructureValidator(targetRegistry, cfg.GetTargetAuth(), cfg.TLSSkipVerify) - validator.SetExpectedFromDigests(NormalizeDigests(referenceDigests)) - - validationResult, err := validator.ValidateMinimal(ctx) - require.NoError(t, err, "Validation error") - - if !validationResult.IsValid() { - t.Logf("Validation issues:\n%s", validationResult.String()) - } - require.True(t, validationResult.IsValid(), "Structure validation failed") - t.Log("Structure validation passed") - - // Step 5: Compare digests - t.Log("Step 5: Comparing digests...") - targetCollector := NewDigestCollector(targetRegistry, cfg.GetTargetAuth(), cfg.TLSSkipVerify) - targetDigests, err := targetCollector.CollectAll(ctx) - require.NoError(t, err, "Failed to collect target digests") - t.Logf("Collected %d target digests", len(targetDigests)) + // ======================================================================== + // STEP 4: Deep comparison of source and target + // ======================================================================== + stepStart = time.Now() + printStep(4, "Deep comparison of registries") - // Normalize both maps for comparison - normalizedReference := NormalizeDigests(referenceDigests) - normalizedTarget := NormalizeDigests(targetDigests) + comparator := NewRegistryComparator( + cfg.SourceRegistry, targetRegistry, + cfg.GetSourceAuth(), cfg.GetTargetAuth(), + cfg.TLSSkipVerify, + ) + comparator.SetProgressCallback(func(msg string) { + t.Logf(" %s", msg) + }) - compareResult := Compare(normalizedReference, normalizedTarget) - if !compareResult.IsMatch() { - t.Logf("Comparison failed:\n%s", compareResult.String()) + comparisonReport, err := comparator.Compare(ctx) + if err != nil { + report.AddStep("Deep comparison", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Comparison failed") } - require.True(t, compareResult.IsMatch(), - "Digest comparison failed: %d mismatches found\n%s", - len(compareResult.Mismatches), - FormatMismatches(compareResult.Mismatches)) - - t.Logf("SUCCESS: All %d digests match!", compareResult.MatchedCount) -} -// TestMirrorE2E_PullOnly tests only the pull operation -func TestMirrorE2E_PullOnly(t *testing.T) { - cfg := GetConfig() - - if !cfg.HasSourceAuth() { - t.Skip("Source authentication not provided") + // Save detailed comparison report + if err := os.WriteFile(comparisonFile, []byte(comparisonReport.DetailedReport()), 0644); err != nil { + t.Logf("Warning: failed to write comparison file: %v", err) + } else { + t.Logf("Detailed comparison written to: %s", comparisonFile) } - bundleDir := t.TempDir() - if cfg.KeepBundle { - bundleDir = filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-pull-%d", time.Now().Unix())) - require.NoError(t, os.MkdirAll(bundleDir, 0755)) - t.Logf("Bundle directory (will be kept): %s", bundleDir) + // Update report with comparison stats + report.SourceImageCount = comparisonReport.TotalSourceImages + report.TargetRepoCount = len(comparisonReport.TargetRepositories) + report.TargetImageCount = comparisonReport.TotalTargetImages + report.MatchedImages = comparisonReport.MatchedImages + report.MissingImages = len(comparisonReport.MissingImages) + report.MismatchedImages = len(comparisonReport.MismatchedImages) + report.SkippedImages = comparisonReport.SkippedImages + report.MatchedLayers = comparisonReport.MatchedLayers + report.MissingLayers = comparisonReport.MissingLayers + report.ComparisonReport = comparisonReport + + // Print summary + t.Log("") + t.Log(comparisonReport.Summary()) + + if !comparisonReport.IsIdentical() { + report.AddStep(fmt.Sprintf("Deep comparison (%d matched, %d missing, %d mismatched)", + comparisonReport.MatchedImages, + len(comparisonReport.MissingImages), + len(comparisonReport.MismatchedImages)), + "FAIL", time.Since(stepStart), + fmt.Errorf("registries differ: %d missing, %d mismatched", + len(comparisonReport.MissingImages), + len(comparisonReport.MismatchedImages))) + + require.True(t, comparisonReport.IsIdentical(), + "Registries are NOT identical!\n\n%s\n\nSee %s for details", + comparisonReport.Summary(), comparisonFile) } - pullCmd := buildPullCommand(cfg, bundleDir) - t.Logf("Running: %s", pullCmd.String()) - - output, err := pullCmd.CombinedOutput() - t.Logf("Output:\n%s", string(output)) - require.NoError(t, err, "Pull failed") - - // Verify bundle was created - bundleFiles, err := os.ReadDir(bundleDir) - require.NoError(t, err) - require.NotEmpty(t, bundleFiles, "Bundle directory is empty") + report.AddStep(fmt.Sprintf("Deep comparison (%d images verified)", comparisonReport.MatchedImages), + "PASS", time.Since(stepStart), nil) + + // ======================================================================== + // SUCCESS + // ======================================================================== + successBox := lipgloss.NewStyle(). + Border(lipgloss.DoubleBorder()). + BorderForeground(green). + Padding(0, 2). + Foreground(green) + + printLine("") + printLine(successBox.Render(fmt.Sprintf( + "SUCCESS: REGISTRIES ARE IDENTICAL\n\nVerified: %d images, %d layers\nAll manifest, config, and layer digests match!", + comparisonReport.MatchedImages, + comparisonReport.MatchedLayers, + ))) +} - t.Logf("Bundle created with %d files:", len(bundleFiles)) - for _, f := range bundleFiles { - info, _ := f.Info() - if info != nil { - t.Logf(" - %s (%d bytes)", f.Name(), info.Size()) - } else { - t.Logf(" - %s", f.Name()) +// getLogDir returns the log directory path for e2e tests. +// Logs are stored in testing/e2e/.logs/-/ +func getLogDir(testName string) string { + // Get project root by finding go.mod + dir, _ := os.Getwd() + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + break + } + parent := filepath.Dir(dir) + if parent == dir { + // Fallback to current dir if go.mod not found + dir, _ = os.Getwd() + break } + dir = parent } + + timestamp := time.Now().Format("20060102-150405") + return filepath.Join(dir, "testing", "e2e", ".logs", fmt.Sprintf("%s-%s", testName, timestamp)) } // setupTargetRegistry sets up the target registry for testing @@ -200,11 +360,12 @@ func setupTargetRegistry(t *testing.T, cfg *Config) (string, string, func()) { t.Helper() if cfg.UseInMemoryRegistry() { - // Use in-memory registry - host, path, _ := mirrorutil.SetupEmptyRegistryRepo(false) - t.Logf("Started in-memory registry at %s%s", host, path) - return host, path, func() { - // In-memory registry is cleaned up when test ends + // Use disk-based test registry + reg := mirrorutil.SetupTestRegistry(false) + repoPath := "/deckhouse/ee" + t.Logf("Started test registry at %s%s", reg.Host, repoPath) + return reg.Host, repoPath, func() { + reg.Close() } } @@ -235,6 +396,11 @@ func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { args = append(args, "--tls-skip-verify") } + // Skip modules (for testing failure scenarios) + if cfg.NoModules { + args = append(args, "--no-modules") + } + args = append(args, bundleDir) cmd := exec.Command(cfg.D8Binary, args...) @@ -269,3 +435,273 @@ func buildPushCommand(cfg *Config, bundleDir, targetRegistry string) *exec.Cmd { return cmd } + +// runCommandWithLog runs command with streaming and saves full output to a log file +func runCommandWithLog(t *testing.T, cmd *exec.Cmd, logFile string) ([]byte, error) { + t.Helper() + + var buf bytes.Buffer + + // Open log file for appending + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + t.Logf("Warning: could not open log file %s: %v", logFile, err) + // Fallback to just streaming + cmd.Stdout = io.MultiWriter(os.Stdout, &buf) + cmd.Stderr = io.MultiWriter(os.Stderr, &buf) + return buf.Bytes(), cmd.Run() + } + defer f.Close() + + // Write command to log + fmt.Fprintf(f, "\n\n========== COMMAND: %s ==========\n", cmd.String()) + fmt.Fprintf(f, "Started: %s\n\n", time.Now().Format(time.RFC3339)) + + // Create multi-writers to write to stdout, buffer, AND log file + cmd.Stdout = io.MultiWriter(os.Stdout, &buf, f) + cmd.Stderr = io.MultiWriter(os.Stderr, &buf, f) + + cmdErr := cmd.Run() + + // Write result to log + if cmdErr != nil { + fmt.Fprintf(f, "\n\n========== COMMAND FAILED: %v ==========\n", cmdErr) + } else { + fmt.Fprintf(f, "\n\n========== COMMAND SUCCEEDED ==========\n") + } + + return buf.Bytes(), cmdErr +} + +// TestReport collects test execution results for final summary +type TestReport struct { + TestName string + StartTime time.Time + EndTime time.Time + SourceRegistry string + TargetRegistry string + LogDir string + + // Source stats + SourceRepoCount int + SourceImageCount int + + // Target stats + TargetRepoCount int + TargetImageCount int + + // Comparison stats + MatchedImages int + MissingImages int + MismatchedImages int + SkippedImages int // Digest-based, .att, .sig tags + + // Deep comparison stats + MatchedLayers int + MissingLayers int + + // Bundle stats + BundleSize int64 + + // Steps + Steps []StepResult + + // Full comparison report + ComparisonReport *ComparisonReport +} + +// StepResult represents a single step in the test +type StepResult struct { + Name string + Status string // "PASS", "FAIL", "SKIP" + Duration time.Duration + Error string +} + +// AddStep adds a step result to the report +func (r *TestReport) AddStep(name, status string, duration time.Duration, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + r.Steps = append(r.Steps, StepResult{ + Name: name, + Status: status, + Duration: duration, + Error: errStr, + }) +} + +// WriteToFile writes the report to a file +func (r *TestReport) WriteToFile(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + fmt.Fprintf(f, "================================================================================\n") + fmt.Fprintf(f, "E2E TEST REPORT: %s\n", r.TestName) + fmt.Fprintf(f, "================================================================================\n\n") + + fmt.Fprintf(f, "EXECUTION:\n") + fmt.Fprintf(f, " Started: %s\n", r.StartTime.Format(time.RFC3339)) + fmt.Fprintf(f, " Finished: %s\n", r.EndTime.Format(time.RFC3339)) + fmt.Fprintf(f, " Duration: %s\n", r.EndTime.Sub(r.StartTime).Round(time.Second)) + fmt.Fprintf(f, " Log dir: %s\n\n", r.LogDir) + + fmt.Fprintf(f, "REGISTRIES:\n") + fmt.Fprintf(f, " Source: %s\n", r.SourceRegistry) + fmt.Fprintf(f, " Target: %s\n\n", r.TargetRegistry) + + fmt.Fprintf(f, "IMAGES TO VERIFY:\n") + fmt.Fprintf(f, " Source: %d images (%d repos)\n", r.SourceImageCount, r.SourceRepoCount) + fmt.Fprintf(f, " Target: %d images (%d repos)\n", r.TargetImageCount, r.TargetRepoCount) + if r.SkippedImages > 0 { + fmt.Fprintf(f, " (excluded %d internal tags from comparison)\n", r.SkippedImages) + } + fmt.Fprintf(f, "\n") + + fmt.Fprintf(f, "BUNDLE:\n") + fmt.Fprintf(f, " Size: %.2f GB\n\n", float64(r.BundleSize)/(1024*1024*1024)) + + fmt.Fprintf(f, "VERIFICATION RESULTS:\n") + fmt.Fprintf(f, " Images matched: %d (manifest + config + layers)\n", r.MatchedImages) + fmt.Fprintf(f, " Layers verified: %d\n", r.MatchedLayers) + fmt.Fprintf(f, " Missing images: %d\n", r.MissingImages) + fmt.Fprintf(f, " Digest mismatch: %d\n", r.MismatchedImages) + fmt.Fprintf(f, " Missing layers: %d\n\n", r.MissingLayers) + + fmt.Fprintf(f, "STEPS:\n") + passCount, failCount := 0, 0 + for _, step := range r.Steps { + if step.Status == "PASS" { + passCount++ + fmt.Fprintf(f, " [PASS] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) + } else if step.Status == "FAIL" { + failCount++ + fmt.Fprintf(f, " [FAIL] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) + if step.Error != "" { + fmt.Fprintf(f, " ERROR: %s\n", step.Error) + } + } else { + fmt.Fprintf(f, " [SKIP] %s\n", step.Name) + } + } + + fmt.Fprintf(f, "\n================================================================================\n") + if failCount > 0 { + fmt.Fprintf(f, "RESULT: FAILED (%d passed, %d failed)\n", passCount, failCount) + } else { + fmt.Fprintf(f, "RESULT: PASSED - REGISTRIES ARE IDENTICAL\n") + fmt.Fprintf(f, " %d repositories verified\n", r.SourceRepoCount) + fmt.Fprintf(f, " %d images verified\n", r.MatchedImages) + } + fmt.Fprintf(f, "================================================================================\n") + + return nil +} + +// Print prints the report to stdout with beautiful lipgloss styling +func (r *TestReport) Print() { + duration := r.EndTime.Sub(r.StartTime) + if r.EndTime.IsZero() { + duration = time.Since(r.StartTime) + } + + var content strings.Builder + + // Header + content.WriteString("\n") + content.WriteString(renderSeparator("═", 80) + "\n") + content.WriteString(" " + titleStyle.Render("E2E TEST REPORT") + "\n") + content.WriteString(renderSeparator("═", 80) + "\n\n") + + // Duration + content.WriteString(" " + labelStyle.Render("Duration: ") + dimStyle.Render(duration.Round(time.Second).String()) + "\n\n") + + // Registries section + content.WriteString(" " + headerStyle.Render("REGISTRIES") + "\n") + content.WriteString(" " + labelStyle.Render("Source: ") + valueStyle.Render(r.SourceRegistry) + "\n") + content.WriteString(" " + labelStyle.Render("Target: ") + valueStyle.Render(r.TargetRegistry) + "\n\n") + + // Images to verify section + content.WriteString(" " + headerStyle.Render("IMAGES TO VERIFY") + "\n") + content.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", + labelStyle.Render("Source:"), + valueStyle.Render(fmt.Sprintf("%d", r.SourceImageCount)), + r.SourceRepoCount)) + content.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", + labelStyle.Render("Target:"), + valueStyle.Render(fmt.Sprintf("%d", r.TargetImageCount)), + r.TargetRepoCount)) + if r.SkippedImages > 0 { + content.WriteString(" " + dimStyle.Render(fmt.Sprintf("(%d internal tags excluded)", r.SkippedImages)) + "\n") + } + content.WriteString("\n") + + // Verification results section + content.WriteString(" " + headerStyle.Render("VERIFICATION") + "\n") + content.WriteString(fmt.Sprintf(" %s %s %s\n", + okBadge, + labelStyle.Render("Images matched: "), + successStyle.Render(fmt.Sprintf("%d", r.MatchedImages))+" "+dimStyle.Render("(manifest + config + layers)"))) + content.WriteString(fmt.Sprintf(" %s %s %s\n", + okBadge, + labelStyle.Render("Layers verified:"), + successStyle.Render(fmt.Sprintf("%d", r.MatchedLayers)))) + + if r.MissingImages > 0 { + content.WriteString(fmt.Sprintf(" %s %s %s\n", + failBadge, + labelStyle.Render("Missing images: "), + errorStyle.Render(fmt.Sprintf("%d", r.MissingImages)))) + } + if r.MismatchedImages > 0 { + content.WriteString(fmt.Sprintf(" %s %s %s\n", + failBadge, + labelStyle.Render("Digest mismatch:"), + errorStyle.Render(fmt.Sprintf("%d", r.MismatchedImages)))) + } + if r.MissingLayers > 0 { + content.WriteString(fmt.Sprintf(" %s %s %s\n", + failBadge, + labelStyle.Render("Missing layers: "), + errorStyle.Render(fmt.Sprintf("%d", r.MissingLayers)))) + } + content.WriteString("\n") + + // Steps section + content.WriteString(" " + headerStyle.Render("STEPS") + "\n") + passCount, failCount := 0, 0 + for _, step := range r.Steps { + dur := dimStyle.Render(fmt.Sprintf("(%s)", step.Duration.Round(time.Millisecond))) + if step.Status == "PASS" { + passCount++ + content.WriteString(fmt.Sprintf(" %s %s %s\n", okBadge, step.Name, dur)) + } else if step.Status == "FAIL" { + failCount++ + content.WriteString(fmt.Sprintf(" %s %s %s\n", failBadge, step.Name, dur)) + if step.Error != "" { + content.WriteString(" " + errorStyle.Render("ERROR: "+step.Error) + "\n") + } + } else { + content.WriteString(fmt.Sprintf(" %s %s\n", skipBadge, step.Name)) + } + } + content.WriteString("\n") + + // Result box + content.WriteString(renderSeparator("─", 80) + "\n") + if failCount > 0 { + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(red) + content.WriteString(" " + resultStyle.Render("RESULT: FAILED") + fmt.Sprintf(" (%d passed, %d failed)\n", passCount, failCount)) + } else { + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(green) + content.WriteString(" " + resultStyle.Render("RESULT: PASSED") + " - REGISTRIES ARE IDENTICAL\n") + content.WriteString(" " + successStyle.Render(fmt.Sprintf("%d images, %d layers", r.MatchedImages, r.MatchedLayers)) + " - all hashes verified\n") + } + content.WriteString(renderSeparator("═", 80) + "\n") + + print("%s", content.String()) +} diff --git a/testing/e2e/mirror/registry_comparator.go b/testing/e2e/mirror/registry_comparator.go new file mode 100644 index 00000000..a8a5a83e --- /dev/null +++ b/testing/e2e/mirror/registry_comparator.go @@ -0,0 +1,936 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "path" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/deckhouse/deckhouse-cli/internal" +) + +// Tags to exclude from comparison (not mirrored by design) +var ( + // Digest-based tags (sha256 hashes used as tags) + digestTagRegex = regexp.MustCompile(`^[a-f0-9]{64}$`) + // SHA256 prefixed tags + sha256TagRegex = regexp.MustCompile(`^sha256-[a-f0-9]{64}`) + // Cosign signature and attestation tags + cosignTagSuffixes = []string{".sig", ".att", ".sbom"} + // Service tags + serviceTags = []string{"d8WriteCheck"} +) + +// shouldSkipTag returns true if the tag should be excluded from comparison +func shouldSkipTag(tag string) bool { + // Skip digest-based tags + if digestTagRegex.MatchString(tag) { + return true + } + // Skip sha256- prefixed tags + if sha256TagRegex.MatchString(tag) { + return true + } + // Skip cosign tags + for _, suffix := range cosignTagSuffixes { + if strings.HasSuffix(tag, suffix) { + return true + } + } + // Skip service tags + for _, svcTag := range serviceTags { + if tag == svcTag { + return true + } + } + return false +} + +// ImageInfo contains detailed information about an image +type ImageInfo struct { + Reference string + Digest string // manifest digest + ConfigDigest string // config digest + Layers []string // layer digests + TotalSize int64 // total size in bytes +} + +// RegistryComparator performs deep comparison between source and target registries +type RegistryComparator struct { + sourceRegistry string + targetRegistry string + sourceAuth authn.Authenticator + targetAuth authn.Authenticator + + nameOpts []name.Option + sourceRemoteOpts []remote.Option + targetRemoteOpts []remote.Option + + // Progress callback + onProgress func(msg string) +} + +// NewRegistryComparator creates a new registry comparator +func NewRegistryComparator( + sourceRegistry, targetRegistry string, + sourceAuth, targetAuth authn.Authenticator, + tlsSkipVerify bool, +) *RegistryComparator { + nameOpts := []name.Option{} + sourceRemoteOpts := []remote.Option{} + targetRemoteOpts := []remote.Option{} + + if tlsSkipVerify { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + sourceRemoteOpts = append(sourceRemoteOpts, remote.WithTransport(transport)) + targetRemoteOpts = append(targetRemoteOpts, remote.WithTransport(transport)) + } + + if sourceAuth != nil && sourceAuth != authn.Anonymous { + sourceRemoteOpts = append(sourceRemoteOpts, remote.WithAuth(sourceAuth)) + } + if targetAuth != nil && targetAuth != authn.Anonymous { + targetRemoteOpts = append(targetRemoteOpts, remote.WithAuth(targetAuth)) + } + + return &RegistryComparator{ + sourceRegistry: sourceRegistry, + targetRegistry: targetRegistry, + sourceAuth: sourceAuth, + targetAuth: targetAuth, + nameOpts: nameOpts, + sourceRemoteOpts: sourceRemoteOpts, + targetRemoteOpts: targetRemoteOpts, + } +} + +// SetProgressCallback sets a callback for progress updates +func (c *RegistryComparator) SetProgressCallback(fn func(msg string)) { + c.onProgress = fn +} + +func (c *RegistryComparator) progress(format string, args ...interface{}) { + if c.onProgress != nil { + c.onProgress(fmt.Sprintf(format, args...)) + } +} + +// ComparisonReport contains detailed comparison results +type ComparisonReport struct { + StartTime time.Time + EndTime time.Time + + SourceRegistry string + TargetRegistry string + + // Repository-level stats + SourceRepositories []string + TargetRepositories []string + MissingRepositories []string // In source but not in target + ExtraRepositories []string // In target but not in source + + // Tag-level stats per repository + RepositoryDetails map[string]*RepositoryComparison + + // Image-level stats + TotalSourceImages int + TotalTargetImages int + SkippedImages int // Digest-based, .att, .sig tags (not mirrored by design) + MatchedImages int + MismatchedImages []ImageMismatch + MissingImages []string // ref in source but not in target + ExtraImages []string // ref in target but not in source + + // Deep comparison stats + DeepCheckedImages int + TotalSourceLayers int + TotalTargetLayers int + MatchedLayers int + MissingLayers int + ConfigMismatches int + LayerMismatches []LayerMismatch +} + +// RepositoryComparison holds comparison for a single repository +type RepositoryComparison struct { + Repository string + SourceTags []string + TargetTags []string + MissingTags []string // In source but not in target + ExtraTags []string // In target but not in source + SkippedTags int // Tags skipped (digest-based, .att, .sig, etc.) + MatchedTags int + TagDetails map[string]*TagComparison +} + +// TagComparison holds comparison for a single tag +type TagComparison struct { + Tag string + SourceDigest string + TargetDigest string + Match bool + SourceConfig string + TargetConfig string + ConfigMatch bool + SourceLayers []string + TargetLayers []string + MissingLayers []string + ExtraLayers []string + LayersMatch bool + DeepChecked bool // true if layers were compared +} + +// ImageMismatch represents a digest mismatch for an image +type ImageMismatch struct { + Reference string + SourceDigest string + TargetDigest string +} + +// LayerMismatch represents a missing or different layer +type LayerMismatch struct { + Reference string + LayerDigest string + Reason string // "missing", "size_mismatch" +} + +// IsIdentical returns true if registries are identical +func (r *ComparisonReport) IsIdentical() bool { + return len(r.MissingRepositories) == 0 && + len(r.MissingImages) == 0 && + len(r.MismatchedImages) == 0 && + len(r.LayerMismatches) == 0 && + r.MissingLayers == 0 && + r.ConfigMismatches == 0 +} + +// Summary returns a summary string +func (r *ComparisonReport) Summary() string { + var sb strings.Builder + + sb.WriteString("REGISTRY COMPARISON SUMMARY\n") + sb.WriteString("===========================\n\n") + + sb.WriteString(fmt.Sprintf("Source: %s\n", r.SourceRegistry)) + sb.WriteString(fmt.Sprintf("Target: %s\n", r.TargetRegistry)) + sb.WriteString(fmt.Sprintf("Duration: %s\n\n", r.EndTime.Sub(r.StartTime).Round(time.Second))) + + sb.WriteString("REPOSITORIES:\n") + sb.WriteString(fmt.Sprintf(" Source: %d\n", len(r.SourceRepositories))) + sb.WriteString(fmt.Sprintf(" Target: %d\n", len(r.TargetRepositories))) + sb.WriteString(fmt.Sprintf(" Missing in target: %d\n", len(r.MissingRepositories))) + sb.WriteString(fmt.Sprintf(" Extra in target: %d\n\n", len(r.ExtraRepositories))) + + sb.WriteString("IMAGES TO VERIFY:\n") + sb.WriteString(fmt.Sprintf(" Source: %d images\n", r.TotalSourceImages)) + sb.WriteString(fmt.Sprintf(" Target: %d images\n", r.TotalTargetImages)) + if r.SkippedImages > 0 { + sb.WriteString(fmt.Sprintf(" (excluded %d internal tags: digest-based, .att, .sig)\n", r.SkippedImages)) + } + sb.WriteString("\n") + sb.WriteString("VERIFICATION RESULTS:\n") + sb.WriteString(fmt.Sprintf(" Matched: %d\n", r.MatchedImages)) + if len(r.MissingImages) > 0 { + sb.WriteString(fmt.Sprintf(" Missing in target: %d\n", len(r.MissingImages))) + } + if len(r.MismatchedImages) > 0 { + sb.WriteString(fmt.Sprintf(" Digest mismatch: %d\n", len(r.MismatchedImages))) + } + if len(r.ExtraImages) > 0 { + sb.WriteString(fmt.Sprintf(" Extra in target: %d\n", len(r.ExtraImages))) + } + + if r.DeepCheckedImages > 0 { + sb.WriteString("DEEP COMPARISON (layers + config):\n") + sb.WriteString(fmt.Sprintf(" Images deep-checked: %d\n", r.DeepCheckedImages)) + sb.WriteString(fmt.Sprintf(" Source layers: %d\n", r.TotalSourceLayers)) + sb.WriteString(fmt.Sprintf(" Target layers: %d\n", r.TotalTargetLayers)) + sb.WriteString(fmt.Sprintf(" Matched layers: %d\n", r.MatchedLayers)) + sb.WriteString(fmt.Sprintf(" Missing layers: %d\n", r.MissingLayers)) + sb.WriteString(fmt.Sprintf(" Config mismatches: %d\n\n", r.ConfigMismatches)) + } + + if r.IsIdentical() { + sb.WriteString("✓ REGISTRIES ARE IDENTICAL (all hashes match)\n") + } else { + sb.WriteString("✗ REGISTRIES DIFFER\n") + } + + return sb.String() +} + +// DetailedReport returns a detailed report string +func (r *ComparisonReport) DetailedReport() string { + var sb strings.Builder + + sb.WriteString(r.Summary()) + sb.WriteString("\n") + + // Missing repositories + if len(r.MissingRepositories) > 0 { + sb.WriteString("MISSING REPOSITORIES:\n") + for _, repo := range r.MissingRepositories { + sb.WriteString(fmt.Sprintf(" - %s\n", repo)) + } + sb.WriteString("\n") + } + + // Missing images (limited to first 100) + if len(r.MissingImages) > 0 { + sb.WriteString(fmt.Sprintf("MISSING IMAGES (%d total):\n", len(r.MissingImages))) + limit := min(100, len(r.MissingImages)) + for i := 0; i < limit; i++ { + sb.WriteString(fmt.Sprintf(" - %s\n", r.MissingImages[i])) + } + if len(r.MissingImages) > limit { + sb.WriteString(fmt.Sprintf(" ... and %d more\n", len(r.MissingImages)-limit)) + } + sb.WriteString("\n") + } + + // Mismatched images (limited to first 50) + if len(r.MismatchedImages) > 0 { + sb.WriteString(fmt.Sprintf("MISMATCHED DIGESTS (%d total):\n", len(r.MismatchedImages))) + limit := min(50, len(r.MismatchedImages)) + for i := 0; i < limit; i++ { + m := r.MismatchedImages[i] + sb.WriteString(fmt.Sprintf(" %s\n", m.Reference)) + sb.WriteString(fmt.Sprintf(" source: %s\n", m.SourceDigest)) + sb.WriteString(fmt.Sprintf(" target: %s\n", m.TargetDigest)) + } + if len(r.MismatchedImages) > limit { + sb.WriteString(fmt.Sprintf(" ... and %d more\n", len(r.MismatchedImages)-limit)) + } + sb.WriteString("\n") + } + + // Layer mismatches + if len(r.LayerMismatches) > 0 { + sb.WriteString(fmt.Sprintf("LAYER MISMATCHES (%d total):\n", len(r.LayerMismatches))) + limit := min(50, len(r.LayerMismatches)) + for i := 0; i < limit; i++ { + m := r.LayerMismatches[i] + sb.WriteString(fmt.Sprintf(" %s\n", m.Reference)) + sb.WriteString(fmt.Sprintf(" layer: %s (%s)\n", m.LayerDigest, m.Reason)) + } + if len(r.LayerMismatches) > limit { + sb.WriteString(fmt.Sprintf(" ... and %d more\n", len(r.LayerMismatches)-limit)) + } + sb.WriteString("\n") + } + + // Per-repository breakdown + sb.WriteString("REPOSITORY BREAKDOWN:\n") + sb.WriteString("---------------------\n") + + // Sort repositories for consistent output + repos := make([]string, 0, len(r.RepositoryDetails)) + for repo := range r.RepositoryDetails { + repos = append(repos, repo) + } + sort.Strings(repos) + + for _, repo := range repos { + detail := r.RepositoryDetails[repo] + status := "✓" + issues := []string{} + + if len(detail.MissingTags) > 0 { + status = "✗" + issues = append(issues, fmt.Sprintf("%d missing", len(detail.MissingTags))) + } + + // Count layer issues + layerIssues := 0 + deepChecked := 0 + totalLayers := 0 + for _, tagDetail := range detail.TagDetails { + if tagDetail.DeepChecked { + deepChecked++ + totalLayers += len(tagDetail.SourceLayers) + layerIssues += len(tagDetail.MissingLayers) + } + } + + if layerIssues > 0 { + status = "✗" + issues = append(issues, fmt.Sprintf("%d layer issues", layerIssues)) + } + + repoName := repo + if repoName == "" { + repoName = "(root)" + } + + sb.WriteString(fmt.Sprintf("%s %s: %d/%d tags", status, repoName, detail.MatchedTags, len(detail.SourceTags))) + if deepChecked > 0 { + sb.WriteString(fmt.Sprintf(", %d layers checked", totalLayers)) + } + if len(issues) > 0 { + sb.WriteString(fmt.Sprintf(" [%s]", strings.Join(issues, ", "))) + } + sb.WriteString("\n") + } + + return sb.String() +} + +// Compare performs a deep comparison between source and target registries +func (c *RegistryComparator) Compare(ctx context.Context) (*ComparisonReport, error) { + report := &ComparisonReport{ + StartTime: time.Now(), + SourceRegistry: c.sourceRegistry, + TargetRegistry: c.targetRegistry, + RepositoryDetails: make(map[string]*RepositoryComparison), + } + + // Step 1: Discover all repositories in source + c.progress("Discovering repositories in source registry...") + sourceRepos, err := c.discoverRepositories(ctx, c.sourceRegistry, c.sourceRemoteOpts) + if err != nil { + return nil, fmt.Errorf("discover source repositories: %w", err) + } + report.SourceRepositories = sourceRepos + c.progress("Found %d repositories in source", len(sourceRepos)) + + // Step 2: Discover all repositories in target + c.progress("Discovering repositories in target registry...") + targetRepos, err := c.discoverRepositories(ctx, c.targetRegistry, c.targetRemoteOpts) + if err != nil { + return nil, fmt.Errorf("discover target repositories: %w", err) + } + report.TargetRepositories = targetRepos + c.progress("Found %d repositories in target", len(targetRepos)) + + // Step 3: Find missing and extra repositories + sourceRepoSet := make(map[string]bool) + for _, r := range sourceRepos { + sourceRepoSet[r] = true + } + targetRepoSet := make(map[string]bool) + for _, r := range targetRepos { + targetRepoSet[r] = true + } + + for _, repo := range sourceRepos { + if !targetRepoSet[repo] { + report.MissingRepositories = append(report.MissingRepositories, repo) + } + } + for _, repo := range targetRepos { + if !sourceRepoSet[repo] { + report.ExtraRepositories = append(report.ExtraRepositories, repo) + } + } + + // Step 4: Compare each repository in detail + c.progress("Comparing repositories...") + for i, repoPath := range sourceRepos { + c.progress("[%d/%d] Comparing %s", i+1, len(sourceRepos), repoPath) + + repoComparison, err := c.compareRepository(ctx, repoPath) + if err != nil { + c.progress("Warning: failed to compare %s: %v", repoPath, err) + continue + } + + report.RepositoryDetails[repoPath] = repoComparison + + // Aggregate stats + report.TotalSourceImages += len(repoComparison.SourceTags) + report.TotalTargetImages += len(repoComparison.TargetTags) + report.SkippedImages += repoComparison.SkippedTags + report.MatchedImages += repoComparison.MatchedTags + + // Collect missing images + for _, tag := range repoComparison.MissingTags { + report.MissingImages = append(report.MissingImages, fmt.Sprintf("%s:%s", repoPath, tag)) + } + + // Collect mismatched images and layer stats + for tag, detail := range repoComparison.TagDetails { + if !detail.Match && detail.TargetDigest != "" { + report.MismatchedImages = append(report.MismatchedImages, ImageMismatch{ + Reference: fmt.Sprintf("%s:%s", repoPath, tag), + SourceDigest: detail.SourceDigest, + TargetDigest: detail.TargetDigest, + }) + } + + // Aggregate layer stats from deep comparison + if detail.DeepChecked { + report.DeepCheckedImages++ + report.TotalSourceLayers += len(detail.SourceLayers) + report.TotalTargetLayers += len(detail.TargetLayers) + report.MissingLayers += len(detail.MissingLayers) + + // Count matched layers + if detail.LayersMatch { + report.MatchedLayers += len(detail.SourceLayers) + } + + if !detail.ConfigMatch { + report.ConfigMismatches++ + } + + // Collect layer mismatches for detailed report + for _, layer := range detail.MissingLayers { + report.LayerMismatches = append(report.LayerMismatches, LayerMismatch{ + Reference: fmt.Sprintf("%s:%s", repoPath, tag), + LayerDigest: layer, + Reason: "missing_in_target", + }) + } + } + } + + // Collect extra images + for _, tag := range repoComparison.ExtraTags { + report.ExtraImages = append(report.ExtraImages, fmt.Sprintf("%s:%s", repoPath, tag)) + } + } + + // Sort results for consistent output + sort.Strings(report.MissingRepositories) + sort.Strings(report.MissingImages) + sort.Strings(report.ExtraImages) + sort.Slice(report.MismatchedImages, func(i, j int) bool { + return report.MismatchedImages[i].Reference < report.MismatchedImages[j].Reference + }) + + report.EndTime = time.Now() + return report, nil +} + +// discoverRepositories discovers all repositories by walking known segments +func (c *RegistryComparator) discoverRepositories(ctx context.Context, registry string, opts []remote.Option) ([]string, error) { + var repos []string + + // Root repository + if c.repositoryExists(ctx, registry, opts) { + repos = append(repos, "") + } + + // Known segments + segments := []string{ + internal.InstallSegment, + internal.InstallStandaloneSegment, + internal.ReleaseChannelSegment, + } + + for _, segment := range segments { + segmentPath := segment + if c.repositoryExists(ctx, path.Join(registry, segmentPath), opts) { + repos = append(repos, segmentPath) + } + } + + // Security segment + securityDBs := []string{ + internal.SecurityTrivyDBSegment, + internal.SecurityTrivyBDUSegment, + internal.SecurityTrivyJavaDBSegment, + internal.SecurityTrivyChecksSegment, + } + for _, db := range securityDBs { + dbPath := path.Join(internal.SecuritySegment, db) + if c.repositoryExists(ctx, path.Join(registry, dbPath), opts) { + repos = append(repos, dbPath) + } + } + + // Modules - need to discover dynamically + modulesPath := path.Join(registry, internal.ModulesSegment) + moduleTags, err := c.listTags(ctx, modulesPath, opts) + if err == nil { + for _, moduleName := range moduleTags { + moduleBasePath := path.Join(internal.ModulesSegment, moduleName) + + // Module root + if c.repositoryExists(ctx, path.Join(registry, moduleBasePath), opts) { + repos = append(repos, moduleBasePath) + } + + // Module release + moduleReleasePath := path.Join(moduleBasePath, "release") + if c.repositoryExists(ctx, path.Join(registry, moduleReleasePath), opts) { + repos = append(repos, moduleReleasePath) + } + } + } + + return repos, nil +} + +// repositoryExists checks if a repository exists and has tags +func (c *RegistryComparator) repositoryExists(ctx context.Context, repo string, opts []remote.Option) bool { + tags, err := c.listTags(ctx, repo, opts) + return err == nil && len(tags) > 0 +} + +// compareRepository compares a single repository between source and target +func (c *RegistryComparator) compareRepository(ctx context.Context, repoPath string) (*RepositoryComparison, error) { + sourceRepo := c.sourceRegistry + targetRepo := c.targetRegistry + if repoPath != "" { + sourceRepo = path.Join(c.sourceRegistry, repoPath) + targetRepo = path.Join(c.targetRegistry, repoPath) + } + + comparison := &RepositoryComparison{ + Repository: repoPath, + TagDetails: make(map[string]*TagComparison), + } + + // Get source tags + allSourceTags, err := c.listTags(ctx, sourceRepo, c.sourceRemoteOpts) + if err != nil { + return nil, fmt.Errorf("list source tags: %w", err) + } + + // Filter out tags that are not mirrored by design + sourceTags := make([]string, 0, len(allSourceTags)) + skippedTags := 0 + for _, tag := range allSourceTags { + if shouldSkipTag(tag) { + skippedTags++ + continue + } + sourceTags = append(sourceTags, tag) + } + comparison.SourceTags = sourceTags + comparison.SkippedTags = skippedTags + + // Get target tags + allTargetTags, err := c.listTags(ctx, targetRepo, c.targetRemoteOpts) + if err != nil { + // Target repo might not exist + allTargetTags = []string{} + } + + // Filter target tags too + targetTags := make([]string, 0, len(allTargetTags)) + for _, tag := range allTargetTags { + if shouldSkipTag(tag) { + continue + } + targetTags = append(targetTags, tag) + } + comparison.TargetTags = targetTags + + // Create sets for comparison + sourceTagSet := make(map[string]bool) + for _, t := range sourceTags { + sourceTagSet[t] = true + } + targetTagSet := make(map[string]bool) + for _, t := range targetTags { + targetTagSet[t] = true + } + + // Find missing and extra tags + for _, tag := range sourceTags { + if !targetTagSet[tag] { + comparison.MissingTags = append(comparison.MissingTags, tag) + } + } + for _, tag := range targetTags { + if !sourceTagSet[tag] { + comparison.ExtraTags = append(comparison.ExtraTags, tag) + } + } + + // Compare images deeply - check manifest, config, and all layers + var wg sync.WaitGroup + var mu sync.Mutex + semaphore := make(chan struct{}, 5) // Limit concurrency (deep comparison is heavier) + + for _, tag := range sourceTags { + if !targetTagSet[tag] { + continue + } + + wg.Add(1) + go func(tag string) { + defer wg.Done() + semaphore <- struct{}{} + defer func() { <-semaphore }() + + sourceRef := sourceRepo + ":" + tag + targetRef := targetRepo + ":" + tag + + // Perform deep comparison + imgComp, err := c.compareImageDeep(ctx, sourceRef, targetRef) + if err != nil { + // Fallback to simple digest comparison + sourceDigest, err1 := c.getDigest(ctx, sourceRef, c.sourceRemoteOpts) + targetDigest, err2 := c.getDigest(ctx, targetRef, c.targetRemoteOpts) + if err1 != nil || err2 != nil { + return + } + + tagComp := &TagComparison{ + Tag: tag, + SourceDigest: sourceDigest, + TargetDigest: targetDigest, + Match: sourceDigest == targetDigest, + DeepChecked: false, + } + + mu.Lock() + comparison.TagDetails[tag] = tagComp + if tagComp.Match { + comparison.MatchedTags++ + } + mu.Unlock() + return + } + + tagComp := &TagComparison{ + Tag: tag, + SourceDigest: imgComp.SourceDigest, + TargetDigest: imgComp.TargetDigest, + Match: imgComp.FullMatch, + SourceConfig: imgComp.SourceConfig, + TargetConfig: imgComp.TargetConfig, + ConfigMatch: imgComp.ConfigMatch, + SourceLayers: imgComp.SourceLayers, + TargetLayers: imgComp.TargetLayers, + MissingLayers: imgComp.MissingLayers, + ExtraLayers: imgComp.ExtraLayers, + LayersMatch: imgComp.LayersMatch, + DeepChecked: true, + } + + mu.Lock() + comparison.TagDetails[tag] = tagComp + if tagComp.Match { + comparison.MatchedTags++ + } + mu.Unlock() + }(tag) + } + + wg.Wait() + + sort.Strings(comparison.MissingTags) + sort.Strings(comparison.ExtraTags) + + return comparison, nil +} + +// listTags lists all tags in a repository +func (c *RegistryComparator) listTags(ctx context.Context, repo string, opts []remote.Option) ([]string, error) { + repoRef, err := name.NewRepository(repo, c.nameOpts...) + if err != nil { + return nil, fmt.Errorf("parse repo %s: %w", repo, err) + } + + tags, err := remote.List(repoRef, opts...) + if err != nil { + return nil, fmt.Errorf("list tags for %s: %w", repo, err) + } + + return tags, nil +} + +// getDigest gets the digest for a specific image reference +func (c *RegistryComparator) getDigest(ctx context.Context, ref string, opts []remote.Option) (string, error) { + imgRef, err := name.ParseReference(ref, c.nameOpts...) + if err != nil { + return "", fmt.Errorf("parse ref %s: %w", ref, err) + } + + desc, err := remote.Head(imgRef, opts...) + if err != nil { + return "", fmt.Errorf("get digest for %s: %w", ref, err) + } + + return desc.Digest.String(), nil +} + +// getImageInfo gets detailed information about an image including all layer digests +func (c *RegistryComparator) getImageInfo(ctx context.Context, ref string, opts []remote.Option) (*ImageInfo, error) { + imgRef, err := name.ParseReference(ref, c.nameOpts...) + if err != nil { + return nil, fmt.Errorf("parse ref %s: %w", ref, err) + } + + // Try to get as an image first + img, err := remote.Image(imgRef, opts...) + if err != nil { + // Might be an index, try that + idx, idxErr := remote.Index(imgRef, opts...) + if idxErr != nil { + return nil, fmt.Errorf("get image %s: %w (also tried index: %v)", ref, err, idxErr) + } + + // For index, get the digest and list manifests + digest, err := idx.Digest() + if err != nil { + return nil, fmt.Errorf("get index digest: %w", err) + } + + manifest, err := idx.IndexManifest() + if err != nil { + return nil, fmt.Errorf("get index manifest: %w", err) + } + + info := &ImageInfo{ + Reference: ref, + Digest: digest.String(), + Layers: make([]string, 0), + } + + // Collect all manifest digests as "layers" for index + for _, m := range manifest.Manifests { + info.Layers = append(info.Layers, m.Digest.String()) + info.TotalSize += m.Size + } + + return info, nil + } + + // Get manifest digest + digest, err := img.Digest() + if err != nil { + return nil, fmt.Errorf("get digest: %w", err) + } + + // Get config digest + configDigest := "" + if cfg, err := img.ConfigName(); err == nil { + configDigest = cfg.String() + } + + // Get all layer digests + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("get layers: %w", err) + } + + info := &ImageInfo{ + Reference: ref, + Digest: digest.String(), + ConfigDigest: configDigest, + Layers: make([]string, 0, len(layers)), + } + + for _, layer := range layers { + layerDigest, err := layer.Digest() + if err != nil { + continue + } + info.Layers = append(info.Layers, layerDigest.String()) + + size, err := layer.Size() + if err == nil { + info.TotalSize += size + } + } + + return info, nil +} + +// compareImageDeep performs deep comparison of two images including all layers +func (c *RegistryComparator) compareImageDeep(ctx context.Context, sourceRef, targetRef string) (*ImageComparison, error) { + sourceInfo, err := c.getImageInfo(ctx, sourceRef, c.sourceRemoteOpts) + if err != nil { + return nil, fmt.Errorf("get source image info: %w", err) + } + + targetInfo, err := c.getImageInfo(ctx, targetRef, c.targetRemoteOpts) + if err != nil { + return nil, fmt.Errorf("get target image info: %w", err) + } + + comparison := &ImageComparison{ + Reference: sourceRef, + SourceDigest: sourceInfo.Digest, + TargetDigest: targetInfo.Digest, + DigestMatch: sourceInfo.Digest == targetInfo.Digest, + SourceLayers: sourceInfo.Layers, + TargetLayers: targetInfo.Layers, + MissingLayers: make([]string, 0), + ExtraLayers: make([]string, 0), + } + + // Compare config digests + if sourceInfo.ConfigDigest != "" && targetInfo.ConfigDigest != "" { + comparison.ConfigMatch = sourceInfo.ConfigDigest == targetInfo.ConfigDigest + comparison.SourceConfig = sourceInfo.ConfigDigest + comparison.TargetConfig = targetInfo.ConfigDigest + } else { + comparison.ConfigMatch = true // Skip if not available + } + + // Compare layers + targetLayerSet := make(map[string]bool) + for _, l := range targetInfo.Layers { + targetLayerSet[l] = true + } + + sourceLayerSet := make(map[string]bool) + for _, l := range sourceInfo.Layers { + sourceLayerSet[l] = true + if !targetLayerSet[l] { + comparison.MissingLayers = append(comparison.MissingLayers, l) + } + } + + for _, l := range targetInfo.Layers { + if !sourceLayerSet[l] { + comparison.ExtraLayers = append(comparison.ExtraLayers, l) + } + } + + comparison.LayersMatch = len(comparison.MissingLayers) == 0 && len(comparison.ExtraLayers) == 0 + comparison.FullMatch = comparison.DigestMatch && comparison.ConfigMatch && comparison.LayersMatch + + return comparison, nil +} + +// ImageComparison holds detailed comparison of a single image +type ImageComparison struct { + Reference string + SourceDigest string + TargetDigest string + DigestMatch bool + SourceConfig string + TargetConfig string + ConfigMatch bool + SourceLayers []string + TargetLayers []string + MissingLayers []string + ExtraLayers []string + LayersMatch bool + FullMatch bool +} + diff --git a/testing/e2e/mirror/structure_validator.go b/testing/e2e/mirror/structure_validator.go deleted file mode 100644 index ae797558..00000000 --- a/testing/e2e/mirror/structure_validator.go +++ /dev/null @@ -1,250 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "context" - "crypto/tls" - "fmt" - "net/http" - "path" - "strings" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/samber/lo" - - "github.com/deckhouse/deckhouse-cli/internal" -) - -// ValidationResult contains the results of structure validation -type ValidationResult struct { - // MissingRepos lists repositories that should exist but don't - MissingRepos []string - - // MissingTags maps repository to list of missing tags - MissingTags map[string][]string - - // ExtraRepos lists unexpected repositories found - ExtraRepos []string - - // Errors contains any errors encountered during validation - Errors []error -} - -// IsValid returns true if validation passed -func (r *ValidationResult) IsValid() bool { - return len(r.MissingRepos) == 0 && - len(r.MissingTags) == 0 && - len(r.Errors) == 0 -} - -// String returns a human-readable summary -func (r *ValidationResult) String() string { - var sb strings.Builder - - if len(r.MissingRepos) > 0 { - sb.WriteString("Missing repositories:\n") - for _, repo := range r.MissingRepos { - sb.WriteString(" - " + repo + "\n") - } - } - - if len(r.MissingTags) > 0 { - sb.WriteString("Missing tags:\n") - for repo, tags := range r.MissingTags { - sb.WriteString(" " + repo + ":\n") - for _, tag := range tags { - sb.WriteString(" - " + tag + "\n") - } - } - } - - if len(r.Errors) > 0 { - sb.WriteString("Errors:\n") - for _, err := range r.Errors { - sb.WriteString(" - " + err.Error() + "\n") - } - } - - if r.IsValid() { - sb.WriteString("Validation passed\n") - } - - return sb.String() -} - -// StructureValidator validates the structure of a mirrored registry -type StructureValidator struct { - registry string - auth authn.Authenticator - - nameOpts []name.Option - remoteOpts []remote.Option - - // Expected structure from reference registry - expectedRepos []string - expectedTags map[string][]string -} - -// NewStructureValidator creates a new structure validator -// tlsSkipVerify: skip TLS certificate verification (for self-signed certs) -func NewStructureValidator(registry string, authenticator authn.Authenticator, tlsSkipVerify bool) *StructureValidator { - nameOpts := []name.Option{} - remoteOpts := []remote.Option{} - - if tlsSkipVerify { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - remoteOpts = append(remoteOpts, remote.WithTransport(transport)) - } - - if authenticator != nil && authenticator != authn.Anonymous { - remoteOpts = append(remoteOpts, remote.WithAuth(authenticator)) - } - - return &StructureValidator{ - registry: registry, - auth: authenticator, - nameOpts: nameOpts, - remoteOpts: remoteOpts, - expectedRepos: make([]string, 0), - expectedTags: make(map[string][]string), - } -} - -// SetExpectedFromDigests sets expected structure from a DigestMap -func (v *StructureValidator) SetExpectedFromDigests(digests DigestMap) { - repoSet := make(map[string]struct{}) - tagsByRepo := make(map[string][]string) - - for ref := range digests { - // Parse ref to extract repo and tag - parts := strings.Split(ref, ":") - if len(parts) != 2 { - continue - } - repo := parts[0] - tag := parts[1] - - repoSet[repo] = struct{}{} - tagsByRepo[repo] = append(tagsByRepo[repo], tag) - } - - v.expectedRepos = lo.Keys(repoSet) - v.expectedTags = tagsByRepo -} - -// Validate checks the registry structure -func (v *StructureValidator) Validate(ctx context.Context) (*ValidationResult, error) { - result := &ValidationResult{ - MissingRepos: make([]string, 0), - MissingTags: make(map[string][]string), - ExtraRepos: make([]string, 0), - Errors: make([]error, 0), - } - - // Validate each expected repository - for _, expectedRepo := range v.expectedRepos { - actualRepo := v.translateRepo(expectedRepo) - - // Check if repo exists by trying to list tags - tags, err := v.listTags(ctx, actualRepo) - if err != nil { - result.MissingRepos = append(result.MissingRepos, actualRepo) - result.Errors = append(result.Errors, fmt.Errorf("repo %s: %w", actualRepo, err)) - continue - } - - // Check for missing tags - expectedTags := v.expectedTags[expectedRepo] - missingTags := lo.Filter(expectedTags, func(tag string, _ int) bool { - return !lo.Contains(tags, tag) - }) - - if len(missingTags) > 0 { - result.MissingTags[actualRepo] = missingTags - } - } - - return result, nil -} - -// ValidateMinimal performs minimal validation (just checks key segments exist) -func (v *StructureValidator) ValidateMinimal(ctx context.Context) (*ValidationResult, error) { - result := &ValidationResult{ - MissingRepos: make([]string, 0), - MissingTags: make(map[string][]string), - Errors: make([]error, 0), - } - - // Check required segments - segments := []string{ - "", // root - internal.InstallSegment, // install - internal.InstallStandaloneSegment, // install-standalone - internal.ReleaseChannelSegment, // release-channel - } - - for _, segment := range segments { - repo := v.registry - if segment != "" { - repo = path.Join(v.registry, segment) - } - - tags, err := v.listTags(ctx, repo) - if err != nil { - result.MissingRepos = append(result.MissingRepos, repo) - result.Errors = append(result.Errors, fmt.Errorf("segment %s: %w", segment, err)) - continue - } - - // Check that at least one release channel tag exists - hasReleaseChannel := lo.ContainsBy(tags, func(tag string) bool { - return lo.Contains(internal.GetAllDefaultReleaseChannels(), tag) - }) - if !hasReleaseChannel { - result.MissingTags[repo] = []string{""} - } - } - - return result, nil -} - -// translateRepo translates a source repo path to target repo path -func (v *StructureValidator) translateRepo(sourceRepo string) string { - // Normalize the source repo to get the path portion - normalizedPath := NormalizeRef(sourceRepo) - // Prepend the target registry - return path.Join(v.registry, normalizedPath) -} - -// listTags lists all tags in a repository -func (v *StructureValidator) listTags(ctx context.Context, repo string) ([]string, error) { - repoRef, err := name.NewRepository(repo, v.nameOpts...) - if err != nil { - return nil, fmt.Errorf("parse repo: %w", err) - } - - tags, err := remote.List(repoRef, v.remoteOpts...) - if err != nil { - return nil, fmt.Errorf("list tags: %w", err) - } - - return tags, nil -} diff --git a/testing/util/mirror/registry.go b/testing/util/mirror/registry.go index 16190a83..9cbe2df1 100644 --- a/testing/util/mirror/registry.go +++ b/testing/util/mirror/registry.go @@ -17,83 +17,42 @@ limitations under the License. package mirror import ( - "context" "io" + "io/fs" golog "log" "net/http/httptest" + "os" + "path/filepath" "strings" - "sync" "github.com/google/go-containerregistry/pkg/registry" - v1 "github.com/google/go-containerregistry/pkg/v1" ) -// ListableBlobHandler wraps a BlobHandler to track ingested blobs -type ListableBlobHandler struct { - registry.BlobHandler - registry.BlobPutHandler - - mu sync.Mutex - ingestedBlobs []string -} - -// Get implements registry.BlobHandler and tracks accessed blobs -func (h *ListableBlobHandler) Get(ctx context.Context, repo string, hash v1.Hash) (io.ReadCloser, error) { - h.mu.Lock() - defer h.mu.Unlock() - h.ingestedBlobs = append(h.ingestedBlobs, hash.String()) - - return h.BlobHandler.Get(ctx, repo, hash) -} - -// ListBlobs returns all blobs that have been accessed -func (h *ListableBlobHandler) ListBlobs() []string { - h.mu.Lock() - defer h.mu.Unlock() - return append([]string{}, h.ingestedBlobs...) -} - -// TestRegistry holds the test registry server and its resources +// TestRegistry is a disk-based container registry for e2e testing. +// Blobs are stored on disk to avoid memory exhaustion when mirroring large images. type TestRegistry struct { - Server *httptest.Server - Host string - RepoPath string - BlobHandler *ListableBlobHandler -} - -// Close stops the test registry server -func (r *TestRegistry) Close() { - if r.Server != nil { - r.Server.Close() - } -} - -// FullPath returns the full registry path including host and repo -func (r *TestRegistry) FullPath() string { - return r.Host + r.RepoPath -} + server *httptest.Server + storageDir string -// SetupEmptyRegistryRepo creates an in-memory registry for testing -// Returns host, repoPath, and a ListableBlobHandler to track blob access -func SetupEmptyRegistryRepo(useTLS bool) ( /*host*/ string /*repoPath*/, string, *ListableBlobHandler) { - reg := SetupTestRegistry(useTLS) - return reg.Host, reg.RepoPath, reg.BlobHandler + Host string // e.g. "127.0.0.1:12345" } -// SetupTestRegistry creates an in-memory registry for testing and returns a TestRegistry -func SetupTestRegistry(useTLS bool) *TestRegistry { - memBlobHandler := registry.NewInMemoryBlobHandler() - bh := &ListableBlobHandler{ - BlobHandler: memBlobHandler, - BlobPutHandler: memBlobHandler.(registry.BlobPutHandler), +// NewTestRegistry creates a new disk-based test registry. +// Storage is created in a temporary directory that will be cleaned up on Close(). +func NewTestRegistry(useTLS bool) (*TestRegistry, error) { + storageDir, err := os.MkdirTemp("", "test-registry-*") + if err != nil { + return nil, err } - registryHandler := registry.New( - registry.WithBlobHandler(bh), + blobHandler := registry.NewDiskBlobHandler(storageDir) + + handler := registry.New( + registry.WithBlobHandler(blobHandler), registry.Logger(golog.New(io.Discard, "", 0)), ) - server := httptest.NewUnstartedServer(registryHandler) + server := httptest.NewUnstartedServer(handler) if useTLS { server.StartTLS() } else { @@ -106,16 +65,71 @@ func SetupTestRegistry(useTLS bool) *TestRegistry { } return &TestRegistry{ - Server: server, - Host: host, - RepoPath: "/deckhouse/ee", - BlobHandler: bh, + server: server, + storageDir: storageDir, + Host: host, + }, nil +} + +// Close stops the registry server and removes all stored data. +func (r *TestRegistry) Close() { + if r.server != nil { + r.server.Close() + } + if r.storageDir != "" { + os.RemoveAll(r.storageDir) } } -// SetupTestRegistryWithPath creates an in-memory registry with a custom repo path -func SetupTestRegistryWithPath(useTLS bool, repoPath string) *TestRegistry { - reg := SetupTestRegistry(useTLS) - reg.RepoPath = repoPath +// StoragePath returns the path to the on-disk blob storage. +// Useful for debugging or inspecting stored data. +func (r *TestRegistry) StoragePath() string { + return r.storageDir +} + +// BlobCount returns the number of blobs currently stored in the registry. +func (r *TestRegistry) BlobCount() int { + count := 0 + _ = filepath.WalkDir(r.storageDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if !d.IsDir() { + count++ + } + return nil + }) + return count +} + +// ListBlobs returns a list of blob digests stored in the registry. +// This is useful for verifying what was pushed. +func (r *TestRegistry) ListBlobs() []string { + var blobs []string + _ = filepath.WalkDir(r.storageDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if !d.IsDir() { + // Extract digest from path (format: storageDir/sha256/abc123...) + rel, _ := filepath.Rel(r.storageDir, path) + parts := strings.Split(rel, string(filepath.Separator)) + if len(parts) == 2 { + blobs = append(blobs, parts[0]+":"+parts[1]) + } + } + return nil + }) + return blobs +} + +// SetupTestRegistry creates a disk-based registry for testing. +// Returns *TestRegistry - use reg.Host to get the address, then append your own repo path. +func SetupTestRegistry(useTLS bool) *TestRegistry { + reg, err := NewTestRegistry(useTLS) + if err != nil { + panic("failed to create test registry: " + err.Error()) + } return reg } + From 87ae40ea53dcf922f16e1ebe262e32faa006a948 Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Fri, 26 Dec 2025 11:26:00 +0300 Subject: [PATCH 05/11] chore: refactor Signed-off-by: Timur Tuktamyshev --- Taskfile.yml | 37 + testing/e2e/mirror/README.md | 6 +- testing/e2e/mirror/commands.go | 129 ++++ testing/e2e/mirror/mirror_e2e_test.go | 825 +++++++--------------- testing/e2e/mirror/output.go | 159 +++++ testing/e2e/mirror/registry_comparator.go | 109 ++- testing/e2e/mirror/report.go | 291 ++++++++ 7 files changed, 934 insertions(+), 622 deletions(-) create mode 100644 testing/e2e/mirror/commands.go create mode 100644 testing/e2e/mirror/output.go create mode 100644 testing/e2e/mirror/report.go diff --git a/Taskfile.yml b/Taskfile.yml index 82be4790..2f34aa3d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -266,6 +266,43 @@ tasks: cmds: - task: _test:go + test:e2e:mirror: + desc: Run E2E tests for d8 mirror (requires credentials) + summary: | + Runs heavy E2E tests for d8 mirror pull/push commands. + + The test performs a complete mirror cycle: + 1. Analyzes source registry + 2. Pulls images to local bundle + 3. Pushes bundle to target registry + 4. Compares all images between source and target + + Required: Set E2E_LICENSE_TOKEN or E2E_SOURCE_USER/E2E_SOURCE_PASSWORD + + Examples: + # With license token + E2E_LICENSE_TOKEN=xxx task test:e2e:mirror + + # With local registry + E2E_SOURCE_REGISTRY=localhost:443/deckhouse \ + E2E_SOURCE_USER=admin \ + E2E_SOURCE_PASSWORD=secret \ + E2E_TLS_SKIP_VERIFY=true \ + task test:e2e:mirror + + # Keep bundle for debugging + E2E_LICENSE_TOKEN=xxx E2E_KEEP_BUNDLE=true task test:e2e:mirror + deps: + - build + cmds: + - go test -v -timeout 120m -tags=e2e ./testing/e2e/mirror/... {{ .CLI_ARGS }} + + test:e2e:mirror:logs:clean: + desc: Clean E2E test logs + cmds: + - rm -rf ./testing/e2e/.logs/* + - echo "E2E logs cleaned" + lint: desc: Run golangci-lint with auto-fix cmds: diff --git a/testing/e2e/mirror/README.md b/testing/e2e/mirror/README.md index 47b58817..e18c2ce3 100644 --- a/testing/e2e/mirror/README.md +++ b/testing/e2e/mirror/README.md @@ -1,6 +1,6 @@ # E2E Tests for d8 mirror -Heavy end-to-end tests for the `d8 mirror pull` and `d8 mirror push` commands. +End-to-end tests for the `d8 mirror pull` and `d8 mirror push` commands. ## Overview @@ -11,7 +11,7 @@ These tests perform a **complete mirror cycle with deep comparison** to ensure s 1. **Analyze source registry** - Discover all repositories and count all images 2. **Pull images** - Execute `d8 mirror pull` to create a bundle 3. **Push images** - Execute `d8 mirror push` to target registry -4. **Deep comparison** - Compare EVERY repository, tag, and digest between source and target +4. **Deep comparison** - Compare every repository, tag, and digest between source and target ### What Gets Compared (Deep Comparison) @@ -29,7 +29,7 @@ This ensures **byte-for-byte identical** registries. - Built `d8` binary (run `task build` from project root) - Valid credentials for the source registry - Network access to the source registry -- Sufficient disk space for the bundle (can be several GB) +- Sufficient disk space for the bundle (can be several GB, aroud 20) ## Running Tests diff --git a/testing/e2e/mirror/commands.go b/testing/e2e/mirror/commands.go new file mode 100644 index 00000000..ad7df44a --- /dev/null +++ b/testing/e2e/mirror/commands.go @@ -0,0 +1,129 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "fmt" + "io" + "os" + "os/exec" + "testing" + "time" +) + +// ============================================================================= +// Command Builders +// ============================================================================= + +// buildPullCommand builds the d8 mirror pull command +func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { + args := []string{ + "mirror", "pull", + "--source", cfg.SourceRegistry, + "--force", // overwrite if exists + } + + // Authentication + if cfg.SourceUser != "" { + args = append(args, "--source-login", cfg.SourceUser) + args = append(args, "--source-password", cfg.SourcePassword) + } else if cfg.LicenseToken != "" { + args = append(args, "--license", cfg.LicenseToken) + } + + // TLS + if cfg.TLSSkipVerify { + args = append(args, "--tls-skip-verify") + } + + // Debug options + if cfg.NoModules { + args = append(args, "--no-modules") + } + + args = append(args, bundleDir) + + cmd := exec.Command(cfg.D8Binary, args...) + cmd.Env = append(os.Environ(), "HOME="+os.Getenv("HOME")) + + return cmd +} + +// buildPushCommand builds the d8 mirror push command +func buildPushCommand(cfg *Config, bundleDir, targetRegistry string) *exec.Cmd { + args := []string{ + "mirror", "push", + bundleDir, + targetRegistry, + } + + // TLS + if cfg.TLSSkipVerify { + args = append(args, "--tls-skip-verify") + } + + // Authentication + if cfg.TargetUser != "" { + args = append(args, "--registry-login", cfg.TargetUser) + args = append(args, "--registry-password", cfg.TargetPassword) + } + + cmd := exec.Command(cfg.D8Binary, args...) + cmd.Env = append(os.Environ(), "HOME="+os.Getenv("HOME")) + + return cmd +} + +// ============================================================================= +// Command Runner +// ============================================================================= + +// runCommandWithLog runs command with streaming output and saves to log file +func runCommandWithLog(t *testing.T, cmd *exec.Cmd, logFile string) error { + t.Helper() + + // Open log file + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + t.Logf("Warning: could not open log file %s: %v", logFile, err) + // Fallback: just stream without logging + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() + } + defer f.Close() + + // Write command header + fmt.Fprintf(f, "\n\n========== COMMAND: %s ==========\n", cmd.String()) + fmt.Fprintf(f, "Started: %s\n\n", time.Now().Format(time.RFC3339)) + + // Stream to stdout and file + cmd.Stdout = io.MultiWriter(os.Stdout, f) + cmd.Stderr = io.MultiWriter(os.Stderr, f) + + // Run + cmdErr := cmd.Run() + + // Write result + if cmdErr != nil { + fmt.Fprintf(f, "\n\n========== COMMAND FAILED: %v ==========\n", cmdErr) + } else { + fmt.Fprintf(f, "\n\n========== COMMAND SUCCEEDED ==========\n") + } + + return cmdErr +} diff --git a/testing/e2e/mirror/mirror_e2e_test.go b/testing/e2e/mirror/mirror_e2e_test.go index 52804ff7..003cc76a 100644 --- a/testing/e2e/mirror/mirror_e2e_test.go +++ b/testing/e2e/mirror/mirror_e2e_test.go @@ -1,3 +1,5 @@ +//go:build e2e + /* Copyright 2024 Flant JSC @@ -17,99 +19,32 @@ limitations under the License. package mirror import ( - "bytes" "context" "fmt" - "io" "os" - "os/exec" "path/filepath" - "strings" "testing" "time" - "github.com/charmbracelet/lipgloss" - "github.com/muesli/termenv" "github.com/stretchr/testify/require" - "golang.org/x/term" mirrorutil "github.com/deckhouse/deckhouse-cli/testing/util/mirror" ) -func init() { - // Force color output - go test buffers stdout which disables color detection - // Check stderr instead (usually unbuffered) or honor FORCE_COLOR env - if term.IsTerminal(int(os.Stderr.Fd())) || os.Getenv("FORCE_COLOR") != "" || os.Getenv("TERM") != "" { - lipgloss.DefaultRenderer().SetColorProfile(termenv.TrueColor) - } -} - -// output is a helper to write to stderr (which preserves colors in go test) -var output = os.Stderr - -func printLine(format string, args ...interface{}) { - fmt.Fprintf(output, format+"\n", args...) -} - -func print(format string, args ...interface{}) { - fmt.Fprintf(output, format, args...) -} - -// Lipgloss styles for beautiful terminal output -var ( - // Colors - cyan = lipgloss.Color("6") - green = lipgloss.Color("2") - red = lipgloss.Color("1") - yellow = lipgloss.Color("3") - blue = lipgloss.Color("4") - white = lipgloss.Color("15") - gray = lipgloss.Color("8") - - // Text styles - titleStyle = lipgloss.NewStyle().Bold(true).Foreground(white) - headerStyle = lipgloss.NewStyle().Bold(true).Foreground(cyan) - labelStyle = lipgloss.NewStyle().Foreground(gray) - valueStyle = lipgloss.NewStyle().Foreground(white) - dimStyle = lipgloss.NewStyle().Foreground(gray) - successStyle = lipgloss.NewStyle().Foreground(green) - errorStyle = lipgloss.NewStyle().Foreground(red) - - // Badge styles - okBadge = lipgloss.NewStyle().Bold(true).Foreground(green).Render("[OK]") - failBadge = lipgloss.NewStyle().Bold(true).Foreground(red).Render("[FAIL]") - skipBadge = lipgloss.NewStyle().Foreground(yellow).Render("[SKIP]") - - // Step styles - stepNumStyle = lipgloss.NewStyle().Bold(true).Foreground(blue) - stepTextStyle = lipgloss.NewStyle().Bold(true).Foreground(white) - - // Separator - separatorStyle = lipgloss.NewStyle().Foreground(cyan) -) - -// printStep prints a formatted step header -func printStep(num int, description string) { - badge := stepNumStyle.Render(fmt.Sprintf("[STEP %d]", num)) - text := stepTextStyle.Render(description) - printLine("\n%s %s", badge, text) -} - -// renderSeparator creates a separator line -func renderSeparator(char string, width int) string { - return separatorStyle.Render(strings.Repeat(char, width)) -} +// ============================================================================= +// E2E Test: Full Mirror Cycle +// ============================================================================= // TestMirrorE2E_FullCycle performs a complete mirror cycle and validates // that source and target registries are identical. // // This is a heavy E2E test that: -// 1. Discovers all repositories in source registry -// 2. Pulls all images to local bundle using d8 mirror pull -// 3. Pushes bundle to target registry using d8 mirror push -// 4. Discovers all repositories in target registry -// 5. Compares EVERY tag and digest between source and target -// 6. Generates detailed comparison report +// 1. Discovers all repositories in source registry +// 2. Pulls all images to local bundle using d8 mirror pull +// 3. Pushes bundle to target registry using d8 mirror push +// 4. Discovers all repositories in target registry +// 5. Compares every tag and digest between source and target +// 6. Generates detailed comparison report // // Run with: // @@ -120,147 +55,257 @@ func renderSeparator(char string, width int) string { func TestMirrorE2E_FullCycle(t *testing.T) { cfg := GetConfig() - // Create log directory in project (gitignored) - logDir := getLogDir("fullcycle") - require.NoError(t, os.MkdirAll(logDir, 0755)) - logFile := filepath.Join(logDir, "test.log") - reportFile := filepath.Join(logDir, "report.txt") - comparisonFile := filepath.Join(logDir, "comparison.txt") - - printLine("") - printLine(renderSeparator("═", 80)) - printLine(" %s", titleStyle.Render("E2E TEST: Mirror Full Cycle")) - printLine(renderSeparator("═", 80)) - printLine(" %s %s", labelStyle.Render("Source:"), valueStyle.Render(cfg.SourceRegistry)) - printLine(" %s %s", labelStyle.Render("Logs: "), dimStyle.Render(logDir)) - printLine(renderSeparator("═", 80)) - printLine("") - + // Skip if no auth provided if !cfg.HasSourceAuth() { t.Skip("Source authentication not provided (use -license-token or -source-user/-source-password)") } - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Minute) - defer cancel() + // Setup test environment + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + // Print header + printTestHeader("Mirror Full Cycle", cfg.SourceRegistry, env.LogDir) + + // Run test steps + runFullCycleTest(t, cfg, env) +} + +// ============================================================================= +// Test Environment +// ============================================================================= + +// testEnv holds all test environment state +type testEnv struct { + LogDir string + LogFile string + ReportFile string + ComparisonFile string + BundleDir string + TargetRegistry string + Report *TestReport + Cleanup func() +} + +// setupTestEnvironment prepares everything needed for the test +func setupTestEnvironment(t *testing.T, cfg *Config) *testEnv { + t.Helper() + + // Create log directory + logDir := getLogDir("fullcycle") + require.NoError(t, os.MkdirAll(logDir, 0755)) + + // Setup target registry + targetHost, targetPath, registryCleanup := setupTargetRegistry(t, cfg) + targetRegistry := targetHost + targetPath + t.Logf("Target registry: %s", targetRegistry) + + // Setup bundle directory + bundleDir := setupBundleDir(t, cfg) // Initialize report report := &TestReport{ TestName: "TestMirrorE2E_FullCycle", StartTime: time.Now(), SourceRegistry: cfg.SourceRegistry, + TargetRegistry: targetRegistry, LogDir: logDir, } - // Ensure report is written at the end - defer func() { - report.EndTime = time.Now() - report.Print() - if err := report.WriteToFile(reportFile); err != nil { - t.Logf("Warning: failed to write report: %v", err) - } else { - t.Logf("Report written to: %s", reportFile) - } - }() + env := &testEnv{ + LogDir: logDir, + LogFile: filepath.Join(logDir, "test.log"), + ReportFile: filepath.Join(logDir, "report.txt"), + ComparisonFile: filepath.Join(logDir, "comparison.txt"), + BundleDir: bundleDir, + TargetRegistry: targetRegistry, + Report: report, + } - // Setup target registry - targetHost, targetPath, cleanup := setupTargetRegistry(t, cfg) - defer cleanup() + // Setup cleanup + env.Cleanup = func() { + registryCleanup() + finalizeReport(t, env) + } - targetRegistry := targetHost + targetPath - report.TargetRegistry = targetRegistry - t.Logf("Target registry: %s", targetRegistry) + return env +} + +// setupBundleDir creates the bundle directory +func setupBundleDir(t *testing.T, cfg *Config) string { + t.Helper() - // Create bundle directory - bundleDir := t.TempDir() if cfg.KeepBundle { - bundleDir = filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-e2e-%d", time.Now().Unix())) + bundleDir := filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-e2e-%d", time.Now().Unix())) require.NoError(t, os.MkdirAll(bundleDir, 0755)) t.Logf("Bundle directory (will be kept): %s", bundleDir) + return bundleDir + } + + bundleDir := t.TempDir() + t.Logf("Bundle directory: %s", bundleDir) + return bundleDir +} + +// setupTargetRegistry sets up the target registry for testing +func setupTargetRegistry(t *testing.T, cfg *Config) (host, path string, cleanup func()) { + t.Helper() + + if cfg.UseInMemoryRegistry() { + reg := mirrorutil.SetupTestRegistry(false) + repoPath := "/deckhouse/ee" + t.Logf("Started test registry at %s%s", reg.Host, repoPath) + return reg.Host, repoPath, reg.Close + } + + return cfg.TargetRegistry, "", func() {} +} + +// finalizeReport writes the final report +func finalizeReport(t *testing.T, env *testEnv) { + t.Helper() + + env.Report.EndTime = time.Now() + env.Report.Print() + + if err := env.Report.WriteToFile(env.ReportFile); err != nil { + t.Logf("Warning: failed to write report: %v", err) } else { - t.Logf("Bundle directory: %s", bundleDir) + t.Logf("Report written to: %s", env.ReportFile) } +} + +// ============================================================================= +// Test Steps +// ============================================================================= + +// runFullCycleTest executes all test steps +func runFullCycleTest(t *testing.T, cfg *Config, env *testEnv) { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Minute) + defer cancel() + + // Step 1: Analyze source registry + runAnalyzeStep(t, cfg, env) + + // Step 2: Pull images + runPullStep(t, cfg, env) - // ======================================================================== - // STEP 1: Analyze source registry - // ======================================================================== + // Step 3: Push images + runPushStep(t, cfg, env) + + // Step 4: Compare registries + runCompareStep(t, ctx, cfg, env) + + // Success! + printSuccessBox(env.Report.MatchedImages, env.Report.MatchedLayers) +} + +// ----------------------------------------------------------------------------- +// Step 1: Analyze Source Registry +// ----------------------------------------------------------------------------- + +func runAnalyzeStep(t *testing.T, cfg *Config, env *testEnv) { + t.Helper() stepStart := time.Now() printStep(1, "Analyzing source registry") - sourceComparator := NewRegistryComparator( + comparator := NewRegistryComparator( cfg.SourceRegistry, "", cfg.GetSourceAuth(), nil, cfg.TLSSkipVerify, ) - sourceComparator.SetProgressCallback(func(msg string) { + comparator.SetProgressCallback(func(msg string) { t.Logf(" %s", msg) }) - sourceRepos, err := sourceComparator.discoverRepositories(ctx, cfg.SourceRegistry, sourceComparator.sourceRemoteOpts) - if err != nil { - report.AddStep("Analyze source registry", "FAIL", time.Since(stepStart), err) - require.NoError(t, err, "Failed to analyze source registry") - } + repos := comparator.discoverRepositories(cfg.SourceRegistry, comparator.sourceRemoteOpts) + env.Report.SourceRepoCount = len(repos) + env.Report.AddStep( + fmt.Sprintf("Analyze source (%d repos)", len(repos)), + "PASS", time.Since(stepStart), nil, + ) + t.Logf("Source registry: %d repositories", len(repos)) +} - report.SourceRepoCount = len(sourceRepos) - report.AddStep(fmt.Sprintf("Analyze source (%d repos)", len(sourceRepos)), - "PASS", time.Since(stepStart), nil) - t.Logf("Source registry: %d repositories", len(sourceRepos)) +// ----------------------------------------------------------------------------- +// Step 2: Pull Images +// ----------------------------------------------------------------------------- - // ======================================================================== - // STEP 2: Execute pull - // ======================================================================== - stepStart = time.Now() +func runPullStep(t *testing.T, cfg *Config, env *testEnv) { + t.Helper() + stepStart := time.Now() printStep(2, "Pulling images to bundle") - pullCmd := buildPullCommand(cfg, bundleDir) - t.Logf("Running: %s", pullCmd.String()) + cmd := buildPullCommand(cfg, env.BundleDir) + t.Logf("Running: %s", cmd.String()) - _, err = runCommandWithLog(t, pullCmd, logFile) + err := runCommandWithLog(t, cmd, env.LogFile) if err != nil { - report.AddStep("Pull images", "FAIL", time.Since(stepStart), err) + env.Report.AddStep("Pull images", "FAIL", time.Since(stepStart), err) require.NoError(t, err, "Pull failed") } - // Log bundle contents - bundleFiles, err := os.ReadDir(bundleDir) + // Calculate bundle size + bundleSize := calculateBundleSize(t, env.BundleDir) + env.Report.BundleSize = bundleSize + + env.Report.AddStep( + fmt.Sprintf("Pull images (%.2f GB bundle)", float64(bundleSize)/(1024*1024*1024)), + "PASS", time.Since(stepStart), nil, + ) + t.Logf("Pull completed: %.2f GB total", float64(bundleSize)/(1024*1024*1024)) +} + +// calculateBundleSize returns total size of bundle files +func calculateBundleSize(t *testing.T, bundleDir string) int64 { + t.Helper() + + files, err := os.ReadDir(bundleDir) require.NoError(t, err) + var totalSize int64 - for _, f := range bundleFiles { + for _, f := range files { if info, err := f.Info(); err == nil { totalSize += info.Size() t.Logf(" %s (%.2f MB)", f.Name(), float64(info.Size())/(1024*1024)) } } - report.BundleSize = totalSize - report.AddStep(fmt.Sprintf("Pull images (%.2f GB bundle)", float64(totalSize)/(1024*1024*1024)), - "PASS", time.Since(stepStart), nil) - t.Logf("Pull completed: %d files, %.2f GB total", len(bundleFiles), float64(totalSize)/(1024*1024*1024)) - - // ======================================================================== - // STEP 3: Execute push - // ======================================================================== - stepStart = time.Now() + return totalSize +} + +// ----------------------------------------------------------------------------- +// Step 3: Push Images +// ----------------------------------------------------------------------------- + +func runPushStep(t *testing.T, cfg *Config, env *testEnv) { + t.Helper() + stepStart := time.Now() printStep(3, "Pushing bundle to target registry") - pushCmd := buildPushCommand(cfg, bundleDir, targetRegistry) - t.Logf("Running: %s", pushCmd.String()) + cmd := buildPushCommand(cfg, env.BundleDir, env.TargetRegistry) + t.Logf("Running: %s", cmd.String()) - _, err = runCommandWithLog(t, pushCmd, logFile) + err := runCommandWithLog(t, cmd, env.LogFile) if err != nil { - report.AddStep("Push to registry", "FAIL", time.Since(stepStart), err) + env.Report.AddStep("Push to registry", "FAIL", time.Since(stepStart), err) require.NoError(t, err, "Push failed") } - report.AddStep("Push to registry", "PASS", time.Since(stepStart), nil) + + env.Report.AddStep("Push to registry", "PASS", time.Since(stepStart), nil) t.Log("Push completed successfully") +} + +// ----------------------------------------------------------------------------- +// Step 4: Compare Registries +// ----------------------------------------------------------------------------- - // ======================================================================== - // STEP 4: Deep comparison of source and target - // ======================================================================== - stepStart = time.Now() +func runCompareStep(t *testing.T, ctx context.Context, cfg *Config, env *testEnv) { + t.Helper() + stepStart := time.Now() printStep(4, "Deep comparison of registries") comparator := NewRegistryComparator( - cfg.SourceRegistry, targetRegistry, + cfg.SourceRegistry, env.TargetRegistry, cfg.GetSourceAuth(), cfg.GetTargetAuth(), cfg.TLSSkipVerify, ) @@ -268,440 +313,98 @@ func TestMirrorE2E_FullCycle(t *testing.T) { t.Logf(" %s", msg) }) - comparisonReport, err := comparator.Compare(ctx) + comparison, err := comparator.Compare(ctx) if err != nil { - report.AddStep("Deep comparison", "FAIL", time.Since(stepStart), err) + env.Report.AddStep("Deep comparison", "FAIL", time.Since(stepStart), err) require.NoError(t, err, "Comparison failed") } - // Save detailed comparison report - if err := os.WriteFile(comparisonFile, []byte(comparisonReport.DetailedReport()), 0644); err != nil { - t.Logf("Warning: failed to write comparison file: %v", err) - } else { - t.Logf("Detailed comparison written to: %s", comparisonFile) - } + // Save detailed comparison + saveComparisonReport(t, env.ComparisonFile, comparison) // Update report with comparison stats - report.SourceImageCount = comparisonReport.TotalSourceImages - report.TargetRepoCount = len(comparisonReport.TargetRepositories) - report.TargetImageCount = comparisonReport.TotalTargetImages - report.MatchedImages = comparisonReport.MatchedImages - report.MissingImages = len(comparisonReport.MissingImages) - report.MismatchedImages = len(comparisonReport.MismatchedImages) - report.SkippedImages = comparisonReport.SkippedImages - report.MatchedLayers = comparisonReport.MatchedLayers - report.MissingLayers = comparisonReport.MissingLayers - report.ComparisonReport = comparisonReport + updateReportWithComparison(env.Report, comparison) // Print summary t.Log("") - t.Log(comparisonReport.Summary()) - - if !comparisonReport.IsIdentical() { - report.AddStep(fmt.Sprintf("Deep comparison (%d matched, %d missing, %d mismatched)", - comparisonReport.MatchedImages, - len(comparisonReport.MissingImages), - len(comparisonReport.MismatchedImages)), + t.Log(comparison.Summary()) + + // Check if identical + if !comparison.IsIdentical() { + env.Report.AddStep( + fmt.Sprintf("Deep comparison (%d matched, %d missing, %d mismatched)", + comparison.MatchedImages, + len(comparison.MissingImages), + len(comparison.MismatchedImages)), "FAIL", time.Since(stepStart), fmt.Errorf("registries differ: %d missing, %d mismatched", - len(comparisonReport.MissingImages), - len(comparisonReport.MismatchedImages))) + len(comparison.MissingImages), + len(comparison.MismatchedImages)), + ) - require.True(t, comparisonReport.IsIdentical(), + require.True(t, comparison.IsIdentical(), "Registries are NOT identical!\n\n%s\n\nSee %s for details", - comparisonReport.Summary(), comparisonFile) - } - - report.AddStep(fmt.Sprintf("Deep comparison (%d images verified)", comparisonReport.MatchedImages), - "PASS", time.Since(stepStart), nil) - - // ======================================================================== - // SUCCESS - // ======================================================================== - successBox := lipgloss.NewStyle(). - Border(lipgloss.DoubleBorder()). - BorderForeground(green). - Padding(0, 2). - Foreground(green) - - printLine("") - printLine(successBox.Render(fmt.Sprintf( - "SUCCESS: REGISTRIES ARE IDENTICAL\n\nVerified: %d images, %d layers\nAll manifest, config, and layer digests match!", - comparisonReport.MatchedImages, - comparisonReport.MatchedLayers, - ))) -} - -// getLogDir returns the log directory path for e2e tests. -// Logs are stored in testing/e2e/.logs/-/ -func getLogDir(testName string) string { - // Get project root by finding go.mod - dir, _ := os.Getwd() - for { - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { - break - } - parent := filepath.Dir(dir) - if parent == dir { - // Fallback to current dir if go.mod not found - dir, _ = os.Getwd() - break - } - dir = parent - } - - timestamp := time.Now().Format("20060102-150405") - return filepath.Join(dir, "testing", "e2e", ".logs", fmt.Sprintf("%s-%s", testName, timestamp)) -} - -// setupTargetRegistry sets up the target registry for testing -// Returns host, path, and cleanup function -func setupTargetRegistry(t *testing.T, cfg *Config) (string, string, func()) { - t.Helper() - - if cfg.UseInMemoryRegistry() { - // Use disk-based test registry - reg := mirrorutil.SetupTestRegistry(false) - repoPath := "/deckhouse/ee" - t.Logf("Started test registry at %s%s", reg.Host, repoPath) - return reg.Host, repoPath, func() { - reg.Close() - } - } - - // Use external registry - return cfg.TargetRegistry, "", func() { - // External registry cleanup is user's responsibility - } -} - -// buildPullCommand builds the d8 mirror pull command -func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { - args := []string{ - "mirror", "pull", - "--source", cfg.SourceRegistry, - "--force", // overwrite if exists - } - - // Add authentication flags - if cfg.SourceUser != "" { - args = append(args, "--source-login", cfg.SourceUser) - args = append(args, "--source-password", cfg.SourcePassword) - } else if cfg.LicenseToken != "" { - args = append(args, "--license", cfg.LicenseToken) - } - - // Add TLS skip verify flag (for self-signed certs) - if cfg.TLSSkipVerify { - args = append(args, "--tls-skip-verify") - } - - // Skip modules (for testing failure scenarios) - if cfg.NoModules { - args = append(args, "--no-modules") - } - - args = append(args, bundleDir) - - cmd := exec.Command(cfg.D8Binary, args...) - cmd.Env = append(os.Environ(), - "HOME="+os.Getenv("HOME"), - ) - - return cmd -} - -// buildPushCommand builds the d8 mirror push command -func buildPushCommand(cfg *Config, bundleDir, targetRegistry string) *exec.Cmd { - args := []string{ - "mirror", "push", - bundleDir, - targetRegistry, - } - - if cfg.TLSSkipVerify { - args = append(args, "--tls-skip-verify") - } - - if cfg.TargetUser != "" { - args = append(args, "--registry-login", cfg.TargetUser) - args = append(args, "--registry-password", cfg.TargetPassword) + comparison.Summary(), env.ComparisonFile) } - cmd := exec.Command(cfg.D8Binary, args...) - cmd.Env = append(os.Environ(), - "HOME="+os.Getenv("HOME"), + env.Report.AddStep( + fmt.Sprintf("Deep comparison (%d images verified)", comparison.MatchedImages), + "PASS", time.Since(stepStart), nil, ) - - return cmd } -// runCommandWithLog runs command with streaming and saves full output to a log file -func runCommandWithLog(t *testing.T, cmd *exec.Cmd, logFile string) ([]byte, error) { +// saveComparisonReport writes the detailed comparison to file +func saveComparisonReport(t *testing.T, path string, comparison *ComparisonReport) { t.Helper() - var buf bytes.Buffer - - // Open log file for appending - f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - t.Logf("Warning: could not open log file %s: %v", logFile, err) - // Fallback to just streaming - cmd.Stdout = io.MultiWriter(os.Stdout, &buf) - cmd.Stderr = io.MultiWriter(os.Stderr, &buf) - return buf.Bytes(), cmd.Run() - } - defer f.Close() - - // Write command to log - fmt.Fprintf(f, "\n\n========== COMMAND: %s ==========\n", cmd.String()) - fmt.Fprintf(f, "Started: %s\n\n", time.Now().Format(time.RFC3339)) - - // Create multi-writers to write to stdout, buffer, AND log file - cmd.Stdout = io.MultiWriter(os.Stdout, &buf, f) - cmd.Stderr = io.MultiWriter(os.Stderr, &buf, f) - - cmdErr := cmd.Run() - - // Write result to log - if cmdErr != nil { - fmt.Fprintf(f, "\n\n========== COMMAND FAILED: %v ==========\n", cmdErr) + if err := os.WriteFile(path, []byte(comparison.DetailedReport()), 0644); err != nil { + t.Logf("Warning: failed to write comparison file: %v", err) } else { - fmt.Fprintf(f, "\n\n========== COMMAND SUCCEEDED ==========\n") + t.Logf("Detailed comparison written to: %s", path) } - - return buf.Bytes(), cmdErr } -// TestReport collects test execution results for final summary -type TestReport struct { - TestName string - StartTime time.Time - EndTime time.Time - SourceRegistry string - TargetRegistry string - LogDir string - - // Source stats - SourceRepoCount int - SourceImageCount int - - // Target stats - TargetRepoCount int - TargetImageCount int - - // Comparison stats - MatchedImages int - MissingImages int - MismatchedImages int - SkippedImages int // Digest-based, .att, .sig tags - - // Deep comparison stats - MatchedLayers int - MissingLayers int - - // Bundle stats - BundleSize int64 - - // Steps - Steps []StepResult - - // Full comparison report - ComparisonReport *ComparisonReport +// updateReportWithComparison updates test report with comparison results +func updateReportWithComparison(report *TestReport, comparison *ComparisonReport) { + report.SourceImageCount = comparison.TotalSourceImages + report.TargetRepoCount = len(comparison.TargetRepositories) + report.TargetImageCount = comparison.TotalTargetImages + report.MatchedImages = comparison.MatchedImages + report.MissingImages = len(comparison.MissingImages) + report.MismatchedImages = len(comparison.MismatchedImages) + report.SkippedImages = comparison.SkippedImages + report.MatchedLayers = comparison.MatchedLayers + report.MissingLayers = comparison.MissingLayers + report.ComparisonReport = comparison } -// StepResult represents a single step in the test -type StepResult struct { - Name string - Status string // "PASS", "FAIL", "SKIP" - Duration time.Duration - Error string -} +// ============================================================================= +// Helpers +// ============================================================================= -// AddStep adds a step result to the report -func (r *TestReport) AddStep(name, status string, duration time.Duration, err error) { - errStr := "" - if err != nil { - errStr = err.Error() - } - r.Steps = append(r.Steps, StepResult{ - Name: name, - Status: status, - Duration: duration, - Error: errStr, - }) -} - -// WriteToFile writes the report to a file -func (r *TestReport) WriteToFile(path string) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - fmt.Fprintf(f, "================================================================================\n") - fmt.Fprintf(f, "E2E TEST REPORT: %s\n", r.TestName) - fmt.Fprintf(f, "================================================================================\n\n") - - fmt.Fprintf(f, "EXECUTION:\n") - fmt.Fprintf(f, " Started: %s\n", r.StartTime.Format(time.RFC3339)) - fmt.Fprintf(f, " Finished: %s\n", r.EndTime.Format(time.RFC3339)) - fmt.Fprintf(f, " Duration: %s\n", r.EndTime.Sub(r.StartTime).Round(time.Second)) - fmt.Fprintf(f, " Log dir: %s\n\n", r.LogDir) - - fmt.Fprintf(f, "REGISTRIES:\n") - fmt.Fprintf(f, " Source: %s\n", r.SourceRegistry) - fmt.Fprintf(f, " Target: %s\n\n", r.TargetRegistry) - - fmt.Fprintf(f, "IMAGES TO VERIFY:\n") - fmt.Fprintf(f, " Source: %d images (%d repos)\n", r.SourceImageCount, r.SourceRepoCount) - fmt.Fprintf(f, " Target: %d images (%d repos)\n", r.TargetImageCount, r.TargetRepoCount) - if r.SkippedImages > 0 { - fmt.Fprintf(f, " (excluded %d internal tags from comparison)\n", r.SkippedImages) - } - fmt.Fprintf(f, "\n") - - fmt.Fprintf(f, "BUNDLE:\n") - fmt.Fprintf(f, " Size: %.2f GB\n\n", float64(r.BundleSize)/(1024*1024*1024)) - - fmt.Fprintf(f, "VERIFICATION RESULTS:\n") - fmt.Fprintf(f, " Images matched: %d (manifest + config + layers)\n", r.MatchedImages) - fmt.Fprintf(f, " Layers verified: %d\n", r.MatchedLayers) - fmt.Fprintf(f, " Missing images: %d\n", r.MissingImages) - fmt.Fprintf(f, " Digest mismatch: %d\n", r.MismatchedImages) - fmt.Fprintf(f, " Missing layers: %d\n\n", r.MissingLayers) - - fmt.Fprintf(f, "STEPS:\n") - passCount, failCount := 0, 0 - for _, step := range r.Steps { - if step.Status == "PASS" { - passCount++ - fmt.Fprintf(f, " [PASS] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) - } else if step.Status == "FAIL" { - failCount++ - fmt.Fprintf(f, " [FAIL] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) - if step.Error != "" { - fmt.Fprintf(f, " ERROR: %s\n", step.Error) - } - } else { - fmt.Fprintf(f, " [SKIP] %s\n", step.Name) - } - } - - fmt.Fprintf(f, "\n================================================================================\n") - if failCount > 0 { - fmt.Fprintf(f, "RESULT: FAILED (%d passed, %d failed)\n", passCount, failCount) - } else { - fmt.Fprintf(f, "RESULT: PASSED - REGISTRIES ARE IDENTICAL\n") - fmt.Fprintf(f, " %d repositories verified\n", r.SourceRepoCount) - fmt.Fprintf(f, " %d images verified\n", r.MatchedImages) - } - fmt.Fprintf(f, "================================================================================\n") - - return nil +// getLogDir returns the log directory path for e2e tests +// Logs are stored in testing/e2e/.logs/-/ +func getLogDir(testName string) string { + projectRoot := findProjectRoot() + timestamp := time.Now().Format("20060102-150405") + return filepath.Join(projectRoot, "testing", "e2e", ".logs", fmt.Sprintf("%s-%s", testName, timestamp)) } -// Print prints the report to stdout with beautiful lipgloss styling -func (r *TestReport) Print() { - duration := r.EndTime.Sub(r.StartTime) - if r.EndTime.IsZero() { - duration = time.Since(r.StartTime) - } +// findProjectRoot finds the project root by looking for go.mod +func findProjectRoot() string { + dir, _ := os.Getwd() - var content strings.Builder - - // Header - content.WriteString("\n") - content.WriteString(renderSeparator("═", 80) + "\n") - content.WriteString(" " + titleStyle.Render("E2E TEST REPORT") + "\n") - content.WriteString(renderSeparator("═", 80) + "\n\n") - - // Duration - content.WriteString(" " + labelStyle.Render("Duration: ") + dimStyle.Render(duration.Round(time.Second).String()) + "\n\n") - - // Registries section - content.WriteString(" " + headerStyle.Render("REGISTRIES") + "\n") - content.WriteString(" " + labelStyle.Render("Source: ") + valueStyle.Render(r.SourceRegistry) + "\n") - content.WriteString(" " + labelStyle.Render("Target: ") + valueStyle.Render(r.TargetRegistry) + "\n\n") - - // Images to verify section - content.WriteString(" " + headerStyle.Render("IMAGES TO VERIFY") + "\n") - content.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", - labelStyle.Render("Source:"), - valueStyle.Render(fmt.Sprintf("%d", r.SourceImageCount)), - r.SourceRepoCount)) - content.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", - labelStyle.Render("Target:"), - valueStyle.Render(fmt.Sprintf("%d", r.TargetImageCount)), - r.TargetRepoCount)) - if r.SkippedImages > 0 { - content.WriteString(" " + dimStyle.Render(fmt.Sprintf("(%d internal tags excluded)", r.SkippedImages)) + "\n") - } - content.WriteString("\n") - - // Verification results section - content.WriteString(" " + headerStyle.Render("VERIFICATION") + "\n") - content.WriteString(fmt.Sprintf(" %s %s %s\n", - okBadge, - labelStyle.Render("Images matched: "), - successStyle.Render(fmt.Sprintf("%d", r.MatchedImages))+" "+dimStyle.Render("(manifest + config + layers)"))) - content.WriteString(fmt.Sprintf(" %s %s %s\n", - okBadge, - labelStyle.Render("Layers verified:"), - successStyle.Render(fmt.Sprintf("%d", r.MatchedLayers)))) - - if r.MissingImages > 0 { - content.WriteString(fmt.Sprintf(" %s %s %s\n", - failBadge, - labelStyle.Render("Missing images: "), - errorStyle.Render(fmt.Sprintf("%d", r.MissingImages)))) - } - if r.MismatchedImages > 0 { - content.WriteString(fmt.Sprintf(" %s %s %s\n", - failBadge, - labelStyle.Render("Digest mismatch:"), - errorStyle.Render(fmt.Sprintf("%d", r.MismatchedImages)))) - } - if r.MissingLayers > 0 { - content.WriteString(fmt.Sprintf(" %s %s %s\n", - failBadge, - labelStyle.Render("Missing layers: "), - errorStyle.Render(fmt.Sprintf("%d", r.MissingLayers)))) - } - content.WriteString("\n") - - // Steps section - content.WriteString(" " + headerStyle.Render("STEPS") + "\n") - passCount, failCount := 0, 0 - for _, step := range r.Steps { - dur := dimStyle.Render(fmt.Sprintf("(%s)", step.Duration.Round(time.Millisecond))) - if step.Status == "PASS" { - passCount++ - content.WriteString(fmt.Sprintf(" %s %s %s\n", okBadge, step.Name, dur)) - } else if step.Status == "FAIL" { - failCount++ - content.WriteString(fmt.Sprintf(" %s %s %s\n", failBadge, step.Name, dur)) - if step.Error != "" { - content.WriteString(" " + errorStyle.Render("ERROR: "+step.Error) + "\n") - } - } else { - content.WriteString(fmt.Sprintf(" %s %s\n", skipBadge, step.Name)) + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir } - } - content.WriteString("\n") - // Result box - content.WriteString(renderSeparator("─", 80) + "\n") - if failCount > 0 { - resultStyle := lipgloss.NewStyle().Bold(true).Foreground(red) - content.WriteString(" " + resultStyle.Render("RESULT: FAILED") + fmt.Sprintf(" (%d passed, %d failed)\n", passCount, failCount)) - } else { - resultStyle := lipgloss.NewStyle().Bold(true).Foreground(green) - content.WriteString(" " + resultStyle.Render("RESULT: PASSED") + " - REGISTRIES ARE IDENTICAL\n") - content.WriteString(" " + successStyle.Render(fmt.Sprintf("%d images, %d layers", r.MatchedImages, r.MatchedLayers)) + " - all hashes verified\n") + parent := filepath.Dir(dir) + if parent == dir { + // Fallback to current dir + dir, _ = os.Getwd() + return dir + } + dir = parent } - content.WriteString(renderSeparator("═", 80) + "\n") - - print("%s", content.String()) } diff --git a/testing/e2e/mirror/output.go b/testing/e2e/mirror/output.go new file mode 100644 index 00000000..34ad8a7c --- /dev/null +++ b/testing/e2e/mirror/output.go @@ -0,0 +1,159 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "fmt" + "os" + "strings" + "sync" + + "github.com/charmbracelet/lipgloss" + "github.com/muesli/termenv" + "golang.org/x/term" +) + +// ============================================================================= +// Colors +// ============================================================================= + +var ( + colorCyan = lipgloss.Color("6") + colorGreen = lipgloss.Color("2") + colorRed = lipgloss.Color("1") + colorYellow = lipgloss.Color("3") + colorBlue = lipgloss.Color("4") + colorWhite = lipgloss.Color("15") + colorGray = lipgloss.Color("8") +) + +// ============================================================================= +// Text Styles +// ============================================================================= + +var ( + styleTitle = lipgloss.NewStyle().Bold(true).Foreground(colorWhite) + styleHeader = lipgloss.NewStyle().Bold(true).Foreground(colorCyan) + styleLabel = lipgloss.NewStyle().Foreground(colorGray) + styleValue = lipgloss.NewStyle().Foreground(colorWhite) + styleDim = lipgloss.NewStyle().Foreground(colorGray) + styleSuccess = lipgloss.NewStyle().Foreground(colorGreen) + styleError = lipgloss.NewStyle().Foreground(colorRed) +) + +// ============================================================================= +// Badges +// ============================================================================= + +var ( + badgeOK = lipgloss.NewStyle().Bold(true).Foreground(colorGreen).Render("[OK]") + badgeFail = lipgloss.NewStyle().Bold(true).Foreground(colorRed).Render("[FAIL]") + badgeSkip = lipgloss.NewStyle().Foreground(colorYellow).Render("[SKIP]") +) + +// ============================================================================= +// Step Styles +// ============================================================================= + +var ( + styleStepNum = lipgloss.NewStyle().Bold(true).Foreground(colorBlue) + styleStepText = lipgloss.NewStyle().Bold(true).Foreground(colorWhite) +) + +// ============================================================================= +// Output Functions +// ============================================================================= + +// output is the destination for styled output (stderr preserves colors in go test) +var output = os.Stderr + +var colorInitOnce sync.Once + +// ensureColorInit initializes color profile for lipgloss (replaces init()) +func ensureColorInit() { + colorInitOnce.Do(func() { + // Force color output - go test buffers stdout which disables color detection + // Check stderr instead (usually unbuffered) or honor FORCE_COLOR env + if term.IsTerminal(int(os.Stderr.Fd())) || os.Getenv("FORCE_COLOR") != "" || os.Getenv("TERM") != "" { + lipgloss.DefaultRenderer().SetColorProfile(termenv.TrueColor) + } + }) +} + +// writeLinef writes a formatted line to output +func writeLinef(format string, args ...interface{}) { + ensureColorInit() + fmt.Fprintf(output, format+"\n", args...) +} + +// writeRawf writes formatted text to output without newline +func writeRawf(format string, args ...interface{}) { + ensureColorInit() + fmt.Fprintf(output, format, args...) +} + +// printStep prints a formatted step header +func printStep(num int, description string) { + badge := styleStepNum.Render(fmt.Sprintf("[STEP %d]", num)) + text := styleStepText.Render(description) + writeLinef("\n%s %s", badge, text) +} + +// ============================================================================= +// Separators +// ============================================================================= + +var styleSeparator = lipgloss.NewStyle().Foreground(colorCyan) + +const separatorWidth = 80 + +// separator creates a separator line +func separator(char string) string { + return styleSeparator.Render(strings.Repeat(char, separatorWidth)) +} + +// ============================================================================= +// Test Header/Footer +// ============================================================================= + +// printTestHeader prints the test header with configuration info +func printTestHeader(testName, sourceRegistry, logDir string) { + writeLinef("") + writeLinef(separator("═")) + writeLinef(" %s", styleTitle.Render("E2E TEST: "+testName)) + writeLinef(separator("═")) + writeLinef(" %s %s", styleLabel.Render("Source:"), styleValue.Render(sourceRegistry)) + writeLinef(" %s %s", styleLabel.Render("Logs: "), styleDim.Render(logDir)) + writeLinef(separator("═")) + writeLinef("") +} + +// printSuccessBox prints a success message in a styled box +func printSuccessBox(matchedImages, matchedLayers int) { + box := lipgloss.NewStyle(). + Border(lipgloss.DoubleBorder()). + BorderForeground(colorGreen). + Padding(0, 2). + Foreground(colorGreen) + + writeLinef("") + writeLinef(box.Render(fmt.Sprintf( + "SUCCESS: REGISTRIES ARE IDENTICAL\n\nVerified: %d images, %d layers\nAll manifest, config, and layer digests match!", + matchedImages, + matchedLayers, + ))) +} diff --git a/testing/e2e/mirror/registry_comparator.go b/testing/e2e/mirror/registry_comparator.go index a8a5a83e..bf1ff286 100644 --- a/testing/e2e/mirror/registry_comparator.go +++ b/testing/e2e/mirror/registry_comparator.go @@ -39,7 +39,7 @@ import ( var ( // Digest-based tags (sha256 hashes used as tags) digestTagRegex = regexp.MustCompile(`^[a-f0-9]{64}$`) - // SHA256 prefixed tags + // SHA256 prefixed tags sha256TagRegex = regexp.MustCompile(`^sha256-[a-f0-9]{64}`) // Cosign signature and attestation tags cosignTagSuffixes = []string{".sig", ".att", ".sbom"} @@ -136,7 +136,7 @@ func (c *RegistryComparator) SetProgressCallback(fn func(msg string)) { c.onProgress = fn } -func (c *RegistryComparator) progress(format string, args ...interface{}) { +func (c *RegistryComparator) logProgressf(format string, args ...interface{}) { if c.onProgress != nil { c.onProgress(fmt.Sprintf(format, args...)) } @@ -169,25 +169,25 @@ type ComparisonReport struct { ExtraImages []string // ref in target but not in source // Deep comparison stats - DeepCheckedImages int - TotalSourceLayers int - TotalTargetLayers int - MatchedLayers int - MissingLayers int - ConfigMismatches int - LayerMismatches []LayerMismatch + DeepCheckedImages int + TotalSourceLayers int + TotalTargetLayers int + MatchedLayers int + MissingLayers int + ConfigMismatches int + LayerMismatches []LayerMismatch } // RepositoryComparison holds comparison for a single repository type RepositoryComparison struct { - Repository string - SourceTags []string - TargetTags []string - MissingTags []string // In source but not in target - ExtraTags []string // In target but not in source - SkippedTags int // Tags skipped (digest-based, .att, .sig, etc.) - MatchedTags int - TagDetails map[string]*TagComparison + Repository string + SourceTags []string + TargetTags []string + MissingTags []string // In source but not in target + ExtraTags []string // In target but not in source + SkippedTags int // Tags skipped (digest-based, .att, .sig, etc.) + MatchedTags int + TagDetails map[string]*TagComparison } // TagComparison holds comparison for a single tag @@ -412,22 +412,16 @@ func (c *RegistryComparator) Compare(ctx context.Context) (*ComparisonReport, er } // Step 1: Discover all repositories in source - c.progress("Discovering repositories in source registry...") - sourceRepos, err := c.discoverRepositories(ctx, c.sourceRegistry, c.sourceRemoteOpts) - if err != nil { - return nil, fmt.Errorf("discover source repositories: %w", err) - } + c.logProgressf("Discovering repositories in source registry...") + sourceRepos := c.discoverRepositories(c.sourceRegistry, c.sourceRemoteOpts) report.SourceRepositories = sourceRepos - c.progress("Found %d repositories in source", len(sourceRepos)) + c.logProgressf("Found %d repositories in source", len(sourceRepos)) // Step 2: Discover all repositories in target - c.progress("Discovering repositories in target registry...") - targetRepos, err := c.discoverRepositories(ctx, c.targetRegistry, c.targetRemoteOpts) - if err != nil { - return nil, fmt.Errorf("discover target repositories: %w", err) - } + c.logProgressf("Discovering repositories in target registry...") + targetRepos := c.discoverRepositories(c.targetRegistry, c.targetRemoteOpts) report.TargetRepositories = targetRepos - c.progress("Found %d repositories in target", len(targetRepos)) + c.logProgressf("Found %d repositories in target", len(targetRepos)) // Step 3: Find missing and extra repositories sourceRepoSet := make(map[string]bool) @@ -451,13 +445,13 @@ func (c *RegistryComparator) Compare(ctx context.Context) (*ComparisonReport, er } // Step 4: Compare each repository in detail - c.progress("Comparing repositories...") + c.logProgressf("Comparing repositories...") for i, repoPath := range sourceRepos { - c.progress("[%d/%d] Comparing %s", i+1, len(sourceRepos), repoPath) + c.logProgressf("[%d/%d] Comparing %s", i+1, len(sourceRepos), repoPath) - repoComparison, err := c.compareRepository(ctx, repoPath) + repoComparison, err := c.compareRepository(repoPath) if err != nil { - c.progress("Warning: failed to compare %s: %v", repoPath, err) + c.logProgressf("Warning: failed to compare %s: %v", repoPath, err) continue } @@ -530,11 +524,11 @@ func (c *RegistryComparator) Compare(ctx context.Context) (*ComparisonReport, er } // discoverRepositories discovers all repositories by walking known segments -func (c *RegistryComparator) discoverRepositories(ctx context.Context, registry string, opts []remote.Option) ([]string, error) { +func (c *RegistryComparator) discoverRepositories(registry string, opts []remote.Option) []string { var repos []string // Root repository - if c.repositoryExists(ctx, registry, opts) { + if c.repositoryExists(registry, opts) { repos = append(repos, "") } @@ -547,7 +541,7 @@ func (c *RegistryComparator) discoverRepositories(ctx context.Context, registry for _, segment := range segments { segmentPath := segment - if c.repositoryExists(ctx, path.Join(registry, segmentPath), opts) { + if c.repositoryExists(path.Join(registry, segmentPath), opts) { repos = append(repos, segmentPath) } } @@ -561,42 +555,42 @@ func (c *RegistryComparator) discoverRepositories(ctx context.Context, registry } for _, db := range securityDBs { dbPath := path.Join(internal.SecuritySegment, db) - if c.repositoryExists(ctx, path.Join(registry, dbPath), opts) { + if c.repositoryExists(path.Join(registry, dbPath), opts) { repos = append(repos, dbPath) } } // Modules - need to discover dynamically modulesPath := path.Join(registry, internal.ModulesSegment) - moduleTags, err := c.listTags(ctx, modulesPath, opts) + moduleTags, err := c.listTags(modulesPath, opts) if err == nil { for _, moduleName := range moduleTags { moduleBasePath := path.Join(internal.ModulesSegment, moduleName) // Module root - if c.repositoryExists(ctx, path.Join(registry, moduleBasePath), opts) { + if c.repositoryExists(path.Join(registry, moduleBasePath), opts) { repos = append(repos, moduleBasePath) } // Module release moduleReleasePath := path.Join(moduleBasePath, "release") - if c.repositoryExists(ctx, path.Join(registry, moduleReleasePath), opts) { + if c.repositoryExists(path.Join(registry, moduleReleasePath), opts) { repos = append(repos, moduleReleasePath) } } } - return repos, nil + return repos } // repositoryExists checks if a repository exists and has tags -func (c *RegistryComparator) repositoryExists(ctx context.Context, repo string, opts []remote.Option) bool { - tags, err := c.listTags(ctx, repo, opts) +func (c *RegistryComparator) repositoryExists(repo string, opts []remote.Option) bool { + tags, err := c.listTags(repo, opts) return err == nil && len(tags) > 0 } // compareRepository compares a single repository between source and target -func (c *RegistryComparator) compareRepository(ctx context.Context, repoPath string) (*RepositoryComparison, error) { +func (c *RegistryComparator) compareRepository(repoPath string) (*RepositoryComparison, error) { sourceRepo := c.sourceRegistry targetRepo := c.targetRegistry if repoPath != "" { @@ -610,11 +604,11 @@ func (c *RegistryComparator) compareRepository(ctx context.Context, repoPath str } // Get source tags - allSourceTags, err := c.listTags(ctx, sourceRepo, c.sourceRemoteOpts) + allSourceTags, err := c.listTags(sourceRepo, c.sourceRemoteOpts) if err != nil { return nil, fmt.Errorf("list source tags: %w", err) } - + // Filter out tags that are not mirrored by design sourceTags := make([]string, 0, len(allSourceTags)) skippedTags := 0 @@ -629,12 +623,12 @@ func (c *RegistryComparator) compareRepository(ctx context.Context, repoPath str comparison.SkippedTags = skippedTags // Get target tags - allTargetTags, err := c.listTags(ctx, targetRepo, c.targetRemoteOpts) + allTargetTags, err := c.listTags(targetRepo, c.targetRemoteOpts) if err != nil { // Target repo might not exist allTargetTags = []string{} } - + // Filter target tags too targetTags := make([]string, 0, len(allTargetTags)) for _, tag := range allTargetTags { @@ -687,11 +681,11 @@ func (c *RegistryComparator) compareRepository(ctx context.Context, repoPath str targetRef := targetRepo + ":" + tag // Perform deep comparison - imgComp, err := c.compareImageDeep(ctx, sourceRef, targetRef) + imgComp, err := c.compareImageDeep(sourceRef, targetRef) if err != nil { // Fallback to simple digest comparison - sourceDigest, err1 := c.getDigest(ctx, sourceRef, c.sourceRemoteOpts) - targetDigest, err2 := c.getDigest(ctx, targetRef, c.targetRemoteOpts) + sourceDigest, err1 := c.getDigest(sourceRef, c.sourceRemoteOpts) + targetDigest, err2 := c.getDigest(targetRef, c.targetRemoteOpts) if err1 != nil || err2 != nil { return } @@ -747,7 +741,7 @@ func (c *RegistryComparator) compareRepository(ctx context.Context, repoPath str } // listTags lists all tags in a repository -func (c *RegistryComparator) listTags(ctx context.Context, repo string, opts []remote.Option) ([]string, error) { +func (c *RegistryComparator) listTags(repo string, opts []remote.Option) ([]string, error) { repoRef, err := name.NewRepository(repo, c.nameOpts...) if err != nil { return nil, fmt.Errorf("parse repo %s: %w", repo, err) @@ -762,7 +756,7 @@ func (c *RegistryComparator) listTags(ctx context.Context, repo string, opts []r } // getDigest gets the digest for a specific image reference -func (c *RegistryComparator) getDigest(ctx context.Context, ref string, opts []remote.Option) (string, error) { +func (c *RegistryComparator) getDigest(ref string, opts []remote.Option) (string, error) { imgRef, err := name.ParseReference(ref, c.nameOpts...) if err != nil { return "", fmt.Errorf("parse ref %s: %w", ref, err) @@ -777,7 +771,7 @@ func (c *RegistryComparator) getDigest(ctx context.Context, ref string, opts []r } // getImageInfo gets detailed information about an image including all layer digests -func (c *RegistryComparator) getImageInfo(ctx context.Context, ref string, opts []remote.Option) (*ImageInfo, error) { +func (c *RegistryComparator) getImageInfo(ref string, opts []remote.Option) (*ImageInfo, error) { imgRef, err := name.ParseReference(ref, c.nameOpts...) if err != nil { return nil, fmt.Errorf("parse ref %s: %w", ref, err) @@ -860,13 +854,13 @@ func (c *RegistryComparator) getImageInfo(ctx context.Context, ref string, opts } // compareImageDeep performs deep comparison of two images including all layers -func (c *RegistryComparator) compareImageDeep(ctx context.Context, sourceRef, targetRef string) (*ImageComparison, error) { - sourceInfo, err := c.getImageInfo(ctx, sourceRef, c.sourceRemoteOpts) +func (c *RegistryComparator) compareImageDeep(sourceRef, targetRef string) (*ImageComparison, error) { + sourceInfo, err := c.getImageInfo(sourceRef, c.sourceRemoteOpts) if err != nil { return nil, fmt.Errorf("get source image info: %w", err) } - targetInfo, err := c.getImageInfo(ctx, targetRef, c.targetRemoteOpts) + targetInfo, err := c.getImageInfo(targetRef, c.targetRemoteOpts) if err != nil { return nil, fmt.Errorf("get target image info: %w", err) } @@ -933,4 +927,3 @@ type ImageComparison struct { LayersMatch bool FullMatch bool } - diff --git a/testing/e2e/mirror/report.go b/testing/e2e/mirror/report.go new file mode 100644 index 00000000..4fb1b85d --- /dev/null +++ b/testing/e2e/mirror/report.go @@ -0,0 +1,291 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/charmbracelet/lipgloss" +) + +// ============================================================================= +// Test Report +// ============================================================================= + +// TestReport collects test execution results for final summary +type TestReport struct { + TestName string + StartTime time.Time + EndTime time.Time + SourceRegistry string + TargetRegistry string + LogDir string + + // Source stats + SourceRepoCount int + SourceImageCount int + + // Target stats + TargetRepoCount int + TargetImageCount int + + // Comparison stats + MatchedImages int + MissingImages int + MismatchedImages int + SkippedImages int // Digest-based, .att, .sig tags + + // Deep comparison stats + MatchedLayers int + MissingLayers int + + // Bundle stats + BundleSize int64 + + // Steps + Steps []StepResult + + // Full comparison report + ComparisonReport *ComparisonReport +} + +// StepResult represents a single step in the test +type StepResult struct { + Name string + Status string // "PASS", "FAIL", "SKIP" + Duration time.Duration + Error string +} + +// ============================================================================= +// Report Methods +// ============================================================================= + +// AddStep adds a step result to the report +func (r *TestReport) AddStep(name, status string, duration time.Duration, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + r.Steps = append(r.Steps, StepResult{ + Name: name, + Status: status, + Duration: duration, + Error: errStr, + }) +} + +// WriteToFile writes the report to a file in plain text format +func (r *TestReport) WriteToFile(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + w := func(format string, args ...interface{}) { + fmt.Fprintf(f, format, args...) + } + + // Header + w("================================================================================\n") + w("E2E TEST REPORT: %s\n", r.TestName) + w("================================================================================\n\n") + + // Execution info + w("EXECUTION:\n") + w(" Started: %s\n", r.StartTime.Format(time.RFC3339)) + w(" Finished: %s\n", r.EndTime.Format(time.RFC3339)) + w(" Duration: %s\n", r.EndTime.Sub(r.StartTime).Round(time.Second)) + w(" Log dir: %s\n\n", r.LogDir) + + // Registries + w("REGISTRIES:\n") + w(" Source: %s\n", r.SourceRegistry) + w(" Target: %s\n\n", r.TargetRegistry) + + // Images + w("IMAGES TO VERIFY:\n") + w(" Source: %d images (%d repos)\n", r.SourceImageCount, r.SourceRepoCount) + w(" Target: %d images (%d repos)\n", r.TargetImageCount, r.TargetRepoCount) + if r.SkippedImages > 0 { + w(" (excluded %d internal tags from comparison)\n", r.SkippedImages) + } + w("\n") + + // Bundle + w("BUNDLE:\n") + w(" Size: %.2f GB\n\n", float64(r.BundleSize)/(1024*1024*1024)) + + // Verification results + w("VERIFICATION RESULTS:\n") + w(" Images matched: %d (manifest + config + layers)\n", r.MatchedImages) + w(" Layers verified: %d\n", r.MatchedLayers) + w(" Missing images: %d\n", r.MissingImages) + w(" Digest mismatch: %d\n", r.MismatchedImages) + w(" Missing layers: %d\n\n", r.MissingLayers) + + // Steps + w("STEPS:\n") + passCount, failCount := r.countSteps() + for _, step := range r.Steps { + switch step.Status { + case "PASS": + w(" [PASS] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) + case "FAIL": + w(" [FAIL] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) + if step.Error != "" { + w(" ERROR: %s\n", step.Error) + } + default: + w(" [SKIP] %s\n", step.Name) + } + } + + // Result + w("\n================================================================================\n") + if failCount > 0 { + w("RESULT: FAILED (%d passed, %d failed)\n", passCount, failCount) + } else { + w("RESULT: PASSED - REGISTRIES ARE IDENTICAL\n") + w(" %d repositories verified\n", r.SourceRepoCount) + w(" %d images verified\n", r.MatchedImages) + } + w("================================================================================\n") + + return nil +} + +// Print prints the report to stderr with beautiful lipgloss styling +func (r *TestReport) Print() { + duration := r.EndTime.Sub(r.StartTime) + if r.EndTime.IsZero() { + duration = time.Since(r.StartTime) + } + + var b strings.Builder + + // Header + b.WriteString("\n") + b.WriteString(separator("═") + "\n") + b.WriteString(" " + styleTitle.Render("E2E TEST REPORT") + "\n") + b.WriteString(separator("═") + "\n\n") + + // Duration + b.WriteString(" " + styleLabel.Render("Duration: ") + styleDim.Render(duration.Round(time.Second).String()) + "\n\n") + + // Registries + b.WriteString(" " + styleHeader.Render("REGISTRIES") + "\n") + b.WriteString(" " + styleLabel.Render("Source: ") + styleValue.Render(r.SourceRegistry) + "\n") + b.WriteString(" " + styleLabel.Render("Target: ") + styleValue.Render(r.TargetRegistry) + "\n\n") + + // Images to verify + b.WriteString(" " + styleHeader.Render("IMAGES TO VERIFY") + "\n") + b.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", + styleLabel.Render("Source:"), + styleValue.Render(fmt.Sprintf("%d", r.SourceImageCount)), + r.SourceRepoCount)) + b.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", + styleLabel.Render("Target:"), + styleValue.Render(fmt.Sprintf("%d", r.TargetImageCount)), + r.TargetRepoCount)) + if r.SkippedImages > 0 { + b.WriteString(" " + styleDim.Render(fmt.Sprintf("(%d internal tags excluded)", r.SkippedImages)) + "\n") + } + b.WriteString("\n") + + // Verification results + b.WriteString(" " + styleHeader.Render("VERIFICATION") + "\n") + b.WriteString(fmt.Sprintf(" %s %s %s\n", + badgeOK, + styleLabel.Render("Images matched: "), + styleSuccess.Render(fmt.Sprintf("%d", r.MatchedImages))+" "+styleDim.Render("(manifest + config + layers)"))) + b.WriteString(fmt.Sprintf(" %s %s %s\n", + badgeOK, + styleLabel.Render("Layers verified:"), + styleSuccess.Render(fmt.Sprintf("%d", r.MatchedLayers)))) + + if r.MissingImages > 0 { + b.WriteString(fmt.Sprintf(" %s %s %s\n", + badgeFail, + styleLabel.Render("Missing images: "), + styleError.Render(fmt.Sprintf("%d", r.MissingImages)))) + } + if r.MismatchedImages > 0 { + b.WriteString(fmt.Sprintf(" %s %s %s\n", + badgeFail, + styleLabel.Render("Digest mismatch:"), + styleError.Render(fmt.Sprintf("%d", r.MismatchedImages)))) + } + if r.MissingLayers > 0 { + b.WriteString(fmt.Sprintf(" %s %s %s\n", + badgeFail, + styleLabel.Render("Missing layers: "), + styleError.Render(fmt.Sprintf("%d", r.MissingLayers)))) + } + b.WriteString("\n") + + // Steps + b.WriteString(" " + styleHeader.Render("STEPS") + "\n") + passCount, failCount := r.countSteps() + for _, step := range r.Steps { + dur := styleDim.Render(fmt.Sprintf("(%s)", step.Duration.Round(time.Millisecond))) + switch step.Status { + case "PASS": + b.WriteString(fmt.Sprintf(" %s %s %s\n", badgeOK, step.Name, dur)) + case "FAIL": + b.WriteString(fmt.Sprintf(" %s %s %s\n", badgeFail, step.Name, dur)) + if step.Error != "" { + b.WriteString(" " + styleError.Render("ERROR: "+step.Error) + "\n") + } + default: + b.WriteString(fmt.Sprintf(" %s %s\n", badgeSkip, step.Name)) + } + } + b.WriteString("\n") + + // Result box + b.WriteString(separator("─") + "\n") + if failCount > 0 { + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(colorRed) + b.WriteString(" " + resultStyle.Render("RESULT: FAILED") + fmt.Sprintf(" (%d passed, %d failed)\n", passCount, failCount)) + } else { + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(colorGreen) + b.WriteString(" " + resultStyle.Render("RESULT: PASSED") + " - REGISTRIES ARE IDENTICAL\n") + b.WriteString(" " + styleSuccess.Render(fmt.Sprintf("%d images, %d layers", r.MatchedImages, r.MatchedLayers)) + " - all hashes verified\n") + } + b.WriteString(separator("═") + "\n") + + writeRawf("%s", b.String()) +} + +// countSteps counts passed and failed steps +func (r *TestReport) countSteps() (int, int) { + var passed, failed int + for _, step := range r.Steps { + switch step.Status { + case "PASS": + passed++ + case "FAIL": + failed++ + } + } + return passed, failed +} From 9960f8016da91a675c764cacb3101e11b5dda7af Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Fri, 26 Dec 2025 11:35:48 +0300 Subject: [PATCH 06/11] fix: lint Signed-off-by: Timur Tuktamyshev --- .golangci.yaml | 1 + testing/util/mirror/registry.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index dcff2e5a..d4fe847e 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -103,6 +103,7 @@ linters: - examples$ - _test\.go$ - deepcopy\.go$ + - testing/ formatters: enable: - gci diff --git a/testing/util/mirror/registry.go b/testing/util/mirror/registry.go index 9cbe2df1..5e0efb43 100644 --- a/testing/util/mirror/registry.go +++ b/testing/util/mirror/registry.go @@ -132,4 +132,3 @@ func SetupTestRegistry(useTLS bool) *TestRegistry { } return reg } - From 099b2945dea18bf60b9f93ac188f082df0b2c8f3 Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Fri, 26 Dec 2025 11:56:14 +0300 Subject: [PATCH 07/11] feat: new pull to e2e Signed-off-by: Timur Tuktamyshev --- Taskfile.yml | 3 +- testing/e2e/mirror/README.md | 135 +++++++++++++++++++-------------- testing/e2e/mirror/commands.go | 5 ++ testing/e2e/mirror/config.go | 9 +++ 4 files changed, 92 insertions(+), 60 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 2f34aa3d..6cc01610 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -292,9 +292,8 @@ tasks: # Keep bundle for debugging E2E_LICENSE_TOKEN=xxx E2E_KEEP_BUNDLE=true task test:e2e:mirror - deps: - - build cmds: + - task build - go test -v -timeout 120m -tags=e2e ./testing/e2e/mirror/... {{ .CLI_ARGS }} test:e2e:mirror:logs:clean: diff --git a/testing/e2e/mirror/README.md b/testing/e2e/mirror/README.md index e18c2ce3..eb4e30f1 100644 --- a/testing/e2e/mirror/README.md +++ b/testing/e2e/mirror/README.md @@ -29,40 +29,43 @@ This ensures **byte-for-byte identical** registries. - Built `d8` binary (run `task build` from project root) - Valid credentials for the source registry - Network access to the source registry -- Sufficient disk space for the bundle (can be several GB, aroud 20) +- Sufficient disk space for the bundle (can be ~20 GB) ## Running Tests -### Basic Usage +### Using Taskfile (Recommended) ```bash -# Run with license token (official Deckhouse registry) -go test -v ./testing/e2e/mirror/... \ - -license-token=YOUR_LICENSE_TOKEN - -# Run with local registry (self-signed cert) -go test -v ./testing/e2e/mirror/... \ +# With environment variables +E2E_SOURCE_REGISTRY=localhost:443/deckhouse \ +E2E_SOURCE_USER=admin \ +E2E_SOURCE_PASSWORD=secret \ +E2E_TLS_SKIP_VERIFY=true \ +task test:e2e:mirror + +# With command-line flags +task test:e2e:mirror -- \ -source-registry=localhost:443/deckhouse \ -source-user=admin \ - -source-password=secret \ + -source-password=admin \ -tls-skip-verify + +# With license token (official Deckhouse registry) +E2E_LICENSE_TOKEN=xxx task test:e2e:mirror ``` -### Full Configuration +### Using go test directly ```bash -go test -v ./testing/e2e/mirror/... \ - -source-registry=my-registry.local/deckhouse \ +# Note: requires -tags=e2e flag +go test -v -tags=e2e -timeout=120m ./testing/e2e/mirror/... \ + -source-registry=localhost:443/deckhouse \ -source-user=admin \ -source-password=secret \ - -target-registry=my-target.local:5000/deckhouse \ - -target-user=admin \ - -target-password=secret \ - -tls-skip-verify \ - -keep-bundle + -tls-skip-verify ``` -### Environment Variables +### Configuration Options | Flag | Environment Variable | Default | Description | |------|---------------------|---------|-------------| @@ -76,47 +79,66 @@ go test -v ./testing/e2e/mirror/... \ | `-tls-skip-verify` | `E2E_TLS_SKIP_VERIFY` | `false` | Skip TLS verification | | `-keep-bundle` | `E2E_KEEP_BUNDLE` | `false` | Keep bundle after test | | `-d8-binary` | `E2E_D8_BINARY` | `bin/d8` | Path to d8 binary | +| `-new-pull` | `E2E_NEW_PULL` | `false` | Use new pull implementation | -## Test Output - -### Log Directory Structure +## Test Artifacts -Test logs are stored in `testing/e2e/.logs/-/`: +Tests produce detailed artifacts in `testing/e2e/.logs/-/`: ``` -testing/e2e/.logs/fullcycle-20251225-123456/ +testing/e2e/.logs/fullcycle-20251226-114128/ ├── test.log # Full command output (pull/push) ├── report.txt # Test summary report └── comparison.txt # Detailed registry comparison ``` -### Sample Report Output +### Cleaning Up Logs + +```bash +task test:e2e:mirror:logs:clean +``` + +### Sample Report (report.txt) ``` ================================================================================ E2E TEST REPORT: TestMirrorE2E_FullCycle ================================================================================ -Duration: 25m30s +EXECUTION: + Started: 2025-12-26T11:41:28+03:00 + Finished: 2025-12-26T11:46:28+03:00 + Duration: 5m1s REGISTRIES: - Source: localhost:443/deckhouse (15 repos, 1847 images) - Target: 127.0.0.1:54321/deckhouse/ee (15 repos, 1847 images) + Source: localhost:443/deckhouse-etalon + Target: 127.0.0.1:61594/deckhouse/ee -COMPARISON: - ✓ Matched: 1847 images (manifest digest) - ✓ Deep checked: 1847 images - ✓ Layers: 15234 verified +IMAGES TO VERIFY: + Source: 324 images (82 repos) + Target: 324 images (82 repos) + (excluded 1071 internal tags from comparison) + +BUNDLE: + Size: 22.52 GB + +VERIFICATION RESULTS: + Images matched: 324 (manifest + config + layers) + Layers verified: 1172 + Missing images: 0 + Digest mismatch: 0 + Missing layers: 0 STEPS: - ✓ Analyze source (15 repos, 1847 images) (45s) - ✓ Pull images (5.23 GB bundle) (12m15s) - ✓ Push to registry (8m30s) - ✓ Deep comparison (1847 images verified) (4m00s) + [PASS] Analyze source (82 repos) (330ms) + [PASS] Pull images (22.52 GB bundle) (2m59.826s) + [PASS] Push to registry (1m57.742s) + [PASS] Deep comparison (324 images verified) (2.266s) --------------------------------------------------------------------------------- +================================================================================ RESULT: PASSED - REGISTRIES ARE IDENTICAL - 1847 images, 15234 layers - all hashes verified + 82 repositories verified + 324 images verified ================================================================================ ``` @@ -130,27 +152,26 @@ REGISTRY COMPARISON SUMMARY Source: localhost:443/deckhouse Target: 127.0.0.1:54321/deckhouse/ee -Duration: 4m0s +Duration: 2s REPOSITORIES: - Source: 15 - Target: 15 + Source: 82 + Target: 82 Missing in target: 0 Extra in target: 0 -IMAGES (manifest digest comparison): - Source: 1847 - Target: 1847 - Matched: 1847 - Missing: 0 - Mismatched: 0 - Extra: 0 +IMAGES TO VERIFY: + Source: 324 images + Target: 324 images + (excluded 1071 internal tags: digest-based, .att, .sig) +VERIFICATION RESULTS: + Matched: 324 DEEP COMPARISON (layers + config): - Images deep-checked: 1847 - Source layers: 15234 - Target layers: 15234 - Matched layers: 15234 + Images deep-checked: 324 + Source layers: 1172 + Target layers: 1172 + Matched layers: 1172 Missing layers: 0 Config mismatches: 0 @@ -158,12 +179,12 @@ DEEP COMPARISON (layers + config): REPOSITORY BREAKDOWN: --------------------- -✓ (root): 523/523 tags, 4521 layers checked +✓ (root): 6/6 tags, 66 layers checked ✓ install: 6/6 tags, 48 layers checked ✓ install-standalone: 78/78 tags, 624 layers checked ✓ release-channel: 6/6 tags, 12 layers checked ✓ security/trivy-db: 1/1 tags, 2 layers checked -✓ modules/deckhouse-admin: 15/15 tags, 120 layers checked +✓ modules/deckhouse-admin: 5/5 tags, 10 layers checked ... ``` @@ -177,9 +198,9 @@ The test has a **120-minute timeout** to handle large registries. Set credentials: ```bash -E2E_LICENSE_TOKEN=your_token go test -v ./testing/e2e/mirror/... +E2E_LICENSE_TOKEN=your_token task test:e2e:mirror # or -E2E_SOURCE_USER=admin E2E_SOURCE_PASSWORD=secret go test -v ./testing/e2e/mirror/... +E2E_SOURCE_USER=admin E2E_SOURCE_PASSWORD=secret task test:e2e:mirror ``` ### "Pull failed" or "Push failed" @@ -198,9 +219,7 @@ Check `comparison.txt` for details: ### Viewing Bundle Contents ```bash -go test -v ./testing/e2e/mirror/... \ - -license-token=TOKEN \ - -keep-bundle +E2E_KEEP_BUNDLE=true task test:e2e:mirror -- ... # Bundle location shown in output ``` diff --git a/testing/e2e/mirror/commands.go b/testing/e2e/mirror/commands.go index ad7df44a..94e9054b 100644 --- a/testing/e2e/mirror/commands.go +++ b/testing/e2e/mirror/commands.go @@ -60,6 +60,11 @@ func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { cmd := exec.Command(cfg.D8Binary, args...) cmd.Env = append(os.Environ(), "HOME="+os.Getenv("HOME")) + // Experimental options (via env) + if cfg.NewPull { + cmd.Env = append(cmd.Env, "NEW_PULL=true") + } + return cmd } diff --git a/testing/e2e/mirror/config.go b/testing/e2e/mirror/config.go index c5a0443e..a0396887 100644 --- a/testing/e2e/mirror/config.go +++ b/testing/e2e/mirror/config.go @@ -65,6 +65,11 @@ var ( noModules = flag.Bool("no-modules", getEnvOrDefault("E2E_NO_MODULES", "") == "true", "Skip modules during pull (for testing failure scenarios)") + + // Experimental options + newPull = flag.Bool("new-pull", + getEnvOrDefault("E2E_NEW_PULL", "") == "true", + "Use new pull implementation") ) func getEnvOrDefault(key, defaultValue string) string { @@ -91,6 +96,9 @@ type Config struct { // Debug/test options NoModules bool // Skip modules during pull (for testing failure scenarios) + + // Experimental options + NewPull bool // Use new pull implementation } // GetConfig returns the current test configuration from flags @@ -108,6 +116,7 @@ func GetConfig() *Config { KeepBundle: *keepBundle, D8Binary: *d8Binary, NoModules: *noModules, + NewPull: *newPull, } } From 634e637a2a217fc1d37e161f773cafb0328d57b5 Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Fri, 26 Dec 2025 12:37:24 +0300 Subject: [PATCH 08/11] fix: add segments to client Signed-off-by: Timur Tuktamyshev --- pkg/libmirror/images/digests.go | 11 +++++++++++ pkg/libmirror/layouts/layouts.go | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/pkg/libmirror/images/digests.go b/pkg/libmirror/images/digests.go index 9748850a..b6bbccb1 100644 --- a/pkg/libmirror/images/digests.go +++ b/pkg/libmirror/images/digests.go @@ -200,8 +200,19 @@ func FindVexImage( if splitIndex == -1 { return "", fmt.Errorf("invalid vex image name format: %s", vexImageName) } + imagePath := vexImageName[:splitIndex] tag := vexImageName[splitIndex+1:] + // Add missing path segments to client if VEX image is in a subpath + imageSegmentsRaw := strings.TrimPrefix(imagePath, client.GetRegistry()) + imageSegmentsRaw = strings.TrimPrefix(imageSegmentsRaw, "/") + if imageSegmentsRaw != "" { + for _, segment := range strings.Split(imageSegmentsRaw, "/") { + client = client.WithSegment(segment) + logger.Debugf("Segment: %s", segment) + } + } + err = client.CheckImageExists(context.TODO(), tag) if errors.Is(err, regclient.ErrImageNotFound) { // Image not found, which is expected for non-vulnerable images diff --git a/pkg/libmirror/layouts/layouts.go b/pkg/libmirror/layouts/layouts.go index 2e4cc61f..3280b0f5 100644 --- a/pkg/libmirror/layouts/layouts.go +++ b/pkg/libmirror/layouts/layouts.go @@ -498,8 +498,19 @@ func FindVexImage( if splitIndex == -1 { return "", fmt.Errorf("invalid vex image name format: %s", vexImageName) } + imagePath := vexImageName[:splitIndex] tag := vexImageName[splitIndex+1:] + // Add missing path segments to client if VEX image is in a subpath + imageSegmentsRaw := strings.TrimPrefix(imagePath, client.GetRegistry()) + imageSegmentsRaw = strings.TrimPrefix(imageSegmentsRaw, "/") + if imageSegmentsRaw != "" { + for _, segment := range strings.Split(imageSegmentsRaw, "/") { + client = client.WithSegment(segment) + logger.Debugf("Segment: %s", segment) + } + } + err = client.CheckImageExists(context.TODO(), tag) if errors.Is(err, regclient.ErrImageNotFound) { // Image not found, which is expected for non-vulnerable images From 58e1797ddb9d4724c88048f0886851836186e474 Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Tue, 30 Dec 2025 11:48:11 +0300 Subject: [PATCH 09/11] feat: new e2e flow Signed-off-by: Timur Tuktamyshev --- Taskfile.yml | 49 +- testing/e2e/mirror/README.md | 265 +++-- testing/e2e/mirror/fullcycle_test.go | 210 ++++ testing/e2e/mirror/helpers_test.go | 215 ++++ testing/e2e/mirror/{ => internal}/commands.go | 63 +- testing/e2e/mirror/{ => internal}/config.go | 122 ++- testing/e2e/mirror/internal/output.go | 144 +++ testing/e2e/mirror/internal/report.go | 277 ++++++ testing/e2e/mirror/internal/source.go | 444 +++++++++ testing/e2e/mirror/internal/verify.go | 677 +++++++++++++ testing/e2e/mirror/mirror_e2e_test.go | 410 -------- testing/e2e/mirror/modules_test.go | 84 ++ testing/e2e/mirror/output.go | 159 --- testing/e2e/mirror/platform_test.go | 118 +++ testing/e2e/mirror/registry_comparator.go | 929 ------------------ testing/e2e/mirror/report.go | 291 ------ testing/e2e/mirror/security_test.go | 91 ++ testing/e2e/mirror/testenv.go | 241 +++++ 18 files changed, 2762 insertions(+), 2027 deletions(-) create mode 100644 testing/e2e/mirror/fullcycle_test.go create mode 100644 testing/e2e/mirror/helpers_test.go rename testing/e2e/mirror/{ => internal}/commands.go (66%) rename testing/e2e/mirror/{ => internal}/config.go (61%) create mode 100644 testing/e2e/mirror/internal/output.go create mode 100644 testing/e2e/mirror/internal/report.go create mode 100644 testing/e2e/mirror/internal/source.go create mode 100644 testing/e2e/mirror/internal/verify.go delete mode 100644 testing/e2e/mirror/mirror_e2e_test.go create mode 100644 testing/e2e/mirror/modules_test.go delete mode 100644 testing/e2e/mirror/output.go create mode 100644 testing/e2e/mirror/platform_test.go delete mode 100644 testing/e2e/mirror/registry_comparator.go delete mode 100644 testing/e2e/mirror/report.go create mode 100644 testing/e2e/mirror/security_test.go create mode 100644 testing/e2e/mirror/testenv.go diff --git a/Taskfile.yml b/Taskfile.yml index 6cc01610..31d5f06e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -267,40 +267,39 @@ tasks: - task: _test:go test:e2e:mirror: - desc: Run E2E tests for d8 mirror (requires credentials) + desc: Run full cycle E2E mirror test (platform + modules + security) summary: | - Runs heavy E2E tests for d8 mirror pull/push commands. + E2E test for d8 mirror pull/push commands. + Required: E2E_LICENSE_TOKEN env variable - The test performs a complete mirror cycle: - 1. Analyzes source registry - 2. Pulls images to local bundle - 3. Pushes bundle to target registry - 4. Compares all images between source and target - - Required: Set E2E_LICENSE_TOKEN or E2E_SOURCE_USER/E2E_SOURCE_PASSWORD - - Examples: - # With license token - E2E_LICENSE_TOKEN=xxx task test:e2e:mirror - - # With local registry - E2E_SOURCE_REGISTRY=localhost:443/deckhouse \ - E2E_SOURCE_USER=admin \ - E2E_SOURCE_PASSWORD=secret \ - E2E_TLS_SKIP_VERIFY=true \ - task test:e2e:mirror - - # Keep bundle for debugging - E2E_LICENSE_TOKEN=xxx E2E_KEEP_BUNDLE=true task test:e2e:mirror + Example: E2E_LICENSE_TOKEN=xxx task test:e2e:mirror + cmds: + - task build + - go test -v -count=1 -timeout 180m -tags=e2e -run 'TestFullCycleE2E' ./testing/e2e/mirror {{ .CLI_ARGS }} + + test:e2e:mirror:platform: + desc: Run platform E2E test + cmds: + - task build + - go test -v -count=1 -timeout 120m -tags=e2e -run 'TestPlatform' ./testing/e2e/mirror {{ .CLI_ARGS }} + + test:e2e:mirror:modules: + desc: Run all modules E2E test + cmds: + - task build + - go test -v -count=1 -timeout 120m -tags=e2e -run 'TestModulesE2E$' ./testing/e2e/mirror {{ .CLI_ARGS }} + + + test:e2e:mirror:security: + desc: Run security E2E test cmds: - task build - - go test -v -timeout 120m -tags=e2e ./testing/e2e/mirror/... {{ .CLI_ARGS }} + - go test -v -count=1 -timeout 30m -tags=e2e -run 'TestSecurityE2E' ./testing/e2e/mirror {{ .CLI_ARGS }} test:e2e:mirror:logs:clean: desc: Clean E2E test logs cmds: - rm -rf ./testing/e2e/.logs/* - - echo "E2E logs cleaned" lint: desc: Run golangci-lint with auto-fix diff --git a/testing/e2e/mirror/README.md b/testing/e2e/mirror/README.md index eb4e30f1..48fc88ad 100644 --- a/testing/e2e/mirror/README.md +++ b/testing/e2e/mirror/README.md @@ -4,65 +4,81 @@ End-to-end tests for the `d8 mirror pull` and `d8 mirror push` commands. ## Overview -These tests perform a **complete mirror cycle with deep comparison** to ensure source and target registries are **100% identical**. +These tests perform **complete mirror cycles with verification** to ensure: +1. All expected images are downloaded from source +2. All images are correctly pushed to target registry +3. All images match between source and target (deep comparison) + +## Test Types + +| Test | Description | Timeout | Command | +|------|-------------|---------|---------| +| **Full Cycle** | Platform + Modules + Security | 3h | `task test:e2e:mirror` | +| **Platform** | Deckhouse core only | 2h | `task test:e2e:mirror:platform` | +| **Platform Stable** | Only stable channel | 2h | `task test:e2e:mirror:platform` + `E2E_DECKHOUSE_TAG=stable` | +| **Modules** | All modules | 2h | `task test:e2e:mirror:modules` | +| **Single Module** | One module (fast) | 2h | `task test:e2e:mirror:modules` + `E2E_INCLUDE_MODULES=module-name` | +| **Security** | Security DBs only | 30m | `task test:e2e:mirror:security` | + +## Verification Approach + +### Step 1: Read Expected Images from Source +Before pulling, we independently read what SHOULD be downloaded: +- Release channel versions from source registry +- `images_digests.json` from each installer image +- Module list and versions + +### Step 2: Pull & Push +Execute `d8 mirror pull` and `d8 mirror push` + +### Step 3: Verify +Compare expected images with what's actually in target: +- All expected digests must exist in target +- All images in target must match source (manifest, config, layers) + +This catches: +- **Pull bugs** - if pull forgets to download an image +- **Push bugs** - if push fails to upload an image +- **Data corruption** - if any digest doesn't match -### Test Steps - -1. **Analyze source registry** - Discover all repositories and count all images -2. **Pull images** - Execute `d8 mirror pull` to create a bundle -3. **Push images** - Execute `d8 mirror push` to target registry -4. **Deep comparison** - Compare every repository, tag, and digest between source and target - -### What Gets Compared (Deep Comparison) - -- **Repository level**: All repositories in source must exist in target -- **Tag level**: All tags in each repository must exist in target -- **Manifest digest**: Every image manifest digest must match (SHA256) -- **Config digest**: Image config blob digest is verified -- **Layer digests**: ALL layer digests of every image are compared -- **Layer count**: Number of layers must match +## Running Tests -This ensures **byte-for-byte identical** registries. +### Quick Start -## Requirements +```bash +# Full cycle with license token +E2E_LICENSE_TOKEN=xxx task test:e2e:mirror -- Built `d8` binary (run `task build` from project root) -- Valid credentials for the source registry -- Network access to the source registry -- Sufficient disk space for the bundle (can be ~20 GB) +# Platform only (faster) +E2E_LICENSE_TOKEN=xxx E2E_DECKHOUSE_TAG=stable task test:e2e:mirror:platform -## Running Tests +# Single module (fastest) +E2E_LICENSE_TOKEN=xxx E2E_INCLUDE_MODULES=module-name task test:e2e:mirror:modules +``` -### Using Taskfile (Recommended) +### Using Environment Variables ```bash -# With environment variables +# Official registry with license +E2E_LICENSE_TOKEN=your_license_token \ +task test:e2e:mirror + +# Local registry E2E_SOURCE_REGISTRY=localhost:443/deckhouse \ E2E_SOURCE_USER=admin \ E2E_SOURCE_PASSWORD=secret \ E2E_TLS_SKIP_VERIFY=true \ -task test:e2e:mirror - -# With command-line flags -task test:e2e:mirror -- \ - -source-registry=localhost:443/deckhouse \ - -source-user=admin \ - -source-password=admin \ - -tls-skip-verify - -# With license token (official Deckhouse registry) -E2E_LICENSE_TOKEN=xxx task test:e2e:mirror -``` +task test:e2e:mirror:platform -### Using go test directly +# Specific release channel +E2E_LICENSE_TOKEN=xxx \ +E2E_DECKHOUSE_TAG=stable \ +task test:e2e:mirror:platform -```bash -# Note: requires -tags=e2e flag -go test -v -tags=e2e -timeout=120m ./testing/e2e/mirror/... \ - -source-registry=localhost:443/deckhouse \ - -source-user=admin \ - -source-password=secret \ - -tls-skip-verify +# Specific modules only +E2E_LICENSE_TOKEN=xxx \ +E2E_INCLUDE_MODULES="pod-reloader,neuvector" \ +task test:e2e:mirror:modules ``` ### Configuration Options @@ -73,20 +89,26 @@ go test -v -tags=e2e -timeout=120m ./testing/e2e/mirror/... \ | `-source-user` | `E2E_SOURCE_USER` | | Source registry username | | `-source-password` | `E2E_SOURCE_PASSWORD` | | Source registry password | | `-license-token` | `E2E_LICENSE_TOKEN` | | License token | -| `-target-registry` | `E2E_TARGET_REGISTRY` | (local disk-based registry) | Target registry | +| `-target-registry` | `E2E_TARGET_REGISTRY` | `""` (in-memory) | Target registry | | `-target-user` | `E2E_TARGET_USER` | | Target registry username | | `-target-password` | `E2E_TARGET_PASSWORD` | | Target registry password | | `-tls-skip-verify` | `E2E_TLS_SKIP_VERIFY` | `false` | Skip TLS verification | +| `-deckhouse-tag` | `E2E_DECKHOUSE_TAG` | | Specific tag/channel (e.g., `stable`, `v1.65.8`) | +| `-no-modules` | `E2E_NO_MODULES` | `false` | Skip modules | +| `-no-platform` | `E2E_NO_PLATFORM` | `false` | Skip platform | +| `-no-security` | `E2E_NO_SECURITY` | `false` | Skip security DBs | +| `-include-modules` | `E2E_INCLUDE_MODULES` | | Comma-separated module list | | `-keep-bundle` | `E2E_KEEP_BUNDLE` | `false` | Keep bundle after test | +| `-existing-bundle` | `E2E_EXISTING_BUNDLE` | | Path to existing bundle (skip pull) | | `-d8-binary` | `E2E_D8_BINARY` | `bin/d8` | Path to d8 binary | -| `-new-pull` | `E2E_NEW_PULL` | `false` | Use new pull implementation | +| `-new-pull` | `E2E_NEW_PULL` | `false` | Use experimental pull | ## Test Artifacts -Tests produce detailed artifacts in `testing/e2e/.logs/-/`: +Tests produce artifacts in `testing/e2e/.logs/-/`: ``` -testing/e2e/.logs/fullcycle-20251226-114128/ +testing/e2e/.logs/TestFullCycleE2E-20251226-114128/ ├── test.log # Full command output (pull/push) ├── report.txt # Test summary report └── comparison.txt # Detailed registry comparison @@ -98,128 +120,63 @@ testing/e2e/.logs/fullcycle-20251226-114128/ task test:e2e:mirror:logs:clean ``` -### Sample Report (report.txt) - -``` -================================================================================ -E2E TEST REPORT: TestMirrorE2E_FullCycle -================================================================================ - -EXECUTION: - Started: 2025-12-26T11:41:28+03:00 - Finished: 2025-12-26T11:46:28+03:00 - Duration: 5m1s - -REGISTRIES: - Source: localhost:443/deckhouse-etalon - Target: 127.0.0.1:61594/deckhouse/ee - -IMAGES TO VERIFY: - Source: 324 images (82 repos) - Target: 324 images (82 repos) - (excluded 1071 internal tags from comparison) - -BUNDLE: - Size: 22.52 GB - -VERIFICATION RESULTS: - Images matched: 324 (manifest + config + layers) - Layers verified: 1172 - Missing images: 0 - Digest mismatch: 0 - Missing layers: 0 - -STEPS: - [PASS] Analyze source (82 repos) (330ms) - [PASS] Pull images (22.52 GB bundle) (2m59.826s) - [PASS] Push to registry (1m57.742s) - [PASS] Deep comparison (324 images verified) (2.266s) - -================================================================================ -RESULT: PASSED - REGISTRIES ARE IDENTICAL - 82 repositories verified - 324 images verified -================================================================================ -``` +## Requirements -### Comparison Report (comparison.txt) +- Built `d8` binary (run `task build`) +- Valid credentials for source registry +- Network access +- Disk space for bundle (20-50 GB depending on scope) -Contains detailed breakdown per repository with layer-level verification: +## What Gets Verified -``` -REGISTRY COMPARISON SUMMARY -=========================== - -Source: localhost:443/deckhouse -Target: 127.0.0.1:54321/deckhouse/ee -Duration: 2s - -REPOSITORIES: - Source: 82 - Target: 82 - Missing in target: 0 - Extra in target: 0 - -IMAGES TO VERIFY: - Source: 324 images - Target: 324 images - (excluded 1071 internal tags: digest-based, .att, .sig) - -VERIFICATION RESULTS: - Matched: 324 -DEEP COMPARISON (layers + config): - Images deep-checked: 324 - Source layers: 1172 - Target layers: 1172 - Matched layers: 1172 - Missing layers: 0 - Config mismatches: 0 - -✓ REGISTRIES ARE IDENTICAL (all hashes match) - -REPOSITORY BREAKDOWN: ---------------------- -✓ (root): 6/6 tags, 66 layers checked -✓ install: 6/6 tags, 48 layers checked -✓ install-standalone: 78/78 tags, 624 layers checked -✓ release-channel: 6/6 tags, 12 layers checked -✓ security/trivy-db: 1/1 tags, 2 layers checked -✓ modules/deckhouse-admin: 5/5 tags, 10 layers checked -... -``` +### Platform Test +1. Release channels exist (alpha, beta, stable, rock-solid) +2. Install images for each version +3. All digests from `images_digests.json` exist in target -## Timeouts +### Modules Test +1. Module list matches expected +2. Each module has release tags +3. Module images match source -The test has a **120-minute timeout** to handle large registries. +### Security Test +1. All security databases exist (trivy-db, trivy-bdu, etc.) +2. Tags match source ## Troubleshooting ### "Source authentication not provided" - -Set credentials: ```bash E2E_LICENSE_TOKEN=your_token task test:e2e:mirror -# or -E2E_SOURCE_USER=admin E2E_SOURCE_PASSWORD=secret task test:e2e:mirror ``` -### "Pull failed" or "Push failed" - -1. Check `d8` binary exists (`task build`) +### "Pull failed" +1. Check `d8` binary: `task build` 2. Check network access -3. Use `-tls-skip-verify` for self-signed certs -4. Check credentials - -### "Registries differ" +3. For self-signed certs: `E2E_TLS_SKIP_VERIFY=true` -Check `comparison.txt` for details: -- **Missing images**: Images in source but not in target -- **Mismatched digests**: Images exist but have different content +### "Verification failed" +Check `comparison.txt`: +- **Missing in target**: Pull or push didn't transfer the image +- **Digest mismatch**: Data corruption or version skew -### Viewing Bundle Contents +### Running Against Local Registry +```bash +E2E_SOURCE_REGISTRY=localhost:5000/deckhouse \ +E2E_SOURCE_USER=admin \ +E2E_SOURCE_PASSWORD=admin \ +E2E_TLS_SKIP_VERIFY=true \ +task test:e2e:mirror:platform +``` +### Keep Bundle for Debugging ```bash -E2E_KEEP_BUNDLE=true task test:e2e:mirror -- ... +E2E_KEEP_BUNDLE=true E2E_LICENSE_TOKEN=xxx task test:e2e:mirror:platform +# Bundle location shown in test output +``` -# Bundle location shown in output +### Use Existing Bundle (Skip Pull) +```bash +E2E_EXISTING_BUNDLE=/path/to/bundle E2E_LICENSE_TOKEN=xxx task test:e2e:mirror:platform +# Test will skip pull step and use existing bundle ``` diff --git a/testing/e2e/mirror/fullcycle_test.go b/testing/e2e/mirror/fullcycle_test.go new file mode 100644 index 00000000..c2661d5a --- /dev/null +++ b/testing/e2e/mirror/fullcycle_test.go @@ -0,0 +1,210 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestFullCycleE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + runFullCycleTest(t, cfg, env) + printFinalReport(t, env) +} + +func runFullCycleTest(t *testing.T, cfg *internal.Config, env *testEnv) { + ctx, cancel := context.WithTimeout(context.Background(), internal.FullCycleTestTimeout) + defer cancel() + + internal.PrintHeader("FULL CYCLE E2E TEST") + + internal.PrintStep(1, "Reading expected images from source registry") + expected := readAllExpectedImages(t, ctx, cfg) + env.Report.ExpectedModules = getModuleNames(expected) + + if cfg.HasExistingBundle() { + t.Logf("Using existing bundle: %s (skipping pull)", env.BundleDir) + env.Report.AddStep("Pull (existing bundle)", "SKIP", 0, nil) + } else { + internal.PrintStep(2, "Pulling images to bundle") + runPullStep(t, cfg, env) + } + + internal.PrintStep(3, "Pushing bundle to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying expected images in target") + runVerificationStep(t, ctx, cfg, env, expected) + + internal.PrintSuccessBox(env.Report.MatchedImages, env.Report.FoundAttTags) +} + +func readAllExpectedImages(t *testing.T, ctx context.Context, cfg *internal.Config) *internal.ExpectedImages { + t.Helper() + + reader := createSourceReader(t, cfg) + result := &internal.ExpectedImages{} + + if !cfg.NoPlatform { + t.Log("Reading platform images...") + channels, err := reader.ReadReleaseChannels(ctx) + if err != nil { + t.Logf("Warning: failed to read release channels: %v", err) + } else { + platform, err := reader.ReadPlatformDigests(ctx, channels) + if err != nil { + t.Logf("Warning: failed to read platform digests: %v", err) + } else { + result.Platform = platform + t.Logf("Platform: %d versions, %d digests", len(platform.Versions), len(platform.ImageDigests)) + } + } + } + + if !cfg.NoModules { + t.Log("Reading modules...") + modules, err := reader.ReadModulesList(ctx) + if err != nil { + t.Logf("Warning: failed to read modules: %v", err) + } else { + modules = filterModules(modules, cfg.IncludeModules) + + for _, moduleName := range modules { + info, err := reader.ReadModuleDigests(ctx, moduleName) + if err != nil { + t.Logf("Warning: failed to read module %s: %v", moduleName, err) + continue + } + result.Modules = append(result.Modules, info) + } + t.Logf("Modules: %d", len(result.Modules)) + } + } + + if !cfg.NoSecurity { + t.Log("Reading security databases...") + security, err := reader.ReadSecurityDigests(ctx) + if err != nil { + t.Logf("Warning: failed to read security: %v", err) + } else { + result.Security = security + t.Logf("Security: %d databases", len(security.Databases)) + } + } + + return result +} + +func getModuleNames(expected *internal.ExpectedImages) []string { + if expected == nil || len(expected.Modules) == 0 { + return nil + } + names := make([]string, len(expected.Modules)) + for i, m := range expected.Modules { + names[i] = m.Name + } + return names +} + +func runVerificationStep(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expected *internal.ExpectedImages) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifyFull(ctx, cfg.DeckhouseTag, cfg.IncludeModules) + if err != nil { + env.Report.AddStep("Verification", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Verification failed") + } + + saveVerificationReport(t, env.ComparisonFile, result) + + env.Report.TotalImages = len(result.ExpectedDigests) + env.Report.MatchedImages = len(result.FoundDigests) + env.Report.MissingImages = len(result.MissingDigests) + env.Report.ExpectedAttTags = len(result.ExpectedAttTags) + env.Report.FoundAttTags = len(result.FoundAttTags) + env.Report.MissingAttTags = len(result.MissingAttTags) + env.Report.ModulesExpected = result.ModulesExpected + env.Report.ModulesFound = result.ModulesFound + env.Report.ModulesMissing = len(result.ModulesMissing) + env.Report.SecurityExpected = result.SecurityExpected + env.Report.SecurityFound = result.SecurityFound + env.Report.SecurityMissing = len(result.SecurityMissing) + env.Report.SourceImageCount = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + env.Report.TargetImageCount = len(result.FoundDigests) + len(result.FoundAttTags) + + t.Log("") + t.Log(result.Summary()) + + var failures []string + if len(result.MissingDigests) > 0 { + failures = append(failures, fmt.Sprintf("missing %d digests in target", len(result.MissingDigests))) + } + if len(result.MissingAttTags) > 0 { + failures = append(failures, fmt.Sprintf("missing %d .att tags in target", len(result.MissingAttTags))) + } + + if len(failures) > 0 { + env.Report.AddStep( + fmt.Sprintf("Verification (%d/%d digests, %d/%d .att)", + len(result.FoundDigests), len(result.ExpectedDigests), + len(result.FoundAttTags), len(result.ExpectedAttTags)), + "FAIL", time.Since(stepStart), + fmt.Errorf("%v", failures), + ) + + require.Empty(t, failures, + "Mirror verification FAILED!\n\n%s\n\nSee %s for details", + result.Summary(), env.ComparisonFile) + } + + env.Report.AddStep( + fmt.Sprintf("Verification (%d digests, %d .att tags)", + len(result.FoundDigests), len(result.FoundAttTags)), + "PASS", time.Since(stepStart), nil, + ) +} + +func printFinalReport(t *testing.T, env *testEnv) { + env.Report.EndTime = time.Now() + report := env.Report.String() + + t.Log("") + t.Log(report) + + saveReport(t, env.ReportFile, env.Report) + t.Logf("Report written to: %s", env.ReportFile) +} + diff --git a/testing/e2e/mirror/helpers_test.go b/testing/e2e/mirror/helpers_test.go new file mode 100644 index 00000000..15a2afb1 --- /dev/null +++ b/testing/e2e/mirror/helpers_test.go @@ -0,0 +1,215 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func createVerifier(t *testing.T, cfg *internal.Config, env *testEnv) *internal.DigestVerifier { + t.Helper() + + sourceReader := internal.NewSourceReader(cfg.SourceRegistry, cfg.GetSourceAuth(), cfg.TLSSkipVerify) + verifier := internal.NewDigestVerifier( + sourceReader, + env.TargetRegistry, + cfg.GetTargetAuth(), + cfg.TLSSkipVerify, + ) + verifier.SetProgressCallback(func(msg string) { + t.Logf(" %s", msg) + }) + return verifier +} + +func createSourceReader(t *testing.T, cfg *internal.Config) *internal.SourceReader { + t.Helper() + + reader := internal.NewSourceReader(cfg.SourceRegistry, cfg.GetSourceAuth(), cfg.TLSSkipVerify) + reader.SetProgressCallback(func(msg string) { + t.Logf(" %s", msg) + }) + return reader +} + +func filterModules(modules []string, include []string) []string { + if len(include) == 0 { + return modules + } + includeSet := make(map[string]bool, len(include)) + for _, m := range include { + includeSet[m] = true + } + var filtered []string + for _, m := range modules { + if includeSet[m] { + filtered = append(filtered, m) + } + } + return filtered +} + +func verifyPlatformImages(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, deckhouseTag string) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifyPlatform(ctx, deckhouseTag) + require.NoError(t, err, "Verification failed") + + saveVerificationReport(t, env.ComparisonFile, result) + + env.Report.TotalImages = len(result.ExpectedDigests) + env.Report.MatchedImages = len(result.FoundDigests) + env.Report.MissingImages = len(result.MissingDigests) + env.Report.ExpectedAttTags = len(result.ExpectedAttTags) + env.Report.FoundAttTags = len(result.FoundAttTags) + env.Report.MissingAttTags = len(result.MissingAttTags) + env.Report.SourceImageCount = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + env.Report.TargetImageCount = len(result.FoundDigests) + len(result.FoundAttTags) + + t.Log("") + t.Log(result.Summary()) + + var failures []string + if len(result.MissingDigests) > 0 { + failures = append(failures, fmt.Sprintf("missing %d digests in target", len(result.MissingDigests))) + } + if len(result.MissingAttTags) > 0 { + failures = append(failures, fmt.Sprintf("missing %d .att tags in target", len(result.MissingAttTags))) + } + + if len(failures) > 0 { + env.Report.AddStep( + fmt.Sprintf("Verification (%d/%d digests, %d/%d .att)", + len(result.FoundDigests), len(result.ExpectedDigests), + len(result.FoundAttTags), len(result.ExpectedAttTags)), + "FAIL", time.Since(stepStart), + fmt.Errorf("%v", failures), + ) + + require.Empty(t, failures, + "Platform verification FAILED!\n\n%s\n\nSee %s for details", + result.Summary(), env.ComparisonFile) + return + } + + env.Report.AddStep( + fmt.Sprintf("Verification (%d digests, %d .att tags)", + len(result.FoundDigests), len(result.FoundAttTags)), + "PASS", time.Since(stepStart), nil, + ) +} + +func verifyModulesImages(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expectedModules []string) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifyModules(ctx, expectedModules) + require.NoError(t, err, "Modules verification failed") + + t.Logf("Found %d/%d modules in target", result.ModulesFound, result.ModulesExpected) + + for _, missing := range result.ModulesMissing { + t.Logf(" ✗ %s", missing) + } + + if len(result.ModulesMissing) > 0 { + env.Report.AddStep( + fmt.Sprintf("Modules Verification (%d/%d found)", + result.ModulesFound, result.ModulesExpected), + "FAIL", time.Since(stepStart), + fmt.Errorf("missing %d modules: %v", len(result.ModulesMissing), result.ModulesMissing), + ) + require.Empty(t, result.ModulesMissing, "Some modules are missing in target") + return + } + + env.Report.ModulesExpected = result.ModulesExpected + env.Report.ModulesFound = result.ModulesFound + env.Report.ModulesMissing = len(result.ModulesMissing) + + env.Report.AddStep( + fmt.Sprintf("Modules Verification (%d modules)", result.ModulesFound), + "PASS", time.Since(stepStart), nil, + ) + t.Log("Modules verification passed") +} + +func verifySecurityImages(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + t.Helper() + stepStart := time.Now() + + verifier := createVerifier(t, cfg, env) + result, err := verifier.VerifySecurity(ctx) + require.NoError(t, err, "Security verification failed") + + t.Logf("Found %d/%d security databases in target", result.SecurityFound, result.SecurityExpected) + + for _, missing := range result.SecurityMissing { + t.Logf(" ✗ %s", missing) + } + + if len(result.SecurityMissing) > 0 { + env.Report.AddStep( + fmt.Sprintf("Security Verification (%d/%d found)", + result.SecurityFound, result.SecurityExpected), + "FAIL", time.Since(stepStart), + fmt.Errorf("missing %d security databases: %v", len(result.SecurityMissing), result.SecurityMissing), + ) + require.Empty(t, result.SecurityMissing, "Some security databases are missing in target") + return + } + + env.Report.SecurityExpected = result.SecurityExpected + env.Report.SecurityFound = result.SecurityFound + env.Report.SecurityMissing = len(result.SecurityMissing) + + env.Report.AddStep( + fmt.Sprintf("Security Verification (%d databases)", result.SecurityFound), + "PASS", time.Since(stepStart), nil, + ) + t.Log("Security verification passed") +} + +func saveVerificationReport(t *testing.T, path string, result *internal.VerificationResult) { + t.Helper() + + report := result.DetailedReport() + err := internal.WriteFile(path, []byte(report)) + if err != nil { + t.Logf("Warning: failed to write verification report: %v", err) + } +} + +func saveReport(t *testing.T, path string, report *internal.TestReport) { + t.Helper() + if err := report.WriteToFile(path); err != nil { + t.Logf("Warning: failed to write report: %v", err) + } +} + diff --git a/testing/e2e/mirror/commands.go b/testing/e2e/mirror/internal/commands.go similarity index 66% rename from testing/e2e/mirror/commands.go rename to testing/e2e/mirror/internal/commands.go index 94e9054b..f9c3f27e 100644 --- a/testing/e2e/mirror/commands.go +++ b/testing/e2e/mirror/internal/commands.go @@ -1,5 +1,5 @@ /* -Copyright 2024 Flant JSC +Copyright 2025 Flant JSC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mirror +package internal import ( "fmt" @@ -25,19 +25,13 @@ import ( "time" ) -// ============================================================================= -// Command Builders -// ============================================================================= - -// buildPullCommand builds the d8 mirror pull command -func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { +func BuildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { args := []string{ "mirror", "pull", "--source", cfg.SourceRegistry, - "--force", // overwrite if exists + "--force", } - // Authentication if cfg.SourceUser != "" { args = append(args, "--source-login", cfg.SourceUser) args = append(args, "--source-password", cfg.SourcePassword) @@ -45,22 +39,31 @@ func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { args = append(args, "--license", cfg.LicenseToken) } - // TLS if cfg.TLSSkipVerify { args = append(args, "--tls-skip-verify") } - // Debug options + if cfg.DeckhouseTag != "" { + args = append(args, "--deckhouse-tag", cfg.DeckhouseTag) + } if cfg.NoModules { args = append(args, "--no-modules") } + if cfg.NoPlatform { + args = append(args, "--no-platform") + } + if cfg.NoSecurity { + args = append(args, "--no-security-db") + } + for _, module := range cfg.IncludeModules { + args = append(args, "--include-module", module) + } args = append(args, bundleDir) cmd := exec.Command(cfg.D8Binary, args...) - cmd.Env = append(os.Environ(), "HOME="+os.Getenv("HOME")) + cmd.Env = os.Environ() - // Experimental options (via env) if cfg.NewPull { cmd.Env = append(cmd.Env, "NEW_PULL=true") } @@ -68,62 +71,59 @@ func buildPullCommand(cfg *Config, bundleDir string) *exec.Cmd { return cmd } -// buildPushCommand builds the d8 mirror push command -func buildPushCommand(cfg *Config, bundleDir, targetRegistry string) *exec.Cmd { +func BuildPushCommand(cfg *Config, bundleDir, targetRegistry string) *exec.Cmd { args := []string{ "mirror", "push", bundleDir, targetRegistry, } - // TLS if cfg.TLSSkipVerify { args = append(args, "--tls-skip-verify") } - // Authentication if cfg.TargetUser != "" { args = append(args, "--registry-login", cfg.TargetUser) args = append(args, "--registry-password", cfg.TargetPassword) } cmd := exec.Command(cfg.D8Binary, args...) - cmd.Env = append(os.Environ(), "HOME="+os.Getenv("HOME")) + cmd.Env = os.Environ() + + // Ensure HOME is set - some tools (like ssh) require it + if os.Getenv("HOME") == "" { + if homeDir, err := os.UserHomeDir(); err == nil { + cmd.Env = append(cmd.Env, "HOME="+homeDir) + } + } + + if cfg.NewPull { + cmd.Env = append(cmd.Env, "NEW_PULL=true") + } return cmd } -// ============================================================================= -// Command Runner -// ============================================================================= - -// runCommandWithLog runs command with streaming output and saves to log file -func runCommandWithLog(t *testing.T, cmd *exec.Cmd, logFile string) error { +func RunCommandWithLog(t *testing.T, cmd *exec.Cmd, logFile string) error { t.Helper() - // Open log file f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { t.Logf("Warning: could not open log file %s: %v", logFile, err) - // Fallback: just stream without logging cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } defer f.Close() - // Write command header fmt.Fprintf(f, "\n\n========== COMMAND: %s ==========\n", cmd.String()) fmt.Fprintf(f, "Started: %s\n\n", time.Now().Format(time.RFC3339)) - // Stream to stdout and file cmd.Stdout = io.MultiWriter(os.Stdout, f) cmd.Stderr = io.MultiWriter(os.Stderr, f) - // Run cmdErr := cmd.Run() - // Write result if cmdErr != nil { fmt.Fprintf(f, "\n\n========== COMMAND FAILED: %v ==========\n", cmdErr) } else { @@ -132,3 +132,4 @@ func runCommandWithLog(t *testing.T, cmd *exec.Cmd, logFile string) error { return cmdErr } + diff --git a/testing/e2e/mirror/config.go b/testing/e2e/mirror/internal/config.go similarity index 61% rename from testing/e2e/mirror/config.go rename to testing/e2e/mirror/internal/config.go index a0396887..9edd9e72 100644 --- a/testing/e2e/mirror/config.go +++ b/testing/e2e/mirror/internal/config.go @@ -1,5 +1,5 @@ /* -Copyright 2024 Flant JSC +Copyright 2025 Flant JSC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,18 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package mirror +package internal import ( "flag" "os" + "path/filepath" + "strings" + "time" "github.com/google/go-containerregistry/pkg/authn" ) -// Test configuration flags +const ( + SecurityTestTimeout = 30 * time.Minute + PlatformTestTimeout = 2 * time.Hour + ModulesTestTimeout = 2 * time.Hour + FullCycleTestTimeout = 3 * time.Hour +) + var ( - // Source registry configuration sourceRegistry = flag.String("source-registry", getEnvOrDefault("E2E_SOURCE_REGISTRY", "registry.deckhouse.ru/deckhouse/fe"), "Reference registry to pull from") @@ -39,7 +47,6 @@ var ( getEnvOrDefault("E2E_LICENSE_TOKEN", ""), "License token for source registry authentication (shortcut for source-user=license-token)") - // Target registry configuration targetRegistry = flag.String("target-registry", getEnvOrDefault("E2E_TARGET_REGISTRY", ""), "Target registry to push to (empty = use in-memory registry)") @@ -50,23 +57,35 @@ var ( getEnvOrDefault("E2E_TARGET_PASSWORD", ""), "Target registry password") - // Test options tlsSkipVerify = flag.Bool("tls-skip-verify", getEnvOrDefault("E2E_TLS_SKIP_VERIFY", "") == "true", "Skip TLS certificate verification (for self-signed certs)") keepBundle = flag.Bool("keep-bundle", getEnvOrDefault("E2E_KEEP_BUNDLE", "") == "true", "Keep bundle directory after test") + existingBundle = flag.String("existing-bundle", + getEnvOrDefault("E2E_EXISTING_BUNDLE", ""), + "Path to existing bundle directory (skip pull step)") d8Binary = flag.String("d8-binary", - getEnvOrDefault("E2E_D8_BINARY", "../../../bin/d8"), + getEnvOrDefault("E2E_D8_BINARY", "bin/d8"), "Path to d8 binary") - // Debug/test options + deckhouseTag = flag.String("deckhouse-tag", + getEnvOrDefault("E2E_DECKHOUSE_TAG", ""), + "Specific Deckhouse tag or release channel (e.g., 'stable', 'v1.65.8')") noModules = flag.Bool("no-modules", getEnvOrDefault("E2E_NO_MODULES", "") == "true", - "Skip modules during pull (for testing failure scenarios)") + "Skip modules during pull") + noPlatform = flag.Bool("no-platform", + getEnvOrDefault("E2E_NO_PLATFORM", "") == "true", + "Skip platform during pull") + noSecurity = flag.Bool("no-security", + getEnvOrDefault("E2E_NO_SECURITY", "") == "true", + "Skip security databases during pull") + includeModules = flag.String("include-modules", + getEnvOrDefault("E2E_INCLUDE_MODULES", ""), + "Comma-separated list of modules to include (empty = all)") - // Experimental options newPull = flag.Bool("new-pull", getEnvOrDefault("E2E_NEW_PULL", "") == "true", "Use new pull implementation") @@ -79,7 +98,6 @@ func getEnvOrDefault(key, defaultValue string) string { return defaultValue } -// Config holds the parsed test configuration type Config struct { SourceRegistry string SourceUser string @@ -90,21 +108,22 @@ type Config struct { TargetUser string TargetPassword string - TLSSkipVerify bool - KeepBundle bool - D8Binary string + TLSSkipVerify bool + KeepBundle bool + ExistingBundle string + D8Binary string - // Debug/test options - NoModules bool // Skip modules during pull (for testing failure scenarios) + DeckhouseTag string + NoModules bool + NoPlatform bool + NoSecurity bool + IncludeModules []string - // Experimental options - NewPull bool // Use new pull implementation + NewPull bool } -// GetConfig returns the current test configuration from flags func GetConfig() *Config { - // flag.Parse() is called automatically by go test - return &Config{ + cfg := &Config{ SourceRegistry: *sourceRegistry, SourceUser: *sourceUser, SourcePassword: *sourcePassword, @@ -114,22 +133,67 @@ func GetConfig() *Config { TargetPassword: *targetPassword, TLSSkipVerify: *tlsSkipVerify, KeepBundle: *keepBundle, - D8Binary: *d8Binary, + ExistingBundle: *existingBundle, + D8Binary: resolveD8Binary(*d8Binary), + DeckhouseTag: *deckhouseTag, NoModules: *noModules, + NoPlatform: *noPlatform, + NoSecurity: *noSecurity, NewPull: *newPull, } + + if *includeModules != "" { + cfg.IncludeModules = parseCommaSeparated(*includeModules) + } + + return cfg +} + +func resolveD8Binary(path string) string { + if filepath.IsAbs(path) { + return path + } + projectRoot := FindProjectRoot() + return filepath.Join(projectRoot, path) +} + +func FindProjectRoot() string { + dir, _ := os.Getwd() + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + dir, _ = os.Getwd() + return dir + } + dir = parent + } +} + + + +func parseCommaSeparated(s string) []string { + if s == "" { + return nil + } + var parts []string + for _, part := range strings.Split(s, ",") { + if trimmed := strings.TrimSpace(part); trimmed != "" { + parts = append(parts, trimmed) + } + } + return parts } -// GetSourceAuth returns authenticator for source registry func (c *Config) GetSourceAuth() authn.Authenticator { - // Explicit credentials take precedence if c.SourceUser != "" { return authn.FromConfig(authn.AuthConfig{ Username: c.SourceUser, Password: c.SourcePassword, }) } - // License token is a shortcut for source-user=license-token if c.LicenseToken != "" { return authn.FromConfig(authn.AuthConfig{ Username: "license-token", @@ -139,12 +203,10 @@ func (c *Config) GetSourceAuth() authn.Authenticator { return authn.Anonymous } -// HasSourceAuth returns true if any source authentication is configured func (c *Config) HasSourceAuth() bool { return c.SourceUser != "" || c.LicenseToken != "" } -// GetTargetAuth returns authenticator for target registry func (c *Config) GetTargetAuth() authn.Authenticator { if c.TargetUser != "" { return authn.FromConfig(authn.AuthConfig{ @@ -155,7 +217,11 @@ func (c *Config) GetTargetAuth() authn.Authenticator { return authn.Anonymous } -// UseInMemoryRegistry returns true if we should use in-memory registry func (c *Config) UseInMemoryRegistry() bool { return c.TargetRegistry == "" } + +func (c *Config) HasExistingBundle() bool { + return c.ExistingBundle != "" +} + diff --git a/testing/e2e/mirror/internal/output.go b/testing/e2e/mirror/internal/output.go new file mode 100644 index 00000000..6549bcab --- /dev/null +++ b/testing/e2e/mirror/internal/output.go @@ -0,0 +1,144 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "fmt" + "os" + "strings" + "sync" + + "github.com/charmbracelet/lipgloss" + "github.com/muesli/termenv" + "golang.org/x/term" +) + +var ( + ColorCyan = lipgloss.Color("6") + ColorGreen = lipgloss.Color("2") + ColorRed = lipgloss.Color("1") + ColorYellow = lipgloss.Color("3") + ColorBlue = lipgloss.Color("4") + ColorWhite = lipgloss.Color("15") + ColorGray = lipgloss.Color("8") +) + +type styles struct { + Title lipgloss.Style + Header lipgloss.Style + Label lipgloss.Style + Value lipgloss.Style + Dim lipgloss.Style + Success lipgloss.Style + Error lipgloss.Style + StepNum lipgloss.Style + StepText lipgloss.Style + Separator lipgloss.Style +} + +func initStyles() styles { + return styles{ + Title: lipgloss.NewStyle().Bold(true).Foreground(ColorWhite), + Header: lipgloss.NewStyle().Bold(true).Foreground(ColorCyan), + Label: lipgloss.NewStyle().Foreground(ColorGray), + Value: lipgloss.NewStyle().Foreground(ColorWhite), + Dim: lipgloss.NewStyle().Foreground(ColorGray), + Success: lipgloss.NewStyle().Foreground(ColorGreen), + Error: lipgloss.NewStyle().Foreground(ColorRed), + StepNum: lipgloss.NewStyle().Bold(true).Foreground(ColorBlue), + StepText: lipgloss.NewStyle().Bold(true).Foreground(ColorWhite), + Separator: lipgloss.NewStyle().Foreground(ColorCyan), + } +} + +var defaultStyles = initStyles() + +var ( + StyleTitle = defaultStyles.Title + StyleHeader = defaultStyles.Header + StyleLabel = defaultStyles.Label + StyleValue = defaultStyles.Value + StyleDim = defaultStyles.Dim + StyleSuccess = defaultStyles.Success + StyleError = defaultStyles.Error +) + +var ( + BadgeOK = defaultStyles.Success.Copy().Bold(true).Render("[OK]") + BadgeFail = defaultStyles.Error.Copy().Bold(true).Render("[FAIL]") + BadgeSkip = lipgloss.NewStyle().Foreground(ColorYellow).Render("[SKIP]") +) + +var output = os.Stderr +var colorInitOnce sync.Once + +func EnsureColorInit() { + colorInitOnce.Do(func() { + if term.IsTerminal(int(os.Stderr.Fd())) || os.Getenv("FORCE_COLOR") != "" || os.Getenv("TERM") != "" { + lipgloss.DefaultRenderer().SetColorProfile(termenv.TrueColor) + } + }) +} + +func WriteLinef(format string, args ...interface{}) { + EnsureColorInit() + fmt.Fprintf(output, format+"\n", args...) +} + +func WriteRawf(format string, args ...interface{}) { + EnsureColorInit() + fmt.Fprintf(output, format, args...) +} + +func PrintStep(num int, description string) { + badge := defaultStyles.StepNum.Render(fmt.Sprintf("[STEP %d]", num)) + text := defaultStyles.StepText.Render(description) + WriteLinef("\n%s %s", badge, text) +} + +const separatorWidth = 80 + +func Separator(char string) string { + return defaultStyles.Separator.Render(strings.Repeat(char, separatorWidth)) +} + +func PrintHeader(title string) { + WriteLinef("") + WriteLinef(Separator("═")) + WriteLinef(" %s", StyleTitle.Render(title)) + WriteLinef(Separator("═")) +} + +func PrintSuccessBox(matchedDigests, matchedAttTags int) { + box := lipgloss.NewStyle(). + Border(lipgloss.DoubleBorder()). + BorderForeground(ColorGreen). + Padding(0, 2). + Foreground(ColorGreen) + + WriteLinef("") + WriteLinef(box.Render(fmt.Sprintf( + "SUCCESS: ALL EXPECTED IMAGES VERIFIED\n\nVerified: %d digests, %d .att tags\nAll expected images are present in target registry!", + matchedDigests, + matchedAttTags, + ))) +} + +func WriteFile(path string, data []byte) error { + return os.WriteFile(path, data, 0644) +} + diff --git a/testing/e2e/mirror/internal/report.go b/testing/e2e/mirror/internal/report.go new file mode 100644 index 00000000..f69ffda5 --- /dev/null +++ b/testing/e2e/mirror/internal/report.go @@ -0,0 +1,277 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/charmbracelet/lipgloss" +) + +type TestReport struct { + TestName string + StartTime time.Time + EndTime time.Time + SourceRegistry string + TargetRegistry string + LogDir string + + SourceImageCount int + TargetImageCount int + + TotalImages int + MatchedImages int + MissingImages int + + ExpectedAttTags int + FoundAttTags int + MissingAttTags int + + ModulesExpected int + ModulesFound int + ModulesMissing int + + SecurityExpected int + SecurityFound int + SecurityMissing int + + BundleSize int64 + ExpectedModules []string + + Steps []StepResult +} + +type StepResult struct { + Name string + Status string + Duration time.Duration + Error string +} + +func (r *TestReport) AddStep(name, status string, duration time.Duration, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + r.Steps = append(r.Steps, StepResult{ + Name: name, + Status: status, + Duration: duration, + Error: errStr, + }) +} + +func (r *TestReport) WriteToFile(path string) error { + content := r.String() + return os.WriteFile(path, []byte(content), 0644) +} + +func (r *TestReport) formatSection(useColors bool, label string, found, expected, missing int) string { + if expected == 0 && missing == 0 { + return "" + } + + var badge, value string + if useColors { + if missing > 0 { + badge = BadgeFail + value = StyleError.Render(fmt.Sprintf("%d / %d", found, expected)) + } else { + badge = BadgeOK + value = StyleSuccess.Render(fmt.Sprintf("%d / %d", found, expected)) + } + label = StyleLabel.Render(label) + return fmt.Sprintf(" %s %s %s\n", badge, label, value) + } + + status := "[OK]" + if missing > 0 { + status = "[FAIL]" + } + labelText := strings.TrimSuffix(label, ":") + result := fmt.Sprintf(" %s %s: %d / %d\n", status, labelText, found, expected) + if missing > 0 { + result += fmt.Sprintf(" Missing %s: %d\n", strings.ToLower(labelText), missing) + } + return result +} + +func (r *TestReport) formatStep(useColors bool, step StepResult) string { + dur := step.Duration.Round(time.Millisecond).String() + if useColors { + dur = StyleDim.Render(fmt.Sprintf("(%s)", dur)) + switch step.Status { + case "PASS": + return fmt.Sprintf(" %s %s %s\n", BadgeOK, step.Name, dur) + case "FAIL": + result := fmt.Sprintf(" %s %s %s\n", BadgeFail, step.Name, dur) + if step.Error != "" { + result += " " + StyleError.Render("ERROR: "+step.Error) + "\n" + } + return result + default: + return fmt.Sprintf(" %s %s\n", BadgeSkip, step.Name) + } + } + + switch step.Status { + case "PASS": + return fmt.Sprintf(" [PASS] %s (%s)\n", step.Name, dur) + case "FAIL": + result := fmt.Sprintf(" [FAIL] %s (%s)\n", step.Name, dur) + if step.Error != "" { + result += fmt.Sprintf(" ERROR: %s\n", step.Error) + } + return result + default: + return fmt.Sprintf(" [SKIP] %s\n", step.Name) + } +} + +func (r *TestReport) formatSummary(useColors bool, passCount, failCount int) string { + var parts []string + if r.MatchedImages > 0 { + parts = append(parts, fmt.Sprintf("%d platform digests", r.MatchedImages)) + } + if r.FoundAttTags > 0 { + parts = append(parts, fmt.Sprintf("%d attestations", r.FoundAttTags)) + } + if r.ModulesFound > 0 { + parts = append(parts, fmt.Sprintf("%d modules", r.ModulesFound)) + } + if r.SecurityFound > 0 { + parts = append(parts, fmt.Sprintf("%d security databases", r.SecurityFound)) + } + + if useColors { + if failCount > 0 { + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(ColorRed) + return " " + resultStyle.Render("RESULT: FAILED") + fmt.Sprintf(" (%d passed, %d failed)\n", passCount, failCount) + } + resultStyle := lipgloss.NewStyle().Bold(true).Foreground(ColorGreen) + result := " " + resultStyle.Render("RESULT: PASSED") + "\n" + if len(parts) > 0 { + result += " " + StyleSuccess.Render(strings.Join(parts, ", ")+" verified") + "\n" + } + return result + } + + if failCount > 0 { + return fmt.Sprintf("RESULT: FAILED (%d passed, %d failed)\n", passCount, failCount) + } + result := "RESULT: PASSED\n" + if len(parts) > 0 { + result += fmt.Sprintf(" %s verified\n", strings.Join(parts, ", ")) + } + return result +} + +func (r *TestReport) format(useColors bool) string { + duration := r.EndTime.Sub(r.StartTime) + if r.EndTime.IsZero() { + duration = time.Since(r.StartTime) + } + + var b strings.Builder + + if useColors { + b.WriteString("\n") + b.WriteString(Separator("═") + "\n") + b.WriteString(" " + StyleTitle.Render("E2E TEST REPORT") + "\n") + b.WriteString(Separator("═") + "\n\n") + b.WriteString(" " + StyleLabel.Render("Duration: ") + StyleDim.Render(duration.Round(time.Second).String()) + "\n\n") + b.WriteString(" " + StyleHeader.Render("REGISTRIES") + "\n") + b.WriteString(" " + StyleLabel.Render("Source: ") + StyleValue.Render(r.SourceRegistry) + "\n") + b.WriteString(" " + StyleLabel.Render("Target: ") + StyleValue.Render(r.TargetRegistry) + "\n\n") + b.WriteString(" " + StyleHeader.Render("VERIFICATION") + "\n") + } else { + b.WriteString("================================================================================\n") + b.WriteString(fmt.Sprintf("E2E TEST REPORT: %s\n", r.TestName)) + b.WriteString("================================================================================\n\n") + b.WriteString("EXECUTION:\n") + b.WriteString(fmt.Sprintf(" Started: %s\n", r.StartTime.Format(time.RFC3339))) + b.WriteString(fmt.Sprintf(" Finished: %s\n", r.EndTime.Format(time.RFC3339))) + b.WriteString(fmt.Sprintf(" Duration: %s\n", duration.Round(time.Second))) + b.WriteString(fmt.Sprintf(" Log dir: %s\n\n", r.LogDir)) + b.WriteString("REGISTRIES:\n") + b.WriteString(fmt.Sprintf(" Source: %s\n", r.SourceRegistry)) + b.WriteString(fmt.Sprintf(" Target: %s\n\n", r.TargetRegistry)) + b.WriteString("VERIFICATION:\n") + } + + if section := r.formatSection(useColors, "Platform digests: ", r.MatchedImages, r.TotalImages, r.MissingImages); section != "" { + b.WriteString(section) + } + if section := r.formatSection(useColors, "Attestation tags: ", r.FoundAttTags, r.ExpectedAttTags, r.MissingAttTags); section != "" { + b.WriteString(section) + } + if r.ModulesExpected > 0 { + b.WriteString(r.formatSection(useColors, "Modules verified: ", r.ModulesFound, r.ModulesExpected, r.ModulesMissing)) + } + if r.SecurityExpected > 0 { + b.WriteString(r.formatSection(useColors, "Security verified:", r.SecurityFound, r.SecurityExpected, r.SecurityMissing)) + } + b.WriteString("\n") + + if useColors { + b.WriteString(" " + StyleHeader.Render("STEPS") + "\n") + } else { + b.WriteString("STEPS:\n") + } + passCount, failCount := r.countSteps() + for _, step := range r.Steps { + b.WriteString(r.formatStep(useColors, step)) + } + b.WriteString("\n") + + if useColors { + b.WriteString(Separator("─") + "\n") + b.WriteString(r.formatSummary(useColors, passCount, failCount)) + b.WriteString(Separator("═") + "\n") + } else { + b.WriteString("================================================================================\n") + b.WriteString(r.formatSummary(useColors, passCount, failCount)) + b.WriteString("================================================================================\n") + } + + return b.String() +} + +func (r *TestReport) Print() { + WriteRawf("%s", r.format(true)) +} + +func (r *TestReport) countSteps() (int, int) { + var passed, failed int + for _, step := range r.Steps { + switch step.Status { + case "PASS": + passed++ + case "FAIL": + failed++ + } + } + return passed, failed +} + +func (r *TestReport) String() string { + return r.format(false) +} + diff --git a/testing/e2e/mirror/internal/source.go b/testing/e2e/mirror/internal/source.go new file mode 100644 index 00000000..0b9b3a20 --- /dev/null +++ b/testing/e2e/mirror/internal/source.go @@ -0,0 +1,444 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "archive/tar" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "path" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + + d8internal "github.com/deckhouse/deckhouse-cli/internal" +) + +func InsecureTransport() http.RoundTripper { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + return transport +} + +type SourceReader struct { + registry string + auth authn.Authenticator + opts []remote.Option + progressFn func(string) +} + +func NewSourceReader(registry string, auth authn.Authenticator, tlsSkipVerify bool) *SourceReader { + opts := []remote.Option{remote.WithAuth(auth)} + if tlsSkipVerify { + opts = append(opts, remote.WithTransport(InsecureTransport())) + } + + return &SourceReader{ + registry: registry, + auth: auth, + opts: opts, + } +} + +func (r *SourceReader) SetProgressCallback(fn func(string)) { + r.progressFn = fn +} + +func (r *SourceReader) Registry() string { + return r.registry +} + +func (r *SourceReader) RemoteOpts() []remote.Option { + return r.opts +} + +func (r *SourceReader) progress(format string, args ...interface{}) { + if r.progressFn != nil { + r.progressFn(fmt.Sprintf(format, args...)) + } +} + +type ReleaseChannelInfo struct { + Channel string + Version string +} + +func (r *SourceReader) ReadReleaseChannels(ctx context.Context) ([]ReleaseChannelInfo, error) { + channels := d8internal.GetAllDefaultReleaseChannels() + result := make([]ReleaseChannelInfo, 0, len(channels)) + + for _, channel := range channels { + r.progress("Reading release channel: %s", channel) + + ref := path.Join(r.registry, d8internal.ReleaseChannelSegment) + ":" + channel + version, err := r.readReleaseChannelVersion(ctx, ref) + if err != nil { + r.progress(" Warning: failed to read %s: %v", channel, err) + continue + } + + result = append(result, ReleaseChannelInfo{ + Channel: channel, + Version: version, + }) + r.progress(" %s -> %s", channel, version) + } + + return result, nil +} + +func (r *SourceReader) readReleaseChannelVersion(ctx context.Context, ref string) (string, error) { + imgRef, err := name.ParseReference(ref) + if err != nil { + return "", fmt.Errorf("parse reference: %w", err) + } + + img, err := remote.Image(imgRef, r.opts...) + if err != nil { + return "", fmt.Errorf("get image: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return "", fmt.Errorf("get layers: %w", err) + } + + for _, layer := range layers { + rc, err := layer.Uncompressed() + if err != nil { + continue + } + + version, err := extractVersionFromTar(rc) + rc.Close() + if err == nil && version != "" { + return version, nil + } + } + + return "", fmt.Errorf("version.json not found in image") +} + +func extractVersionFromTar(rc io.Reader) (string, error) { + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + + if strings.HasSuffix(hdr.Name, "version.json") { + var meta struct { + Version string `json:"version"` + } + if err := json.NewDecoder(tr).Decode(&meta); err != nil { + return "", err + } + return meta.Version, nil + } + } + return "", nil +} + +type PlatformDigests struct { + Versions []string + InstallDigests map[string]string + ImageDigests []string +} + +func (r *SourceReader) ReadPlatformDigests(ctx context.Context, channels []ReleaseChannelInfo) (*PlatformDigests, error) { + result := &PlatformDigests{ + InstallDigests: make(map[string]string), + ImageDigests: make([]string, 0), + } + + versionSet := make(map[string]bool) + for _, ch := range channels { + versionSet[ch.Version] = true + } + + for version := range versionSet { + result.Versions = append(result.Versions, version) + } + sort.Strings(result.Versions) + + digestSet := make(map[string]bool) + + for _, version := range result.Versions { + r.progress("Reading install:%s digests...", version) + + tag := version + if !strings.HasPrefix(tag, "v") { + if _, err := semver.NewVersion(version); err == nil { + tag = "v" + tag + } + } + + installRef := path.Join(r.registry, d8internal.InstallSegment) + ":" + tag + digests, err := r.readInstallDigests(ctx, installRef) + if err != nil { + r.progress(" Warning: failed to read install:%s: %v", tag, err) + continue + } + + r.progress(" Found %d digests", len(digests)) + for _, d := range digests { + digestSet[d] = true + } + } + + for d := range digestSet { + result.ImageDigests = append(result.ImageDigests, d) + } + sort.Strings(result.ImageDigests) + + return result, nil +} + +func (r *SourceReader) readInstallDigests(ctx context.Context, ref string) ([]string, error) { + imgRef, err := name.ParseReference(ref) + if err != nil { + return nil, fmt.Errorf("parse reference: %w", err) + } + + desc, err := remote.Get(imgRef, r.opts...) + if err != nil { + return nil, fmt.Errorf("get descriptor: %w", err) + } + + var img v1.Image + + if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + if err != nil { + return nil, fmt.Errorf("get index: %w", err) + } + + manifest, err := idx.IndexManifest() + if err != nil { + return nil, fmt.Errorf("get index manifest: %w", err) + } + + if len(manifest.Manifests) == 0 { + return nil, fmt.Errorf("index has no manifests") + } + + img, err = idx.Image(manifest.Manifests[0].Digest) + if err != nil { + return nil, fmt.Errorf("get image from index: %w", err) + } + } else { + img, err = desc.Image() + if err != nil { + return nil, fmt.Errorf("get image: %w", err) + } + } + + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("get layers: %w", err) + } + + for _, layer := range layers { + rc, err := layer.Uncompressed() + if err != nil { + continue + } + + digests, err := extractDigestsFromTar(rc) + rc.Close() + if err == nil && len(digests) > 0 { + return digests, nil + } + } + + return nil, fmt.Errorf("images_digests.json not found") +} + +func extractDigestsFromTar(rc io.Reader) ([]string, error) { + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if strings.HasSuffix(hdr.Name, "deckhouse/candi/images_digests.json") || + hdr.Name == "deckhouse/candi/images_digests.json" { + var digestsByModule map[string]map[string]string + if err := json.NewDecoder(tr).Decode(&digestsByModule); err != nil { + return nil, err + } + + var result []string + for _, images := range digestsByModule { + for _, digest := range images { + result = append(result, digest) + } + } + return result, nil + } + } + return nil, nil +} + +type ModuleInfo struct { + Name string + Versions []string + Digests []string +} + +func (r *SourceReader) listTags(ctx context.Context, repoPath string) ([]string, error) { + ref, err := name.ParseReference(repoPath + ":latest") + if err != nil { + return nil, fmt.Errorf("parse reference: %w", err) + } + + repo := ref.Context() + opts := append(r.opts, remote.WithContext(ctx)) + tags, err := remote.List(repo, opts...) + if err != nil { + return nil, fmt.Errorf("list tags: %w", err) + } + + return tags, nil +} + +func (r *SourceReader) ReadModulesList(ctx context.Context) ([]string, error) { + r.progress("Discovering modules...") + + modulesRef := path.Join(r.registry, d8internal.ModulesSegment) + tags, err := r.listTags(ctx, modulesRef) + if err != nil { + return nil, fmt.Errorf("list modules: %w", err) + } + + r.progress("Found %d modules", len(tags)) + return tags, nil +} + +func (r *SourceReader) ReadModuleDigests(ctx context.Context, moduleName string) (*ModuleInfo, error) { + r.progress("Reading module %s...", moduleName) + + info := &ModuleInfo{ + Name: moduleName, + Versions: make([]string, 0), + Digests: make([]string, 0), + } + + moduleReleaseRef := path.Join(r.registry, d8internal.ModulesSegment, moduleName, "release") + tags, err := r.listTags(ctx, moduleReleaseRef) + if err != nil { + r.progress(" No release tags found") + return info, nil + } + + info.Versions = tags + r.progress(" Found %d release tags", len(tags)) + + return info, nil +} + +type SecurityDigests struct { + Databases map[string][]string +} + +func (r *SourceReader) ReadSecurityDigests(ctx context.Context) (*SecurityDigests, error) { + r.progress("Reading security databases...") + + result := &SecurityDigests{ + Databases: make(map[string][]string), + } + + databases := []string{ + d8internal.SecurityTrivyDBSegment, + d8internal.SecurityTrivyBDUSegment, + d8internal.SecurityTrivyJavaDBSegment, + d8internal.SecurityTrivyChecksSegment, + } + + for _, db := range databases { + dbRef := path.Join(r.registry, d8internal.SecuritySegment, db) + tags, err := r.listTags(ctx, dbRef) + if err != nil { + r.progress(" %s: not found", db) + continue + } + + result.Databases[db] = tags + r.progress(" %s: %d tags", db, len(tags)) + } + + return result, nil +} + +type ExpectedImages struct { + Platform *PlatformDigests + Modules []*ModuleInfo + Security *SecurityDigests +} + +func (r *SourceReader) ReadAllExpected(ctx context.Context) (*ExpectedImages, error) { + result := &ExpectedImages{} + + channels, err := r.ReadReleaseChannels(ctx) + if err != nil { + return nil, fmt.Errorf("read release channels: %w", err) + } + + result.Platform, err = r.ReadPlatformDigests(ctx, channels) + if err != nil { + return nil, fmt.Errorf("read platform digests: %w", err) + } + + modules, err := r.ReadModulesList(ctx) + if err != nil { + r.progress("Warning: failed to read modules: %v", err) + } else { + for _, moduleName := range modules { + info, err := r.ReadModuleDigests(ctx, moduleName) + if err != nil { + r.progress("Warning: failed to read module %s: %v", moduleName, err) + continue + } + result.Modules = append(result.Modules, info) + } + } + + result.Security, err = r.ReadSecurityDigests(ctx) + if err != nil { + r.progress("Warning: failed to read security: %v", err) + } + + return result, nil +} + diff --git a/testing/e2e/mirror/internal/verify.go b/testing/e2e/mirror/internal/verify.go new file mode 100644 index 00000000..5378c73a --- /dev/null +++ b/testing/e2e/mirror/internal/verify.go @@ -0,0 +1,677 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "fmt" + "path" + "strings" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + + d8internal "github.com/deckhouse/deckhouse-cli/internal" +) + +type DigestVerifier struct { + sourceReader *SourceReader + targetReg string + targetAuth authn.Authenticator + targetOpts []remote.Option + progressFn func(string) +} + +func NewDigestVerifier( + sourceReader *SourceReader, + targetReg string, + targetAuth authn.Authenticator, + tlsSkipVerify bool, +) *DigestVerifier { + opts := []remote.Option{remote.WithAuth(targetAuth)} + if tlsSkipVerify { + opts = append(opts, remote.WithTransport(InsecureTransport())) + } + + return &DigestVerifier{ + sourceReader: sourceReader, + targetReg: targetReg, + targetAuth: targetAuth, + targetOpts: opts, + } +} + +func (v *DigestVerifier) SetProgressCallback(fn func(string)) { + v.progressFn = fn + v.sourceReader.SetProgressCallback(fn) +} + +func (v *DigestVerifier) logProgressf(format string, args ...interface{}) { + if v.progressFn != nil { + v.progressFn(fmt.Sprintf(format, args...)) + } +} + +type VerificationResult struct { + StartTime time.Time + EndTime time.Time + + ExpectedDigests []string + ExpectedAttTags []string + ReleaseChannels []ReleaseChannelInfo + Versions []string + + FoundDigests []string + MissingDigests []string + FoundAttTags []string + MissingAttTags []string + + ModulesExpected int + ModulesFound int + ModulesMissing []string + + SecurityExpected int + SecurityFound int + SecurityMissing []string + + Errors []string + + TotalExpected int + TotalFound int + TotalMissing int +} + +func (r *VerificationResult) IsSuccess() bool { + return len(r.MissingDigests) == 0 && + len(r.MissingAttTags) == 0 && + len(r.ModulesMissing) == 0 && + len(r.SecurityMissing) == 0 +} + +func (r *VerificationResult) Summary() string { + var sb strings.Builder + + sb.WriteString("VERIFICATION SUMMARY\n") + sb.WriteString("====================\n\n") + + sb.WriteString(fmt.Sprintf("Duration: %v\n", r.EndTime.Sub(r.StartTime).Round(time.Second))) + sb.WriteString(fmt.Sprintf("Release channels: %d\n", len(r.ReleaseChannels))) + sb.WriteString(fmt.Sprintf("Versions: %d\n", len(r.Versions))) + sb.WriteString("\n") + + sb.WriteString("EXPECTED FROM SOURCE:\n") + sb.WriteString(fmt.Sprintf(" Platform digests: %d\n", len(r.ExpectedDigests))) + sb.WriteString(fmt.Sprintf(" Attestation tags (.att): %d\n", len(r.ExpectedAttTags))) + if r.ModulesExpected > 0 { + sb.WriteString(fmt.Sprintf(" Modules: %d\n", r.ModulesExpected)) + } + if r.SecurityExpected > 0 { + sb.WriteString(fmt.Sprintf(" Security databases: %d\n", r.SecurityExpected)) + } + sb.WriteString("\n") + + sb.WriteString("VERIFICATION RESULTS:\n") + sb.WriteString(fmt.Sprintf(" ✓ Platform digests: %d / %d\n", len(r.FoundDigests), len(r.ExpectedDigests))) + sb.WriteString(fmt.Sprintf(" ✗ Missing digests: %d\n", len(r.MissingDigests))) + sb.WriteString(fmt.Sprintf(" ✓ Attestation tags: %d / %d\n", len(r.FoundAttTags), len(r.ExpectedAttTags))) + sb.WriteString(fmt.Sprintf(" ✗ Missing .att tags: %d\n", len(r.MissingAttTags))) + if r.ModulesExpected > 0 { + sb.WriteString(fmt.Sprintf(" ✓ Modules: %d / %d\n", r.ModulesFound, r.ModulesExpected)) + if len(r.ModulesMissing) > 0 { + sb.WriteString(fmt.Sprintf(" ✗ Missing modules: %d\n", len(r.ModulesMissing))) + } + } + if r.SecurityExpected > 0 { + sb.WriteString(fmt.Sprintf(" ✓ Security databases: %d / %d\n", r.SecurityFound, r.SecurityExpected)) + if len(r.SecurityMissing) > 0 { + sb.WriteString(fmt.Sprintf(" ✗ Missing security: %d\n", len(r.SecurityMissing))) + } + } + sb.WriteString("\n") + + if r.IsSuccess() { + sb.WriteString("STATUS: ✓ PASSED - All expected images found in target\n") + } else { + sb.WriteString("STATUS: ✗ FAILED - Some images missing in target\n") + } + + return sb.String() +} + +func (r *VerificationResult) DetailedReport() string { + var sb strings.Builder + + sb.WriteString(r.Summary()) + sb.WriteString("\n") + + sb.WriteString("RELEASE CHANNELS:\n") + for _, ch := range r.ReleaseChannels { + sb.WriteString(fmt.Sprintf(" %s -> %s\n", ch.Channel, ch.Version)) + } + sb.WriteString("\n") + + if len(r.MissingDigests) > 0 { + sb.WriteString("MISSING DIGESTS:\n") + for _, d := range r.MissingDigests { + sb.WriteString(fmt.Sprintf(" - %s\n", d)) + } + sb.WriteString("\n") + } + + if len(r.MissingAttTags) > 0 { + sb.WriteString("MISSING ATTESTATION TAGS:\n") + for _, t := range r.MissingAttTags { + sb.WriteString(fmt.Sprintf(" - %s\n", t)) + } + sb.WriteString("\n") + } + + if len(r.ModulesMissing) > 0 { + sb.WriteString("MISSING MODULES:\n") + for _, m := range r.ModulesMissing { + sb.WriteString(fmt.Sprintf(" - %s\n", m)) + } + sb.WriteString("\n") + } + + if len(r.SecurityMissing) > 0 { + sb.WriteString("MISSING SECURITY DATABASES:\n") + for _, s := range r.SecurityMissing { + sb.WriteString(fmt.Sprintf(" - %s\n", s)) + } + sb.WriteString("\n") + } + + if len(r.Errors) > 0 { + sb.WriteString("ERRORS:\n") + for _, e := range r.Errors { + sb.WriteString(fmt.Sprintf(" - %s\n", e)) + } + sb.WriteString("\n") + } + + return sb.String() +} + +func (v *DigestVerifier) VerifyPlatform(ctx context.Context, deckhouseTag string) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + v.logProgressf("Reading release channels from source...") + var channels []ReleaseChannelInfo + var err error + + if deckhouseTag != "" { + v.logProgressf(" Using specified tag: %s", deckhouseTag) + channels = []ReleaseChannelInfo{{Channel: deckhouseTag, Version: deckhouseTag}} + } else { + channels, err = v.sourceReader.ReadReleaseChannels(ctx) + if err != nil { + return nil, fmt.Errorf("read release channels: %w", err) + } + } + result.ReleaseChannels = channels + v.logProgressf(" Found %d release channels", len(channels)) + + v.logProgressf("Reading platform digests from install images...") + platformDigests, err := v.sourceReader.ReadPlatformDigests(ctx, channels) + if err != nil { + return nil, fmt.Errorf("read platform digests: %w", err) + } + + result.Versions = platformDigests.Versions + result.ExpectedDigests = platformDigests.ImageDigests + v.logProgressf(" Found %d unique digests across %d versions", len(result.ExpectedDigests), len(result.Versions)) + + if len(result.ExpectedDigests) == 0 { + tag := "unknown" + if len(channels) > 0 { + tag = channels[0].Version + } + return nil, fmt.Errorf("found 0 platform digests - verification cannot proceed (check install image for tag %s)", tag) + } + + v.logProgressf("Finding .att tags for expected digests...") + allAttTags := v.getAttTagsFromSource(ctx) + result.ExpectedAttTags = v.filterAttTagsForDigests(allAttTags, result.ExpectedDigests) + v.logProgressf(" Found %d .att tags in source (%d total, %d match our digests)", + len(result.ExpectedAttTags), len(allAttTags), len(result.ExpectedAttTags)) + + v.logProgressf("Verifying digests in target registry...") + v.verifyDigests(ctx, result) + v.logProgressf(" Found: %d, Missing: %d", len(result.FoundDigests), len(result.MissingDigests)) + + v.logProgressf("Verifying .att tags in target registry...") + v.verifyAttTags(ctx, result) + v.logProgressf(" Found: %d, Missing: %d", len(result.FoundAttTags), len(result.MissingAttTags)) + + result.EndTime = time.Now() + result.TotalExpected = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + result.TotalFound = len(result.FoundDigests) + len(result.FoundAttTags) + result.TotalMissing = len(result.MissingDigests) + len(result.MissingAttTags) + + return result, nil +} + + +func (v *DigestVerifier) VerifyModules(ctx context.Context, moduleNames []string) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + v.logProgressf("Getting module list...") + var modules []string + var err error + + if len(moduleNames) > 0 && moduleNames[0] != "" { + modules = moduleNames + } else { + modules, err = v.sourceReader.ReadModulesList(ctx) + if err != nil { + return nil, fmt.Errorf("read modules list: %w", err) + } + } + v.logProgressf(" Found %d modules to verify", len(modules)) + + result.ModulesExpected = len(modules) + + releaseChannels := d8internal.GetAllDefaultReleaseChannels() + sourceOpts := v.sourceReader.RemoteOpts() + + v.logProgressf("Verifying modules...") + + for _, moduleName := range modules { + v.logProgressf(" Checking module: %s", moduleName) + + sourceReleaseRepo := path.Join(v.sourceReader.Registry(), d8internal.ModulesSegment, moduleName, "release") + targetReleaseRepo := path.Join(v.targetReg, d8internal.ModulesSegment, moduleName, "release") + + targetRef, err := name.ParseReference(targetReleaseRepo + ":latest") + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("module %s: invalid target reference: %v", moduleName, err)) + continue + } + + targetOpts := append(v.targetOpts, remote.WithContext(ctx)) + targetTags, err := remote.List(targetRef.Context(), targetOpts...) + if err != nil { + v.logProgressf(" Target: no tags found (module not mirrored)") + result.ModulesMissing = append(result.ModulesMissing, moduleName) + continue + } + + // Filter to only release channels + targetChannels := []string{} + for _, tag := range targetTags { + for _, channel := range releaseChannels { + if tag == channel { + targetChannels = append(targetChannels, tag) + break + } + } + } + + if len(targetChannels) == 0 { + v.logProgressf(" Target: no release channels found (module not properly mirrored)") + result.ModulesMissing = append(result.ModulesMissing, moduleName) + continue + } + + v.logProgressf(" Target has %d channels: %v", len(targetChannels), targetChannels) + + matchedChannels := []string{} + mismatchedChannels := []string{} + sourceNotFoundChannels := []string{} + + for _, channel := range targetChannels { + targetTagRef := targetReleaseRepo + ":" + channel + targetImgRef, err := name.ParseReference(targetTagRef) + if err != nil { + continue + } + + targetDesc, err := remote.Head(targetImgRef, targetOpts...) + if err != nil { + continue // Already listed, should exist + } + targetDigest := targetDesc.Digest.String() + + sourceTagRef := sourceReleaseRepo + ":" + channel + sourceImgRef, err := name.ParseReference(sourceTagRef) + if err != nil { + sourceNotFoundChannels = append(sourceNotFoundChannels, channel) + continue + } + + sourceOptsWithCtx := append(sourceOpts, remote.WithContext(ctx)) + sourceDesc, err := remote.Head(sourceImgRef, sourceOptsWithCtx...) + if err != nil { + // Channel exists in target but not in source - might be removed upstream + v.logProgressf(" ⚠ %s: exists in target but not in source (may be removed upstream)", channel) + sourceNotFoundChannels = append(sourceNotFoundChannels, channel) + continue + } + sourceDigest := sourceDesc.Digest.String() + + if sourceDigest != targetDigest { + v.logProgressf(" ✗ %s: DIGEST MISMATCH!", channel) + v.logProgressf(" Source: %s", sourceDigest) + v.logProgressf(" Target: %s", targetDigest) + mismatchedChannels = append(mismatchedChannels, channel) + result.Errors = append(result.Errors, + fmt.Sprintf("module %s/%s: digest mismatch (source=%s, target=%s)", + moduleName, channel, sourceDigest[:19], targetDigest[:19])) + } else { + v.logProgressf(" ✓ %s: digest match %s", channel, sourceDigest[:19]) + matchedChannels = append(matchedChannels, channel) + } + } + + if len(mismatchedChannels) > 0 { + v.logProgressf(" Result: %d matched, %d MISMATCHED, %d source-not-found", + len(matchedChannels), len(mismatchedChannels), len(sourceNotFoundChannels)) + result.ModulesFound++ + } else if len(matchedChannels) > 0 { + v.logProgressf(" ✓ All %d channels verified with matching digests", len(matchedChannels)) + result.ModulesFound++ + } else if len(sourceNotFoundChannels) > 0 { + v.logProgressf(" ⚠ All %d channels exist only in target (removed from source?)", len(sourceNotFoundChannels)) + result.ModulesFound++ + } + } + + result.EndTime = time.Now() + return result, nil +} + +func (v *DigestVerifier) VerifySecurity(ctx context.Context) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + expectedTags := map[string]string{ + d8internal.SecurityTrivyDBSegment: "2", + d8internal.SecurityTrivyBDUSegment: "1", + d8internal.SecurityTrivyJavaDBSegment: "1", + d8internal.SecurityTrivyChecksSegment: "0", + } + + result.SecurityExpected = len(expectedTags) + sourceOpts := v.sourceReader.RemoteOpts() + sourceReg := v.sourceReader.Registry() + + v.logProgressf("Verifying security databases...") + + for db, expectedTag := range expectedTags { + v.logProgressf(" Checking %s:%s...", db, expectedTag) + + sourceRef := path.Join(sourceReg, d8internal.SecuritySegment, db) + ":" + expectedTag + sourceImgRef, err := name.ParseReference(sourceRef) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("db %s: invalid source reference: %v", db, err)) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + + sourceOptsWithCtx := append(sourceOpts, remote.WithContext(ctx)) + sourceDesc, err := remote.Head(sourceImgRef, sourceOptsWithCtx...) + if err != nil { + v.logProgressf(" ⚠ Cannot read source %s:%s: %v (skipping)", db, expectedTag, err) + result.SecurityExpected-- + continue + } + sourceDigest := sourceDesc.Digest.String() + v.logProgressf(" Source digest: %s", sourceDigest[:19]) + + targetRef := path.Join(v.targetReg, d8internal.SecuritySegment, db) + ":" + expectedTag + targetImgRef, err := name.ParseReference(targetRef) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("db %s: invalid target reference: %v", db, err)) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + + targetOpts := append(v.targetOpts, remote.WithContext(ctx)) + targetDesc, err := remote.Head(targetImgRef, targetOpts...) + if err != nil { + v.logProgressf(" ✗ %s:%s NOT FOUND in target", db, expectedTag) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + targetDigest := targetDesc.Digest.String() + + if sourceDigest != targetDigest { + v.logProgressf(" ✗ %s:%s DIGEST MISMATCH!", db, expectedTag) + v.logProgressf(" Source: %s", sourceDigest) + v.logProgressf(" Target: %s", targetDigest) + result.Errors = append(result.Errors, + fmt.Sprintf("db %s: digest mismatch (source=%s, target=%s)", + db, sourceDigest[:19], targetDigest[:19])) + result.SecurityMissing = append(result.SecurityMissing, db) + continue + } + + v.logProgressf(" ✓ %s:%s digest match %s", db, expectedTag, sourceDigest[:19]) + result.SecurityFound++ + } + + result.EndTime = time.Now() + return result, nil +} + +type verifyItem struct { + ref string + onErr func(string, error) + onFound func(string) + onMissing func(string) +} + +func (v *DigestVerifier) verifyInParallel(ctx context.Context, items []verifyItem) { + var wg sync.WaitGroup + var mu sync.Mutex + sem := make(chan struct{}, 10) + + for _, item := range items { + wg.Add(1) + go func(it verifyItem) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + imgRef, err := name.ParseReference(it.ref) + if err != nil { + mu.Lock() + it.onErr(it.ref, err) + mu.Unlock() + return + } + + opts := append(v.targetOpts, remote.WithContext(ctx)) + _, err = remote.Head(imgRef, opts...) + mu.Lock() + if err != nil { + it.onMissing(it.ref) + } else { + it.onFound(it.ref) + } + mu.Unlock() + }(item) + } + + wg.Wait() +} + +func (v *DigestVerifier) verifyDigests(ctx context.Context, result *VerificationResult) { + items := make([]verifyItem, 0, len(result.ExpectedDigests)) + for _, digest := range result.ExpectedDigests { + digest := digest // capture loop variable + ref := v.targetReg + "@" + digest + items = append(items, verifyItem{ + ref: ref, + onErr: func(ref string, err error) { + result.Errors = append(result.Errors, fmt.Sprintf("invalid digest ref %s: %v", digest, err)) + }, + onFound: func(ref string) { + result.FoundDigests = append(result.FoundDigests, digest) + }, + onMissing: func(ref string) { + result.MissingDigests = append(result.MissingDigests, digest) + }, + }) + } + v.verifyInParallel(ctx, items) +} + +func (v *DigestVerifier) getAttTagsFromSource(ctx context.Context) []string { + sourceReg := v.sourceReader.Registry() + sourceOpts := append(v.sourceReader.RemoteOpts(), remote.WithContext(ctx)) + + ref, err := name.ParseReference(sourceReg + ":latest") + if err != nil { + v.logProgressf(" Warning: failed to parse source registry: %v", err) + return nil + } + + repo := ref.Context() + tags, err := remote.List(repo, sourceOpts...) + if err != nil { + v.logProgressf(" Warning: failed to list source tags: %v", err) + return nil + } + + var attTags []string + for _, tag := range tags { + if strings.HasSuffix(tag, ".att") { + attTags = append(attTags, tag) + } + } + + return attTags +} + +func (v *DigestVerifier) filterAttTagsForDigests(attTags []string, digests []string) []string { + expectedPrefixes := make(map[string]bool) + for _, digest := range digests { + if strings.HasPrefix(digest, "sha256:") { + hash := strings.TrimPrefix(digest, "sha256:") + prefix := "sha256-" + hash + expectedPrefixes[prefix] = true + } else { + digestPreview := digest + if len(digest) > 20 { + digestPreview = digest[:20] + } + v.logProgressf(" Warning: non-sha256 digest found: %s (skipping .att tag matching)", digestPreview) + } + } + + var filtered []string + for _, tag := range attTags { + if strings.HasSuffix(tag, ".att") { + prefix := strings.TrimSuffix(tag, ".att") + if expectedPrefixes[prefix] { + filtered = append(filtered, tag) + } + } + } + + return filtered +} + +func (v *DigestVerifier) verifyAttTags(ctx context.Context, result *VerificationResult) { + items := make([]verifyItem, 0, len(result.ExpectedAttTags)) + for _, attTag := range result.ExpectedAttTags { + attTag := attTag // capture loop variable + ref := v.targetReg + ":" + attTag + items = append(items, verifyItem{ + ref: ref, + onErr: func(ref string, err error) { + result.Errors = append(result.Errors, fmt.Sprintf("invalid att ref %s: %v", attTag, err)) + }, + onFound: func(ref string) { + result.FoundAttTags = append(result.FoundAttTags, attTag) + }, + onMissing: func(ref string) { + result.MissingAttTags = append(result.MissingAttTags, attTag) + }, + }) + } + v.verifyInParallel(ctx, items) +} + +func (v *DigestVerifier) VerifyFull(ctx context.Context, deckhouseTag string, moduleNames []string) (*VerificationResult, error) { + result := &VerificationResult{ + StartTime: time.Now(), + } + + v.logProgressf("=== PLATFORM VERIFICATION ===") + platformResult, err := v.VerifyPlatform(ctx, deckhouseTag) + if err != nil { + return nil, fmt.Errorf("platform verification: %w", err) + } + mergeResults(result, platformResult) + + v.logProgressf("\n=== MODULES VERIFICATION ===") + modulesResult, err := v.VerifyModules(ctx, moduleNames) + if err != nil { + return nil, fmt.Errorf("modules verification: %w", err) + } + mergeResults(result, modulesResult) + + v.logProgressf("\n=== SECURITY VERIFICATION ===") + securityResult, err := v.VerifySecurity(ctx) + if err != nil { + return nil, fmt.Errorf("security verification: %w", err) + } + mergeResults(result, securityResult) + + result.EndTime = time.Now() + result.TotalExpected = len(result.ExpectedDigests) + len(result.ExpectedAttTags) + + result.ModulesExpected + result.SecurityExpected + result.TotalFound = len(result.FoundDigests) + len(result.FoundAttTags) + + result.ModulesFound + result.SecurityFound + result.TotalMissing = len(result.MissingDigests) + len(result.MissingAttTags) + + len(result.ModulesMissing) + len(result.SecurityMissing) + + return result, nil +} + +func mergeResults(dst, src *VerificationResult) { + dst.ExpectedDigests = append(dst.ExpectedDigests, src.ExpectedDigests...) + dst.ExpectedAttTags = append(dst.ExpectedAttTags, src.ExpectedAttTags...) + dst.ReleaseChannels = append(dst.ReleaseChannels, src.ReleaseChannels...) + dst.Versions = append(dst.Versions, src.Versions...) + dst.FoundDigests = append(dst.FoundDigests, src.FoundDigests...) + dst.MissingDigests = append(dst.MissingDigests, src.MissingDigests...) + dst.FoundAttTags = append(dst.FoundAttTags, src.FoundAttTags...) + dst.MissingAttTags = append(dst.MissingAttTags, src.MissingAttTags...) + + dst.ModulesExpected += src.ModulesExpected + dst.ModulesFound += src.ModulesFound + dst.ModulesMissing = append(dst.ModulesMissing, src.ModulesMissing...) + + dst.SecurityExpected += src.SecurityExpected + dst.SecurityFound += src.SecurityFound + dst.SecurityMissing = append(dst.SecurityMissing, src.SecurityMissing...) + + dst.Errors = append(dst.Errors, src.Errors...) +} + diff --git a/testing/e2e/mirror/mirror_e2e_test.go b/testing/e2e/mirror/mirror_e2e_test.go deleted file mode 100644 index 003cc76a..00000000 --- a/testing/e2e/mirror/mirror_e2e_test.go +++ /dev/null @@ -1,410 +0,0 @@ -//go:build e2e - -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "context" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/require" - - mirrorutil "github.com/deckhouse/deckhouse-cli/testing/util/mirror" -) - -// ============================================================================= -// E2E Test: Full Mirror Cycle -// ============================================================================= - -// TestMirrorE2E_FullCycle performs a complete mirror cycle and validates -// that source and target registries are identical. -// -// This is a heavy E2E test that: -// 1. Discovers all repositories in source registry -// 2. Pulls all images to local bundle using d8 mirror pull -// 3. Pushes bundle to target registry using d8 mirror push -// 4. Discovers all repositories in target registry -// 5. Compares every tag and digest between source and target -// 6. Generates detailed comparison report -// -// Run with: -// -// go test -v ./testing/e2e/mirror/... \ -// -source-registry=localhost:443/deckhouse \ -// -source-user=admin -source-password=admin \ -// -tls-skip-verify -func TestMirrorE2E_FullCycle(t *testing.T) { - cfg := GetConfig() - - // Skip if no auth provided - if !cfg.HasSourceAuth() { - t.Skip("Source authentication not provided (use -license-token or -source-user/-source-password)") - } - - // Setup test environment - env := setupTestEnvironment(t, cfg) - defer env.Cleanup() - - // Print header - printTestHeader("Mirror Full Cycle", cfg.SourceRegistry, env.LogDir) - - // Run test steps - runFullCycleTest(t, cfg, env) -} - -// ============================================================================= -// Test Environment -// ============================================================================= - -// testEnv holds all test environment state -type testEnv struct { - LogDir string - LogFile string - ReportFile string - ComparisonFile string - BundleDir string - TargetRegistry string - Report *TestReport - Cleanup func() -} - -// setupTestEnvironment prepares everything needed for the test -func setupTestEnvironment(t *testing.T, cfg *Config) *testEnv { - t.Helper() - - // Create log directory - logDir := getLogDir("fullcycle") - require.NoError(t, os.MkdirAll(logDir, 0755)) - - // Setup target registry - targetHost, targetPath, registryCleanup := setupTargetRegistry(t, cfg) - targetRegistry := targetHost + targetPath - t.Logf("Target registry: %s", targetRegistry) - - // Setup bundle directory - bundleDir := setupBundleDir(t, cfg) - - // Initialize report - report := &TestReport{ - TestName: "TestMirrorE2E_FullCycle", - StartTime: time.Now(), - SourceRegistry: cfg.SourceRegistry, - TargetRegistry: targetRegistry, - LogDir: logDir, - } - - env := &testEnv{ - LogDir: logDir, - LogFile: filepath.Join(logDir, "test.log"), - ReportFile: filepath.Join(logDir, "report.txt"), - ComparisonFile: filepath.Join(logDir, "comparison.txt"), - BundleDir: bundleDir, - TargetRegistry: targetRegistry, - Report: report, - } - - // Setup cleanup - env.Cleanup = func() { - registryCleanup() - finalizeReport(t, env) - } - - return env -} - -// setupBundleDir creates the bundle directory -func setupBundleDir(t *testing.T, cfg *Config) string { - t.Helper() - - if cfg.KeepBundle { - bundleDir := filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-e2e-%d", time.Now().Unix())) - require.NoError(t, os.MkdirAll(bundleDir, 0755)) - t.Logf("Bundle directory (will be kept): %s", bundleDir) - return bundleDir - } - - bundleDir := t.TempDir() - t.Logf("Bundle directory: %s", bundleDir) - return bundleDir -} - -// setupTargetRegistry sets up the target registry for testing -func setupTargetRegistry(t *testing.T, cfg *Config) (host, path string, cleanup func()) { - t.Helper() - - if cfg.UseInMemoryRegistry() { - reg := mirrorutil.SetupTestRegistry(false) - repoPath := "/deckhouse/ee" - t.Logf("Started test registry at %s%s", reg.Host, repoPath) - return reg.Host, repoPath, reg.Close - } - - return cfg.TargetRegistry, "", func() {} -} - -// finalizeReport writes the final report -func finalizeReport(t *testing.T, env *testEnv) { - t.Helper() - - env.Report.EndTime = time.Now() - env.Report.Print() - - if err := env.Report.WriteToFile(env.ReportFile); err != nil { - t.Logf("Warning: failed to write report: %v", err) - } else { - t.Logf("Report written to: %s", env.ReportFile) - } -} - -// ============================================================================= -// Test Steps -// ============================================================================= - -// runFullCycleTest executes all test steps -func runFullCycleTest(t *testing.T, cfg *Config, env *testEnv) { - ctx, cancel := context.WithTimeout(context.Background(), 120*time.Minute) - defer cancel() - - // Step 1: Analyze source registry - runAnalyzeStep(t, cfg, env) - - // Step 2: Pull images - runPullStep(t, cfg, env) - - // Step 3: Push images - runPushStep(t, cfg, env) - - // Step 4: Compare registries - runCompareStep(t, ctx, cfg, env) - - // Success! - printSuccessBox(env.Report.MatchedImages, env.Report.MatchedLayers) -} - -// ----------------------------------------------------------------------------- -// Step 1: Analyze Source Registry -// ----------------------------------------------------------------------------- - -func runAnalyzeStep(t *testing.T, cfg *Config, env *testEnv) { - t.Helper() - stepStart := time.Now() - printStep(1, "Analyzing source registry") - - comparator := NewRegistryComparator( - cfg.SourceRegistry, "", - cfg.GetSourceAuth(), nil, - cfg.TLSSkipVerify, - ) - comparator.SetProgressCallback(func(msg string) { - t.Logf(" %s", msg) - }) - - repos := comparator.discoverRepositories(cfg.SourceRegistry, comparator.sourceRemoteOpts) - env.Report.SourceRepoCount = len(repos) - env.Report.AddStep( - fmt.Sprintf("Analyze source (%d repos)", len(repos)), - "PASS", time.Since(stepStart), nil, - ) - t.Logf("Source registry: %d repositories", len(repos)) -} - -// ----------------------------------------------------------------------------- -// Step 2: Pull Images -// ----------------------------------------------------------------------------- - -func runPullStep(t *testing.T, cfg *Config, env *testEnv) { - t.Helper() - stepStart := time.Now() - printStep(2, "Pulling images to bundle") - - cmd := buildPullCommand(cfg, env.BundleDir) - t.Logf("Running: %s", cmd.String()) - - err := runCommandWithLog(t, cmd, env.LogFile) - if err != nil { - env.Report.AddStep("Pull images", "FAIL", time.Since(stepStart), err) - require.NoError(t, err, "Pull failed") - } - - // Calculate bundle size - bundleSize := calculateBundleSize(t, env.BundleDir) - env.Report.BundleSize = bundleSize - - env.Report.AddStep( - fmt.Sprintf("Pull images (%.2f GB bundle)", float64(bundleSize)/(1024*1024*1024)), - "PASS", time.Since(stepStart), nil, - ) - t.Logf("Pull completed: %.2f GB total", float64(bundleSize)/(1024*1024*1024)) -} - -// calculateBundleSize returns total size of bundle files -func calculateBundleSize(t *testing.T, bundleDir string) int64 { - t.Helper() - - files, err := os.ReadDir(bundleDir) - require.NoError(t, err) - - var totalSize int64 - for _, f := range files { - if info, err := f.Info(); err == nil { - totalSize += info.Size() - t.Logf(" %s (%.2f MB)", f.Name(), float64(info.Size())/(1024*1024)) - } - } - return totalSize -} - -// ----------------------------------------------------------------------------- -// Step 3: Push Images -// ----------------------------------------------------------------------------- - -func runPushStep(t *testing.T, cfg *Config, env *testEnv) { - t.Helper() - stepStart := time.Now() - printStep(3, "Pushing bundle to target registry") - - cmd := buildPushCommand(cfg, env.BundleDir, env.TargetRegistry) - t.Logf("Running: %s", cmd.String()) - - err := runCommandWithLog(t, cmd, env.LogFile) - if err != nil { - env.Report.AddStep("Push to registry", "FAIL", time.Since(stepStart), err) - require.NoError(t, err, "Push failed") - } - - env.Report.AddStep("Push to registry", "PASS", time.Since(stepStart), nil) - t.Log("Push completed successfully") -} - -// ----------------------------------------------------------------------------- -// Step 4: Compare Registries -// ----------------------------------------------------------------------------- - -func runCompareStep(t *testing.T, ctx context.Context, cfg *Config, env *testEnv) { - t.Helper() - stepStart := time.Now() - printStep(4, "Deep comparison of registries") - - comparator := NewRegistryComparator( - cfg.SourceRegistry, env.TargetRegistry, - cfg.GetSourceAuth(), cfg.GetTargetAuth(), - cfg.TLSSkipVerify, - ) - comparator.SetProgressCallback(func(msg string) { - t.Logf(" %s", msg) - }) - - comparison, err := comparator.Compare(ctx) - if err != nil { - env.Report.AddStep("Deep comparison", "FAIL", time.Since(stepStart), err) - require.NoError(t, err, "Comparison failed") - } - - // Save detailed comparison - saveComparisonReport(t, env.ComparisonFile, comparison) - - // Update report with comparison stats - updateReportWithComparison(env.Report, comparison) - - // Print summary - t.Log("") - t.Log(comparison.Summary()) - - // Check if identical - if !comparison.IsIdentical() { - env.Report.AddStep( - fmt.Sprintf("Deep comparison (%d matched, %d missing, %d mismatched)", - comparison.MatchedImages, - len(comparison.MissingImages), - len(comparison.MismatchedImages)), - "FAIL", time.Since(stepStart), - fmt.Errorf("registries differ: %d missing, %d mismatched", - len(comparison.MissingImages), - len(comparison.MismatchedImages)), - ) - - require.True(t, comparison.IsIdentical(), - "Registries are NOT identical!\n\n%s\n\nSee %s for details", - comparison.Summary(), env.ComparisonFile) - } - - env.Report.AddStep( - fmt.Sprintf("Deep comparison (%d images verified)", comparison.MatchedImages), - "PASS", time.Since(stepStart), nil, - ) -} - -// saveComparisonReport writes the detailed comparison to file -func saveComparisonReport(t *testing.T, path string, comparison *ComparisonReport) { - t.Helper() - - if err := os.WriteFile(path, []byte(comparison.DetailedReport()), 0644); err != nil { - t.Logf("Warning: failed to write comparison file: %v", err) - } else { - t.Logf("Detailed comparison written to: %s", path) - } -} - -// updateReportWithComparison updates test report with comparison results -func updateReportWithComparison(report *TestReport, comparison *ComparisonReport) { - report.SourceImageCount = comparison.TotalSourceImages - report.TargetRepoCount = len(comparison.TargetRepositories) - report.TargetImageCount = comparison.TotalTargetImages - report.MatchedImages = comparison.MatchedImages - report.MissingImages = len(comparison.MissingImages) - report.MismatchedImages = len(comparison.MismatchedImages) - report.SkippedImages = comparison.SkippedImages - report.MatchedLayers = comparison.MatchedLayers - report.MissingLayers = comparison.MissingLayers - report.ComparisonReport = comparison -} - -// ============================================================================= -// Helpers -// ============================================================================= - -// getLogDir returns the log directory path for e2e tests -// Logs are stored in testing/e2e/.logs/-/ -func getLogDir(testName string) string { - projectRoot := findProjectRoot() - timestamp := time.Now().Format("20060102-150405") - return filepath.Join(projectRoot, "testing", "e2e", ".logs", fmt.Sprintf("%s-%s", testName, timestamp)) -} - -// findProjectRoot finds the project root by looking for go.mod -func findProjectRoot() string { - dir, _ := os.Getwd() - - for { - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { - return dir - } - - parent := filepath.Dir(dir) - if parent == dir { - // Fallback to current dir - dir, _ = os.Getwd() - return dir - } - dir = parent - } -} diff --git a/testing/e2e/mirror/modules_test.go b/testing/e2e/mirror/modules_test.go new file mode 100644 index 00000000..90645a55 --- /dev/null +++ b/testing/e2e/mirror/modules_test.go @@ -0,0 +1,84 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestModulesE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + cfg.NoPlatform = true + cfg.NoSecurity = true + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), internal.ModulesTestTimeout) + defer cancel() + + runModulesTest(t, ctx, cfg, env) +} + +func runModulesTest(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + internal.PrintHeader("MODULES E2E TEST") + + internal.PrintStep(1, "Reading expected modules from source") + expectedModules := readExpectedModules(t, ctx, cfg) + + internal.PrintStep(2, "Pulling modules") + runPullStep(t, cfg, env) + + internal.PrintStep(3, "Pushing to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying modules in target") + verifyModulesInTarget(t, ctx, cfg, env, expectedModules) + + fmt.Printf("\n✅ Modules test passed: %d modules\n", len(expectedModules)) +} + +func readExpectedModules(t *testing.T, ctx context.Context, cfg *internal.Config) []string { + t.Helper() + + reader := createSourceReader(t, cfg) + modules, err := reader.ReadModulesList(ctx) + require.NoError(t, err, "Failed to read modules list") + + modules = filterModules(modules, cfg.IncludeModules) + + t.Logf("Expected %d modules: %v", len(modules), modules) + return modules +} + +func verifyModulesInTarget(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expectedModules []string) { + t.Helper() + verifyModulesImages(t, ctx, cfg, env, expectedModules) +} diff --git a/testing/e2e/mirror/output.go b/testing/e2e/mirror/output.go deleted file mode 100644 index 34ad8a7c..00000000 --- a/testing/e2e/mirror/output.go +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "fmt" - "os" - "strings" - "sync" - - "github.com/charmbracelet/lipgloss" - "github.com/muesli/termenv" - "golang.org/x/term" -) - -// ============================================================================= -// Colors -// ============================================================================= - -var ( - colorCyan = lipgloss.Color("6") - colorGreen = lipgloss.Color("2") - colorRed = lipgloss.Color("1") - colorYellow = lipgloss.Color("3") - colorBlue = lipgloss.Color("4") - colorWhite = lipgloss.Color("15") - colorGray = lipgloss.Color("8") -) - -// ============================================================================= -// Text Styles -// ============================================================================= - -var ( - styleTitle = lipgloss.NewStyle().Bold(true).Foreground(colorWhite) - styleHeader = lipgloss.NewStyle().Bold(true).Foreground(colorCyan) - styleLabel = lipgloss.NewStyle().Foreground(colorGray) - styleValue = lipgloss.NewStyle().Foreground(colorWhite) - styleDim = lipgloss.NewStyle().Foreground(colorGray) - styleSuccess = lipgloss.NewStyle().Foreground(colorGreen) - styleError = lipgloss.NewStyle().Foreground(colorRed) -) - -// ============================================================================= -// Badges -// ============================================================================= - -var ( - badgeOK = lipgloss.NewStyle().Bold(true).Foreground(colorGreen).Render("[OK]") - badgeFail = lipgloss.NewStyle().Bold(true).Foreground(colorRed).Render("[FAIL]") - badgeSkip = lipgloss.NewStyle().Foreground(colorYellow).Render("[SKIP]") -) - -// ============================================================================= -// Step Styles -// ============================================================================= - -var ( - styleStepNum = lipgloss.NewStyle().Bold(true).Foreground(colorBlue) - styleStepText = lipgloss.NewStyle().Bold(true).Foreground(colorWhite) -) - -// ============================================================================= -// Output Functions -// ============================================================================= - -// output is the destination for styled output (stderr preserves colors in go test) -var output = os.Stderr - -var colorInitOnce sync.Once - -// ensureColorInit initializes color profile for lipgloss (replaces init()) -func ensureColorInit() { - colorInitOnce.Do(func() { - // Force color output - go test buffers stdout which disables color detection - // Check stderr instead (usually unbuffered) or honor FORCE_COLOR env - if term.IsTerminal(int(os.Stderr.Fd())) || os.Getenv("FORCE_COLOR") != "" || os.Getenv("TERM") != "" { - lipgloss.DefaultRenderer().SetColorProfile(termenv.TrueColor) - } - }) -} - -// writeLinef writes a formatted line to output -func writeLinef(format string, args ...interface{}) { - ensureColorInit() - fmt.Fprintf(output, format+"\n", args...) -} - -// writeRawf writes formatted text to output without newline -func writeRawf(format string, args ...interface{}) { - ensureColorInit() - fmt.Fprintf(output, format, args...) -} - -// printStep prints a formatted step header -func printStep(num int, description string) { - badge := styleStepNum.Render(fmt.Sprintf("[STEP %d]", num)) - text := styleStepText.Render(description) - writeLinef("\n%s %s", badge, text) -} - -// ============================================================================= -// Separators -// ============================================================================= - -var styleSeparator = lipgloss.NewStyle().Foreground(colorCyan) - -const separatorWidth = 80 - -// separator creates a separator line -func separator(char string) string { - return styleSeparator.Render(strings.Repeat(char, separatorWidth)) -} - -// ============================================================================= -// Test Header/Footer -// ============================================================================= - -// printTestHeader prints the test header with configuration info -func printTestHeader(testName, sourceRegistry, logDir string) { - writeLinef("") - writeLinef(separator("═")) - writeLinef(" %s", styleTitle.Render("E2E TEST: "+testName)) - writeLinef(separator("═")) - writeLinef(" %s %s", styleLabel.Render("Source:"), styleValue.Render(sourceRegistry)) - writeLinef(" %s %s", styleLabel.Render("Logs: "), styleDim.Render(logDir)) - writeLinef(separator("═")) - writeLinef("") -} - -// printSuccessBox prints a success message in a styled box -func printSuccessBox(matchedImages, matchedLayers int) { - box := lipgloss.NewStyle(). - Border(lipgloss.DoubleBorder()). - BorderForeground(colorGreen). - Padding(0, 2). - Foreground(colorGreen) - - writeLinef("") - writeLinef(box.Render(fmt.Sprintf( - "SUCCESS: REGISTRIES ARE IDENTICAL\n\nVerified: %d images, %d layers\nAll manifest, config, and layer digests match!", - matchedImages, - matchedLayers, - ))) -} diff --git a/testing/e2e/mirror/platform_test.go b/testing/e2e/mirror/platform_test.go new file mode 100644 index 00000000..47187e39 --- /dev/null +++ b/testing/e2e/mirror/platform_test.go @@ -0,0 +1,118 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestPlatformE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + cfg.NoModules = true + cfg.NoSecurity = true + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), internal.PlatformTestTimeout) + defer cancel() + + runPlatformTest(t, ctx, cfg, env) +} + +func runPlatformTest(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + internal.PrintHeader("PLATFORM E2E TEST") + + internal.PrintStep(1, "Reading expected platform images from source") + expected := readExpectedPlatformImages(t, ctx, cfg) + + if cfg.HasExistingBundle() { + t.Logf("Using existing bundle: %s (skipping pull)", env.BundleDir) + env.Report.AddStep("Pull (existing bundle)", "SKIP", 0, nil) + } else { + internal.PrintStep(2, "Pulling platform images") + runPullStep(t, cfg, env) + } + + internal.PrintStep(3, "Pushing to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying expected images in target") + verifyExpectedInTarget(t, ctx, cfg, env, expected) + + internal.PrintSuccessBox(env.Report.MatchedImages, env.Report.FoundAttTags) +} + +type ExpectedPlatformImages struct { + Channels []internal.ReleaseChannelInfo + Versions []string + Digests []string +} + +func readExpectedPlatformImages(t *testing.T, ctx context.Context, cfg *internal.Config) *ExpectedPlatformImages { + t.Helper() + + reader := createSourceReader(t, cfg) + result := &ExpectedPlatformImages{} + + channels, err := reader.ReadReleaseChannels(ctx) + require.NoError(t, err, "Failed to read release channels") + result.Channels = channels + + t.Logf("Found %d release channels:", len(channels)) + for _, ch := range channels { + t.Logf(" %s -> %s", ch.Channel, ch.Version) + } + + if cfg.DeckhouseTag != "" { + for _, ch := range channels { + if ch.Channel == cfg.DeckhouseTag { + channels = []internal.ReleaseChannelInfo{ch} + break + } + } + } + + platform, err := reader.ReadPlatformDigests(ctx, channels) + require.NoError(t, err, "Failed to read platform digests") + + result.Versions = platform.Versions + result.Digests = platform.ImageDigests + + t.Logf("Expected: %d versions, %d digests", len(result.Versions), len(result.Digests)) + + return result +} + +func verifyExpectedInTarget(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expected *ExpectedPlatformImages) { + t.Helper() + verifyPlatformImages(t, ctx, cfg, env, cfg.DeckhouseTag) + t.Logf("Platform verification passed: %d digests, %d .att tags", env.Report.MatchedImages, env.Report.FoundAttTags) +} + diff --git a/testing/e2e/mirror/registry_comparator.go b/testing/e2e/mirror/registry_comparator.go deleted file mode 100644 index bf1ff286..00000000 --- a/testing/e2e/mirror/registry_comparator.go +++ /dev/null @@ -1,929 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "context" - "crypto/tls" - "fmt" - "net/http" - "path" - "regexp" - "sort" - "strings" - "sync" - "time" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - - "github.com/deckhouse/deckhouse-cli/internal" -) - -// Tags to exclude from comparison (not mirrored by design) -var ( - // Digest-based tags (sha256 hashes used as tags) - digestTagRegex = regexp.MustCompile(`^[a-f0-9]{64}$`) - // SHA256 prefixed tags - sha256TagRegex = regexp.MustCompile(`^sha256-[a-f0-9]{64}`) - // Cosign signature and attestation tags - cosignTagSuffixes = []string{".sig", ".att", ".sbom"} - // Service tags - serviceTags = []string{"d8WriteCheck"} -) - -// shouldSkipTag returns true if the tag should be excluded from comparison -func shouldSkipTag(tag string) bool { - // Skip digest-based tags - if digestTagRegex.MatchString(tag) { - return true - } - // Skip sha256- prefixed tags - if sha256TagRegex.MatchString(tag) { - return true - } - // Skip cosign tags - for _, suffix := range cosignTagSuffixes { - if strings.HasSuffix(tag, suffix) { - return true - } - } - // Skip service tags - for _, svcTag := range serviceTags { - if tag == svcTag { - return true - } - } - return false -} - -// ImageInfo contains detailed information about an image -type ImageInfo struct { - Reference string - Digest string // manifest digest - ConfigDigest string // config digest - Layers []string // layer digests - TotalSize int64 // total size in bytes -} - -// RegistryComparator performs deep comparison between source and target registries -type RegistryComparator struct { - sourceRegistry string - targetRegistry string - sourceAuth authn.Authenticator - targetAuth authn.Authenticator - - nameOpts []name.Option - sourceRemoteOpts []remote.Option - targetRemoteOpts []remote.Option - - // Progress callback - onProgress func(msg string) -} - -// NewRegistryComparator creates a new registry comparator -func NewRegistryComparator( - sourceRegistry, targetRegistry string, - sourceAuth, targetAuth authn.Authenticator, - tlsSkipVerify bool, -) *RegistryComparator { - nameOpts := []name.Option{} - sourceRemoteOpts := []remote.Option{} - targetRemoteOpts := []remote.Option{} - - if tlsSkipVerify { - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - sourceRemoteOpts = append(sourceRemoteOpts, remote.WithTransport(transport)) - targetRemoteOpts = append(targetRemoteOpts, remote.WithTransport(transport)) - } - - if sourceAuth != nil && sourceAuth != authn.Anonymous { - sourceRemoteOpts = append(sourceRemoteOpts, remote.WithAuth(sourceAuth)) - } - if targetAuth != nil && targetAuth != authn.Anonymous { - targetRemoteOpts = append(targetRemoteOpts, remote.WithAuth(targetAuth)) - } - - return &RegistryComparator{ - sourceRegistry: sourceRegistry, - targetRegistry: targetRegistry, - sourceAuth: sourceAuth, - targetAuth: targetAuth, - nameOpts: nameOpts, - sourceRemoteOpts: sourceRemoteOpts, - targetRemoteOpts: targetRemoteOpts, - } -} - -// SetProgressCallback sets a callback for progress updates -func (c *RegistryComparator) SetProgressCallback(fn func(msg string)) { - c.onProgress = fn -} - -func (c *RegistryComparator) logProgressf(format string, args ...interface{}) { - if c.onProgress != nil { - c.onProgress(fmt.Sprintf(format, args...)) - } -} - -// ComparisonReport contains detailed comparison results -type ComparisonReport struct { - StartTime time.Time - EndTime time.Time - - SourceRegistry string - TargetRegistry string - - // Repository-level stats - SourceRepositories []string - TargetRepositories []string - MissingRepositories []string // In source but not in target - ExtraRepositories []string // In target but not in source - - // Tag-level stats per repository - RepositoryDetails map[string]*RepositoryComparison - - // Image-level stats - TotalSourceImages int - TotalTargetImages int - SkippedImages int // Digest-based, .att, .sig tags (not mirrored by design) - MatchedImages int - MismatchedImages []ImageMismatch - MissingImages []string // ref in source but not in target - ExtraImages []string // ref in target but not in source - - // Deep comparison stats - DeepCheckedImages int - TotalSourceLayers int - TotalTargetLayers int - MatchedLayers int - MissingLayers int - ConfigMismatches int - LayerMismatches []LayerMismatch -} - -// RepositoryComparison holds comparison for a single repository -type RepositoryComparison struct { - Repository string - SourceTags []string - TargetTags []string - MissingTags []string // In source but not in target - ExtraTags []string // In target but not in source - SkippedTags int // Tags skipped (digest-based, .att, .sig, etc.) - MatchedTags int - TagDetails map[string]*TagComparison -} - -// TagComparison holds comparison for a single tag -type TagComparison struct { - Tag string - SourceDigest string - TargetDigest string - Match bool - SourceConfig string - TargetConfig string - ConfigMatch bool - SourceLayers []string - TargetLayers []string - MissingLayers []string - ExtraLayers []string - LayersMatch bool - DeepChecked bool // true if layers were compared -} - -// ImageMismatch represents a digest mismatch for an image -type ImageMismatch struct { - Reference string - SourceDigest string - TargetDigest string -} - -// LayerMismatch represents a missing or different layer -type LayerMismatch struct { - Reference string - LayerDigest string - Reason string // "missing", "size_mismatch" -} - -// IsIdentical returns true if registries are identical -func (r *ComparisonReport) IsIdentical() bool { - return len(r.MissingRepositories) == 0 && - len(r.MissingImages) == 0 && - len(r.MismatchedImages) == 0 && - len(r.LayerMismatches) == 0 && - r.MissingLayers == 0 && - r.ConfigMismatches == 0 -} - -// Summary returns a summary string -func (r *ComparisonReport) Summary() string { - var sb strings.Builder - - sb.WriteString("REGISTRY COMPARISON SUMMARY\n") - sb.WriteString("===========================\n\n") - - sb.WriteString(fmt.Sprintf("Source: %s\n", r.SourceRegistry)) - sb.WriteString(fmt.Sprintf("Target: %s\n", r.TargetRegistry)) - sb.WriteString(fmt.Sprintf("Duration: %s\n\n", r.EndTime.Sub(r.StartTime).Round(time.Second))) - - sb.WriteString("REPOSITORIES:\n") - sb.WriteString(fmt.Sprintf(" Source: %d\n", len(r.SourceRepositories))) - sb.WriteString(fmt.Sprintf(" Target: %d\n", len(r.TargetRepositories))) - sb.WriteString(fmt.Sprintf(" Missing in target: %d\n", len(r.MissingRepositories))) - sb.WriteString(fmt.Sprintf(" Extra in target: %d\n\n", len(r.ExtraRepositories))) - - sb.WriteString("IMAGES TO VERIFY:\n") - sb.WriteString(fmt.Sprintf(" Source: %d images\n", r.TotalSourceImages)) - sb.WriteString(fmt.Sprintf(" Target: %d images\n", r.TotalTargetImages)) - if r.SkippedImages > 0 { - sb.WriteString(fmt.Sprintf(" (excluded %d internal tags: digest-based, .att, .sig)\n", r.SkippedImages)) - } - sb.WriteString("\n") - sb.WriteString("VERIFICATION RESULTS:\n") - sb.WriteString(fmt.Sprintf(" Matched: %d\n", r.MatchedImages)) - if len(r.MissingImages) > 0 { - sb.WriteString(fmt.Sprintf(" Missing in target: %d\n", len(r.MissingImages))) - } - if len(r.MismatchedImages) > 0 { - sb.WriteString(fmt.Sprintf(" Digest mismatch: %d\n", len(r.MismatchedImages))) - } - if len(r.ExtraImages) > 0 { - sb.WriteString(fmt.Sprintf(" Extra in target: %d\n", len(r.ExtraImages))) - } - - if r.DeepCheckedImages > 0 { - sb.WriteString("DEEP COMPARISON (layers + config):\n") - sb.WriteString(fmt.Sprintf(" Images deep-checked: %d\n", r.DeepCheckedImages)) - sb.WriteString(fmt.Sprintf(" Source layers: %d\n", r.TotalSourceLayers)) - sb.WriteString(fmt.Sprintf(" Target layers: %d\n", r.TotalTargetLayers)) - sb.WriteString(fmt.Sprintf(" Matched layers: %d\n", r.MatchedLayers)) - sb.WriteString(fmt.Sprintf(" Missing layers: %d\n", r.MissingLayers)) - sb.WriteString(fmt.Sprintf(" Config mismatches: %d\n\n", r.ConfigMismatches)) - } - - if r.IsIdentical() { - sb.WriteString("✓ REGISTRIES ARE IDENTICAL (all hashes match)\n") - } else { - sb.WriteString("✗ REGISTRIES DIFFER\n") - } - - return sb.String() -} - -// DetailedReport returns a detailed report string -func (r *ComparisonReport) DetailedReport() string { - var sb strings.Builder - - sb.WriteString(r.Summary()) - sb.WriteString("\n") - - // Missing repositories - if len(r.MissingRepositories) > 0 { - sb.WriteString("MISSING REPOSITORIES:\n") - for _, repo := range r.MissingRepositories { - sb.WriteString(fmt.Sprintf(" - %s\n", repo)) - } - sb.WriteString("\n") - } - - // Missing images (limited to first 100) - if len(r.MissingImages) > 0 { - sb.WriteString(fmt.Sprintf("MISSING IMAGES (%d total):\n", len(r.MissingImages))) - limit := min(100, len(r.MissingImages)) - for i := 0; i < limit; i++ { - sb.WriteString(fmt.Sprintf(" - %s\n", r.MissingImages[i])) - } - if len(r.MissingImages) > limit { - sb.WriteString(fmt.Sprintf(" ... and %d more\n", len(r.MissingImages)-limit)) - } - sb.WriteString("\n") - } - - // Mismatched images (limited to first 50) - if len(r.MismatchedImages) > 0 { - sb.WriteString(fmt.Sprintf("MISMATCHED DIGESTS (%d total):\n", len(r.MismatchedImages))) - limit := min(50, len(r.MismatchedImages)) - for i := 0; i < limit; i++ { - m := r.MismatchedImages[i] - sb.WriteString(fmt.Sprintf(" %s\n", m.Reference)) - sb.WriteString(fmt.Sprintf(" source: %s\n", m.SourceDigest)) - sb.WriteString(fmt.Sprintf(" target: %s\n", m.TargetDigest)) - } - if len(r.MismatchedImages) > limit { - sb.WriteString(fmt.Sprintf(" ... and %d more\n", len(r.MismatchedImages)-limit)) - } - sb.WriteString("\n") - } - - // Layer mismatches - if len(r.LayerMismatches) > 0 { - sb.WriteString(fmt.Sprintf("LAYER MISMATCHES (%d total):\n", len(r.LayerMismatches))) - limit := min(50, len(r.LayerMismatches)) - for i := 0; i < limit; i++ { - m := r.LayerMismatches[i] - sb.WriteString(fmt.Sprintf(" %s\n", m.Reference)) - sb.WriteString(fmt.Sprintf(" layer: %s (%s)\n", m.LayerDigest, m.Reason)) - } - if len(r.LayerMismatches) > limit { - sb.WriteString(fmt.Sprintf(" ... and %d more\n", len(r.LayerMismatches)-limit)) - } - sb.WriteString("\n") - } - - // Per-repository breakdown - sb.WriteString("REPOSITORY BREAKDOWN:\n") - sb.WriteString("---------------------\n") - - // Sort repositories for consistent output - repos := make([]string, 0, len(r.RepositoryDetails)) - for repo := range r.RepositoryDetails { - repos = append(repos, repo) - } - sort.Strings(repos) - - for _, repo := range repos { - detail := r.RepositoryDetails[repo] - status := "✓" - issues := []string{} - - if len(detail.MissingTags) > 0 { - status = "✗" - issues = append(issues, fmt.Sprintf("%d missing", len(detail.MissingTags))) - } - - // Count layer issues - layerIssues := 0 - deepChecked := 0 - totalLayers := 0 - for _, tagDetail := range detail.TagDetails { - if tagDetail.DeepChecked { - deepChecked++ - totalLayers += len(tagDetail.SourceLayers) - layerIssues += len(tagDetail.MissingLayers) - } - } - - if layerIssues > 0 { - status = "✗" - issues = append(issues, fmt.Sprintf("%d layer issues", layerIssues)) - } - - repoName := repo - if repoName == "" { - repoName = "(root)" - } - - sb.WriteString(fmt.Sprintf("%s %s: %d/%d tags", status, repoName, detail.MatchedTags, len(detail.SourceTags))) - if deepChecked > 0 { - sb.WriteString(fmt.Sprintf(", %d layers checked", totalLayers)) - } - if len(issues) > 0 { - sb.WriteString(fmt.Sprintf(" [%s]", strings.Join(issues, ", "))) - } - sb.WriteString("\n") - } - - return sb.String() -} - -// Compare performs a deep comparison between source and target registries -func (c *RegistryComparator) Compare(ctx context.Context) (*ComparisonReport, error) { - report := &ComparisonReport{ - StartTime: time.Now(), - SourceRegistry: c.sourceRegistry, - TargetRegistry: c.targetRegistry, - RepositoryDetails: make(map[string]*RepositoryComparison), - } - - // Step 1: Discover all repositories in source - c.logProgressf("Discovering repositories in source registry...") - sourceRepos := c.discoverRepositories(c.sourceRegistry, c.sourceRemoteOpts) - report.SourceRepositories = sourceRepos - c.logProgressf("Found %d repositories in source", len(sourceRepos)) - - // Step 2: Discover all repositories in target - c.logProgressf("Discovering repositories in target registry...") - targetRepos := c.discoverRepositories(c.targetRegistry, c.targetRemoteOpts) - report.TargetRepositories = targetRepos - c.logProgressf("Found %d repositories in target", len(targetRepos)) - - // Step 3: Find missing and extra repositories - sourceRepoSet := make(map[string]bool) - for _, r := range sourceRepos { - sourceRepoSet[r] = true - } - targetRepoSet := make(map[string]bool) - for _, r := range targetRepos { - targetRepoSet[r] = true - } - - for _, repo := range sourceRepos { - if !targetRepoSet[repo] { - report.MissingRepositories = append(report.MissingRepositories, repo) - } - } - for _, repo := range targetRepos { - if !sourceRepoSet[repo] { - report.ExtraRepositories = append(report.ExtraRepositories, repo) - } - } - - // Step 4: Compare each repository in detail - c.logProgressf("Comparing repositories...") - for i, repoPath := range sourceRepos { - c.logProgressf("[%d/%d] Comparing %s", i+1, len(sourceRepos), repoPath) - - repoComparison, err := c.compareRepository(repoPath) - if err != nil { - c.logProgressf("Warning: failed to compare %s: %v", repoPath, err) - continue - } - - report.RepositoryDetails[repoPath] = repoComparison - - // Aggregate stats - report.TotalSourceImages += len(repoComparison.SourceTags) - report.TotalTargetImages += len(repoComparison.TargetTags) - report.SkippedImages += repoComparison.SkippedTags - report.MatchedImages += repoComparison.MatchedTags - - // Collect missing images - for _, tag := range repoComparison.MissingTags { - report.MissingImages = append(report.MissingImages, fmt.Sprintf("%s:%s", repoPath, tag)) - } - - // Collect mismatched images and layer stats - for tag, detail := range repoComparison.TagDetails { - if !detail.Match && detail.TargetDigest != "" { - report.MismatchedImages = append(report.MismatchedImages, ImageMismatch{ - Reference: fmt.Sprintf("%s:%s", repoPath, tag), - SourceDigest: detail.SourceDigest, - TargetDigest: detail.TargetDigest, - }) - } - - // Aggregate layer stats from deep comparison - if detail.DeepChecked { - report.DeepCheckedImages++ - report.TotalSourceLayers += len(detail.SourceLayers) - report.TotalTargetLayers += len(detail.TargetLayers) - report.MissingLayers += len(detail.MissingLayers) - - // Count matched layers - if detail.LayersMatch { - report.MatchedLayers += len(detail.SourceLayers) - } - - if !detail.ConfigMatch { - report.ConfigMismatches++ - } - - // Collect layer mismatches for detailed report - for _, layer := range detail.MissingLayers { - report.LayerMismatches = append(report.LayerMismatches, LayerMismatch{ - Reference: fmt.Sprintf("%s:%s", repoPath, tag), - LayerDigest: layer, - Reason: "missing_in_target", - }) - } - } - } - - // Collect extra images - for _, tag := range repoComparison.ExtraTags { - report.ExtraImages = append(report.ExtraImages, fmt.Sprintf("%s:%s", repoPath, tag)) - } - } - - // Sort results for consistent output - sort.Strings(report.MissingRepositories) - sort.Strings(report.MissingImages) - sort.Strings(report.ExtraImages) - sort.Slice(report.MismatchedImages, func(i, j int) bool { - return report.MismatchedImages[i].Reference < report.MismatchedImages[j].Reference - }) - - report.EndTime = time.Now() - return report, nil -} - -// discoverRepositories discovers all repositories by walking known segments -func (c *RegistryComparator) discoverRepositories(registry string, opts []remote.Option) []string { - var repos []string - - // Root repository - if c.repositoryExists(registry, opts) { - repos = append(repos, "") - } - - // Known segments - segments := []string{ - internal.InstallSegment, - internal.InstallStandaloneSegment, - internal.ReleaseChannelSegment, - } - - for _, segment := range segments { - segmentPath := segment - if c.repositoryExists(path.Join(registry, segmentPath), opts) { - repos = append(repos, segmentPath) - } - } - - // Security segment - securityDBs := []string{ - internal.SecurityTrivyDBSegment, - internal.SecurityTrivyBDUSegment, - internal.SecurityTrivyJavaDBSegment, - internal.SecurityTrivyChecksSegment, - } - for _, db := range securityDBs { - dbPath := path.Join(internal.SecuritySegment, db) - if c.repositoryExists(path.Join(registry, dbPath), opts) { - repos = append(repos, dbPath) - } - } - - // Modules - need to discover dynamically - modulesPath := path.Join(registry, internal.ModulesSegment) - moduleTags, err := c.listTags(modulesPath, opts) - if err == nil { - for _, moduleName := range moduleTags { - moduleBasePath := path.Join(internal.ModulesSegment, moduleName) - - // Module root - if c.repositoryExists(path.Join(registry, moduleBasePath), opts) { - repos = append(repos, moduleBasePath) - } - - // Module release - moduleReleasePath := path.Join(moduleBasePath, "release") - if c.repositoryExists(path.Join(registry, moduleReleasePath), opts) { - repos = append(repos, moduleReleasePath) - } - } - } - - return repos -} - -// repositoryExists checks if a repository exists and has tags -func (c *RegistryComparator) repositoryExists(repo string, opts []remote.Option) bool { - tags, err := c.listTags(repo, opts) - return err == nil && len(tags) > 0 -} - -// compareRepository compares a single repository between source and target -func (c *RegistryComparator) compareRepository(repoPath string) (*RepositoryComparison, error) { - sourceRepo := c.sourceRegistry - targetRepo := c.targetRegistry - if repoPath != "" { - sourceRepo = path.Join(c.sourceRegistry, repoPath) - targetRepo = path.Join(c.targetRegistry, repoPath) - } - - comparison := &RepositoryComparison{ - Repository: repoPath, - TagDetails: make(map[string]*TagComparison), - } - - // Get source tags - allSourceTags, err := c.listTags(sourceRepo, c.sourceRemoteOpts) - if err != nil { - return nil, fmt.Errorf("list source tags: %w", err) - } - - // Filter out tags that are not mirrored by design - sourceTags := make([]string, 0, len(allSourceTags)) - skippedTags := 0 - for _, tag := range allSourceTags { - if shouldSkipTag(tag) { - skippedTags++ - continue - } - sourceTags = append(sourceTags, tag) - } - comparison.SourceTags = sourceTags - comparison.SkippedTags = skippedTags - - // Get target tags - allTargetTags, err := c.listTags(targetRepo, c.targetRemoteOpts) - if err != nil { - // Target repo might not exist - allTargetTags = []string{} - } - - // Filter target tags too - targetTags := make([]string, 0, len(allTargetTags)) - for _, tag := range allTargetTags { - if shouldSkipTag(tag) { - continue - } - targetTags = append(targetTags, tag) - } - comparison.TargetTags = targetTags - - // Create sets for comparison - sourceTagSet := make(map[string]bool) - for _, t := range sourceTags { - sourceTagSet[t] = true - } - targetTagSet := make(map[string]bool) - for _, t := range targetTags { - targetTagSet[t] = true - } - - // Find missing and extra tags - for _, tag := range sourceTags { - if !targetTagSet[tag] { - comparison.MissingTags = append(comparison.MissingTags, tag) - } - } - for _, tag := range targetTags { - if !sourceTagSet[tag] { - comparison.ExtraTags = append(comparison.ExtraTags, tag) - } - } - - // Compare images deeply - check manifest, config, and all layers - var wg sync.WaitGroup - var mu sync.Mutex - semaphore := make(chan struct{}, 5) // Limit concurrency (deep comparison is heavier) - - for _, tag := range sourceTags { - if !targetTagSet[tag] { - continue - } - - wg.Add(1) - go func(tag string) { - defer wg.Done() - semaphore <- struct{}{} - defer func() { <-semaphore }() - - sourceRef := sourceRepo + ":" + tag - targetRef := targetRepo + ":" + tag - - // Perform deep comparison - imgComp, err := c.compareImageDeep(sourceRef, targetRef) - if err != nil { - // Fallback to simple digest comparison - sourceDigest, err1 := c.getDigest(sourceRef, c.sourceRemoteOpts) - targetDigest, err2 := c.getDigest(targetRef, c.targetRemoteOpts) - if err1 != nil || err2 != nil { - return - } - - tagComp := &TagComparison{ - Tag: tag, - SourceDigest: sourceDigest, - TargetDigest: targetDigest, - Match: sourceDigest == targetDigest, - DeepChecked: false, - } - - mu.Lock() - comparison.TagDetails[tag] = tagComp - if tagComp.Match { - comparison.MatchedTags++ - } - mu.Unlock() - return - } - - tagComp := &TagComparison{ - Tag: tag, - SourceDigest: imgComp.SourceDigest, - TargetDigest: imgComp.TargetDigest, - Match: imgComp.FullMatch, - SourceConfig: imgComp.SourceConfig, - TargetConfig: imgComp.TargetConfig, - ConfigMatch: imgComp.ConfigMatch, - SourceLayers: imgComp.SourceLayers, - TargetLayers: imgComp.TargetLayers, - MissingLayers: imgComp.MissingLayers, - ExtraLayers: imgComp.ExtraLayers, - LayersMatch: imgComp.LayersMatch, - DeepChecked: true, - } - - mu.Lock() - comparison.TagDetails[tag] = tagComp - if tagComp.Match { - comparison.MatchedTags++ - } - mu.Unlock() - }(tag) - } - - wg.Wait() - - sort.Strings(comparison.MissingTags) - sort.Strings(comparison.ExtraTags) - - return comparison, nil -} - -// listTags lists all tags in a repository -func (c *RegistryComparator) listTags(repo string, opts []remote.Option) ([]string, error) { - repoRef, err := name.NewRepository(repo, c.nameOpts...) - if err != nil { - return nil, fmt.Errorf("parse repo %s: %w", repo, err) - } - - tags, err := remote.List(repoRef, opts...) - if err != nil { - return nil, fmt.Errorf("list tags for %s: %w", repo, err) - } - - return tags, nil -} - -// getDigest gets the digest for a specific image reference -func (c *RegistryComparator) getDigest(ref string, opts []remote.Option) (string, error) { - imgRef, err := name.ParseReference(ref, c.nameOpts...) - if err != nil { - return "", fmt.Errorf("parse ref %s: %w", ref, err) - } - - desc, err := remote.Head(imgRef, opts...) - if err != nil { - return "", fmt.Errorf("get digest for %s: %w", ref, err) - } - - return desc.Digest.String(), nil -} - -// getImageInfo gets detailed information about an image including all layer digests -func (c *RegistryComparator) getImageInfo(ref string, opts []remote.Option) (*ImageInfo, error) { - imgRef, err := name.ParseReference(ref, c.nameOpts...) - if err != nil { - return nil, fmt.Errorf("parse ref %s: %w", ref, err) - } - - // Try to get as an image first - img, err := remote.Image(imgRef, opts...) - if err != nil { - // Might be an index, try that - idx, idxErr := remote.Index(imgRef, opts...) - if idxErr != nil { - return nil, fmt.Errorf("get image %s: %w (also tried index: %v)", ref, err, idxErr) - } - - // For index, get the digest and list manifests - digest, err := idx.Digest() - if err != nil { - return nil, fmt.Errorf("get index digest: %w", err) - } - - manifest, err := idx.IndexManifest() - if err != nil { - return nil, fmt.Errorf("get index manifest: %w", err) - } - - info := &ImageInfo{ - Reference: ref, - Digest: digest.String(), - Layers: make([]string, 0), - } - - // Collect all manifest digests as "layers" for index - for _, m := range manifest.Manifests { - info.Layers = append(info.Layers, m.Digest.String()) - info.TotalSize += m.Size - } - - return info, nil - } - - // Get manifest digest - digest, err := img.Digest() - if err != nil { - return nil, fmt.Errorf("get digest: %w", err) - } - - // Get config digest - configDigest := "" - if cfg, err := img.ConfigName(); err == nil { - configDigest = cfg.String() - } - - // Get all layer digests - layers, err := img.Layers() - if err != nil { - return nil, fmt.Errorf("get layers: %w", err) - } - - info := &ImageInfo{ - Reference: ref, - Digest: digest.String(), - ConfigDigest: configDigest, - Layers: make([]string, 0, len(layers)), - } - - for _, layer := range layers { - layerDigest, err := layer.Digest() - if err != nil { - continue - } - info.Layers = append(info.Layers, layerDigest.String()) - - size, err := layer.Size() - if err == nil { - info.TotalSize += size - } - } - - return info, nil -} - -// compareImageDeep performs deep comparison of two images including all layers -func (c *RegistryComparator) compareImageDeep(sourceRef, targetRef string) (*ImageComparison, error) { - sourceInfo, err := c.getImageInfo(sourceRef, c.sourceRemoteOpts) - if err != nil { - return nil, fmt.Errorf("get source image info: %w", err) - } - - targetInfo, err := c.getImageInfo(targetRef, c.targetRemoteOpts) - if err != nil { - return nil, fmt.Errorf("get target image info: %w", err) - } - - comparison := &ImageComparison{ - Reference: sourceRef, - SourceDigest: sourceInfo.Digest, - TargetDigest: targetInfo.Digest, - DigestMatch: sourceInfo.Digest == targetInfo.Digest, - SourceLayers: sourceInfo.Layers, - TargetLayers: targetInfo.Layers, - MissingLayers: make([]string, 0), - ExtraLayers: make([]string, 0), - } - - // Compare config digests - if sourceInfo.ConfigDigest != "" && targetInfo.ConfigDigest != "" { - comparison.ConfigMatch = sourceInfo.ConfigDigest == targetInfo.ConfigDigest - comparison.SourceConfig = sourceInfo.ConfigDigest - comparison.TargetConfig = targetInfo.ConfigDigest - } else { - comparison.ConfigMatch = true // Skip if not available - } - - // Compare layers - targetLayerSet := make(map[string]bool) - for _, l := range targetInfo.Layers { - targetLayerSet[l] = true - } - - sourceLayerSet := make(map[string]bool) - for _, l := range sourceInfo.Layers { - sourceLayerSet[l] = true - if !targetLayerSet[l] { - comparison.MissingLayers = append(comparison.MissingLayers, l) - } - } - - for _, l := range targetInfo.Layers { - if !sourceLayerSet[l] { - comparison.ExtraLayers = append(comparison.ExtraLayers, l) - } - } - - comparison.LayersMatch = len(comparison.MissingLayers) == 0 && len(comparison.ExtraLayers) == 0 - comparison.FullMatch = comparison.DigestMatch && comparison.ConfigMatch && comparison.LayersMatch - - return comparison, nil -} - -// ImageComparison holds detailed comparison of a single image -type ImageComparison struct { - Reference string - SourceDigest string - TargetDigest string - DigestMatch bool - SourceConfig string - TargetConfig string - ConfigMatch bool - SourceLayers []string - TargetLayers []string - MissingLayers []string - ExtraLayers []string - LayersMatch bool - FullMatch bool -} diff --git a/testing/e2e/mirror/report.go b/testing/e2e/mirror/report.go deleted file mode 100644 index 4fb1b85d..00000000 --- a/testing/e2e/mirror/report.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mirror - -import ( - "fmt" - "os" - "strings" - "time" - - "github.com/charmbracelet/lipgloss" -) - -// ============================================================================= -// Test Report -// ============================================================================= - -// TestReport collects test execution results for final summary -type TestReport struct { - TestName string - StartTime time.Time - EndTime time.Time - SourceRegistry string - TargetRegistry string - LogDir string - - // Source stats - SourceRepoCount int - SourceImageCount int - - // Target stats - TargetRepoCount int - TargetImageCount int - - // Comparison stats - MatchedImages int - MissingImages int - MismatchedImages int - SkippedImages int // Digest-based, .att, .sig tags - - // Deep comparison stats - MatchedLayers int - MissingLayers int - - // Bundle stats - BundleSize int64 - - // Steps - Steps []StepResult - - // Full comparison report - ComparisonReport *ComparisonReport -} - -// StepResult represents a single step in the test -type StepResult struct { - Name string - Status string // "PASS", "FAIL", "SKIP" - Duration time.Duration - Error string -} - -// ============================================================================= -// Report Methods -// ============================================================================= - -// AddStep adds a step result to the report -func (r *TestReport) AddStep(name, status string, duration time.Duration, err error) { - errStr := "" - if err != nil { - errStr = err.Error() - } - r.Steps = append(r.Steps, StepResult{ - Name: name, - Status: status, - Duration: duration, - Error: errStr, - }) -} - -// WriteToFile writes the report to a file in plain text format -func (r *TestReport) WriteToFile(path string) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - w := func(format string, args ...interface{}) { - fmt.Fprintf(f, format, args...) - } - - // Header - w("================================================================================\n") - w("E2E TEST REPORT: %s\n", r.TestName) - w("================================================================================\n\n") - - // Execution info - w("EXECUTION:\n") - w(" Started: %s\n", r.StartTime.Format(time.RFC3339)) - w(" Finished: %s\n", r.EndTime.Format(time.RFC3339)) - w(" Duration: %s\n", r.EndTime.Sub(r.StartTime).Round(time.Second)) - w(" Log dir: %s\n\n", r.LogDir) - - // Registries - w("REGISTRIES:\n") - w(" Source: %s\n", r.SourceRegistry) - w(" Target: %s\n\n", r.TargetRegistry) - - // Images - w("IMAGES TO VERIFY:\n") - w(" Source: %d images (%d repos)\n", r.SourceImageCount, r.SourceRepoCount) - w(" Target: %d images (%d repos)\n", r.TargetImageCount, r.TargetRepoCount) - if r.SkippedImages > 0 { - w(" (excluded %d internal tags from comparison)\n", r.SkippedImages) - } - w("\n") - - // Bundle - w("BUNDLE:\n") - w(" Size: %.2f GB\n\n", float64(r.BundleSize)/(1024*1024*1024)) - - // Verification results - w("VERIFICATION RESULTS:\n") - w(" Images matched: %d (manifest + config + layers)\n", r.MatchedImages) - w(" Layers verified: %d\n", r.MatchedLayers) - w(" Missing images: %d\n", r.MissingImages) - w(" Digest mismatch: %d\n", r.MismatchedImages) - w(" Missing layers: %d\n\n", r.MissingLayers) - - // Steps - w("STEPS:\n") - passCount, failCount := r.countSteps() - for _, step := range r.Steps { - switch step.Status { - case "PASS": - w(" [PASS] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) - case "FAIL": - w(" [FAIL] %s (%s)\n", step.Name, step.Duration.Round(time.Millisecond)) - if step.Error != "" { - w(" ERROR: %s\n", step.Error) - } - default: - w(" [SKIP] %s\n", step.Name) - } - } - - // Result - w("\n================================================================================\n") - if failCount > 0 { - w("RESULT: FAILED (%d passed, %d failed)\n", passCount, failCount) - } else { - w("RESULT: PASSED - REGISTRIES ARE IDENTICAL\n") - w(" %d repositories verified\n", r.SourceRepoCount) - w(" %d images verified\n", r.MatchedImages) - } - w("================================================================================\n") - - return nil -} - -// Print prints the report to stderr with beautiful lipgloss styling -func (r *TestReport) Print() { - duration := r.EndTime.Sub(r.StartTime) - if r.EndTime.IsZero() { - duration = time.Since(r.StartTime) - } - - var b strings.Builder - - // Header - b.WriteString("\n") - b.WriteString(separator("═") + "\n") - b.WriteString(" " + styleTitle.Render("E2E TEST REPORT") + "\n") - b.WriteString(separator("═") + "\n\n") - - // Duration - b.WriteString(" " + styleLabel.Render("Duration: ") + styleDim.Render(duration.Round(time.Second).String()) + "\n\n") - - // Registries - b.WriteString(" " + styleHeader.Render("REGISTRIES") + "\n") - b.WriteString(" " + styleLabel.Render("Source: ") + styleValue.Render(r.SourceRegistry) + "\n") - b.WriteString(" " + styleLabel.Render("Target: ") + styleValue.Render(r.TargetRegistry) + "\n\n") - - // Images to verify - b.WriteString(" " + styleHeader.Render("IMAGES TO VERIFY") + "\n") - b.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", - styleLabel.Render("Source:"), - styleValue.Render(fmt.Sprintf("%d", r.SourceImageCount)), - r.SourceRepoCount)) - b.WriteString(fmt.Sprintf(" %s %s images (%d repos)\n", - styleLabel.Render("Target:"), - styleValue.Render(fmt.Sprintf("%d", r.TargetImageCount)), - r.TargetRepoCount)) - if r.SkippedImages > 0 { - b.WriteString(" " + styleDim.Render(fmt.Sprintf("(%d internal tags excluded)", r.SkippedImages)) + "\n") - } - b.WriteString("\n") - - // Verification results - b.WriteString(" " + styleHeader.Render("VERIFICATION") + "\n") - b.WriteString(fmt.Sprintf(" %s %s %s\n", - badgeOK, - styleLabel.Render("Images matched: "), - styleSuccess.Render(fmt.Sprintf("%d", r.MatchedImages))+" "+styleDim.Render("(manifest + config + layers)"))) - b.WriteString(fmt.Sprintf(" %s %s %s\n", - badgeOK, - styleLabel.Render("Layers verified:"), - styleSuccess.Render(fmt.Sprintf("%d", r.MatchedLayers)))) - - if r.MissingImages > 0 { - b.WriteString(fmt.Sprintf(" %s %s %s\n", - badgeFail, - styleLabel.Render("Missing images: "), - styleError.Render(fmt.Sprintf("%d", r.MissingImages)))) - } - if r.MismatchedImages > 0 { - b.WriteString(fmt.Sprintf(" %s %s %s\n", - badgeFail, - styleLabel.Render("Digest mismatch:"), - styleError.Render(fmt.Sprintf("%d", r.MismatchedImages)))) - } - if r.MissingLayers > 0 { - b.WriteString(fmt.Sprintf(" %s %s %s\n", - badgeFail, - styleLabel.Render("Missing layers: "), - styleError.Render(fmt.Sprintf("%d", r.MissingLayers)))) - } - b.WriteString("\n") - - // Steps - b.WriteString(" " + styleHeader.Render("STEPS") + "\n") - passCount, failCount := r.countSteps() - for _, step := range r.Steps { - dur := styleDim.Render(fmt.Sprintf("(%s)", step.Duration.Round(time.Millisecond))) - switch step.Status { - case "PASS": - b.WriteString(fmt.Sprintf(" %s %s %s\n", badgeOK, step.Name, dur)) - case "FAIL": - b.WriteString(fmt.Sprintf(" %s %s %s\n", badgeFail, step.Name, dur)) - if step.Error != "" { - b.WriteString(" " + styleError.Render("ERROR: "+step.Error) + "\n") - } - default: - b.WriteString(fmt.Sprintf(" %s %s\n", badgeSkip, step.Name)) - } - } - b.WriteString("\n") - - // Result box - b.WriteString(separator("─") + "\n") - if failCount > 0 { - resultStyle := lipgloss.NewStyle().Bold(true).Foreground(colorRed) - b.WriteString(" " + resultStyle.Render("RESULT: FAILED") + fmt.Sprintf(" (%d passed, %d failed)\n", passCount, failCount)) - } else { - resultStyle := lipgloss.NewStyle().Bold(true).Foreground(colorGreen) - b.WriteString(" " + resultStyle.Render("RESULT: PASSED") + " - REGISTRIES ARE IDENTICAL\n") - b.WriteString(" " + styleSuccess.Render(fmt.Sprintf("%d images, %d layers", r.MatchedImages, r.MatchedLayers)) + " - all hashes verified\n") - } - b.WriteString(separator("═") + "\n") - - writeRawf("%s", b.String()) -} - -// countSteps counts passed and failed steps -func (r *TestReport) countSteps() (int, int) { - var passed, failed int - for _, step := range r.Steps { - switch step.Status { - case "PASS": - passed++ - case "FAIL": - failed++ - } - } - return passed, failed -} diff --git a/testing/e2e/mirror/security_test.go b/testing/e2e/mirror/security_test.go new file mode 100644 index 00000000..09b73240 --- /dev/null +++ b/testing/e2e/mirror/security_test.go @@ -0,0 +1,91 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" +) + +func TestSecurityE2E(t *testing.T) { + cfg := internal.GetConfig() + + if !cfg.HasSourceAuth() { + t.Skip("Skipping: no source authentication configured (set E2E_LICENSE_TOKEN)") + } + + cfg.NoModules = true + cfg.NoPlatform = true + + env := setupTestEnvironment(t, cfg) + defer env.Cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), internal.SecurityTestTimeout) + defer cancel() + + runSecurityTest(t, ctx, cfg, env) +} + +func runSecurityTest(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv) { + internal.PrintHeader("SECURITY E2E TEST") + + internal.PrintStep(1, "Reading expected security databases from source") + expected := readExpectedSecurityImages(t, ctx, cfg) + + internal.PrintStep(2, "Pulling security databases") + runPullStep(t, cfg, env) + + internal.PrintStep(3, "Pushing to target registry") + runPushStep(t, cfg, env) + + internal.PrintStep(4, "Verifying security databases in target") + verifySecurityInTarget(t, ctx, cfg, env, expected) + + totalTags := 0 + for _, tags := range expected.Databases { + totalTags += len(tags) + } + fmt.Printf("\n✅ Security test passed: %d databases, %d tags\n", len(expected.Databases), totalTags) +} + +func readExpectedSecurityImages(t *testing.T, ctx context.Context, cfg *internal.Config) *internal.SecurityDigests { + t.Helper() + + reader := createSourceReader(t, cfg) + security, err := reader.ReadSecurityDigests(ctx) + require.NoError(t, err, "Failed to read security databases") + + t.Logf("Found %d security databases:", len(security.Databases)) + for db, tags := range security.Databases { + t.Logf(" %s: %d tags", db, len(tags)) + } + + return security +} + +func verifySecurityInTarget(t *testing.T, ctx context.Context, cfg *internal.Config, env *testEnv, expected *internal.SecurityDigests) { + t.Helper() + verifySecurityImages(t, ctx, cfg, env) +} + diff --git a/testing/e2e/mirror/testenv.go b/testing/e2e/mirror/testenv.go new file mode 100644 index 00000000..1ef53f52 --- /dev/null +++ b/testing/e2e/mirror/testenv.go @@ -0,0 +1,241 @@ +//go:build e2e + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/deckhouse/deckhouse-cli/testing/e2e/mirror/internal" + mirrorutil "github.com/deckhouse/deckhouse-cli/testing/util/mirror" +) + +type testEnv struct { + LogDir string + LogFile string + ReportFile string + ComparisonFile string + BundleDir string + TargetRegistry string + Report *internal.TestReport + Cleanup func() +} + +func setupTestEnvironment(t *testing.T, cfg *internal.Config) *testEnv { + t.Helper() + + logDir := getLogDir(t.Name()) + require.NoError(t, os.MkdirAll(logDir, 0755)) + + targetHost, targetPath, registryCleanup := setupTargetRegistry(t, cfg) + targetRegistry := targetHost + targetPath + t.Logf("Target registry: %s", targetRegistry) + + bundleDir := setupBundleDir(t, cfg) + + report := &internal.TestReport{ + TestName: t.Name(), + StartTime: time.Now(), + SourceRegistry: cfg.SourceRegistry, + TargetRegistry: targetRegistry, + LogDir: logDir, + } + + env := &testEnv{ + LogDir: logDir, + LogFile: filepath.Join(logDir, "test.log"), + ReportFile: filepath.Join(logDir, "report.txt"), + ComparisonFile: filepath.Join(logDir, "comparison.txt"), + BundleDir: bundleDir, + TargetRegistry: targetRegistry, + Report: report, + } + + env.Cleanup = func() { + registryCleanup() + finalizeReport(t, env) + } + + return env +} + +func setupBundleDir(t *testing.T, cfg *internal.Config) string { + t.Helper() + + if cfg.ExistingBundle != "" { + if _, err := os.Stat(cfg.ExistingBundle); os.IsNotExist(err) { + t.Fatalf("Existing bundle directory not found: %s", cfg.ExistingBundle) + } + t.Logf("Using existing bundle: %s", cfg.ExistingBundle) + return cfg.ExistingBundle + } + + if cfg.KeepBundle { + bundleDir := filepath.Join(os.TempDir(), fmt.Sprintf("d8-mirror-e2e-%d", time.Now().Unix())) + require.NoError(t, os.MkdirAll(bundleDir, 0755)) + t.Logf("Bundle directory (will be kept): %s", bundleDir) + return bundleDir + } + + bundleDir := t.TempDir() + t.Logf("Bundle directory: %s", bundleDir) + return bundleDir +} + +func setupTargetRegistry(t *testing.T, cfg *internal.Config) (host, path string, cleanup func()) { + t.Helper() + + if cfg.UseInMemoryRegistry() { + reg := mirrorutil.SetupTestRegistry(false) + repoPath := "/deckhouse/ee" + t.Logf("Started test registry at %s%s", reg.Host, repoPath) + return reg.Host, repoPath, reg.Close + } + + return cfg.TargetRegistry, "", func() {} +} + +func finalizeReport(t *testing.T, env *testEnv) { + t.Helper() + + env.Report.EndTime = time.Now() + env.Report.Print() + + if err := env.Report.WriteToFile(env.ReportFile); err != nil { + t.Logf("Warning: failed to write report: %v", err) + } else { + t.Logf("Report written to: %s", env.ReportFile) + } +} + +func runPullStep(t *testing.T, cfg *internal.Config, env *testEnv) { + t.Helper() + stepStart := time.Now() + + cmd := internal.BuildPullCommand(cfg, env.BundleDir) + t.Logf("Running: %s", cmd.String()) + + err := internal.RunCommandWithLog(t, cmd, env.LogFile) + if err != nil { + env.Report.AddStep("Pull images", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Pull failed") + } + + bundleInfo := validateBundle(t, env.BundleDir, cfg) + env.Report.BundleSize = bundleInfo.TotalSize + if len(bundleInfo.Modules) > 0 { + env.Report.ExpectedModules = bundleInfo.Modules + } + + env.Report.AddStep( + fmt.Sprintf("Pull images (%.2f GB bundle)", float64(bundleInfo.TotalSize)/(1024*1024*1024)), + "PASS", time.Since(stepStart), nil, + ) + t.Logf("Pull completed: %.2f GB total", float64(bundleInfo.TotalSize)/(1024*1024*1024)) +} + +func runPushStep(t *testing.T, cfg *internal.Config, env *testEnv) { + t.Helper() + stepStart := time.Now() + + cmd := internal.BuildPushCommand(cfg, env.BundleDir, env.TargetRegistry) + t.Logf("Running: %s", cmd.String()) + + err := internal.RunCommandWithLog(t, cmd, env.LogFile) + if err != nil { + env.Report.AddStep("Push to registry", "FAIL", time.Since(stepStart), err) + require.NoError(t, err, "Push failed") + } + + env.Report.AddStep("Push to registry", "PASS", time.Since(stepStart), nil) + t.Log("Push completed successfully") +} + +type BundleInfo struct { + TotalSize int64 + Modules []string + HasPlatform bool + HasSecurity bool +} + +func validateBundle(t *testing.T, bundleDir string, cfg *internal.Config) *BundleInfo { + t.Helper() + + files, err := os.ReadDir(bundleDir) + require.NoError(t, err, "Failed to read bundle directory") + + info := &BundleInfo{} + + for _, f := range files { + if f.IsDir() { + continue + } + + finfo, err := f.Info() + require.NoError(t, err) + info.TotalSize += finfo.Size() + + name := f.Name() + t.Logf(" %s (%.2f MB)", name, float64(finfo.Size())/(1024*1024)) + + switch { + case name == "platform.tar" || strings.HasPrefix(name, "platform."): + info.HasPlatform = true + case name == "security.tar" || strings.HasPrefix(name, "security."): + info.HasSecurity = true + case strings.HasPrefix(name, "module-") && strings.Contains(name, ".tar"): + moduleName := strings.TrimPrefix(name, "module-") + moduleName = strings.Split(moduleName, ".")[0] + if moduleName != "" && !slices.Contains(info.Modules, moduleName) { + info.Modules = append(info.Modules, moduleName) + } + } + } + + if !cfg.NoPlatform { + require.True(t, info.HasPlatform, "Bundle missing platform.tar - pull may have failed!") + } + if !cfg.NoSecurity { + require.True(t, info.HasSecurity, "Bundle missing security.tar - pull may have failed!") + } + if !cfg.NoModules && len(cfg.IncludeModules) == 0 { + require.NotEmpty(t, info.Modules, "Bundle has no modules - pull may have failed!") + } + + if len(info.Modules) > 0 { + t.Logf("Bundle contains %d modules: %v", len(info.Modules), info.Modules) + } + + return info +} + +func getLogDir(testName string) string { + projectRoot := internal.FindProjectRoot() + timestamp := time.Now().Format("20060102-150405") + safeName := strings.ReplaceAll(testName, "/", "-") + return filepath.Join(projectRoot, "testing", "e2e", ".logs", fmt.Sprintf("%s-%s", safeName, timestamp)) +} + From e0486c3e8e9bca3020564b7e1b8056a0229b5089 Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Mon, 12 Jan 2026 14:08:29 +0300 Subject: [PATCH 10/11] feat: module verification Signed-off-by: Timur Tuktamyshev --- pkg/libmirror/images/digests_test.go | 1 + testing/e2e/mirror/fullcycle_test.go | 12 ++ testing/e2e/mirror/helpers_test.go | 39 +++++- testing/e2e/mirror/internal/report.go | 18 ++- testing/e2e/mirror/internal/source.go | 192 ++++++++++++++++++++++++-- testing/e2e/mirror/internal/verify.go | 150 ++++++++++++++++---- 6 files changed, 361 insertions(+), 51 deletions(-) diff --git a/pkg/libmirror/images/digests_test.go b/pkg/libmirror/images/digests_test.go index 7359667a..acfdf04c 100644 --- a/pkg/libmirror/images/digests_test.go +++ b/pkg/libmirror/images/digests_test.go @@ -48,6 +48,7 @@ func TestExtractImageDigestsFromDeckhouseInstaller(t *testing.T) { installersLayout := createOCILayoutWithInstallerImage(t, "nonexistent.registry.com/deckhouse", installerTag, expectedImages) client := mock.NewRegistryClientMock(t) client.CheckImageExistsMock.Return(nil) + client.GetRegistryMock.Return("nonexistent.registry.com/deckhouse") images, err := ExtractImageDigestsFromDeckhouseInstaller( ¶ms.PullParams{BaseParams: params.BaseParams{ DeckhouseRegistryRepo: "nonexistent.registry.com/deckhouse", diff --git a/testing/e2e/mirror/fullcycle_test.go b/testing/e2e/mirror/fullcycle_test.go index c2661d5a..95f93a82 100644 --- a/testing/e2e/mirror/fullcycle_test.go +++ b/testing/e2e/mirror/fullcycle_test.go @@ -159,6 +159,12 @@ func runVerificationStep(t *testing.T, ctx context.Context, cfg *internal.Config env.Report.ModulesExpected = result.ModulesExpected env.Report.ModulesFound = result.ModulesFound env.Report.ModulesMissing = len(result.ModulesMissing) + env.Report.ModuleVersionsTotal = result.ModuleVersionsTotal + env.Report.ModuleVersionsFound = result.ModuleVersionsFound + env.Report.ModuleVersionsMissing = len(result.ModuleVersionsMissing) + env.Report.ModuleDigestsTotal = result.ModuleDigestsTotal + env.Report.ModuleDigestsFound = result.ModuleDigestsFound + env.Report.ModuleDigestsMissing = len(result.ModuleDigestsMissing) env.Report.SecurityExpected = result.SecurityExpected env.Report.SecurityFound = result.SecurityFound env.Report.SecurityMissing = len(result.SecurityMissing) @@ -175,6 +181,12 @@ func runVerificationStep(t *testing.T, ctx context.Context, cfg *internal.Config if len(result.MissingAttTags) > 0 { failures = append(failures, fmt.Sprintf("missing %d .att tags in target", len(result.MissingAttTags))) } + if len(result.ModuleVersionsMissing) > 0 { + failures = append(failures, fmt.Sprintf("missing %d module versions in target", len(result.ModuleVersionsMissing))) + } + if len(result.ModuleDigestsMissing) > 0 { + failures = append(failures, fmt.Sprintf("missing %d module digests in target", len(result.ModuleDigestsMissing))) + } if len(failures) > 0 { env.Report.AddStep( diff --git a/testing/e2e/mirror/helpers_test.go b/testing/e2e/mirror/helpers_test.go index 15a2afb1..720b6aec 100644 --- a/testing/e2e/mirror/helpers_test.go +++ b/testing/e2e/mirror/helpers_test.go @@ -133,28 +133,54 @@ func verifyModulesImages(t *testing.T, ctx context.Context, cfg *internal.Config require.NoError(t, err, "Modules verification failed") t.Logf("Found %d/%d modules in target", result.ModulesFound, result.ModulesExpected) + t.Logf("Found %d/%d module versions in target", result.ModuleVersionsFound, result.ModuleVersionsTotal) + t.Logf("Found %d/%d module digests in target", result.ModuleDigestsFound, result.ModuleDigestsTotal) for _, missing := range result.ModulesMissing { - t.Logf(" ✗ %s", missing) + t.Logf(" ✗ module: %s", missing) + } + for _, missing := range result.ModuleVersionsMissing { + t.Logf(" ✗ version: %s", missing) + } + for _, missing := range result.ModuleDigestsMissing { + t.Logf(" ✗ digest: %s", missing) } + var failures []string if len(result.ModulesMissing) > 0 { + failures = append(failures, fmt.Sprintf("%d modules missing", len(result.ModulesMissing))) + } + if len(result.ModuleVersionsMissing) > 0 { + failures = append(failures, fmt.Sprintf("%d versions missing", len(result.ModuleVersionsMissing))) + } + if len(result.ModuleDigestsMissing) > 0 { + failures = append(failures, fmt.Sprintf("%d digests missing", len(result.ModuleDigestsMissing))) + } + + if len(failures) > 0 { env.Report.AddStep( - fmt.Sprintf("Modules Verification (%d/%d found)", - result.ModulesFound, result.ModulesExpected), + fmt.Sprintf("Modules Verification (%d modules, %d versions)", + result.ModulesFound, result.ModuleVersionsFound), "FAIL", time.Since(stepStart), - fmt.Errorf("missing %d modules: %v", len(result.ModulesMissing), result.ModulesMissing), + fmt.Errorf("%v", failures), ) - require.Empty(t, result.ModulesMissing, "Some modules are missing in target") + require.Empty(t, failures, "Modules verification failed: %v", failures) return } env.Report.ModulesExpected = result.ModulesExpected env.Report.ModulesFound = result.ModulesFound env.Report.ModulesMissing = len(result.ModulesMissing) + env.Report.ModuleVersionsTotal = result.ModuleVersionsTotal + env.Report.ModuleVersionsFound = result.ModuleVersionsFound + env.Report.ModuleVersionsMissing = len(result.ModuleVersionsMissing) + env.Report.ModuleDigestsTotal = result.ModuleDigestsTotal + env.Report.ModuleDigestsFound = result.ModuleDigestsFound + env.Report.ModuleDigestsMissing = len(result.ModuleDigestsMissing) env.Report.AddStep( - fmt.Sprintf("Modules Verification (%d modules)", result.ModulesFound), + fmt.Sprintf("Modules Verification (%d modules, %d versions, %d digests)", + result.ModulesFound, result.ModuleVersionsFound, result.ModuleDigestsFound), "PASS", time.Since(stepStart), nil, ) t.Log("Modules verification passed") @@ -212,4 +238,3 @@ func saveReport(t *testing.T, path string, report *internal.TestReport) { t.Logf("Warning: failed to write report: %v", err) } } - diff --git a/testing/e2e/mirror/internal/report.go b/testing/e2e/mirror/internal/report.go index f69ffda5..a4f856a2 100644 --- a/testing/e2e/mirror/internal/report.go +++ b/testing/e2e/mirror/internal/report.go @@ -44,9 +44,15 @@ type TestReport struct { FoundAttTags int MissingAttTags int - ModulesExpected int - ModulesFound int - ModulesMissing int + ModulesExpected int + ModulesFound int + ModulesMissing int + ModuleVersionsTotal int + ModuleVersionsFound int + ModuleVersionsMissing int + ModuleDigestsTotal int + ModuleDigestsFound int + ModuleDigestsMissing int SecurityExpected int SecurityFound int @@ -225,6 +231,12 @@ func (r *TestReport) format(useColors bool) string { if r.ModulesExpected > 0 { b.WriteString(r.formatSection(useColors, "Modules verified: ", r.ModulesFound, r.ModulesExpected, r.ModulesMissing)) } + if r.ModuleVersionsTotal > 0 { + b.WriteString(r.formatSection(useColors, "Module versions: ", r.ModuleVersionsFound, r.ModuleVersionsTotal, r.ModuleVersionsMissing)) + } + if r.ModuleDigestsTotal > 0 { + b.WriteString(r.formatSection(useColors, "Module digests: ", r.ModuleDigestsFound, r.ModuleDigestsTotal, r.ModuleDigestsMissing)) + } if r.SecurityExpected > 0 { b.WriteString(r.formatSection(useColors, "Security verified:", r.SecurityFound, r.SecurityExpected, r.SecurityMissing)) } diff --git a/testing/e2e/mirror/internal/source.go b/testing/e2e/mirror/internal/source.go index 0b9b3a20..dbbde863 100644 --- a/testing/e2e/mirror/internal/source.go +++ b/testing/e2e/mirror/internal/source.go @@ -312,9 +312,10 @@ func extractDigestsFromTar(rc io.Reader) ([]string, error) { } type ModuleInfo struct { - Name string - Versions []string - Digests []string + Name string + ReleaseChannels []ReleaseChannelInfo + Versions []string + ImageDigests []string } func (r *SourceReader) listTags(ctx context.Context, repoPath string) ([]string, error) { @@ -350,24 +351,191 @@ func (r *SourceReader) ReadModuleDigests(ctx context.Context, moduleName string) r.progress("Reading module %s...", moduleName) info := &ModuleInfo{ - Name: moduleName, - Versions: make([]string, 0), - Digests: make([]string, 0), + Name: moduleName, + ReleaseChannels: make([]ReleaseChannelInfo, 0), + Versions: make([]string, 0), + ImageDigests: make([]string, 0), } - moduleReleaseRef := path.Join(r.registry, d8internal.ModulesSegment, moduleName, "release") - tags, err := r.listTags(ctx, moduleReleaseRef) - if err != nil { - r.progress(" No release tags found") + channels := d8internal.GetAllDefaultReleaseChannels() + moduleReleaseBase := path.Join(r.registry, d8internal.ModulesSegment, moduleName, "release") + + versionSet := make(map[string]bool) + for _, channel := range channels { + ref := moduleReleaseBase + ":" + channel + version, err := r.readModuleReleaseChannelVersion(ctx, ref) + if err != nil { + continue + } + + info.ReleaseChannels = append(info.ReleaseChannels, ReleaseChannelInfo{ + Channel: channel, + Version: version, + }) + versionSet[version] = true + } + + if len(info.ReleaseChannels) == 0 { + r.progress(" No release channels found") return info, nil } - info.Versions = tags - r.progress(" Found %d release tags", len(tags)) + r.progress(" Found %d release channels", len(info.ReleaseChannels)) + + for version := range versionSet { + info.Versions = append(info.Versions, version) + } + sort.Strings(info.Versions) + + digestSet := make(map[string]bool) + moduleBase := path.Join(r.registry, d8internal.ModulesSegment, moduleName) + + for _, version := range info.Versions { + moduleRef := moduleBase + ":" + version + digests, err := r.readModuleImageDigests(ctx, moduleRef) + if err != nil { + r.progress(" Warning: failed to read digests for %s:%s: %v", moduleName, version, err) + continue + } + + for _, d := range digests { + digestSet[d] = true + } + } + + for d := range digestSet { + info.ImageDigests = append(info.ImageDigests, d) + } + sort.Strings(info.ImageDigests) + + r.progress(" Found %d versions, %d unique digests", len(info.Versions), len(info.ImageDigests)) return info, nil } +func (r *SourceReader) readModuleReleaseChannelVersion(ctx context.Context, ref string) (string, error) { + imgRef, err := name.ParseReference(ref) + if err != nil { + return "", fmt.Errorf("parse reference: %w", err) + } + + opts := append(r.opts, remote.WithContext(ctx)) + img, err := remote.Image(imgRef, opts...) + if err != nil { + return "", fmt.Errorf("get image: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return "", fmt.Errorf("get layers: %w", err) + } + + for _, layer := range layers { + rc, err := layer.Uncompressed() + if err != nil { + continue + } + + version, err := extractVersionFromTar(rc) + rc.Close() + if err == nil && version != "" { + return version, nil + } + } + + return "", fmt.Errorf("version.json not found in image") +} + +func (r *SourceReader) readModuleImageDigests(ctx context.Context, ref string) ([]string, error) { + imgRef, err := name.ParseReference(ref) + if err != nil { + return nil, fmt.Errorf("parse reference: %w", err) + } + + opts := append(r.opts, remote.WithContext(ctx)) + desc, err := remote.Get(imgRef, opts...) + if err != nil { + return nil, fmt.Errorf("get descriptor: %w", err) + } + + var img v1.Image + + if desc.MediaType.IsIndex() { + idx, err := desc.ImageIndex() + if err != nil { + return nil, fmt.Errorf("get index: %w", err) + } + + manifest, err := idx.IndexManifest() + if err != nil { + return nil, fmt.Errorf("get index manifest: %w", err) + } + + if len(manifest.Manifests) == 0 { + return nil, fmt.Errorf("index has no manifests") + } + + img, err = idx.Image(manifest.Manifests[0].Digest) + if err != nil { + return nil, fmt.Errorf("get image from index: %w", err) + } + } else { + img, err = desc.Image() + if err != nil { + return nil, fmt.Errorf("get image: %w", err) + } + } + + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("get layers: %w", err) + } + + for _, layer := range layers { + rc, err := layer.Uncompressed() + if err != nil { + continue + } + + digests, err := extractModuleDigestsFromTar(rc) + rc.Close() + if err == nil && len(digests) > 0 { + return digests, nil + } + } + + return nil, nil +} + +func extractModuleDigestsFromTar(rc io.Reader) ([]string, error) { + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Name == "images_digests.json" || strings.HasSuffix(hdr.Name, "/images_digests.json") { + var digestsByComponent map[string]map[string]string + if err := json.NewDecoder(tr).Decode(&digestsByComponent); err != nil { + return nil, err + } + + var result []string + for _, images := range digestsByComponent { + for _, digest := range images { + result = append(result, digest) + } + } + return result, nil + } + } + return nil, nil +} + type SecurityDigests struct { Databases map[string][]string } diff --git a/testing/e2e/mirror/internal/verify.go b/testing/e2e/mirror/internal/verify.go index 5378c73a..347ba5c6 100644 --- a/testing/e2e/mirror/internal/verify.go +++ b/testing/e2e/mirror/internal/verify.go @@ -83,9 +83,15 @@ type VerificationResult struct { FoundAttTags []string MissingAttTags []string - ModulesExpected int - ModulesFound int - ModulesMissing []string + ModulesExpected int + ModulesFound int + ModulesMissing []string + ModuleVersionsTotal int + ModuleVersionsFound int + ModuleVersionsMissing []string + ModuleDigestsTotal int + ModuleDigestsFound int + ModuleDigestsMissing []string SecurityExpected int SecurityFound int @@ -102,6 +108,8 @@ func (r *VerificationResult) IsSuccess() bool { return len(r.MissingDigests) == 0 && len(r.MissingAttTags) == 0 && len(r.ModulesMissing) == 0 && + len(r.ModuleVersionsMissing) == 0 && + len(r.ModuleDigestsMissing) == 0 && len(r.SecurityMissing) == 0 } @@ -137,6 +145,18 @@ func (r *VerificationResult) Summary() string { if len(r.ModulesMissing) > 0 { sb.WriteString(fmt.Sprintf(" ✗ Missing modules: %d\n", len(r.ModulesMissing))) } + if r.ModuleVersionsTotal > 0 { + sb.WriteString(fmt.Sprintf(" ✓ Module versions: %d / %d\n", r.ModuleVersionsFound, r.ModuleVersionsTotal)) + if len(r.ModuleVersionsMissing) > 0 { + sb.WriteString(fmt.Sprintf(" ✗ Missing versions: %d\n", len(r.ModuleVersionsMissing))) + } + } + if r.ModuleDigestsTotal > 0 { + sb.WriteString(fmt.Sprintf(" ✓ Module digests: %d / %d\n", r.ModuleDigestsFound, r.ModuleDigestsTotal)) + if len(r.ModuleDigestsMissing) > 0 { + sb.WriteString(fmt.Sprintf(" ✗ Missing digests: %d\n", len(r.ModuleDigestsMissing))) + } + } } if r.SecurityExpected > 0 { sb.WriteString(fmt.Sprintf(" ✓ Security databases: %d / %d\n", r.SecurityFound, r.SecurityExpected)) @@ -191,6 +211,22 @@ func (r *VerificationResult) DetailedReport() string { sb.WriteString("\n") } + if len(r.ModuleVersionsMissing) > 0 { + sb.WriteString("MISSING MODULE VERSIONS:\n") + for _, v := range r.ModuleVersionsMissing { + sb.WriteString(fmt.Sprintf(" - %s\n", v)) + } + sb.WriteString("\n") + } + + if len(r.ModuleDigestsMissing) > 0 { + sb.WriteString("MISSING MODULE DIGESTS:\n") + for _, d := range r.ModuleDigestsMissing { + sb.WriteString(fmt.Sprintf(" - %s\n", d)) + } + sb.WriteString("\n") + } + if len(r.SecurityMissing) > 0 { sb.WriteString("MISSING SECURITY DATABASES:\n") for _, s := range r.SecurityMissing { @@ -295,12 +331,25 @@ func (v *DigestVerifier) VerifyModules(ctx context.Context, moduleNames []string releaseChannels := d8internal.GetAllDefaultReleaseChannels() sourceOpts := v.sourceReader.RemoteOpts() + targetOpts := append(v.targetOpts, remote.WithContext(ctx)) v.logProgressf("Verifying modules...") for _, moduleName := range modules { v.logProgressf(" Checking module: %s", moduleName) + moduleInfo, err := v.sourceReader.ReadModuleDigests(ctx, moduleName) + if err != nil { + v.logProgressf(" Warning: failed to read module info: %v", err) + result.Errors = append(result.Errors, fmt.Sprintf("module %s: failed to read: %v", moduleName, err)) + continue + } + + if len(moduleInfo.ReleaseChannels) == 0 { + v.logProgressf(" No release channels in source, skipping") + continue + } + sourceReleaseRepo := path.Join(v.sourceReader.Registry(), d8internal.ModulesSegment, moduleName, "release") targetReleaseRepo := path.Join(v.targetReg, d8internal.ModulesSegment, moduleName, "release") @@ -310,7 +359,6 @@ func (v *DigestVerifier) VerifyModules(ctx context.Context, moduleNames []string continue } - targetOpts := append(v.targetOpts, remote.WithContext(ctx)) targetTags, err := remote.List(targetRef.Context(), targetOpts...) if err != nil { v.logProgressf(" Target: no tags found (module not mirrored)") @@ -318,7 +366,6 @@ func (v *DigestVerifier) VerifyModules(ctx context.Context, moduleNames []string continue } - // Filter to only release channels targetChannels := []string{} for _, tag := range targetTags { for _, channel := range releaseChannels { @@ -337,9 +384,9 @@ func (v *DigestVerifier) VerifyModules(ctx context.Context, moduleNames []string v.logProgressf(" Target has %d channels: %v", len(targetChannels), targetChannels) - matchedChannels := []string{} - mismatchedChannels := []string{} - sourceNotFoundChannels := []string{} + matchedChannels := 0 + mismatchedChannels := 0 + sourceOptsWithCtx := append(sourceOpts, remote.WithContext(ctx)) for _, channel := range targetChannels { targetTagRef := targetReleaseRepo + ":" + channel @@ -350,58 +397,97 @@ func (v *DigestVerifier) VerifyModules(ctx context.Context, moduleNames []string targetDesc, err := remote.Head(targetImgRef, targetOpts...) if err != nil { - continue // Already listed, should exist + continue } targetDigest := targetDesc.Digest.String() sourceTagRef := sourceReleaseRepo + ":" + channel sourceImgRef, err := name.ParseReference(sourceTagRef) if err != nil { - sourceNotFoundChannels = append(sourceNotFoundChannels, channel) continue } - sourceOptsWithCtx := append(sourceOpts, remote.WithContext(ctx)) sourceDesc, err := remote.Head(sourceImgRef, sourceOptsWithCtx...) if err != nil { - // Channel exists in target but not in source - might be removed upstream - v.logProgressf(" ⚠ %s: exists in target but not in source (may be removed upstream)", channel) - sourceNotFoundChannels = append(sourceNotFoundChannels, channel) + v.logProgressf(" ⚠ %s: exists in target but not in source", channel) continue } sourceDigest := sourceDesc.Digest.String() if sourceDigest != targetDigest { v.logProgressf(" ✗ %s: DIGEST MISMATCH!", channel) - v.logProgressf(" Source: %s", sourceDigest) - v.logProgressf(" Target: %s", targetDigest) - mismatchedChannels = append(mismatchedChannels, channel) + mismatchedChannels++ result.Errors = append(result.Errors, - fmt.Sprintf("module %s/%s: digest mismatch (source=%s, target=%s)", - moduleName, channel, sourceDigest[:19], targetDigest[:19])) + fmt.Sprintf("module %s/%s: digest mismatch", moduleName, channel)) } else { v.logProgressf(" ✓ %s: digest match %s", channel, sourceDigest[:19]) - matchedChannels = append(matchedChannels, channel) + matchedChannels++ } } - if len(mismatchedChannels) > 0 { - v.logProgressf(" Result: %d matched, %d MISMATCHED, %d source-not-found", - len(matchedChannels), len(mismatchedChannels), len(sourceNotFoundChannels)) - result.ModulesFound++ - } else if len(matchedChannels) > 0 { - v.logProgressf(" ✓ All %d channels verified with matching digests", len(matchedChannels)) - result.ModulesFound++ - } else if len(sourceNotFoundChannels) > 0 { - v.logProgressf(" ⚠ All %d channels exist only in target (removed from source?)", len(sourceNotFoundChannels)) + if mismatchedChannels == 0 && matchedChannels > 0 { result.ModulesFound++ } + + v.logProgressf(" Verifying module version images...") + targetModuleRepo := path.Join(v.targetReg, d8internal.ModulesSegment, moduleName) + + for _, version := range moduleInfo.Versions { + result.ModuleVersionsTotal++ + targetVersionRef := targetModuleRepo + ":" + version + targetImgRef, err := name.ParseReference(targetVersionRef) + if err != nil { + result.ModuleVersionsMissing = append(result.ModuleVersionsMissing, moduleName+":"+version) + continue + } + + _, err = remote.Head(targetImgRef, targetOpts...) + if err != nil { + v.logProgressf(" ✗ %s:%s NOT FOUND in target", moduleName, version) + result.ModuleVersionsMissing = append(result.ModuleVersionsMissing, moduleName+":"+version) + } else { + v.logProgressf(" ✓ %s:%s found", moduleName, version) + result.ModuleVersionsFound++ + } + } + + if len(moduleInfo.ImageDigests) > 0 { + v.logProgressf(" Verifying %d module image digests...", len(moduleInfo.ImageDigests)) + v.verifyModuleDigests(ctx, result, moduleName, targetModuleRepo, moduleInfo.ImageDigests) + } } result.EndTime = time.Now() return result, nil } +func (v *DigestVerifier) verifyModuleDigests(ctx context.Context, result *VerificationResult, moduleName, targetRepo string, digests []string) { + items := make([]verifyItem, 0, len(digests)) + + for _, digest := range digests { + digest := digest + result.ModuleDigestsTotal++ + ref := targetRepo + "@" + digest + + items = append(items, verifyItem{ + ref: ref, + onErr: func(ref string, err error) { + result.Errors = append(result.Errors, fmt.Sprintf("module %s digest %s: %v", moduleName, digest[:19], err)) + }, + onFound: func(ref string) { + result.ModuleDigestsFound++ + }, + onMissing: func(ref string) { + result.ModuleDigestsMissing = append(result.ModuleDigestsMissing, moduleName+"@"+digest) + }, + }) + } + + v.verifyInParallel(ctx, items) + v.logProgressf(" Module digests: %d found, %d missing", + result.ModuleDigestsFound, len(result.ModuleDigestsMissing)) +} + func (v *DigestVerifier) VerifySecurity(ctx context.Context) (*VerificationResult, error) { result := &VerificationResult{ StartTime: time.Now(), @@ -667,6 +753,12 @@ func mergeResults(dst, src *VerificationResult) { dst.ModulesExpected += src.ModulesExpected dst.ModulesFound += src.ModulesFound dst.ModulesMissing = append(dst.ModulesMissing, src.ModulesMissing...) + dst.ModuleVersionsTotal += src.ModuleVersionsTotal + dst.ModuleVersionsFound += src.ModuleVersionsFound + dst.ModuleVersionsMissing = append(dst.ModuleVersionsMissing, src.ModuleVersionsMissing...) + dst.ModuleDigestsTotal += src.ModuleDigestsTotal + dst.ModuleDigestsFound += src.ModuleDigestsFound + dst.ModuleDigestsMissing = append(dst.ModuleDigestsMissing, src.ModuleDigestsMissing...) dst.SecurityExpected += src.SecurityExpected dst.SecurityFound += src.SecurityFound From 4c834e0db4c0b9c02300028dd0a06fef388e9aa3 Mon Sep 17 00:00:00 2001 From: Timur Tuktamyshev Date: Mon, 12 Jan 2026 14:10:15 +0300 Subject: [PATCH 11/11] chore: update readme.md Signed-off-by: Timur Tuktamyshev --- testing/e2e/mirror/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/testing/e2e/mirror/README.md b/testing/e2e/mirror/README.md index 48fc88ad..2b313b0f 100644 --- a/testing/e2e/mirror/README.md +++ b/testing/e2e/mirror/README.md @@ -25,8 +25,8 @@ These tests perform **complete mirror cycles with verification** to ensure: ### Step 1: Read Expected Images from Source Before pulling, we independently read what SHOULD be downloaded: - Release channel versions from source registry -- `images_digests.json` from each installer image -- Module list and versions +- `images_digests.json` from each installer image (platform) +- Module list, versions, and `images_digests.json` from each module image ### Step 2: Pull & Push Execute `d8 mirror pull` and `d8 mirror push` @@ -136,8 +136,9 @@ task test:e2e:mirror:logs:clean ### Modules Test 1. Module list matches expected -2. Each module has release tags -3. Module images match source +2. Release channel tags exist and digests match source +3. Module version images exist (`:v1.2.3` tags from release channels) +4. All digests from `images_digests.json` exist in target (if module has them) ### Security Test 1. All security databases exist (trivy-db, trivy-bdu, etc.) @@ -157,7 +158,9 @@ E2E_LICENSE_TOKEN=your_token task test:e2e:mirror ### "Verification failed" Check `comparison.txt`: -- **Missing in target**: Pull or push didn't transfer the image +- **Missing digests**: Pull or push didn't transfer the image +- **Missing module versions**: Module version image not in target +- **Missing module digests**: Module's internal images not transferred - **Digest mismatch**: Data corruption or version skew ### Running Against Local Registry