Skip to content

Split the config maps based on size while being created. So that many config maps are created with max size of 950KB and rule evaluator loads all configmaps to file for rules generator #1801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
balakumarpg wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Split the config maps based on size while being created. So that many config maps are created with max size of 950KB and rule evaluator loads all configmaps to file for rules generator #1801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
64b3f2f
Split the config maps based on size while being created. So that many…
balakumarpg Nov 10, 2025
e873885
refactor(operator): split rules into separate configmaps per type
balakumarpg Nov 10, 2025
a4c7eae
# <type>(<scope>): <description>
balakumarpg Nov 25, 2025
cc43815
style(operator): fix golangci-lint errors and manifest format
balakumarpg Nov 25, 2025
af22eea
fix(operator): restore status updates and fix tests
balakumarpg Nov 25, 2025
9055d24
fix(e2e): update tests for three-configmap architecture
balakumarpg Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions charts/rule-evaluator/templates/deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,21 @@ spec:
- name: config-out
emptyDir: {}
- name: rules
configMap:
name: rules
# Mount exactly one ConfigMap per rule type (3 total)
# - rules: namespace-scoped Rules
# - clusterrules: cluster-scoped ClusterRules
# - globalrules: GlobalRules
projected:
sources:
- configMap:
name: rules
optional: true
- configMap:
name: clusterrules
optional: true
- configMap:
name: globalrules
optional: true
- name: rules-out
emptyDir: {}
affinity:
Expand Down
7 changes: 3 additions & 4 deletions cmd/config-reloader/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ COPY go.sum go.sum
COPY vendor* vendor
COPY cmd cmd

RUN GO111MODULE=on go get github.com/mikefarah/yq/v3 # Install yq tool.

ENV GOEXPERIMENT=boringcrypto
ENV CGO_ENABLED=1
ENV GOFIPS140=off
Expand All @@ -37,10 +35,11 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
gcc-aarch64-linux-gnu libc6-dev-arm64-cross; \
CC=aarch64-linux-gnu-gcc; \
fi && \
VERSION=$(awk -F': ' '/^version:/ {print $2}' charts/values.global.yaml) && \
BUILD_DATE=$(date --iso-8601=seconds) && \
GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=${CC} \
go build \
-ldflags="-X github.com/prometheus/common/version.Version=$(cat charts/values.global.yaml | yq '.version' ) \
-X github.com/prometheus/common/version.BuildDate=$(date --iso-8601=seconds)" \
-ldflags="-X github.com/prometheus/common/version.Version=${VERSION} -X github.com/prometheus/common/version.BuildDate=${BUILD_DATE}" \
-o config-reloader \
cmd/config-reloader/*.go

Expand Down
7 changes: 3 additions & 4 deletions cmd/datasource-syncer/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ COPY go.sum go.sum
COPY vendor* vendor
COPY cmd cmd

RUN GO111MODULE=on go get github.com/mikefarah/yq/v3 # Install yq tool.

ENV GOEXPERIMENT=boringcrypto
ENV CGO_ENABLED=1
ENV GOFIPS140=off
Expand All @@ -37,10 +35,11 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
gcc-aarch64-linux-gnu libc6-dev-arm64-cross; \
CC=aarch64-linux-gnu-gcc; \
fi && \
VERSION=$(awk -F': ' '/^version:/ {print $2}' charts/values.global.yaml) && \
BUILD_DATE=$(date --iso-8601=seconds) && \
GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=${CC} \
go build \
-ldflags="-X github.com/prometheus/common/version.Version=$(cat charts/values.global.yaml | yq '.version' ) \
-X github.com/prometheus/common/version.BuildDate=$(date --iso-8601=seconds)" \
-ldflags="-X github.com/prometheus/common/version.Version=${VERSION} -X github.com/prometheus/common/version.BuildDate=${BUILD_DATE}" \
-o datasource-syncer \
cmd/datasource-syncer/*.go

Expand Down
7 changes: 3 additions & 4 deletions cmd/frontend/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@ WORKDIR /app
COPY charts/values.global.yaml charts/values.global.yaml
COPY --from=assets /app ./

RUN GO111MODULE=on go get github.com/mikefarah/yq/v3 # Install yq tool.

ENV GOEXPERIMENT=boringcrypto
ENV CGO_ENABLED=1
ENV GOFIPS140=off
Expand All @@ -50,10 +48,11 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
gcc-aarch64-linux-gnu libc6-dev-arm64-cross; \
CC=aarch64-linux-gnu-gcc; \
fi && \
VERSION=$(awk -F': ' '/^version:/ {print $2}' charts/values.global.yaml) && \
BUILD_DATE=$(date --iso-8601=seconds) && \
GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=${CC} \
go build \
-ldflags="-X github.com/prometheus/common/version.Version=$(cat charts/values.global.yaml | yq ".version" ) \
-X github.com/prometheus/common/version.BuildDate=$(date --iso-8601=seconds)" \
-ldflags="-X github.com/prometheus/common/version.Version=${VERSION} -X github.com/prometheus/common/version.BuildDate=${BUILD_DATE}" \
-o frontend \
cmd/frontend/*.go

Expand Down
7 changes: 3 additions & 4 deletions cmd/operator/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ COPY vendor* vendor
COPY cmd cmd
COPY pkg pkg

RUN GO111MODULE=on go get github.com/mikefarah/yq/v3 # Install yq tool.

ENV GOEXPERIMENT=boringcrypto
ENV CGO_ENABLED=1
ENV GOFIPS140=off
Expand All @@ -38,10 +36,11 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
gcc-aarch64-linux-gnu libc6-dev-arm64-cross; \
CC=aarch64-linux-gnu-gcc; \
fi && \
VERSION=$(awk -F': ' '/^version:/ {print $2}' charts/values.global.yaml) && \
BUILD_DATE=$(date --iso-8601=seconds) && \
GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=${CC} \
go build \
-ldflags="-X github.com/prometheus/common/version.Version=$(cat charts/values.global.yaml | yq '.version' ) \
-X github.com/prometheus/common/version.BuildDate=$(date --iso-8601=seconds)" \
-ldflags="-X github.com/prometheus/common/version.Version=${VERSION} -X github.com/prometheus/common/version.BuildDate=${BUILD_DATE}" \
-o operator \
cmd/operator/*.go

Expand Down
7 changes: 3 additions & 4 deletions cmd/rule-evaluator/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ COPY cmd cmd
COPY pkg pkg
COPY internal internal

RUN GO111MODULE=on go get github.com/mikefarah/yq/v3 # Install yq tool.

ENV GOEXPERIMENT=boringcrypto
ENV CGO_ENABLED=1
ENV GOFIPS140=off
Expand All @@ -39,10 +37,11 @@ RUN if [ "${TARGETARCH}" = "arm64" ] && [ "${BUILDARCH}" != "arm64" ]; then \
gcc-aarch64-linux-gnu libc6-dev-arm64-cross; \
CC=aarch64-linux-gnu-gcc; \
fi && \
VERSION=$(awk -F': ' '/^version:/ {print $2}' charts/values.global.yaml) && \
BUILD_DATE=$(date --iso-8601=seconds) && \
GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=${CC} \
go build \
-ldflags="-X github.com/prometheus/common/version.Version=$(cat charts/values.global.yaml | yq '.version' ) \
-X github.com/prometheus/common/version.BuildDate=$(date --iso-8601=seconds)" \
-ldflags="-X github.com/prometheus/common/version.Version=${VERSION} -X github.com/prometheus/common/version.BuildDate=${BUILD_DATE}" \
-o rule-evaluator \
cmd/rule-evaluator/*.go

Expand Down
59 changes: 38 additions & 21 deletions e2e/ruler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,30 +561,47 @@ func testCreateRules(
var diff string

err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) {
var cm corev1.ConfigMap
if err := kubeClient.Get(ctx, client.ObjectKey{Namespace: systemNamespace, Name: "rules-generated"}, &cm); err != nil {
if apierrors.IsNotFound(err) {
return false, nil
}
return false, fmt.Errorf("get ConfigMap: %w", err)
}
data := cm.Data
if features.Config.Compression == monitoringv1.CompressionGzip {
// When compression is enabled, we expect the config map with recording
// rules to be compressed with gzip. Decompress all files for validation.
for key, compressedData := range cm.BinaryData {
r, err := gzip.NewReader(bytes.NewReader(compressedData))
if err != nil {
t.Fatal(err)
// Collect data from all three ConfigMaps: rules, clusterrules, globalrules
data := make(map[string]string)

// List of ConfigMaps to check
configMapNames := []string{"rules", "clusterrules", "globalrules"}

for _, cmName := range configMapNames {
var cm corev1.ConfigMap
if err := kubeClient.Get(ctx, client.ObjectKey{Namespace: systemNamespace, Name: cmName}, &cm); err != nil {
if apierrors.IsNotFound(err) {
// ConfigMap doesn't exist yet, continue waiting
continue
}
decompressed, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
return false, fmt.Errorf("get ConfigMap %s: %w", cmName, err)
}

// Merge data from this ConfigMap
if features.Config.Compression == monitoringv1.CompressionGzip {
// When compression is enabled, decompress all files for validation
for key, compressedData := range cm.BinaryData {
r, err := gzip.NewReader(bytes.NewReader(compressedData))
if err != nil {
t.Fatal(err)
}
decompressed, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
}
if _, ok := data[key]; ok {
t.Errorf("duplicate ConfigMap key %q", key)
}
data[key] = string(decompressed)
}
if _, ok := data[key]; ok {
t.Errorf("duplicate ConfigMap key %q", key)
} else {
// Uncompressed data
for key, value := range cm.Data {
if _, ok := data[key]; ok {
t.Errorf("duplicate ConfigMap key %q", key)
}
data[key] = value
}
data[key] = string(decompressed)
}
}

Expand Down
17 changes: 15 additions & 2 deletions manifests/rule-evaluator.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,8 +213,21 @@ spec:
- name: config-out
emptyDir: {}
- name: rules
configMap:
name: rules
# Mount exactly one ConfigMap per rule type (3 total)
# - rules: namespace-scoped Rules
# - clusterrules: cluster-scoped ClusterRules
# - globalrules: GlobalRules
projected:
sources:
- configMap:
name: rules
optional: true
- configMap:
name: clusterrules
optional: true
- configMap:
name: globalrules
optional: true
- name: rules-out
emptyDir: {}
affinity:
Expand Down
Loading