Merge upstream/master into overrides_api

Signed-off-by: Bogdan Stancu <stancu.bogdan.nicolae@gmail.com>

Author: bogdan-st

.github/workflows/scripts/install-docker.sh

@@ -2,10 +2,30 @@
 
 set -x
 VER="28.0.4"
-curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
+
+# Detect architecture
+ARCH=$(uname -m)
+case $ARCH in
+  x86_64)
+    DOCKER_ARCH="x86_64"
+    BUILDX_ARCH="amd64"
+    ;;
+  aarch64)
+    DOCKER_ARCH="aarch64"
+    BUILDX_ARCH="arm64"
+    ;;
+  *)
+    echo "Unsupported architecture: $ARCH"
+    exit 1
+    ;;
+esac
+
+echo "Installing Docker $VER for architecture: $ARCH (docker: $DOCKER_ARCH, buildx: $BUILDX_ARCH)"
+
+curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/$DOCKER_ARCH/docker-$VER.tgz
 tar -xz -C /tmp -f /tmp/docker-$VER.tgz
 mkdir -vp ~/.docker/cli-plugins/
-curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
+curl --silent -L "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-$BUILDX_ARCH" > ~/.docker/cli-plugins/docker-buildx
 chmod a+x ~/.docker/cli-plugins/docker-buildx
 mv /tmp/docker/* /usr/bin
 docker run --privileged --rm tonistiigi/binfmt --install all

.github/workflows/test-build-deploy.yml

@@ -42,6 +42,8 @@ jobs:
         run: make BUILD_IN_CONTAINER=false check-doc
       - name: Check White Noise.
         run: make BUILD_IN_CONTAINER=false check-white-noise
+      - name: Check Modernize
+        run: make BUILD_IN_CONTAINER=false check-modernize
 
   test:
     runs-on: ubuntu-24.04
@@ -150,19 +152,60 @@ jobs:
 
   integration:
     needs: build
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 50
     strategy:
       fail-fast: false
       matrix:
-        tags:
-          - requires_docker
-          - integration_alertmanager
-          - integration_backward_compatibility
-          - integration_memberlist
-          - integration_querier
-          - integration_ruler
-          - integration_query_fuzz
-          - integration_remote_write_v2
+        include:
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: requires_docker
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: integration_alertmanager
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: integration_backward_compatibility
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: integration_memberlist
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: integration_querier
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: integration_ruler
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: integration_query_fuzz
+          - runner: ubuntu-24.04
+            arch: amd64
+            tags: integration_remote_write_v2
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: requires_docker
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: integration_alertmanager
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: integration_memberlist
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: integration_ruler
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: integration_remote_write_v2
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: integration_query_fuzz
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: integration_backward_compatibility
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+            tags: integration_querier
     steps:
       - name: Upgrade golang
         uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
@@ -195,42 +238,43 @@ jobs:
           docker pull consul:1.8.4
           docker pull gcr.io/etcd-development/etcd:v3.4.7
           if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then
-            docker pull quay.io/cortexproject/cortex:v1.13.1
-            docker pull quay.io/cortexproject/cortex:v1.13.2
-            docker pull quay.io/cortexproject/cortex:v1.14.0
-            docker pull quay.io/cortexproject/cortex:v1.14.1
-            docker pull quay.io/cortexproject/cortex:v1.15.0
-            docker pull quay.io/cortexproject/cortex:v1.15.1
-            docker pull quay.io/cortexproject/cortex:v1.15.2
-            docker pull quay.io/cortexproject/cortex:v1.15.3
-            docker pull quay.io/cortexproject/cortex:v1.16.0
             docker pull quay.io/cortexproject/cortex:v1.16.1
-            docker pull quay.io/cortexproject/cortex:v1.17.0
-            docker pull quay.io/cortexproject/cortex:v1.17.1
-            docker pull quay.io/cortexproject/cortex:v1.18.0
+            docker pull quay.io/cortexproject/cortex:v1.17.2
             docker pull quay.io/cortexproject/cortex:v1.18.1
+            docker pull quay.io/cortexproject/cortex:v1.19.1
+            docker pull quay.io/cortexproject/cortex:v1.20.0
           elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then
-            docker pull quay.io/cortexproject/cortex:v1.18.1 
-            docker pull quay.io/prometheus/prometheus:v3.5.0
+            docker pull quay.io/cortexproject/cortex:v1.20.0 
+            docker pull quay.io/prometheus/prometheus:v3.6.0
           fi
           docker pull memcached:1.6.1
           docker pull redis:7.0.4-alpine
         env:
           TEST_TAGS: ${{ matrix.tags }}
       - name: Integration Tests
+        timeout-minutes: 45
         run: |
           export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}"
           export IMAGE_TAG=$(make image-tag)
-          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64"
+          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:${IMAGE_TAG}-${{ matrix.arch }}"
           export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex"
-          echo "Running integration tests with image: $CORTEX_IMAGE"
+          echo "Running integration tests on ${{ matrix.arch }} with image: ${CORTEX_IMAGE}"
           go test -tags=slicelabels,integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/...
         env:
           IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }}
 
   integration-configs-db:
     needs: build
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: amd64
+          - runner: ubuntu-24.04-arm
+            arch: arm64
     steps:
       - name: Checkout Repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -243,15 +287,17 @@ jobs:
       - name: Extract Docker Images Archive
         run: tar -xvf images.tar -C /
       - name: Run Integration Configs Tests
+        timeout-minutes: 15
         # Github Actions does not support TTY in their default runners yet
         run: |
           touch build-image/.uptodate
           MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
+          echo "Running configs integration tests on ${{ matrix.arch }}"
           make BUILD_IMAGE=quay.io/cortexproject/build-image:master-59491e9aae TTY='' configs-integration-test
 
   deploy_website:
     needs: [build, test]
-    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
+    if: github.ref == 'refs/heads/master' && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-24.04
     container:
       image: quay.io/cortexproject/build-image:master-59491e9aae

ADOPTERS.md

@@ -21,5 +21,6 @@ This is the list of organisations that are using Cortex in **production environm
 * [Planetary Quantum](https://www.planetary-quantum.com)
 * [Platform9](https://platform9.com/)
 * [REWE Digital](https://rewe-digital.com/)
+* [Swiggy](https://www.swiggy.in/)
 * [SysEleven](https://www.syseleven.de/)
 * [Twilio](https://www.twilio.com/)

CHANGELOG.md

@@ -1,17 +1,34 @@
 # Changelog
 
 ## master / unreleased
+
+* [FEATURE] StoreGateway: Introduces a new parquet mode. #7046
+* [ENHANCEMENT] Alertmanager: Upgrade alertmanger to 0.29.0 and add a new incidentIO integration. #7092
+* [ENHANCEMENT] Querier: Add a `-querier.parquet-queryable-shard-cache-ttl` flag to add TTL to parquet shard cache. #7098
+* [ENHANCEMENT] Ingester: Add `enable_matcher_optimization` config to apply low selectivity matchers lazily. #7063
+* [ENHANCEMENT] Distributor: Add a label references validation for remote write v2 request. #7074
+* [ENHANCEMENT] Distributor: Add count, spans, and buckets validations for native histogram. #7072
+* [ENHANCEMENT] Alertmanager/Ruler: Introduce a user scanner to reduce the number of list calls to object storage. #6999
+* [ENHANCEMENT] Ruler: Add DecodingConcurrency config flag for Thanos Engine. #7118
+* [ENHANCEMENT] Compactor: Avoid double compaction by cleaning partition files in 2 cycles. #7129
+* [BUGFIX] Ring: Change DynamoDB KV to retry indefinitely for WatchKey. #7088
+* [BUGFIX] Ruler: Add XFunctions validation support. #7111
+* [BUGFIX] Distributor: Fix panic on health check failure when using stream push. #7116
+
+## 1.20.0 2025-11-10
+
 * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603
 * [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718
 * [CHANGE] Validate a tenantID when to use a single tenant resolver. #6727
+* [CHANGE] Ring: Add zone label to ring_members metric. #6900
 * [FEATURE] Distributor: Add an experimental `-distributor.otlp.enable-type-and-unit-labels` flag to add `__type__` and `__unit__` labels for OTLP metrics. #6969
 * [FEATURE] Distributor: Add an experimental `-distributor.otlp.allow-delta-temporality` flag to ingest delta temporality otlp metrics. #6934
 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458
 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526
 * [FEATURE] Update prometheus alertmanager version to v0.28.0 and add new integration msteamsv2, jira, and rocketchat. #6590
 * [FEATURE] Ingester/StoreGateway: Add `ResourceMonitor` module in Cortex, and add `ResourceBasedLimiter` in Ingesters and StoreGateways. #6674
 * [FEATURE] Support Prometheus remote write 2.0. #6330
-* [FEATURE] Ingester: Support out-of-order native histogram ingestion. It automatically enabled when `-ingester.out-of-order-time-window > 0` and `-blocks-storage.tsdb.enable-native-histograms=true`. #6626 #6663
+* [FEATURE] Ingester: Support out-of-order native histogram ingestion. It is automatically enabled when `-ingester.out-of-order-time-window > 0` and `-blocks-storage.tsdb.enable-native-histograms=true`. #6626 #6663
 * [FEATURE] Ruler: Add support for percentage based sharding for rulers. #6680
 * [FEATURE] Ruler: Add support for group labels. #6665
 * [FEATURE] Query federation: Introduce a regex tenant resolver to allow regex in `X-Scope-OrgID` value. #6713
@@ -20,12 +37,17 @@
 * [FEATURE] Experimental Support Parquet format: Implement parquet converter service to convert a TSDB block into Parquet and Parquet Queryable. #6716 #6743
 * [FEATURE] Distributor/Ingester: Implemented experimental feature to use gRPC stream connection for push requests. This can be enabled by setting `-distributor.use-stream-push=true`. #6580
 * [FEATURE] Compactor: Add support for percentage based sharding for compactors. #6738
-* [FEATURE] Querier: Allow choosing PromQL engine via header. #6777
+* [FEATURE] Querier: Allow choosing PromQL engine via header `X-PromQL-EngineType`. #6777
 * [FEATURE] Querier: Support for configuring query optimizers and enabling XFunctions in the Thanos engine. #6873
 * [FEATURE] Query Frontend: Add support /api/v1/format_query API for formatting queries. #6893
 * [FEATURE] Query Frontend: Add support for /api/v1/parse_query API (experimental) to parse a PromQL expression and return it as a JSON-formatted AST (abstract syntax tree). #6978
+* [ENHANCEMENT] Upgrade the Prometheus version to 3.6.0 and add a `-name-validation-scheme` flag to support UTF-8. #7040 #7056
+* [ENHANCEMENT] Distributor: Emit an error with a 400 status code when empty labels are found before the relabelling or label dropping process. #7052
+* [ENHANCEMENT] Parquet Storage: Add support for additional sort columns during Parquet file generation #7003
+* [ENHANCEMENT] Modernizes the entire codebase by using go modernize tool. #7005
 * [ENHANCEMENT] Overrides Exporter: Expose all fields that can be converted to float64. Also, the label value `max_local_series_per_metric` got renamed to `max_series_per_metric`, and `max_local_series_per_user` got renamed to `max_series_per_user`. #6979
 * [ENHANCEMENT] Ingester: Add `cortex_ingester_tsdb_wal_replay_unknown_refs_total` and `cortex_ingester_tsdb_wbl_replay_unknown_refs_total` metrics to track unknown series references during wal/wbl replaying. #6945
+* [ENHANCEMENT] Distributor: Introduce a Protobuf model for Prometheus Remote Write 2.0 and a pool to improve performance. #6917
 * [ENHANCEMENT] Ruler: Emit an error message when the rule synchronization fails. #6902
 * [ENHANCEMENT] Querier: Support snappy and zstd response compression for `-querier.response-compression` flag. #6848
 * [ENHANCEMENT] Tenant Federation: Add a # of query result limit logic when the `-tenant-federation.regex-matcher-enabled` is enabled. #6845
@@ -35,9 +57,8 @@
 * [ENHANCEMENT] Ingester: Add a `cortex_ingester_active_native_histogram_series` metric to track # of active NH series. #6695
 * [ENHANCEMENT] Query Frontend: Add new limit `-frontend.max-query-response-size` for total query response size after decompression in query frontend. #6607
 * [ENHANCEMENT] Alertmanager: Add nflog and silences maintenance metrics. #6659
-* [ENHANCEMENT] Querier: limit label APIs to query only ingesters if `start` param is not been specified. #6618
+* [ENHANCEMENT] Querier: limit label APIs to query only ingesters if `start` param is not specified. #6618
 * [ENHANCEMENT] Alertmanager: Add new limits `-alertmanager.max-silences-count` and `-alertmanager.max-silences-size-bytes` for limiting silences per tenant. #6605
-* [ENHANCEMENT] Update prometheus version to v3.1.0. #6583
 * [ENHANCEMENT] Add `compactor.auto-forget-delay` for compactor to auto forget compactors after X minutes without heartbeat. #6533
 * [ENHANCEMENT] StoreGateway: Emit more histogram buckets on the `cortex_querier_storegateway_refetches_per_query` metric. #6570
 * [ENHANCEMENT] Querier: Apply bytes limiter to LabelNames and LabelValuesForLabelNames. #6568
@@ -66,7 +87,6 @@
 * [ENHANCEMENT] Distributor: Add native histograms max sample size bytes limit validation. #6834
 * [ENHANCEMENT] Querier: Support caching parquet labels file in parquet queryable. #6835
 * [ENHANCEMENT] Querier: Support query limits in parquet queryable. #6870
-* [ENHANCEMENT] Ring: Add zone label to ring_members metric. #6900
 * [ENHANCEMENT] Ingester: Add new metric `cortex_ingester_push_errors_total` to track reasons for ingester request failures. #6901
 * [ENHANCEMENT] Ring: Expose `detailed_metrics_enabled` for all rings. Default true. #6926
 * [ENHANCEMENT] Parquet Storage: Allow Parquet Queryable to disable fallback to Store Gateway. #6920
@@ -76,6 +96,10 @@
 * [ENHANCEMENT] Upgrade build image and Go version to 1.24.6. #6970 #6976
 * [ENHANCEMENT] Implement versioned transactions for writes to DynamoDB ring. #6986
 * [ENHANCEMENT] Add source metadata to requests(api vs ruler) #6947
+* [ENHANCEMENT] Add new metric `cortex_discarded_series` and `cortex_discarded_series_per_labelset` to track number of series that have a discarded sample. #6995
+* [ENHANCEMENT] Ingester: Add `cortex_ingester_tsdb_head_stale_series` metric to keep track of number of stale series on head. #7071
+* [ENHANCEMENT] Expose more Go runtime metrics. #7070
+* [ENHANCEMENT] Distributor: Filter out label with empty value. #7069
 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517
 * [BUGFIX] Ingester: Fix labelset data race condition. #6573
 * [BUGFIX] Compactor: Cleaner should not put deletion marker for blocks with no-compact marker. #6576
@@ -95,6 +119,16 @@
 * [BUGFIX] Compactor: Delete the prefix `blocks_meta` from the metadata fetcher metrics. #6832
 * [BUGFIX] Store Gateway: Avoid race condition by deduplicating entries in bucket stores user scan. #6863
 * [BUGFIX] Runtime-config: Change to check tenant limit validation when loading runtime config only for `all`, `distributor`, `querier`, and `ruler` targets. #6880
+* [BUGFIX] Distributor: Fix `/distributor/all_user_stats` api to work during rolling updates on ingesters. #7026
+* [BUGFIX] Runtime-config: Fix panic when the runtime config is `null`. #7062
+* [BUGFIX] Scheduler: Avoid all queriers reserved for prioritized requests. #7057
+* [BUGFIX] Fix bug where validating metric names uses the wrong validation logic. #7086
+* [BUGFIX] Compactor: Avoid race condition which allow a grouper to not compact all partitions. #7082
+* [BUGFIX] Add alertmanager receiver validation for discord and email. #7097
+
+## 1.19.1 2025-09-20
+
+* [BUGFIX] Frontend: Fix remote read snappy input due to request string logging when query stats enabled. #7025
 
 ## 1.19.0 2025-02-27
 

Makefile

@@ -126,7 +126,7 @@ GOVOLUMES=	-v $(shell pwd)/.cache:/go/cache:delegated,z \
 			-v $(shell pwd)/.pkg:/go/pkg:delegated,z \
 			-v $(shell pwd):/go/src/github.com/cortexproject/cortex:delegated,z
 
-exes $(EXES) protos $(PROTO_GOS) lint test cover shell mod-check check-protos web-build web-pre web-deploy doc: build-image/$(UPTODATE)
+exes $(EXES) protos $(PROTO_GOS) lint test cover shell mod-check check-protos web-build web-pre web-deploy doc modernize: build-image/$(UPTODATE)
 	@mkdir -p $(shell pwd)/.pkg
 	@mkdir -p $(shell pwd)/.cache
 	@echo
@@ -251,6 +251,9 @@ web-build: web-pre
 web-deploy:
 	./tools/website/web-deploy.sh
 
+modernize:
+	go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@v0.20.0 -fix ./...
+
 # Generates the config file documentation.
 doc: clean-doc
 	go run -tags slicelabels ./tools/doc-generator ./docs/configuration/config-file-reference.template > ./docs/configuration/config-file-reference.md
@@ -260,6 +263,7 @@ doc: clean-doc
 	go run -tags slicelabels ./tools/doc-generator ./docs/guides/encryption-at-rest.template           > ./docs/guides/encryption-at-rest.md
 	embedmd -w docs/operations/requests-mirroring-to-secondary-cluster.md
 	embedmd -w docs/guides/overrides-exporter.md
+	go run -tags slicelabels ./tools/doc-generator -json-schema            						       > ./schemas/cortex-config-schema.json
 
 endif
 
@@ -308,6 +312,9 @@ clean-white-noise:
 check-white-noise: clean-white-noise
 	@git diff --exit-code --quiet -- '*.md' || (echo "Please remove trailing whitespaces running 'make clean-white-noise'" && false)
 
+check-modernize: modernize
+	@git diff --exit-code -- . || (echo "Please modernize running 'make modernize'" && false)
+
 web-serve:
 	cd website && hugo --config config.toml --minify -v server
 

RELEASE.md

@@ -35,6 +35,7 @@ Our goal is to provide a new minor release every 6 weeks. This is a new process
 | v1.17.0        | 2024-04-25                                 | Ben Ye (@yeya24)                            |
 | v1.18.0        | 2024-08-16                                 | Daniel Blando (@danielblando)               |
 | v1.19.0        | 2025-01-22                                 | Charlie Le (@charlietle)                    |
+| v1.20.0        | 2025-10-31                                 | Ben Ye (@yeya24)                            |
 
 ## Release shepherd responsibilities
 

VERSION

@@ -1 +1 @@
-1.19.0
+1.20.0