Custom TLS for Exporter (Encryption Only)

With this change we allow users to bring custom certificates and enable
TLS for the exporter. This will be an opt-in feature, PGO will not
automatically generate certs like it does for some other features.

You can enable TLS by using the following spec fields:

spec:
  monitoring:
    pgmonitor:
      exporter:
        customTLSSecret:
          name: hippo.tls

Once TLS is enabled in the exporter, you can configure your Prometheus
instance to scrape over https.

Author: jmckulk

bin/crunchy-postgres-exporter/start.sh

@@ -239,6 +239,10 @@ sed -i \
   /tmp/queries.yml
 
 PG_OPTIONS="--extend.query-path=${QUERY_DIR?}/queries.yml  --web.listen-address=:${POSTGRES_EXPORTER_PORT}"
+if [[ -v WEB_CONFIG_DIR ]]; then
+    # TODO (jmckulk): define path not dir
+    PG_OPTIONS+=" --web.config.file=${WEB_CONFIG_DIR}/web-config.yml"
+fi
 
 echo_info "Starting postgres-exporter.."
 DATA_SOURCE_URI="${EXPORTER_PG_HOST}:${EXPORTER_PG_PORT}/${EXPORTER_PG_DATABASE}?${EXPORTER_PG_PARAMS}" DATA_SOURCE_USER="${EXPORTER_PG_USER}" DATA_SOURCE_PASS="${EXPORTER_PG_PASSWORD}" ${PG_EXP_HOME?}/postgres_exporter ${PG_OPTIONS?} >>/dev/stdout 2>&1 &

build/crd/todos.yaml

@@ -52,6 +52,9 @@
 - op: copy
   from: /work
   path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/monitoring/properties/pgmonitor/properties/exporter/properties/configuration/items/properties/secret/properties/name/description
+- op: copy
+  from: /work
+  path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/monitoring/properties/pgmonitor/properties/exporter/properties/customTLSSecret/properties/name/description
 - op: copy
   from: /work
   path: /spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/proxy/properties/pgBouncer/properties/config/properties/files/items/properties/configMap/properties/name/description

config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -9845,10 +9845,10 @@ spec:
                           configuration:
                             description: 'Projected volumes containing custom PostgreSQL
                               Exporter configuration.  Currently supports the customization
-                              of PostgreSQL Exporter queries. If a "queries.yaml"
-                              file is detected in any volume projected using this
-                              field, it will be loaded using the "extend.query-path"
-                              flag: https://github.com/prometheus-community/postgres_exporter#flags
+                              of PostgreSQL Exporter queries. If a "queries.yml" file
+                              is detected in any volume projected using this field,
+                              it will be loaded using the "extend.query-path" flag:
+                              https://github.com/prometheus-community/postgres_exporter#flags
                               Changing the values of field causes PostgreSQL and the
                               exporter to restart.'
                             items:
@@ -10091,6 +10091,60 @@ spec:
                                   type: object
                               type: object
                             type: array
+                          customTLSSecret:
+                            description: Projected secret containing custom TLS certificates
+                              to encrypt output from the exporter web server
+                            properties:
+                              items:
+                                description: items if unspecified, each key-value
+                                  pair in the Data field of the referenced Secret
+                                  will be projected into the volume as a file whose
+                                  name is the key and content is the value. If specified,
+                                  the listed keys will be projected into the specified
+                                  paths, and unlisted keys will not be present. If
+                                  a key is specified which is not present in the Secret,
+                                  the volume setup will error unless it is marked
+                                  optional. Paths must be relative and may not contain
+                                  the '..' path or start with '..'.
+                                items:
+                                  description: Maps a string key to a path within
+                                    a volume.
+                                  properties:
+                                    key:
+                                      description: key is the key to project.
+                                      type: string
+                                    mode:
+                                      description: 'mode is Optional: mode bits used
+                                        to set permissions on this file. Must be an
+                                        octal value between 0000 and 0777 or a decimal
+                                        value between 0 and 511. YAML accepts both
+                                        octal and decimal values, JSON requires decimal
+                                        values for mode bits. If not specified, the
+                                        volume defaultMode will be used. This might
+                                        be in conflict with other options that affect
+                                        the file mode, like fsGroup, and the result
+                                        can be other mode bits set.'
+                                      format: int32
+                                      type: integer
+                                    path:
+                                      description: path is the relative path of the
+                                        file to map the key to. May not be an absolute
+                                        path. May not contain the path element '..'.
+                                        May not start with the string '..'.
+                                      type: string
+                                  required:
+                                  - key
+                                  - path
+                                  type: object
+                                type: array
+                              name:
+                                description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                                type: string
+                              optional:
+                                description: optional field specify whether the Secret
+                                  or its key must be defined
+                                type: boolean
+                            type: object
                           image:
                             description: The image name to use for crunchy-postgres-exporter
                               containers. The image may also be set using the RELATED_IMAGE_PGEXPORTER

docs/content/references/crd.md

@@ -40,6 +40,32 @@ PGO will detect the change and add the Exporter sidecar to all Postgres Pods tha
 cluster. PGO will also do the work to allow the Exporter to connect to the database and gather
 metrics that can be accessed using the [PGO Monitoring] stack.
 
+### Configuring TLS Encryption for the Exporter
+
+PGO allows you to configure the exporter sidecar to use TLS encryption. If you provide a custom TLS
+Secret via the exporter spec:
+
+```
+  monitoring:
+    pgmonitor:
+      exporter:
+        customTLSSecret:
+          name: hippo.tls
+```
+
+Like other custom TLS Secrets that can be configured with PGO, the Secret will need to be created in
+the same Namespace as your PostgresCluster. It should also contain the TLS key (`tls.key`) and TLS
+certificate (`tls.crt`) needed to enable encryption.
+
+```
+data:
+  tls.crt: <value>
+  tls.key: <value>
+```
+
+After you configure TLS for the exporter, you will need to update your Prometheus deployment to use
+TLS, and your connection to the exporter will be encrypted. Check out the [Prometheus] documentation
+for more information on configuring TLS for [Prometheus].
 
 ## Accessing the Metrics
 

docs/content/tutorial/monitoring.md

@@ -17,7 +17,7 @@ directory=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 crd_build_dir="$directory"/../build/crd
 
 # Generate a Kustomize patch file for removing any TODOs we inherit from the Kubernetes API.
-# Right now there are one TODO in our CRD. This script focuses on removing the specific TODO
+# Right now there is one TODO in our CRD. This script focuses on removing the specific TODO
 # anywhere they are found in the CRD.
 
 # The TODO comes from the following:

hack/create-todo-patch.sh

@@ -171,6 +171,7 @@ func (r *Reconciler) Reconcile(
 		primaryService           *corev1.Service
 		rootCA                   *pki.RootCertificateAuthority
 		monitoringSecret         *corev1.Secret
+		exporterWebConfig        *corev1.ConfigMap
 		err                      error
 	)
 
@@ -304,11 +305,14 @@ func (r *Reconciler) Reconcile(
 	if err == nil {
 		monitoringSecret, err = r.reconcileMonitoringSecret(ctx, cluster)
 	}
+	if err == nil {
+		exporterWebConfig, err = r.reconcileExporterWebConfig(ctx, cluster)
+	}
 	if err == nil {
 		err = r.reconcileInstanceSets(
 			ctx, cluster, clusterConfigMap, clusterReplicationSecret,
 			rootCA, clusterPodService, instanceServiceAccount, instances,
-			patroniLeaderService, primaryCertificate, clusterVolumes)
+			patroniLeaderService, primaryCertificate, clusterVolumes, exporterWebConfig)
 	}
 
 	if err == nil {

internal/controller/postgrescluster/controller.go

@@ -502,6 +502,7 @@ func (r *Reconciler) reconcileInstanceSets(
 	patroniLeaderService *corev1.Service,
 	primaryCertificate *corev1.SecretProjection,
 	clusterVolumes []corev1.PersistentVolumeClaim,
+	exporterWebConfig *corev1.ConfigMap,
 ) error {
 
 	// Go through the observed instances and check if a primary has been determined.
@@ -538,7 +539,7 @@ func (r *Reconciler) reconcileInstanceSets(
 			rootCA, clusterPodService, instanceServiceAccount,
 			patroniLeaderService, primaryCertificate,
 			findAvailableInstanceNames(*set, instances, clusterVolumes),
-			numInstancePods, clusterVolumes)
+			numInstancePods, clusterVolumes, exporterWebConfig)
 
 		if err == nil {
 			err = r.reconcileInstanceSetPodDisruptionBudget(ctx, cluster, set)
@@ -976,6 +977,7 @@ func (r *Reconciler) scaleUpInstances(
 	availableInstanceNames []string,
 	numInstancePods int,
 	clusterVolumes []corev1.PersistentVolumeClaim,
+	exporterWebConfig *corev1.ConfigMap,
 ) ([]*appsv1.StatefulSet, error) {
 	log := logging.FromContext(ctx)
 
@@ -1019,7 +1021,7 @@ func (r *Reconciler) scaleUpInstances(
 			clusterConfigMap, clusterReplicationSecret,
 			rootCA, clusterPodService, instanceServiceAccount,
 			patroniLeaderService, primaryCertificate, instances[i],
-			numInstancePods, clusterVolumes,
+			numInstancePods, clusterVolumes, exporterWebConfig,
 		)
 	}
 	if err == nil {
@@ -1048,6 +1050,7 @@ func (r *Reconciler) reconcileInstance(
 	instance *appsv1.StatefulSet,
 	numInstancePods int,
 	clusterVolumes []corev1.PersistentVolumeClaim,
+	exporterWebConfig *corev1.ConfigMap,
 ) error {
 	log := logging.FromContext(ctx).WithValues("instance", instance.Name)
 	ctx = logging.NewContext(ctx, log)
@@ -1100,7 +1103,7 @@ func (r *Reconciler) reconcileInstance(
 
 	// Add pgMonitor resources to the instance Pod spec
 	if err == nil {
-		err = addPGMonitorToInstancePodSpec(cluster, &instance.Spec.Template)
+		err = addPGMonitorToInstancePodSpec(cluster, &instance.Spec.Template, exporterWebConfig)
 	}
 
 	// add nss_wrapper init container and add nss_wrapper env vars to the database and pgbackrest