diff --git a/api/lagoon/v1beta2/lagoontask_types.go b/api/lagoon/v1beta2/lagoontask_types.go index 6dd13ff1..cad5ea04 100644 --- a/api/lagoon/v1beta2/lagoontask_types.go +++ b/api/lagoon/v1beta2/lagoontask_types.go @@ -96,8 +96,6 @@ func (b TaskType) String() string { // LagoonTaskSpec defines the desired state of LagoonTask type LagoonTaskSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file Key string `json:"key,omitempty"` Task schema.LagoonTaskInfo `json:"task,omitempty"` Project LagoonTaskProject `json:"project,omitempty"` diff --git a/cmd/main.go b/cmd/main.go index 7d10f930..78d5c843 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -51,7 +51,9 @@ import ( "github.com/hashicorp/golang-lru/v2/expirable" k8upv1 "github.com/k8up-io/k8up/v2/api/v1" lagoonv1beta2 "github.com/uselagoon/remote-controller/api/lagoon/v1beta2" + deploymentsctrl "github.com/uselagoon/remote-controller/internal/controllers/deployments" harborctrl "github.com/uselagoon/remote-controller/internal/controllers/harbor" + namespacectrl "github.com/uselagoon/remote-controller/internal/controllers/namespace" lagoonv1beta2ctrl "github.com/uselagoon/remote-controller/internal/controllers/v1beta2" "github.com/uselagoon/remote-controller/internal/messenger" k8upv1alpha1 "github.com/vshn/k8up/api/v1alpha1" @@ -948,6 +950,34 @@ func main() { setupLog.Error(err, "unable to seed controller startup state") } + setupLog.Info("starting namespace controller") + // start the namespace reconciler + if err = (&namespacectrl.NamespaceReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("namespace").WithName("Namespace"), + Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + Messaging: messaging, + LagoonTargetName: lagoonTargetName, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Namespace") + os.Exit(1) + } + + setupLog.Info("starting deployment controller") + // start the namespace reconciler + if err = (&deploymentsctrl.DeploymentsReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("deployments").WithName("Deployments"), + Scheme: mgr.GetScheme(), + EnableMQ: enableMQ, + Messaging: messaging, + LagoonTargetName: lagoonTargetName, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Deployments") + os.Exit(1) + } + setupLog.Info("starting build controller") // v1beta2 is the latest version if err = (&lagoonv1beta2ctrl.LagoonBuildReconciler{ diff --git a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml index 5545d603..ddc2d366 100644 --- a/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml +++ b/config/crd/bases/crd.lagoon.sh_lagoontasks.yaml @@ -181,9 +181,6 @@ spec: - project type: object key: - description: |- - INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - Important: Run "make" to regenerate code after modifying this file type: string misc: description: LagoonMiscInfo defines the resource or backup information diff --git a/go.mod b/go.mod index b0941ea6..fd9688db 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/onsi/ginkgo/v2 v2.25.3 github.com/onsi/gomega v1.38.2 github.com/prometheus/client_golang v1.23.2 - github.com/uselagoon/machinery v0.0.34 + github.com/uselagoon/machinery v0.0.35-0.20251124010253-71bebf6d6966 github.com/vshn/k8up v1.99.99 github.com/xhit/go-str2duration/v2 v2.1.0 golang.org/x/text v0.29.0 diff --git a/go.sum b/go.sum index 48bcd80e..f4a68e1c 100644 --- a/go.sum +++ b/go.sum @@ -1222,6 +1222,8 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/uselagoon/machinery v0.0.34 h1:5DsvXEyMeXmzQhjt11YH7+kZJueabovrwKTv0x7jQV8= github.com/uselagoon/machinery v0.0.34/go.mod h1:G0ujppuNR0BrtAnlmH8xDb9TDfayb4A36aeo0DYg7fQ= +github.com/uselagoon/machinery v0.0.35-0.20251124010253-71bebf6d6966 h1:zfOFGy2aaAn5SnGoOBn/GqmiPzQKY4Crd5A0Y+DXI90= +github.com/uselagoon/machinery v0.0.35-0.20251124010253-71bebf6d6966/go.mod h1:G0ujppuNR0BrtAnlmH8xDb9TDfayb4A36aeo0DYg7fQ= github.com/uudashr/gocognit v0.0.0-20190926065955-1655d0de0517/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= diff --git a/internal/controllers/deployments/deployments.go b/internal/controllers/deployments/deployments.go new file mode 100644 index 00000000..c4a26261 --- /dev/null +++ b/internal/controllers/deployments/deployments.go @@ -0,0 +1,129 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployments + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strconv" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/messenger" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// DeploymentsReconciler reconciles idling +type DeploymentsReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger + LagoonTargetName string +} + +type ServiceState struct { + Name string `json:"name"` + Type string `json:"type"` + Replicas int32 `json:"replicas"` +} + +func (r *DeploymentsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("deployment", req.NamespacedName) + + var deployment appsv1.Deployment + if err := r.Get(ctx, req.NamespacedName, &deployment); err != nil { + return ctrl.Result{}, ignoreNotFound(err) + } + opLog.Info(fmt.Sprintf("deployment %s", deployment.Name)) + opLog.Info(fmt.Sprintf(`{"replicas":%d}`, *deployment.Spec.Replicas)) + // this would be nice to be a lagoon label :) + if val, ok := deployment.Labels["idling.amazee.io/idled"]; ok { + var namespace corev1.Namespace + if err := r.Get(ctx, types.NamespacedName{ + Name: deployment.Namespace, + }, &namespace); err != nil { + return ctrl.Result{}, ignoreNotFound(err) + } + opLog.Info(fmt.Sprintf("deployment %s idle state %v", deployment.Name, val)) + if r.EnableMQ { + environmentName := namespace.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.Labels["lagoon.sh/environmentId"]) + envID := helpers.UintPtr(uint(eID)) + projectName := namespace.Labels["lagoon.sh/project"] + pID, _ := strconv.Atoi(namespace.Labels["lagoon.sh/projectId"]) + projectID := helpers.UintPtr(uint(pID)) + serviceName := deployment.Labels["lagoon.sh/service"] + serviceType := deployment.Labels["lagoon.sh/service-type"] + state := ServiceState{ + Name: serviceName, + Type: serviceType, + Replicas: *deployment.Spec.Replicas, + } + stateJSON, _ := json.Marshal(state) + msg := schema.LagoonMessage{ + Type: "servicestate", + Namespace: namespace.Name, + Meta: &schema.LagoonLogMeta{ + EnvironmentID: envID, + ProjectID: projectID, + Environment: environmentName, + Project: projectName, + Cluster: r.LagoonTargetName, + AdvancedData: base64.StdEncoding.EncodeToString(stateJSON), + }, + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because for some reason, log the error and move on + // this may result in the state being out of sync in lagoon but eventually will be consistent + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + return ctrl.Result{}, nil + } + } + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the watch on the namespace resource with an event filter (see predicates.go) +func (r *DeploymentsReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&appsv1.Deployment{}). + WithEventFilter(DeploymentsPredicates{}). + Complete(r) +} + +// will ignore not found errors +func ignoreNotFound(err error) error { + if apierrors.IsNotFound(err) { + return nil + } + return err +} diff --git a/internal/controllers/deployments/predicates.go b/internal/controllers/deployments/predicates.go new file mode 100644 index 00000000..f97da861 --- /dev/null +++ b/internal/controllers/deployments/predicates.go @@ -0,0 +1,39 @@ +package deployments + +import ( + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// DeploymentsPredicates defines the funcs for predicates +type DeploymentsPredicates struct { + predicate.Funcs +} + +// Create is used when a creation event is received by the controller. +func (n DeploymentsPredicates) Create(e event.CreateEvent) bool { + return false +} + +// Delete is used when a deletion event is received by the controller. +func (n DeploymentsPredicates) Delete(e event.DeleteEvent) bool { + return false +} + +// Update is used when an update event is received by the controller. +func (n DeploymentsPredicates) Update(e event.UpdateEvent) bool { + if _, ok := e.ObjectOld.GetLabels()["lagoon.sh/service"]; ok { + oldDeep := e.ObjectOld.(*appsv1.Deployment) + newDep := e.ObjectNew.(*appsv1.Deployment) + if *oldDeep.Spec.Replicas != *newDep.Spec.Replicas { + return true + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (n DeploymentsPredicates) Generic(e event.GenericEvent) bool { + return false +} diff --git a/internal/controllers/namespace/namespace.go b/internal/controllers/namespace/namespace.go new file mode 100644 index 00000000..23d63bd9 --- /dev/null +++ b/internal/controllers/namespace/namespace.go @@ -0,0 +1,115 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namespace + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strconv" + + "github.com/go-logr/logr" + "github.com/uselagoon/machinery/api/schema" + "github.com/uselagoon/remote-controller/internal/helpers" + "github.com/uselagoon/remote-controller/internal/messenger" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// NamespaceReconciler reconciles idling +type NamespaceReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + EnableMQ bool + Messaging *messenger.Messenger + LagoonTargetName string +} + +type Idled struct { + Idled bool `json:"idled"` +} + +func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + opLog := r.Log.WithValues("namespace", req.NamespacedName) + + var namespace corev1.Namespace + if err := r.Get(ctx, req.NamespacedName, &namespace); err != nil { + return ctrl.Result{}, ignoreNotFound(err) + } + + // this would be nice to be a lagoon label :) + if val, ok := namespace.Labels["idling.amazee.io/idled"]; ok { + idled, _ := strconv.ParseBool(val) + opLog.Info(fmt.Sprintf("environment %s idle state %t", namespace.Name, idled)) + if r.EnableMQ { + environmentName := namespace.Labels["lagoon.sh/environment"] + eID, _ := strconv.Atoi(namespace.Labels["lagoon.sh/environmentId"]) + envID := helpers.UintPtr(uint(eID)) + projectName := namespace.Labels["lagoon.sh/project"] + pID, _ := strconv.Atoi(namespace.Labels["lagoon.sh/projectId"]) + projectID := helpers.UintPtr(uint(pID)) + idling := Idled{ + Idled: idled, + } + idlingJSON, _ := json.Marshal(idling) + msg := schema.LagoonMessage{ + Type: "idling", + Namespace: namespace.Name, + Meta: &schema.LagoonLogMeta{ + EnvironmentID: envID, + ProjectID: projectID, + Environment: environmentName, + Project: projectName, + Cluster: r.LagoonTargetName, + AdvancedData: base64.StdEncoding.EncodeToString(idlingJSON), + }, + } + msgBytes, err := json.Marshal(msg) + if err != nil { + opLog.Error(err, "Unable to encode message as JSON") + } + // @TODO: if we can't publish the message because for some reason, log the error and move on + // this may result in the state being out of sync in lagoon but eventually will be consistent + if err := r.Messaging.Publish("lagoon-tasks:controller", msgBytes); err != nil { + return ctrl.Result{}, nil + } + } + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the watch on the namespace resource with an event filter (see predicates.go) +func (r *NamespaceReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Namespace{}). + WithEventFilter(NamespacePredicates{}). + Complete(r) +} + +// will ignore not found errors +func ignoreNotFound(err error) error { + if apierrors.IsNotFound(err) { + return nil + } + return err +} diff --git a/internal/controllers/namespace/predicates.go b/internal/controllers/namespace/predicates.go new file mode 100644 index 00000000..b0f3c426 --- /dev/null +++ b/internal/controllers/namespace/predicates.go @@ -0,0 +1,38 @@ +package namespace + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// NamespacePredicates defines the funcs for predicates +type NamespacePredicates struct { + predicate.Funcs +} + +// Create is used when a creation event is received by the controller. +func (n NamespacePredicates) Create(e event.CreateEvent) bool { + return false +} + +// Delete is used when a deletion event is received by the controller. +func (n NamespacePredicates) Delete(e event.DeleteEvent) bool { + return false +} + +// Update is used when an update event is received by the controller. +func (n NamespacePredicates) Update(e event.UpdateEvent) bool { + if oldIdled, ok := e.ObjectOld.GetLabels()["idling.amazee.io/idled"]; ok { + if newIdled, ok := e.ObjectNew.GetLabels()["idling.amazee.io/idled"]; ok { + if oldIdled != newIdled { + return true + } + } + } + return false +} + +// Generic is used when any other event is received by the controller. +func (n NamespacePredicates) Generic(e event.GenericEvent) bool { + return false +} diff --git a/internal/controllers/v1beta2/buildpodmonitor_handlers.go b/internal/controllers/v1beta2/buildpodmonitor_handlers.go index 0854b0ea..85cf71d6 100644 --- a/internal/controllers/v1beta2/buildpodmonitor_handlers.go +++ b/internal/controllers/v1beta2/buildpodmonitor_handlers.go @@ -302,6 +302,7 @@ func (r *BuildMonitorReconciler) updateDeploymentAndEnvironmentTask( Name: serviceName, Type: serviceType, Containers: containers, + Replicas: *deployment.Spec.Replicas, }) } msg.Meta.Services = serviceNames diff --git a/internal/messenger/consumer.go b/internal/messenger/consumer.go index aa1b4d71..44238332 100644 --- a/internal/messenger/consumer.go +++ b/internal/messenger/consumer.go @@ -412,6 +412,36 @@ func (m *Messenger) Consumer(targetName string) { _ = message.Ack(false) // ack to remove from queue return } + case "deploytarget:environment:idling": + opLog.Info( + fmt.Sprintf( + "Received environment idling request for project %s, environment %s - %s", + jobSpec.Project.Name, + jobSpec.Environment.Name, + m.genNamespace(jobSpec), + ), + ) + // idle or unidle an environment, optionally forcible scale it so it can't be unidled by the ingress + err := m.ScaleOrIdleEnvironment(ctx, opLog, m.genNamespace(jobSpec), jobSpec) + if err != nil { + _ = message.Ack(false) // ack to remove from queue + return + } + case "deploytarget:environment:service": + opLog.Info( + fmt.Sprintf( + "Received environment service request for project %s, environment %s service - %s", + jobSpec.Project.Name, + jobSpec.Environment.Name, + m.genNamespace(jobSpec), + ), + ) + // idle an environment, optionally forcible scale it so it can't be unidled by the ingress + err := m.EnvironmentServiceState(ctx, opLog, m.genNamespace(jobSpec), jobSpec) + if err != nil { + _ = message.Ack(false) // ack to remove from queue + return + } default: // if we get something that we don't know about, spit out the entire message opLog.Info( diff --git a/internal/messenger/tasks_handler.go b/internal/messenger/tasks_handler.go index f82a5cba..6a83e79e 100644 --- a/internal/messenger/tasks_handler.go +++ b/internal/messenger/tasks_handler.go @@ -5,10 +5,17 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math" + "strconv" + "time" + "github.com/go-logr/logr" lagoonv1beta2 "github.com/uselagoon/remote-controller/api/lagoon/v1beta2" "github.com/uselagoon/remote-controller/internal/helpers" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" ) @@ -92,3 +99,114 @@ func createAdvancedTask(namespace string, jobSpec *lagoonv1beta2.LagoonTaskSpec, } return nil } + +type Idling struct { + Idle bool `json:"idle"` + ForceScale bool `json:"forceScale"` +} + +type Service struct { + Name string `json:"name"` + State string `json:"state"` +} + +func (m *Messenger) ScaleOrIdleEnvironment(ctx context.Context, opLog logr.Logger, ns string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { + namespace := &corev1.Namespace{} + err := m.Client.Get(ctx, types.NamespacedName{ + Name: ns, + }, namespace) + if err != nil { + return err + } + idling := Idling{} + if err := json.Unmarshal(jobSpec.Misc.MiscResource, &idling); err != nil { + opLog.Error(err, + "Unable to unmarshal the idling json.", + ) + return err + } + if idling.Idle { + if idling.ForceScale { + // this would be nice to be a lagoon label :) + namespace.Labels["idling.amazee.io/force-scaled"] = "true" + } else { + // this would be nice to be a lagoon label :) + namespace.Labels["idling.amazee.io/force-idled"] = "true" + } + } else { + // this would be nice to be a lagoon label :) + namespace.Labels["idling.amazee.io/unidle"] = "true" + } + if err := m.Client.Update(context.Background(), namespace); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update namespace %s to set idle state.", + ns, + ), + ) + return err + } + return nil +} + +func (m *Messenger) EnvironmentServiceState(ctx context.Context, opLog logr.Logger, ns string, jobSpec *lagoonv1beta2.LagoonTaskSpec) error { + deployment := &appsv1.Deployment{} + service := Service{} + if err := json.Unmarshal(jobSpec.Misc.MiscResource, &service); err != nil { + opLog.Error(err, + "Unable to unmarshal the service json.", + ) + return err + } + err := m.Client.Get(ctx, types.NamespacedName{ + Name: service.Name, + Namespace: ns, + }, deployment) + if err != nil { + return err + } + update := false + switch service.State { + case "restart": + deployment.Spec.Template.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) + update = true + case "stop": + if *deployment.Spec.Replicas > 0 { + // if the service has replicas, then save the replica count and scale it to 0 + deployment.Annotations["service.lagoon.sh/replicas"] = strconv.FormatInt(int64(*deployment.Spec.Replicas), 10) + replicas := int32(0) + deployment.Spec.Replicas = &replicas + update = true + } + case "start": + if *deployment.Spec.Replicas == 0 { + // if the service has no replicas, set it back to what the previous replica value was + prevReplicas, err := strconv.ParseInt(deployment.Annotations["service.lagoon.sh/replicas"], 10, 32) + if err != nil { + return err + } + replicas := int32(1) + if prevReplicas > 0 && prevReplicas <= math.MaxInt32 { + replicas = int32(prevReplicas) + } + deployment.Spec.Replicas = &replicas + delete(deployment.Annotations, "service.lagoon.sh/replicas") + update = true + } + default: + // nothing to do + return nil + } + if update { + if err := m.Client.Update(ctx, deployment); err != nil { + opLog.Error(err, + fmt.Sprintf( + "Unable to update deployment %s to change its state.", + ns, + ), + ) + return err + } + } + return nil +}