Owls104087 (refactored) fix an intermitent failure in ItKubernetesDomainEvents#testK8SEventsDelete (#3704)

* Generate cluster Created/Deleted/Changed events for unreferenced cluster resources

Co-authored-by: Anthony Lai <anthony.lai@oracle.com>

Author: doxiao

common/src/main/java/oracle/kubernetes/common/logging/MessageKeys.java

@@ -167,6 +167,7 @@ public class MessageKeys {
   public static final String WATCH_CLUSTER_DELETED = "WLSKO-0229";
   public static final String CLUSTER_STATUS = "WLSKO-0230";
   public static final String DOMAIN_INTROSPECTION_INCOMPLETE = "WLSKO-0231";
+  public static final String WATCH_CLUSTER_WITHOUT_DOMAIN = "WLSKO-0232";
 
   // domain status messages
   public static final String MAKE_RIGHT_WILL_RETRY = "WLSDO-0000";

common/src/main/resources/Operator.properties

@@ -175,6 +175,8 @@ WLSKO-0229=Watch event triggered for deletion of WebLogic Cluster {0} in WebLogi
 WLSKO-0230=Status for Cluster Resource with name {0} is now: {1}
 WLSKO-0231=Domain introspection is incomplete. \
   Check the introspector job logs for possible errors. Job logs are -> {0}.
+WLSKO-0232=Watch event triggered for WebLogic Cluster {0}, which is not referenced by any domain. 
+
 # Domain status messages
 
 WLSDO-0000={0}. Will retry next at {1} and approximately every {2} seconds afterward until {3} if the failure is not resolved.

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItKubernetesDomainEvents.java

@@ -454,7 +454,7 @@ void testDomainK8sEventsScalePastMaxAndChangeIntrospectVersion() {
 
       assertTrue(scaleCluster(clusterRes1Name, domainNamespace3, 2), "failed to scale cluster via patching");
       logger.info("verify the ClusterChanged event is generated");
-      checkEvent(opNamespace, domainNamespace3, domainUid, CLUSTER_CHANGED, "Normal", timestamp);
+      checkEvent(opNamespace, domainNamespace3, null, CLUSTER_CHANGED, "Normal", timestamp);
       checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace3);
 
       for (int i = 1; i <= replicaCount; i++) {
@@ -662,23 +662,24 @@ void testIncludeServerOutInPodLog() {
   }
 
   /**
-   * Test DomainDeleted event is logged when domain resource is deleted.
+   * Test DomainDeleted and ClusterDeleted events are logged when domain and cluster resources are deleted.
    */
   @Test
-  @DisplayName("Test domain events for various domain life cycle changes")
+  @DisplayName("Test domain and cluster events for deleting domain and cluster resources")
   void testK8SEventsDelete() {
     OffsetDateTime timestamp = now();
     createDomain(domainNamespace2, domainUid, pvName2, pvcName2);
     deleteDomainCustomResource(domainUid, domainNamespace2);
     deleteClusterCustomResource(cluster1Name, domainNamespace2);
+
     checkPodDoesNotExist(adminServerPodName, domainUid, domainNamespace2);
     checkPodDoesNotExist(managedServerPodNamePrefix + 1, domainUid, domainNamespace2);
     checkPodDoesNotExist(managedServerPodNamePrefix + 2, domainUid, domainNamespace2);
 
     //verify domain deleted event
     checkEvent(opNamespace, domainNamespace2, domainUid, DOMAIN_DELETED, "Normal", timestamp);
     //verify cluster deleted event
-    checkEvent(opNamespace, domainNamespace2, domainUid, CLUSTER_DELETED, "Normal", timestamp);
+    checkEvent(opNamespace, domainNamespace2, null, CLUSTER_DELETED, "Normal", timestamp);
   }
 
   /**

operator/src/main/java/oracle/kubernetes/operator/ClusterResourceStatusUpdater.java

@@ -13,15 +13,14 @@
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 
-import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import oracle.kubernetes.common.logging.MessageKeys;
 import oracle.kubernetes.operator.calls.CallResponse;
 import oracle.kubernetes.operator.calls.UnrecoverableErrorBuilder;
 import oracle.kubernetes.operator.helpers.CallBuilder;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.EventHelper;
+import oracle.kubernetes.operator.helpers.EventHelper.ClusterResourceEventData;
 import oracle.kubernetes.operator.helpers.EventHelper.EventData;
-import oracle.kubernetes.operator.helpers.EventHelper.EventItem;
 import oracle.kubernetes.operator.helpers.ResponseStep;
 import oracle.kubernetes.operator.logging.LoggingFacade;
 import oracle.kubernetes.operator.logging.LoggingFactory;
@@ -226,13 +225,13 @@ private StepAndPacket createReplaceClusterResourceStatusStep() {
 
       // add steps to create events for updating conditions
       Optional.ofNullable(newClusterStatus)
-          .map(ncs -> getClusteStatusConditionEvents(ncs.getConditions())).orElse(Collections.emptyList())
+          .map(ncs -> getClusterStatusConditionEvents(ncs.getConditions())).orElse(Collections.emptyList())
           .stream().map(EventHelper::createClusterResourceEventStep).forEach(result::add);
 
       return new StepAndPacket(Step.chain(result), packet);
     }
 
-    private List<EventData> getClusteStatusConditionEvents(List<ClusterCondition> conditions) {
+    private List<EventData> getClusterStatusConditionEvents(List<ClusterCondition> conditions) {
       List<EventData> list = new ArrayList<>();
       list.addAll(getClusterStatusConditionTrueEvents(conditions));
       list.addAll(getClusterStatusConditionFalseEvents(conditions));
@@ -256,14 +255,14 @@ private List<EventData> getClusterStatusConditionTrueEvents(List<ClusterConditio
 
     private EventData toTrueClusterResourceEvent(ClusterCondition condition) {
       return Optional.ofNullable(condition.getType().getAddedEvent())
-          .map(ClusterResourceEventData::new)
+          .map(eventItem -> new ClusterResourceEventData(eventItem, resource))
           .map(this::initializeClusterResourceEventData)
           .orElse(null);
     }
 
     private EventData toFalseClusterResourceEvent(ClusterCondition removedCondition) {
       return Optional.ofNullable(removedCondition.getType().getRemovedEvent())
-          .map(ClusterResourceEventData::new)
+          .map(eventItem -> new ClusterResourceEventData(eventItem, resource))
           .map(this::initializeClusterResourceEventData)
           .orElse(null);
     }
@@ -286,21 +285,6 @@ private Step createReplaceClusterStatusAsyncStep() {
               createReplacementClusterResource(),
               new ClusterResourceStatusReplaceResponseStep(this));
     }
-
-    private class ClusterResourceEventData extends EventData {
-
-      public ClusterResourceEventData(EventItem eventItem) {
-        super(eventItem);
-      }
-
-      @Override
-      public String getUID() {
-        return Optional.of(resource)
-            .map(ClusterResource::getMetadata)
-            .map(V1ObjectMeta::getUid)
-            .orElse("");
-      }
-    }
   }
 
   private static class ReadClusterResponseStep extends ResponseStep<ClusterResource> {

operator/src/main/java/oracle/kubernetes/operator/DomainProcessor.java

@@ -14,22 +14,34 @@
 import io.kubernetes.client.util.Watch;
 import io.kubernetes.client.util.Watch.Response;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
+import oracle.kubernetes.operator.helpers.EventHelper.EventItem;
 import oracle.kubernetes.weblogic.domain.model.ClusterResource;
 import oracle.kubernetes.weblogic.domain.model.DomainResource;
 
 /**
- * An abstraction for processing a domain.
+ * An abstraction for processing a domain and a cluster.
  */
 public interface DomainProcessor {
 
   /**
    * Ensures that the domain is up-to-date. This may involve validation and introspection of the domain itself,
    * changes to Kubernetes resources such as pods and services.
-   * @param liveInfo an info object that tracks what is know about the domain
+   * @param liveInfo an info object that tracks what is known about the domain
    * @return Make-right operation
    */
   MakeRightDomainOperation createMakeRightOperation(DomainPresenceInfo liveInfo);
 
+  /**
+   * Ensures that a cluster event is generated for a cluster resource no matter whether it is referenced by a domain
+   * or not.
+   *
+   * @param clusterEvent the event that needs to be generated
+   * @param cluster the cluster resource that the event is associated with
+   * @return Make-right operation
+   */
+  MakeRightClusterOperation createMakeRightOperationForClusterEvent(
+      EventItem clusterEvent, ClusterResource cluster);
+
   /**
    * Handles a watch event for clusters in the managed namespaces.
    * @param item a Kubernetes watch event

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorDelegate.java

@@ -5,7 +5,9 @@
 
 import javax.annotation.Nonnull;
 
+import oracle.kubernetes.operator.helpers.ClusterPresenceInfo;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
+import oracle.kubernetes.operator.makeright.MakeRightClusterOperationImpl;
 import oracle.kubernetes.operator.makeright.MakeRightDomainOperationImpl;
 import oracle.kubernetes.operator.work.FiberGate;
 
@@ -47,4 +49,9 @@ public interface DomainProcessorDelegate extends CoreDelegate {
   default MakeRightDomainOperation createMakeRightOperation(MakeRightExecutor executor, DomainPresenceInfo info) {
     return new MakeRightDomainOperationImpl(executor, this, info);
   }
+
+  @Nonnull
+  default MakeRightClusterOperation createMakeRightOperation(MakeRightExecutor executor, ClusterPresenceInfo info) {
+    return new MakeRightClusterOperationImpl(executor, this, info);
+  }
 }