Sit-cfg override fixes:

- MAJOR
-- Move log location sit-cfg override stanzas to just after listen-address (work-around for XML parsing issue seen in domain-in-image wl server log).
-- Add a 'TBD' note in the code for an outage in handling dynamic cluster SSL, NAP, and admin ports (discovered via code inspection).   We need to (A) have the introspect report these, check that the Operator can consume them and set up services accordingly, and (C) add tests to cover this area.
- MINOR
-- Call 'add' instead of 'replace' if overriding a missing Listen address (best practice, replace works but this is not documented as supported).
-- Do not 'replace' or 'add' an unset NAP ListenAddress (corrects mock/unit test issue seen in a particular T3Channel, no override needed as default NAP LA comes from server default LA).
-- Do not add SSL listen-address overide stanzas since there's no listen address field in an SSL stanza (this wasn't causing an issue, discovered during code inspection).
-- Update introspect mock/unit tests to cover even more use cases.

Author: Tom Barnes

operator/src/main/resources/scripts/introspectDomain.py

@@ -603,6 +603,14 @@ def addServerTemplates(self):
     self.undent()
 
   def addServerTemplate(self, serverTemplate):
+    # TBD This looks wrong.  It should match addClusteredServer.
+    #     It adds default port & listen address. But:
+    #       It's missing adding admin port, SSL, and NAPs.
+    #       It reports the wrong listen address (original rather than overridden).
+    #     Probably tests are passing fine, as it's likely (A)
+    #     the Operator.java is only reading the port, and (B) cluster tests
+    #     are only looking at the cluster default port.
+    #     So:  Follow up and fix this, and file test JIRAs for tests.
     name=self.name(serverTemplate)
     self.writeln("- name: " + name)
     if serverTemplate.isListenPortEnabled():
@@ -803,28 +811,34 @@ def customizeServers(self):
     for server in self.env.getDomain().getServers():
       self.customizeServer(server)
 
+  def writeListenAddress(self, originalValue, newValue):
+    repVerb="\"replace\""
+    if originalValue is None or len(originalValue)==0:
+      repVerb="\"add\""
+    self.writeln("<d:listen-address f:combine-mode=" + repVerb + ">" + newValue + "</d:listen-address>")
+
   def customizeServer(self, server):
     name=server.getName()
     listen_address=self.env.toDNS1123Legal(self.env.getDomainUID() + "-" + name)
     self.writeln("<d:server>")
     self.indent()
     self.writeln("<d:name>" + name + "</d:name>")
-    self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
-    if server.getSSL():
-      self.writeln("<d:ssl>")
-      self.indent()
-      self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
-      self.undent()
-      self.writeln("</d:ssl>")
+    self.customizeLog(name, server, false)
+    self.writeListenAddress(server.getListenAddress(),listen_address)
     for nap in server.getNetworkAccessPoints():
+      # Don't bother 'add' a nap listen-address, only do a 'replace'.
+      # If we try 'add' this appears to mess up an attempt to 
+      #   'add' PublicAddress/Port via custom sit-cfg.
+      # FWIW there's theoretically no need to 'add' or 'replace' when empty
+      #   since the runtime default is the server listen-address.
       nap_name=nap.getName()
-      self.writeln("<d:network-access-point>")
-      self.indent()
-      self.writeln("<d:name>" + nap_name + "</d:name>")
-      self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
-      self.undent()
-      self.writeln("</d:network-access-point>")
-    self.customizeLog(name, server, false)
+      if not (nap.getListenAddress() is None) and len(nap.getListenAddress()) > 0:
+        self.writeln("<d:network-access-point>")
+        self.indent()
+        self.writeln("<d:name>" + nap_name + "</d:name>")
+        self.writeListenAddress("force a replace",listen_address)
+        self.undent()
+        self.writeln("</d:network-access-point>")
     self.undent()
     self.writeln("</d:server>")
 
@@ -839,8 +853,8 @@ def customizeServerTemplate(self, template):
     self.writeln("<d:server-template>")
     self.indent()
     self.writeln("<d:name>" + name + "</d:name>")
-    self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
     self.customizeLog(server_name_prefix + "${id}", template, false)
+    self.writeListenAddress(template.getListenAddress(),listen_address)
     self.undent()
     self.writeln("</d:server-template>")
 

operator/src/main/resources/scripts/startNodeManager.sh

@@ -111,6 +111,12 @@ createFolder ${NODEMGR_HOME}
 
 NODEMGR_LOG_HOME=${NODEMGR_LOG_HOME:-${LOG_HOME:-${NODEMGR_HOME}/${DOMAIN_UID}}}
 
+trace "Info: NODEMGR_HOME='${NODEMGR_HOME}'"
+trace "Info: LOG_HOME='${LOG_HOME}'"
+trace "Info: SERVER_NAME='${SERVER_NAME}'"
+trace "Info: DOMAIN_UID='${DOMAIN_UID}'"
+trace "Info: NODEMGR_LOG_HOME='${NODEMGR_LOG_HOME}'"
+
 createFolder ${NODEMGR_LOG_HOME}
 
 nodemgr_log_file=${NODEMGR_LOG_HOME}/${SERVER_NAME}_nodemanager.log

src/integration-tests/introspector/checkBeans.inputt

@@ -1,4 +1,7 @@
+/Log/base_domain,FileName,logs/base_domain.log,/shared/logs/base_domain.log
+
 /Servers/${ADMIN_NAME},ListenAddress,,${DOMAIN_UID}-${ADMIN_NAME}
+
 /Servers/${ADMIN_NAME}/Log/${ADMIN_NAME},FileName,logs/${ADMIN_NAME}.log,/shared/logs/${ADMIN_NAME}.log
 
 /Servers/${ADMIN_NAME}/NetworkAccessPoints/T3Channel1,ListenAddress,unresolvable-dns-name,${DOMAIN_UID}-${ADMIN_NAME}
@@ -17,6 +20,9 @@
 /Servers/standalone1,MaxMessageSize,7777777,1111111
 /Servers/standalone2,MaxMessageSize,10000000,2222222
 
+/ServerTemplates/mycluster-template/Log/mycluster-template,FileName,logs/mycluster-template.log,/shared/logs/managed-server${id}.log
+/ServerTemplates/mycluster-template,ListenAddress,None,domain1-managed-server${id}
+
 /JDBCSystemResources/testDS/JDBCResource/testDS/JDBCDriverParams/testDS,Url,jdbc:oracle:thin:@myoriginalhostname:1521:myoriginalsid,jdbc:oracle:thin:@mynewhost:1521:mynewsid
 /JDBCSystemResources/testDS/JDBCResource/testDS/JDBCDriverParams/testDS,PasswordEncrypted,*,!
 /JDBCSystemResources/testDS/JDBCResource/testDS/JDBCDriverParams/testDS/Properties/testDS/Properties/user,Value,scott,supersecret

src/integration-tests/introspector/introspectTest.sh

@@ -179,14 +179,20 @@ function cleanupMajor() {
 function cleanupMinor() {
   trace "Info: RERUN_INTROSPECT_ONLY==true, skipping cleanup.sh and domain home setup, and only deleting wl pods + introspector job."
 
-  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${ADMIN_NAME} > /dev/null 2>&1
-  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${MANAGED_SERVER_NAME_BASE}1 > /dev/null 2>&1
-  kubectl -n $NAMESPACE delete job ${DOMAIN_UID}-introspect-domain-job > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${ADMIN_NAME}                --grace-period=2 > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${MANAGED_SERVER_NAME_BASE}1 --grace-period=2 > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete job ${DOMAIN_UID}-introspect-domain-job        --grace-period=2 > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}--introspect-domain-pod       --grace-period=2 > /dev/null 2>&1
   rm -fr ${test_home}/jobfiles
   tracen "Info: Waiting for wl pods to completely go away before continuing."
   while [ 1 -eq 1 ]; do
     echo -n "."
-    [ "`kubectl -n ${NAMESPACE} get pods | grep '${DOMAIN_UID}.*server'`" = "" ] && break
+    # echo
+    # echo "Waiting for: '`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*server\"`'"
+    # echo "Waiting for: '`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*introspect\"`'"
+    [ "`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*server\"`" = "" ] \
+     && [ "`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*introspect\"`" = "" ] \
+     && break
     sleep 1
   done
   echo
@@ -339,7 +345,7 @@ function deployTestScriptConfigMap() {
   cp ${SCRIPTPATH}/introspectDomainProxy.sh ${test_home}/test-scripts || exit 1
 
   if [ "$CREATE_DOMAIN" = "true" ]; then
-    rm -f ${test_home}/scripts/createDomain.py
+    rm -f ${test_home}/test-scripts/createDomain.py
     ${SCRIPTPATH}/util_subst.sh -g createDomain.pyt ${test_home}/test-scripts/createDomain.py || exit 1
   fi
 
@@ -498,19 +504,15 @@ function deployIntrospectJobPod() {
 
   trace "Info: Run introspection job, parse its output to files, and put files in configmap '$introspect_output_cm_name'."
 
-  # delete anything left over from a previous invocation of this function
+  # delete anything left over from a previous invocation of this function, assume all pods
+  # have already been cleaned up
+
+  rm -f ${target_yaml}
 
   kubectl -n $NAMESPACE delete cm $introspect_output_cm_name \
     --ignore-not-found  \
     2>&1 | tracePipe "Info: kubectl output: "
 
-  if [ -f "${target_yaml}" ]; then
-    kubectl -n $NAMESPACE delete -f ${target_yaml} \
-      --ignore-not-found \
-      2>&1 | tracePipe "Info: kubectl output: "
-    rm -f ${target_yaml}
-  fi
-
   trace "Info: Deploying job pod '$pod_name' and waiting for it to be ready."
 
   (
@@ -755,11 +757,13 @@ function checkOverrides() {
 #     matches
 #
 
+
 kubectl -n $NAMESPACE delete secret my-secret > /dev/null 2>&1
 kubectl -n $NAMESPACE create secret generic my-secret \
         --from-literal=key1=supersecret  \
         --from-literal=key2=topsecret 2>&1 | tracePipe "Info: kubectl output: "
 
+
 if [ ! "$RERUN_INTROSPECT_ONLY" = "true" ]; then
 
   cleanupMajor
@@ -785,9 +789,13 @@ else
 
   cleanupMinor
 
+  deployDomainConfigMap
+  deployTestScriptConfigMap
   deployCustomOverridesConfigMap
-  deployIntrospectJob
+  #deployIntrospectJob
+  deployIntrospectJobPod
   deployPod ${ADMIN_NAME?}
+  deployPod ${MANAGED_SERVER_NAME_BASE?}1
 
 fi
 

src/integration-tests/introspector/util_subst.sh

@@ -100,7 +100,7 @@ sed -i -e 's;${[a-zA-Z0-9_][a-zA-Z0-9_]*:-\([^}]*\)};\1;g' $tfilename
 # On a failure, helpfully list the problem lines in the input file
 # along with the line numbers.
 
-count="`grep -v 'subst-ignore-missing' $tfilename | grep -v 'env:' | grep -v 'secret:' | grep -c '\${'`"
+count="`grep -v 'subst-ignore-missing' $tfilename | grep -v 'env:' | grep -v 'secret:' | grep -v '\${id}' | grep -c '\${'`"
 if [ $count -gt 0 ]; then
   trace "Debug: Error, found unresolved variables in file '$sfilename':"
   grep -n "\${" $tfilename | sed 's/^/@  line /' | grep -v 'subst-ignore-missing'