Add topology support for hetero systems

Signed-off-by: Ralph Castain <rhc@pmix.org>

Author: rhc54

orte/mca/odls/odls_types.h

@@ -12,7 +12,7 @@
  * Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2018      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -93,6 +93,9 @@ typedef uint8_t orte_daemon_cmd_flag_t;
 /* tell DVM daemons to cleanup resources from job */
 #define ORTE_DAEMON_DVM_CLEANUP_JOB_CMD     (orte_daemon_cmd_flag_t) 34
 
+/* pass node info */
+#define ORTE_DAEMON_PASS_NODE_INFO_CMD      (orte_daemon_cmd_flag_t) 35
+
 /*
  * Struct written up the pipe from the child to the parent.
  */

orte/mca/plm/base/plm_base_launch_support.c

@@ -130,7 +130,11 @@ void orte_plm_base_daemons_reported(int fd, short args, void *cbdata)
     orte_state_caddy_t *caddy = (orte_state_caddy_t*)cbdata;
     orte_topology_t *t;
     orte_node_t *node;
-    int i;
+    int i, rc;
+    uint8_t u8;
+    opal_buffer_t buf;
+    orte_grpcomm_signature_t *sig;
+    orte_daemon_cmd_flag_t command = ORTE_DAEMON_PASS_NODE_INFO_CMD;
 
     ORTE_ACQUIRE_OBJECT(caddy);
 
@@ -177,6 +181,78 @@ void orte_plm_base_daemons_reported(int fd, short args, void *cbdata)
     /* ensure we update the routing plan */
     orte_routed.update_routing_plan(NULL);
 
+    /* prep the buffer */
+    OBJ_CONSTRUCT(&buf, opal_buffer_t);
+    /* load the command */
+    if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &command, 1, ORTE_DAEMON_CMD))) {
+        ORTE_ERROR_LOG(rc);
+        OBJ_DESTRUCT(&buf);
+        ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
+        OBJ_RELEASE(caddy);
+        return;
+    }
+
+
+    /* if we did not execute a tree-spawn, then the daemons do
+     * not currently have a nidmap for the job - in that case,
+     * send one to them */
+    if (!orte_nidmap_communicated) {
+        u8 = 1;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &u8, 1, OPAL_UINT8))) {
+            ORTE_ERROR_LOG(rc);
+            OBJ_DESTRUCT(&buf);
+            ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
+            OBJ_RELEASE(caddy);
+            return;
+        }
+        if (OPAL_SUCCESS != (rc = orte_util_nidmap_create(orte_node_pool, &buf))) {
+            ORTE_ERROR_LOG(rc);
+            OBJ_DESTRUCT(&buf);
+            ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
+            OBJ_RELEASE(caddy);
+            return;
+        }
+        orte_nidmap_communicated = true;
+    } else {
+        u8 = 0;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &u8, 1, OPAL_UINT8))) {
+            ORTE_ERROR_LOG(rc);
+            OBJ_DESTRUCT(&buf);
+            ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
+            OBJ_RELEASE(caddy);
+            return;
+        }
+    }
+
+    /* we always send the topologies and the #slots on each node. Note
+     * that we cannot send the #slots until after the above step since,
+     * for unmanaged allocations, we might have just determined it! */
+    if (OPAL_SUCCESS != (rc = orte_util_pass_node_info(&buf))) {
+        ORTE_ERROR_LOG(rc);
+        OBJ_DESTRUCT(&buf);
+        ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
+        OBJ_RELEASE(caddy);
+        return;
+    }
+
+    /* goes to all daemons */
+    sig = OBJ_NEW(orte_grpcomm_signature_t);
+    sig->signature = (orte_process_name_t*)malloc(sizeof(orte_process_name_t));
+    sig->signature[0].jobid = ORTE_PROC_MY_NAME->jobid;
+    sig->signature[0].vpid = ORTE_VPID_WILDCARD;
+    sig->sz = 1;
+    if (ORTE_SUCCESS != (rc = orte_grpcomm.xcast(sig, ORTE_RML_TAG_DAEMON, &buf))) {
+        ORTE_ERROR_LOG(rc);
+        OBJ_RELEASE(sig);
+        OBJ_DESTRUCT(&buf);
+        ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
+        OBJ_RELEASE(caddy);
+        return;
+    }
+    OBJ_DESTRUCT(&buf);
+    /* maintain accounting */
+    OBJ_RELEASE(sig);
+
     /* progress the job */
     caddy->jdata->state = ORTE_JOB_STATE_DAEMONS_REPORTED;
     ORTE_ACTIVATE_JOB_STATE(caddy->jdata, ORTE_JOB_STATE_VM_READY);

orte/orted/orted_comm.c

@@ -59,6 +59,7 @@
 #include "orte/util/proc_info.h"
 #include "orte/util/session_dir.h"
 #include "orte/util/name_fns.h"
+#include "orte/util/nidmap.h"
 
 #include "orte/mca/errmgr/errmgr.h"
 #include "orte/mca/grpcomm/base/base.h"
@@ -126,7 +127,7 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
     char *coprocessors;
     orte_job_map_t *map;
     int8_t flag;
-    uint8_t *cmpdata;
+    uint8_t *cmpdata, u8;
     size_t cmplen;
 
     /* unpack the command */
@@ -241,6 +242,32 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
         }
         break;
 
+
+    case ORTE_DAEMON_PASS_NODE_INFO_CMD:
+        if (orte_debug_daemons_flag) {
+            opal_output(0, "%s orted_cmd: received pass_node_info",
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
+        }
+        if (!ORTE_PROC_IS_HNP) {
+            n = 1;
+            if (ORTE_SUCCESS != (ret = opal_dss.unpack(buffer, &u8, &n, OPAL_UINT8))) {
+                ORTE_ERROR_LOG(ret);
+                goto CLEANUP;
+            }
+            if (1 == u8) {
+                if (ORTE_SUCCESS != (ret = orte_util_decode_nidmap(buffer))) {
+                    ORTE_ERROR_LOG(ret);
+                    goto CLEANUP;
+                }
+            }
+            if (ORTE_SUCCESS != (ret = orte_util_parse_node_info(buffer))) {
+                ORTE_ERROR_LOG(ret);
+                goto CLEANUP;
+            }
+        }
+        break;
+
+
         /****    ADD_LOCAL_PROCS   ****/
     case ORTE_DAEMON_ADD_LOCAL_PROCS:
     case ORTE_DAEMON_DVM_ADD_PROCS: