schizo/ompi: correctly handle the yield_when_idle option

in schizo/ompi, sets the new OMPI_MCA_mpi_oversubscribe environment
variable according to the node oversubscription state.

This MCA parameter is used to set the default value of the
mpi_yield_when_idle parameter.

This two steps tango is needed so the mpi_yield_when_idle setting
is always honored when set in a config file.

Refs. #6433

Signed-off-by: Gilles Gouaillardet <gilles@rist.or.jp>

Author: ggouaillardet

ompi/runtime/ompi_mpi_params.c

@@ -17,7 +17,7 @@
  * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
+ * Copyright (c) 2016-2019 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -82,6 +82,7 @@ static bool show_default_mca_params = false;
 static bool show_file_mca_params = false;
 static bool show_enviro_mca_params = false;
 static bool show_override_mca_params = false;
+static bool ompi_mpi_oversubscribe = false;
 
 int ompi_mpi_register_params(void)
 {
@@ -108,13 +109,18 @@ int ompi_mpi_register_params(void)
      * opal_progress: decide whether to yield and the event library
      * tick rate
      */
-    /* JMS: Need ORTE data here -- set this to 0 when
-       exactly/under-subscribed, or 1 when oversubscribed */
-    ompi_mpi_yield_when_idle = false;
+    ompi_mpi_oversubscribe = false;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "oversubscribe",
+                                 "Internal MCA parameter set by the runtime environment when oversubscribing nodes",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_mpi_oversubscribe);
+    ompi_mpi_yield_when_idle = ompi_mpi_oversubscribe;
     (void) mca_base_var_register("ompi", "mpi", NULL, "yield_when_idle",
                                  "Yield the processor when waiting for MPI communication (for MPI processes, will default to 1 when oversubscribing nodes)",
                                  MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
+                                 OPAL_INFO_LVL_5,
                                  MCA_BASE_VAR_SCOPE_READONLY,
                                  &ompi_mpi_yield_when_idle);
 

orte/mca/schizo/ompi/schizo_ompi.c

@@ -15,8 +15,8 @@
  * Copyright (c) 2011-2017 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2017      UT-Battelle, LLC. All rights reserved.
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015-2019 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
  * Copyright (c) 2018      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
@@ -908,9 +908,9 @@ static int setup_fork(orte_job_t *jdata,
 
     /* setup yield schedule - do not override any user-supplied directive! */
     if (oversubscribed) {
-        opal_setenv("OMPI_MCA_mpi_yield_when_idle", "1", false, &app->env);
+        opal_setenv("OMPI_MCA_mpi_oversubscribe", "1", true, &app->env);
     } else {
-        opal_setenv("OMPI_MCA_mpi_yield_when_idle", "0", false, &app->env);
+        opal_setenv("OMPI_MCA_mpi_oversubscribe", "0", true, &app->env);
     }
 
     /* set the app_context number into the environment */