Support collecting perf profiles remotely (#275)

If we're running tests on a remote machine with -DTEST_SUITE_REMOTE_HOST and generate a perf profile with `lit --param profile=perf`, currently the profile step will be run on the local machine and fail.

This patch fixes this by running the profile step on the remote, as well as copying the generated profiles back to the host so tools like LNT can inspect them.

There's two parts to this, first we need to run the remote module after the perf (and hpmcount) module so that we can wrap the profile step with ssh etc.

Secondly, we need to scp over the generated .perf_data files like we also do for the profilegen.py module. However today this is also done as part of the profile step which we now need to start running over ssh. So to fix this, this adds a new "profile collect" step that runs after the profile step, but always runs on the local machine.

I've tested this out on a remote build configured with `-DTEST_SUITE_REMOTE_HOST=... -DTEST_SUITE_USE_PERF=ON -DTEST_SUITE_PROFILE_GENERATE=ON -DTEST_SUITE_USE_IR_PGO=ON` and running the tests with `lit --param profile=perf`, and it seems to work.

Author: lukel97

lit.cfg

@@ -25,11 +25,6 @@ config.traditional_output = False
 config.single_source = False
 if "SSH_AUTH_SOCK" in os.environ:
     config.environment["SSH_AUTH_SOCK"] = os.environ["SSH_AUTH_SOCK"]
-if not hasattr(config, "remote_host"):
-    config.remote_host = ""
-config.remote_host = lit_config.params.get("remote_host", config.remote_host)
-if config.remote_host:
-    config.test_modules.append("remote")
 
 # Load previous test results so we can skip tests that did not change.
 previous_results_file = lit_config.params.get("previous", None)
@@ -55,3 +50,9 @@ if lit_config.params.get("profile") == "hpmcount":
 config.substitutions += [
     ("%b", os.path.join(config.test_exec_root, "tools")),
 ]
+
+if not hasattr(config, "remote_host"):
+    config.remote_host = ""
+config.remote_host = lit_config.params.get("remote_host", config.remote_host)
+if config.remote_host:
+    config.test_modules.append("remote")

litsupport/modules/profilegen.py

@@ -27,4 +27,4 @@ def mutatePlan(context, plan):
     profdatafile = context.executable + ".profdata"
     args = ["merge", "-output=%s" % profdatafile] + context.profilefiles
     mergecmd = shellcommand.ShellCommand(context.config.llvm_profdata, args)
-    plan.profilescript += [mergecmd.toCommandline()]
+    plan.profilecollectscript += [mergecmd.toCommandline()]

litsupport/modules/remote.py

@@ -43,14 +43,18 @@ def mutatePlan(context, plan):
     plan.verifyscript = _mutateScript(context, plan.verifyscript)
     for name, script in plan.metricscripts.items():
         plan.metricscripts[name] = _mutateScript(context, script)
+    plan.profilescript = _mutateScript(context, plan.profilescript)
 
     # Merging profile data should happen on the host because that is where
     # the toolchain resides, however we have to retrieve the profile data
     # from the device first, add commands for that to the profile script.
-    for path in plan.profile_files:
+    profile_files = plan.profile_files
+    if context.profilefile:
+        profile_files += [context.profilefile]
+    for path in profile_files:
         assert os.path.isabs(path)
         command = "scp %s:%s %s" % (context.config.remote_host, path, path)
-        plan.profilescript.insert(0, command)
+        plan.profilecollectscript.insert(0, command)
 
     assert context.read_result_file is testplan.default_read_result_file
     context.read_result_file = remote_read_result_file

litsupport/testplan.py

@@ -24,6 +24,7 @@ def __init__(self):
         self.preparescript = []
         self.profile_files = []
         self.profilescript = []
+        self.profilecollectscript = []
 
 
 def mutateScript(context, script, mutator):
@@ -119,6 +120,13 @@ def _executePlan(context, plan):
     if exitCode != 0:
         logging.warning("Profile script '%s' failed", plan.profilescript)
 
+    # Execute steps to manage the generated profile data, on the host.
+    _, _, exitCode, _ = _executeScript(
+        context, plan.profilecollectscript, "profilecollect"
+    )
+    if exitCode != 0:
+        logging.warning("Profile collect script '%s' failed", plan.profilecollectscript)
+
     # Perform various metric extraction steps setup by testing modules.
     for metric_collector in plan.metric_collectors:
         try:
@@ -201,3 +209,4 @@ def __init__(self, test, litConfig, tmpDir, tmpBase):
         self.tmpDir = tmpDir
         self.tmpBase = tmpBase
         self.read_result_file = default_read_result_file
+        self.profilefile = None