languages: handle included files out of band

Extracting included files in the executed script itself requires some
code for each supported language. It's also cumbersome to clean up.
Instead let the invoker handle the dirty details and just add the right
reference in the final script.

Author: jkloetzke

pym/bob/input.py

@@ -136,15 +136,19 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
 def mergeScripts(fragments, glue):
     """Join all scripts of the recipe and its classes.
 
-    The result is a tuple with (setupScript, mainScript, digestScript)
+    The result is a tuple with (setupScript, mainScript, digestScript, includedFiles)
     """
+
     return (
         joinScripts((f[0][0] for f in fragments), glue),
         joinScripts((f[1][0] for f in fragments), glue),
         joinScripts(
             ( joinScripts((f[0][1] for f in fragments), "\n"),
               joinScripts((f[1][1] for f in fragments), "\n"),
-            ), "\n")
+            ), "\n"),
+        { name : content for name, content in
+                         chain.from_iterable(chain(f[0][2].items(), f[1][2].items()) for f in fragments)
+        },
     )
 
 
@@ -790,6 +794,9 @@ def getPostRunCmds(self):
     def getDigestScript(self):
         raise NotImplementedError
 
+    def getIncludedFiles(self):
+        raise NotImplementedError
+
     def getLabel(self):
         raise NotImplementedError
 
@@ -1042,6 +1049,9 @@ def getDigestScript(self):
         """
         return self._coreStep.getDigestScript()
 
+    def getIncludedFiles(self):
+        return self._coreStep.getIncludedFiles()
+
     def isDeterministic(self):
         """Return whether the step is deterministic.
 
@@ -1458,6 +1468,9 @@ def getDigestScript(self):
         else:
             return None
 
+    def getIncludedFiles(self):
+        return self.corePackage.recipe.checkoutIncludedFiles
+
     @property
     def fingerprintMask(self):
         return 0
@@ -1537,7 +1550,7 @@ def hasNetAccess(self):
 class CoreBuildStep(CoreStep):
     __slots__ = []
 
-    def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[]):
+    def __init__(self, corePackage, script=(None, None, None, {}), digestEnv=Env(), env=Env(), args=[]):
         isValid = script[1] is not None
         super().__init__(corePackage, isValid, True, digestEnv, env, args)
 
@@ -1568,6 +1581,9 @@ def getMainScript(self):
     def getDigestScript(self):
         return self.corePackage.recipe.buildDigestScript
 
+    def getIncludedFiles(self):
+        return self.corePackage.recipe.buildIncludedFiles
+
     @property
     def fingerprintMask(self):
         # Remove bits of all tools that are not used in buildStep
@@ -1591,7 +1607,7 @@ def hasNetAccess(self):
 class CorePackageStep(CoreStep):
     __slots__ = []
 
-    def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[]):
+    def __init__(self, corePackage, script=(None, None, None, {}), digestEnv=Env(), env=Env(), args=[]):
         isValid = script[1] is not None
         super().__init__(corePackage, isValid, True, digestEnv, env, args)
 
@@ -1622,6 +1638,9 @@ def getMainScript(self):
     def getDigestScript(self):
         return self.corePackage.recipe.packageDigestScript
 
+    def getIncludedFiles(self):
+        return self.corePackage.recipe.packageIncludedFiles
+
     @property
     def fingerprintMask(self):
         return self.corePackage.fingerprintMask
@@ -1884,7 +1903,7 @@ def resolve(self, text, section):
                 raise ParseError("Bad substiturion in {}: {}".format(section, str(e)))
             return resolver.resolve(ret)
         else:
-            return (None, None)
+            return (None, None, {})
 
 def mergeFilter(left, right):
     if left is None:
@@ -2316,7 +2335,7 @@ def coDet(r):
 
         # the package step must always be valid
         if self.__package[1] is None:
-            self.__package = (None, "", 'da39a3ee5e6b4b0d3255bfef95601890afd80709')
+            self.__package = (None, "", 'da39a3ee5e6b4b0d3255bfef95601890afd80709', {})
 
         # final shared value
         self.__shared = self.__shared == True
@@ -2643,7 +2662,7 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None,
                 directPackages, indirectPackages, states, uidGen(), doFingerprint)
 
         # optional checkout step
-        if self.__checkout != (None, None, None) or self.__checkoutSCMs or self.__checkoutAsserts:
+        if self.__checkout != (None, None, None, {}) or self.__checkoutSCMs or self.__checkoutAsserts:
             checkoutDigestEnv = env.prune(self.__checkoutVars)
             checkoutEnv = ( env.prune(self.__checkoutVars | self.__checkoutVarsWeak)
                 if self.__checkoutVarsWeak else checkoutDigestEnv )
@@ -2653,7 +2672,7 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None,
             srcCoreStep = p.createInvalidCoreCheckoutStep()
 
         # optional build step
-        if self.__build != (None, None, None):
+        if self.__build != (None, None, None, {}):
             buildDigestEnv = env.prune(self.__buildVars)
             buildEnv = ( env.prune(self.__buildVars | self.__buildVarsWeak)
                 if self.__buildVarsWeak else buildDigestEnv )
@@ -2765,6 +2784,10 @@ def checkoutMainScript(self):
     def checkoutDigestScript(self):
         return self.__checkout[2] or ""
 
+    @property
+    def checkoutIncludedFiles(self):
+        return self.__checkout[3]
+
     @property
     def checkoutDeterministic(self):
         return self.__checkoutDeterministic
@@ -2793,6 +2816,10 @@ def buildMainScript(self):
     def buildDigestScript(self):
         return self.__build[2]
 
+    @property
+    def buildIncludedFiles(self):
+        return self.__build[3]
+
     @property
     def buildVars(self):
         return self.__buildVars
@@ -2813,6 +2840,10 @@ def packageMainScript(self):
     def packageDigestScript(self):
         return self.__package[2]
 
+    @property
+    def packageIncludedFiles(self):
+        return self.__package[3]
+
     @property
     def packageVars(self):
         return self.__packageVars

pym/bob/invoker.py

@@ -19,6 +19,7 @@
 import os
 import re
 import shutil
+import stat
 import subprocess
 import sys
 import tempfile
@@ -266,7 +267,6 @@ def __getSandboxCmds(self, tmpDir):
         #FIXME: if verbosity >= 4: cmdArgs.append('-D')
         cmdArgs.extend(["-S", tmpDir])
         cmdArgs.extend(["-H", "bob"])
-        cmdArgs.extend(["-d", "/tmp"])
         sandboxRootFs = os.path.abspath(self.__spec.sandboxRootWorkspace)
         for f in os.listdir(sandboxRootFs):
             cmdArgs.extend(["-M", os.path.join(sandboxRootFs, f), "-m", "/"+f])
@@ -301,6 +301,16 @@ async def executeStep(self, mode, clean=False, keepSandbox=False):
             # also used as ephemeral sandbox container.
             tmpDir = tempfile.mkdtemp()
 
+            # Create /tmp in sandbox container. Receives included files and
+            # needs to be there anyway.
+            os.mkdir(os.path.join(tmpDir, "tmp"), 0o755)
+            for name, content in self.__spec.includedFiles.items():
+                fn = os.path.join(tmpDir, "tmp", name)
+                with open(fn, "wb") as f:
+                    f.write(content)
+                # Set explicit mode to retain Bob 0.19 behaviour.
+                os.chmod(fn, stat.S_IREAD|stat.S_IWRITE)
+
             # prepare workspace
             clean = self.__spec.clean if self.__spec.clean is not None else clean
             if not os.path.isdir(self.__spec.workspaceWorkspacePath):
@@ -468,6 +478,7 @@ async def executeFingerprint(self, keepSandbox=False):
 
                 # Setup workspace
                 cmdArgs = self.__getSandboxCmds(tmpDir)
+                cmdArgs.extend(["-d", "/tmp"])
                 cmdArgs.extend(["-d", "/bob/fingerprint"])
                 cmdArgs.extend(["-W", "/bob/fingerprint"])