Merge pull request #436 from jkloetzke/out-of-tree-builds

Add out-of-tree build support

Author: jkloetzke

contrib/bash-completion/bob

@@ -29,8 +29,8 @@ __bob_complete_dir()
                  compgen -d -P "$2" -S / -- "$1" ) )
 }
 
-__bob_commands="build dev clean graph help jenkins ls project status  query-scm \
-                query-recipe query-path query-meta show"
+__bob_commands="build dev clean graph help init jenkins ls project status \
+                query-scm query-recipe query-path query-meta show"
 
 # Complete a Bob path
 #
@@ -162,6 +162,18 @@ __bob_ls()
    __bob_complete_path "-a --all -c -D -d --direct --no-sandbox -o --origin -p --prefixed -r --recursive --sandbox -u --unsorted"
 }
 
+__bob_init()
+{
+   case "$cur" in
+      -*)
+         __bob_complete_words "-h --help"
+         ;;
+      *)
+         __bob_complete_dir "$cur"
+         ;;
+   esac
+}
+
 __bob_jenkins_add()
 {
    sandbox="--sandbox"

doc/conf.py

@@ -260,6 +260,7 @@ def __getattr__(cls, name):
     ('manpages/bob-clean', 'bob-clean', 'Delete unused src/build/dist paths of release builds', ['Jan Klötzke'], 1),
     ('manpages/bob-dev', 'bob-dev', 'Bob develop mode build', ['Jan Klötzke'], 1),
     ('manpages/bob-graph', 'bob-graph', 'Generate dependency graph', ['Ralf Hubert'], 1),
+    ('manpages/bob-init', 'bob-init', 'Initialize out-of-source build tree', ['Jan Klötzke'], 1),
     ('manpages/bob-jenkins', 'bob-jenkins', 'Configure Jenkins server', ['Jan Klötzke'], 1),
     ('manpages/bob-ls', 'bob-ls', 'List package hierarchy', ['Jan Klötzke'], 1),
     ('manpages/bobpaths', 'bobpaths', 'Specifying paths to Bob packages', ['Jan Klötzke'], 7),

doc/manpages/bob-build-dev.rst

@@ -1,3 +1,12 @@
+Build tree location
+-------------------
+
+The build can be done directly in the project root directory or in a separate
+directory. To build outside of the project directory the build-tree must first
+be initialized with :ref:`bob init <manpage-bob-init>`. Any number of build
+trees may refer to the same project. Inside the external build-tree there may
+be a dedicated ``default.yaml``, overriding settings from the project.
+
 Options
 -------
 
@@ -255,3 +264,4 @@ See also
 --------
 
 :ref:`bobpaths(7) <manpage-bobpaths>` :ref:`bob-status(1) <manpage-bob-status>`
+:ref:`bob-init(1) <manpage-bob-init>`

doc/manpages/bob-init.rst

@@ -0,0 +1,32 @@
+.. _manpage-bob-init:
+
+bob-init
+========
+
+.. only:: not man
+
+   Name
+   ----
+
+   bob-init - Initialize out-of-source build tree
+
+Synopsis
+--------
+
+::
+
+    bob init [-h] PROJECT [BUILD]
+
+
+Description
+-----------
+
+Setup a build directory that builds the project located at ``PROJECT``. By
+default the current directory is initialized. Optionally the build directory
+can be specified as 2nd parameter ``BUILD``. The directories to the build
+directory will be created if they do not exist yet.
+
+See also
+--------
+
+:ref:`bob-build(1) <manpage-build>` :ref:`bob-dev(1) <manpage-dev>`

doc/manpages/index.rst

@@ -11,6 +11,7 @@ Contents:
    bob-clean
    bob-dev
    bob-graph
+   bob-init
    bob-jenkins
    bob-ls
    bobpaths

doc/manual/extending.rst

@@ -56,8 +56,11 @@ internal and might change without notice.
 .. autoclass:: bob.input.PluginState()
    :members:
 
+.. autoclass:: bob.input.RecipeSet()
+   :members:
+
 .. autoclass:: bob.input.Recipe()
-   :members: getName, getPackageName, isRoot
+   :members:
 
 .. autoclass:: bob.input.Package()
    :members:

doc/tutorial/compile.rst

@@ -169,6 +169,20 @@ and test the whole QEMU image. The choice is yours.
    before moving on. Otherwise you risk that your changes are wiped out if Bob
    determines that a clean build is needed (e.g. due to recipe changes).
 
+Out of tree builds
+==================
+
+The :ref:`manpage-dev` and :ref:`manpage-build` commands do not need to be
+executed in the project directory directly. It is also possible to initialize
+an external build directory with :ref:`manpage-bob-init`::
+
+    $ bob init . /path/to/build/directory
+    $ cd /path/to/build/directory
+    $ bob dev vexpress
+
+Each build directory can have its own :ref:`configuration-config-usr` that
+overrides the defaults of the project.
+
 Query SCM status
 ================
 

pym/bob/cmds/misc.py

@@ -6,11 +6,12 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from ..input import RecipeSet
-from ..errors import BuildError
+from ..errors import ParseError, BuildError
 from ..utils import processDefines
 import argparse
 import codecs
 import sys
+import os, os.path
 
 try:
     # test if stdout can handle box drawing characters
@@ -268,3 +269,42 @@ def doQueryRecipe(argv, bobRoot):
 
     for fn in package.getRecipe().getSources():
         print(fn)
+
+def doInit(argv, bobRoot):
+    parser = argparse.ArgumentParser(prog="bob init",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="""Initialize out-of-source build tree.
+
+Create a Bob build tree in the current directory or at the given BUILD
+directory. The recipes, classes, plugins and all other files are taken from the
+project root directory at PROJECT.
+""")
+    parser.add_argument('project', metavar="PROJECT",
+        help="Project root directory")
+    parser.add_argument('build', nargs='?', metavar="BUILD", default=".",
+        help="Build directory (default: .)")
+
+    args = parser.parse_args(argv)
+
+    recipesDir = os.path.join(args.project, "recipes")
+    if not os.path.isdir(recipesDir):
+        raise ParseError("No recipes directory found in " + recipesDir)
+
+    try:
+        os.makedirs(args.build, exist_ok=True)
+        if os.path.samefile(args.project, args.build):
+            print("The project directory does not need to be initialized.",
+                  file=sys.stderr)
+            return
+    except OSError as e:
+        raise ParseError("Error creating build directory: " + str(e))
+
+    projectLink = os.path.join(args.build, ".bob-project")
+    if os.path.exists(projectLink):
+        raise ParseError("Build tree already initialized!")
+
+    try:
+        with open(projectLink, "w") as f:
+            f.write(os.path.abspath(args.project))
+    except OSError as e:
+        raise ParseError("Cannot create project link: " + str(e))

pym/bob/input.py

@@ -2214,6 +2214,15 @@ def __resolveClassesOrder(self, cls, stack, visited, isRecipe=False):
         return ret
 
     def getLayer(self):
+        """Get layer to which this recipe belongs.
+
+        Returns a list of the layer hierarchy. The root layer is represented
+        by an empty list. If the recipe belongs to a nested layer the layers
+        are named from top to bottom. Example:
+        ``layers/foo/layers/bar/recipes/baz.yaml`` -> ``['foo', 'bar']``.
+
+        :rtype: List[str]
+        """
         return self.__layer
 
     def resolveClasses(self, rootEnv):
@@ -2359,6 +2368,7 @@ def coDet(r):
             self.__root = rootEnv.evaluate(self.__root, "root")
 
     def getRecipeSet(self):
+        """Get the :class:`RecipeSet` to which the recipe belongs"""
         return self.__recipeSet
 
     def getSources(self):
@@ -2388,15 +2398,21 @@ def isRoot(self):
         return self.__root == True
 
     def isRelocatable(self):
-        """Returns True if the packages of this recipe are relocatable."""
+        """Returns True if the packages of this recipe are relocatable.
+
+        :meta private:
+        """
         return self.__relocatable
 
     def isShared(self):
         return self.__shared
 
     def jobServer(self):
-        """Returns True if the jobserver should be used to schedule
-        builds for this recipe."""
+        """Returns True if the jobserver should be used to schedule builds for
+        this recipe.
+
+        :meta private:
+        """
         return self.__jobServer
 
     def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None,
@@ -2972,6 +2988,10 @@ def validate(self, data):
         raise schema.SchemaError(None, "Mount entry must be a string or a two/three items list!")
 
 class RecipeSet:
+    """The RecipeSet corresponds to the project root directory.
+
+    It holds global information about the project.
+    """
 
     BUILD_DEV_SCHEMA = schema.Schema(
         {
@@ -3402,12 +3422,20 @@ def getProjectGenerators(self):
         return self.__projectGenerators
 
     def envWhiteList(self):
+        """The set of all white listed environment variables
+
+        :rtype: Set[str]
+        """
         return set(self.__whiteList)
 
     def archiveSpec(self):
         return self.__archive
 
     def defaultEnv(self):
+        """The default environment that each root recipe inherits
+
+        :rtype: Mapping[str, str]
+        """
         return self.__defaultEnv
 
     def scmDefaults(self):
@@ -3457,15 +3485,24 @@ def loadYaml(self, path, schema, default={}, preValidate=lambda x: None):
             return schema[0].validate(default)
 
     def parse(self, envOverrides={}, platform=getPlatformString()):
-        if not os.path.isdir("recipes"):
-            raise ParseError("No recipes directory found.")
+        recipesRoot = ""
+        if os.path.isfile(".bob-project"):
+            try:
+                with open(".bob-project") as f:
+                    recipesRoot = f.read()
+            except OSError as e:
+                raise ParseError("Broken project link: " + str(e))
+        recipesDir = os.path.join(recipesRoot, "recipes")
+        if not os.path.isdir(recipesDir):
+            raise ParseError("No recipes directory found in " + recipesDir)
         self.__cache.open()
         try:
-            self.__parse(envOverrides, platform)
+            self.__parse(envOverrides, platform, recipesRoot)
         finally:
             self.__cache.close()
+        self.__projectRoot = recipesRoot or os.getcwd()
 
-    def __parse(self, envOverrides, platform):
+    def __parse(self, envOverrides, platform, recipesRoot=""):
         self.__pluginPropDeps = b''
         self.__pluginSettingsDeps = b''
         self.__createSchemas()
@@ -3477,7 +3514,11 @@ def __parse(self, envOverrides, platform):
                 os.path.join(os.path.expanduser("~"), '.config')), 'bob', 'default.yaml'), True)
 
         # Begin with root layer
-        self.__parseLayer([], "9999")
+        self.__parseLayer([], "9999", recipesRoot)
+
+        # Out-of-tree builds may have a dedicated default.yaml
+        if recipesRoot:
+            self.__parseUserConfig("default.yaml", True)
 
         # config files overrule everything else
         for c in self.__configFiles:
@@ -3520,8 +3561,8 @@ def __parse(self, envOverrides, platform):
         self.__rootRecipe = Recipe.createVirtualRoot(self, sorted(filteredRoots), self.__properties)
         self.__addRecipe(self.__rootRecipe)
 
-    def __parseLayer(self, layer, maxVer):
-        rootDir = os.path.join("", *(os.path.join("layers", l) for l in layer))
+    def __parseLayer(self, layer, maxVer, recipesRoot):
+        rootDir = os.path.join(recipesRoot, *(os.path.join("layers", l) for l in layer))
         if not os.path.isdir(rootDir or "."):
             raise ParseError("Layer '{}' does not exist!".format("/".join(layer)))
 
@@ -3562,7 +3603,7 @@ def preValidate(data):
         # First parse any sub-layers. Their settings have a lower precedence
         # and may be overwritten by higher layers.
         for l in config.get("layers", []):
-            self.__parseLayer(layer + [l], maxVer)
+            self.__parseLayer(layer + [l], maxVer, recipesRoot)
 
         # Load plugins and re-create schemas as new keys may have been added
         self.__loadPlugins(rootDir, layer, config.get("plugins", []))
@@ -3831,6 +3872,15 @@ def sandboxFingerprints(self):
             self.__sandboxFingerprints = self.getPolicy("sandboxFingerprints")
             return self.__sandboxFingerprints
 
+    def getProjectRoot(self):
+        """Get project root directory.
+
+        The project root is where the recipes, classes and layers are located.
+        In case of out-of-tree builds it will be distinct from the build
+        directory.
+        """
+        return self.__projectRoot
+
 
 class YamlCache:
     def __if_expression_constructor(loader, node):

pym/bob/scripts.py

@@ -44,6 +44,11 @@ def __help(*args, **kwargs):
     doHelp(availableCommands.keys(), *args, **kwargs)
     return 0
 
+def __init(*args, **kwargs):
+    from .cmds.misc import doInit
+    doInit(*args, **kwargs)
+    return 0
+
 def __jenkins(*args, **kwargs):
      from .cmds.jenkins import doJenkins
      doJenkins(*args, **kwargs)
@@ -110,6 +115,7 @@ def __invoke(*args, **kwargs):
     "clean"         : ('hl', __clean, "Delete unused src/build/dist paths of release builds"),
     "graph"         : ('hl', __graph, "Make a interactive dependency graph"),
     "help"          : ('hl', __help, "Display help information about command"),
+    "init"          : ('hl', __init, "Initialize build tree"),
     "jenkins"       : ('hl', __jenkins, "Configure Jenkins server"),
     "ls"            : ('hl', __ls, "List package hierarchy"),
     "project"       : ('hl', __project, "Create project files"),