[Static Linux SDK] Build script improvements. (#417)

* [Static Linux SDK] Build script improvements.

When running on an aarch64 host, we need to download a different version
of the Swift compiler in order to do the build.  If we run the build on
a macOS host, we also have to tell the update-checkout script to fetch
all the required dependencies (even Linux-only ones).

Also tell Foundation where swift-collections lives, to avoid it fetching
another copy.

* [CI] Run Docker builds in the right directory.

Make sure we run the Docker builds in the directory containing the
Dockerfile.  Also, prefer lists of arguments rather than strings,
especially when including filenames.

Author: al45tair

ci_test.py

@@ -16,19 +16,22 @@
 import urllib.request
 import json
 import subprocess
+import shlex
 import sys
 import os
 
 
 def run_command(cmd, log_file=None):
-    print("Running: {}".format(cmd))
+    if isinstance(cmd, str):
+        cmd = shlex.split(cmd)
+    print("Running: {}".format(shlex.join(cmd)))
     sys.stdout.flush()
     if log_file:
         file = open(log_file, "w")
-        p = subprocess.Popen(cmd, shell=True, stdout=file, stderr=file)
     else:
-        p = subprocess.Popen(cmd, shell=True)
-      
+        file = None
+    p = subprocess.Popen(cmd, stdout=file, stderr=file)
+
     (output, err) = p.communicate()
     return p.wait()
 
@@ -62,30 +65,47 @@ def main():
     results = {}
     suite_status = True
     dockerfiles = get_dockerfiles()
+    root_dir = os.path.dirname(os.path.realpath(__file__))
     for dockerfile in dockerfiles:
-        docker_dir = os.path.dirname(os.path.realpath(__file__))
+        # Make sure everything is relative
+        dockerfile = os.path.relpath(os.path.realpath(dockerfile), root_dir)
+
+        docker_dir, docker_name = os.path.split(dockerfile)
+
         print("Testing {}".format(dockerfile))
         sys.stdout.flush()
-        log_file = dockerfile.replace(docker_dir,"").replace("/", "_")
+        log_file = dockerfile.replace("/", "_")
         log_file = "{}.log".format(log_file)
-        cmd = "docker build --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile)
+        cmd = [
+            'docker', 'build', '--no-cache=true',
+            '-f', dockerfile,
+            docker_dir
+        ]
         if "buildx" in dockerfile:
             # if "buildx" is part of the path, we want to use the new buildx build system and build
             # for both amd64 and arm64.
-            cmd = "docker buildx create --use"
-            run_command(cmd, log_file)
-            cmd = "docker buildx inspect --bootstrap"
-            run_command(cmd, log_file)
-            cmd = "docker buildx build --platform linux/arm64,linux/amd64 --no-cache=true -f {dockerfile} .".format(dockerfile=dockerfile)
-        status = run_command(cmd, log_file)
+            run_command("docker buildx create --use", log_file=log_file)
+            run_command("docker buildx inspect --bootstrap", log_file=log_file)
+
+            cmd = [
+                'docker', 'buildx', 'build',
+                '--platform', 'linux/arm64,linux/amd64',
+                '--no-cache=true',
+                '-f', dockerfile,
+                docker_dir
+            ]
+
+        status = run_command(cmd, log_file=log_file)
         results[dockerfile] = status
         if status != 0:
             suite_status = False
             results[dockerfile] = "FAILED"
         else:
             results[dockerfile] = "PASSED"
 
-        cmd = "mv {log} {results}{log}".format(log=log_file, results=results[dockerfile])
+        cmd = [
+            'mv', log_file, results[dockerfile] + log_file
+        ]
         run_command(cmd)
         print("[{}] - {}".format(results[dockerfile], dockerfile))
         sys.stdout.flush()

swift-ci/sdks/static-linux/Dockerfile

@@ -24,6 +24,9 @@ ARG BORINGSSL_VERSION=fips-20220613
 ARG ICU_VERSION=maint/maint-69
 ARG ZLIB_VERSION=1.3.1
 
+# Architecture to build on (empty means x86-64)
+ARG OS_ARCH_SUFFIX=
+
 # ............................................................................
 
 # Install development tools
@@ -62,7 +65,7 @@ ENV SWIFT_SIGNING_KEY=$SWIFT_SIGNING_KEY \
     OS_MAJOR_VER=$OS_MAJOR_VER \
     OS_MINOR_VER=$OS_MINOR_VER \
     OS_VER=$SWIFT_PLATFORM$OS_MAJOR_VER.$OS_MINOR_VER \
-    SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER"
+    SWIFT_WEBROOT="$SWIFT_WEBROOT/$SWIFT_PLATFORM$OS_MAJOR_VER$OS_MINOR_VER$OS_ARCH_SUFFIX"
 
 COPY scripts/install-swift.sh /scripts/install-swift.sh
 RUN chmod ugo+x /scripts/install-swift.sh

swift-ci/sdks/static-linux/build

@@ -14,18 +14,33 @@
 #
 # ===----------------------------------------------------------------------===
 
-DOCKER=docker
+if [[ "$DOCKER" == "" ]]; then
+    DOCKER=docker
+fi
+
+case $(arch) in
+    arm64|aarch64)
+        OS_ARCH_SUFFIX=-aarch64
+        ;;
+    amd64|x86_64)
+        OS_ARCH_SUFFIX=
+        ;;
+    *)
+        echo "Unknown architecture $(arch)"
+        exit 1
+        ;;
+esac
 
 # Build the Docker image
-$(DOCKER) build -t static-swift-linux .
+$DOCKER build --build-arg OS_ARCH_SUFFIX=$OS_ARCH_SUFFIX -t static-swift-linux .
 
 # Check-out the sources
 scripts/fetch-source.sh --clone-with-ssh --source-dir source
 
 mkdir -p products
 
 # Run the build
-$(DOCKER) run -it --rm \
+$DOCKER run -it --rm \
           -v ./source:/source \
           -v ./products:/products \
           static-swift-linux \

swift-ci/sdks/static-linux/scripts/build.sh

@@ -144,6 +144,20 @@ while [ "$#" -gt 0 ]; do
     shift
 done
 
+# Work out the host architecture
+case $(arch) in
+    arm64|aarch64)
+        host_arch=arm64
+        ;;
+    amd64|x86_64)
+        host_arch=x86_64
+        ;;
+    *)
+        echo "Unknown host architecture $(arch)"
+        exit 1
+        ;;
+esac
+
 # Change the commas for spaces
 archs="${archs//,/ }"
 
@@ -662,7 +676,7 @@ EOF
         --compiler-vendor=apple \
         --bootstrapping hosttools \
         --build-linux-static --install-swift \
-        --stdlib-deployment-targets linux-x86_64,linux-static-$arch \
+        --stdlib-deployment-targets linux-$host_arch,linux-static-$arch \
         --build-stdlib-deployment-targets all \
         --musl-path=${build_dir}/sdk_root \
         --linux-static-arch=$arch \
@@ -724,6 +738,7 @@ EOF
           -DFOUNDATION_PATH_TO_LIBDISPATCH_BUILD=${build_dir}/$arch/dispatch \
           -D_SwiftFoundation_SourceDIR=${source_dir}/swift-project/swift-foundation \
           -D_SwiftFoundationICU_SourceDIR=${source_dir}/swift-project/swift-foundation-icu \
+          -D_SwiftCollections_SourceDIR=${source_dir}/swift-project/swift-collections \
           -DCMAKE_Swift_COMPILER_WORKS=YES \
           -Ddispatch_DIR=${build_dir}/$arch/dispatch/cmake/modules
 

swift-ci/sdks/static-linux/scripts/fetch-source.sh

@@ -164,7 +164,7 @@ cd swift
 # Get its dependencies
 header "Fetching Swift Dependencies"
 
-extra_args=--skip-history
+extra_args="--skip-history --all-repositories"
 if [[ $SWIFT_VERSION == scheme:* ]]; then
     utils/update-checkout ${clone_arg} --scheme ${SWIFT_VERSION#scheme:} ${extra_args}
 elif [[ $SWIFT_VERSION == tag:* ]]; then