diff --git a/Pipfile b/Pipfile index 2c820966..19207778 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ emrichen = "*" shutils = "*" "ruamel.yaml" = "*" "roam" = "*" +bson = "*" [requires] python_version = "3.8" diff --git a/appimagebuilder/__main__.py b/appimagebuilder/__main__.py index d81cae47..18842660 100755 --- a/appimagebuilder/__main__.py +++ b/appimagebuilder/__main__.py @@ -12,18 +12,6 @@ # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. -# Copyright 2020 Alexis Lopez Zubieta -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. - import logging from appimagebuilder import recipe diff --git a/appimagebuilder/commands/create_appimage.py b/appimagebuilder/commands/create_appimage.py index c9137a50..a2b243f1 100644 --- a/appimagebuilder/commands/create_appimage.py +++ b/appimagebuilder/commands/create_appimage.py @@ -9,8 +9,11 @@ # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. -from appimagebuilder.modules.appimage import AppImageCreator +import os + +from appimagebuilder.modules.prime.type_2 import Type2Creator from appimagebuilder.commands.command import Command +from appimagebuilder.modules.prime.type_3 import Type3Creator from appimagebuilder.recipe.roamer import Roamer @@ -23,5 +26,52 @@ def id(self): super().id() def __call__(self, *args, **kwargs): - creator = AppImageCreator(self.recipe) + appimage_format = self.recipe.AppImage.format() or 2 + self.app_dir = self.recipe.AppDir.path() + + self.target_arch = self.recipe.AppImage.arch() + self.app_name = self.recipe.AppDir.app_info.name() + self.app_version = self.recipe.AppDir.app_info.version() + + fallback_file_name = os.path.join( + os.getcwd(), + "%s-%s-%s.AppImage" % (self.app_name, self.app_version, self.target_arch), + ) + self.file_name = self.recipe.AppDir.app_info.file_name() or fallback_file_name + + if appimage_format == 2: + self._create_type_2_appimage() + return + + if appimage_format == 3: + self._create_type_3_appimage() + return + + raise RuntimeError(f"Unknown AppImage format {appimage_format}") + + def _create_type_2_appimage(self): + update_information = self.recipe.AppImage["update-information"]() or "None" + + sign_key = self.recipe.AppImage["sign-key"] or "None" + if sign_key == "None": + sign_key = None + creator = Type2Creator( + self.app_dir, + self.target_arch, + update_information, + sign_key, + self.file_name, + ) creator.create() + + def _create_type_3_appimage(self): + update_information = self.recipe.AppImage["update-information"]() or "None" + creator = Type3Creator(self.app_dir) + creator.create( + self.file_name, + { + "update-information": update_information + }, + gnupg_key=self.recipe.AppImage["sign-key"]() or None, + compression_method="zstd", + ) diff --git a/appimagebuilder/modules/prime/__init__.py b/appimagebuilder/modules/prime/__init__.py new file mode 100644 index 00000000..6f1f3c22 --- /dev/null +++ b/appimagebuilder/modules/prime/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2021 Alexis Lopez Zubieta +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. diff --git a/appimagebuilder/modules/prime/common.py b/appimagebuilder/modules/prime/common.py new file mode 100644 index 00000000..11f53782 --- /dev/null +++ b/appimagebuilder/modules/prime/common.py @@ -0,0 +1,20 @@ +# Copyright 2021 Alexis Lopez Zubieta +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +import logging +import os +from urllib import request + + +def download_if_required(url, path): + if not os.path.exists(path): + logging.info("Downloading: %s" % url) + request.urlretrieve(url, path) diff --git a/appimagebuilder/modules/appimage.py b/appimagebuilder/modules/prime/type_2.py similarity index 72% rename from appimagebuilder/modules/appimage.py rename to appimagebuilder/modules/prime/type_2.py index e7722695..f45d6d4d 100644 --- a/appimagebuilder/modules/appimage.py +++ b/appimagebuilder/modules/prime/type_2.py @@ -1,3 +1,15 @@ +# Copyright 2021 Alexis Lopez Zubieta +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + # Copyright 2020 Alexis Lopez Zubieta # # Permission is hereby granted, free of charge, to any person obtaining a @@ -14,15 +26,14 @@ from urllib import request from appimagebuilder.gateways.appimagetool import AppImageToolCommand +from appimagebuilder.modules.prime import common -class AppImageCreator: - def __init__(self, recipe): - self.app_dir = recipe.AppDir.path() - self.target_arch = recipe.AppImage.arch() - self.app_name = recipe.AppDir.app_info.name() - self.app_version = recipe.AppDir.app_info.version() - self.update_information = recipe.AppImage["update-information"]() or "None" +class Type2Creator: + def __init__(self, appdir, target_arch, update_information, sign_key, output_filename): + self.app_dir = appdir + self.target_arch = target_arch + self.update_information = update_information self.guess_update_information = False if self.update_information == "None": @@ -34,22 +45,18 @@ def __init__(self, recipe): self.update_information = None self.guess_update_information = True - self.sing_key = recipe.AppImage["sign-key"]() or "None" + self.sing_key = sign_key if self.sing_key == "None": self.sing_key = None - fallback_file_name = os.path.join( - os.getcwd(), - "%s-%s-%s.AppImage" % (self.app_name, self.app_version, self.target_arch), - ) - self.target_file = recipe.AppImage.file_name() or fallback_file_name + self.target_file = output_filename def create(self): self._assert_target_architecture() runtime_url = self._get_runtime_url() runtime_path = self._get_runtime_path() - self._download_runtime_if_required(runtime_path, runtime_url) + common.download_if_required(runtime_url, runtime_path) self._generate_appimage(runtime_path) @@ -62,11 +69,6 @@ def _generate_appimage(self, runtime_path): appimage_tool.runtime_file = runtime_path appimage_tool.run() - def _download_runtime_if_required(self, runtime_path, runtime_url): - if not os.path.exists(runtime_path): - logging.info("Downloading runtime: %s" % runtime_url) - request.urlretrieve(runtime_url, runtime_path) - def _get_runtime_path(self): os.makedirs("appimage-builder-cache", exist_ok=True) runtime_path = "appimage-builder-cache/runtime-%s" % self.target_arch diff --git a/appimagebuilder/modules/prime/type_3.py b/appimagebuilder/modules/prime/type_3.py new file mode 100644 index 00000000..c5bb187e --- /dev/null +++ b/appimagebuilder/modules/prime/type_3.py @@ -0,0 +1,187 @@ +# Copyright 2021 Alexis Lopez Zubieta +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +import logging +import os +import pathlib +import subprocess +import tempfile + +import bson + +from appimagebuilder.modules.prime import common +from appimagebuilder.utils import file_utils +from appimagebuilder.utils import shell, elf + + +class Type3Creator: + def __init__(self, app_dir, cache_dir="appimage-builder-cache"): + self.logger = logging.getLogger() + self.app_dir = pathlib.Path(app_dir).absolute() + self.cache_dir = pathlib.Path(cache_dir) + self.runtime_project_url = ( + "https://github.com/AppImageCrafters/appimage-runtime" + ) + + self.required_tool_paths = shell.resolve_commands_paths(["mksquashfs", "gpg"]) + + def create( + self, output_filename, metadata=None, gnupg_key=None, compression_method="gzip" + ): + if metadata is None: + metadata = {} + + self.logger.warning( + "Type 3 AppImages are still experimental and under development!" + ) + + squashfs_path = self._squash_appdir(compression_method) + + runtime_path = self._resolve_executable() + + file_utils.extend_file(runtime_path, squashfs_path, output_filename) + + payload_offset = os.path.getsize(runtime_path) + metadata_offset = os.path.getsize(output_filename) + self._append_metadata(output_filename, metadata) + signatures_offset = os.path.getsize(output_filename) + self._fill_header( + output_filename, payload_offset, metadata_offset, signatures_offset + ) + + if gnupg_key: + self._sign_bundle(output_filename, gnupg_key, signatures_offset) + + # remove squashfs + squashfs_path.unlink() + + file_utils.set_permissions_rx_all(output_filename) + + def _squash_appdir(self, compression_method): + squashfs_path = self.cache_dir / "AppDir.sqfs" + + self.logger.info("Squashing AppDir") + command = "{mksquashfs} {AppDir} {squashfs_path} -reproducible -comp {compression} ".format( + AppDir=self.app_dir, + squashfs_path=squashfs_path, + compression=compression_method, + **self.required_tool_paths, + ) + _proc = subprocess.run( + command, + stderr=subprocess.PIPE, + shell=True, + ) + + shell.assert_successful_result(_proc) + return squashfs_path + + def _resolve_executable(self): + launcher_arch = elf.get_arch(self.app_dir / "AppRun") + url = self._get_runtime_url(launcher_arch) + path = self._get_runtime_path(launcher_arch) + common.download_if_required(url, path.__str__()) + + return path + + def _fill_header( + self, output_filename, payload_offset, metadata_offset, signature_offset + ): + with open(output_filename, "r+b") as f: + f.seek(0x410, 0) + f.write(payload_offset.to_bytes(8, "little")) + f.write(metadata_offset.to_bytes(8, "little")) + f.write(signature_offset.to_bytes(8, "little")) + + def _get_runtime_path(self, arch): + self.cache_dir.parent.mkdir(parents=True, exist_ok=True) + runtime_path = self.cache_dir / f"runtime-{arch}" + + return runtime_path + + def _get_runtime_url(self, arch): + runtime_url_template = ( + self.runtime_project_url + + "/releases/download/continuous/runtime-Release-%s" + ) + runtime_url = runtime_url_template % arch + return runtime_url + + def _append_metadata(self, output_filename, metadata): + raw = bson.dumps(metadata) + + with open(output_filename, "r+b") as fd: + fd.seek(0, 2) + fd.write(raw) + + def _sign_bundle(self, output_filename, keyid, signatures_offset): + signature = self._generate_bundle_signature_using_gpg( + keyid, output_filename, signatures_offset + ) + + encoded_signatures = bson.dumps( + { + "signatures": [ + { + "method": "gpg", + "keyid": keyid, + "data": signature, + } + ] + } + ) + + with open(output_filename, "r+b") as fd: + fd.seek(signatures_offset, 0) + fd.write(encoded_signatures) + + def _generate_bundle_signature_using_gpg(self, keyid, filename, limit): + # file chunks will be written here + input_path = tempfile.NamedTemporaryFile().name + os.mkfifo(input_path) + + # sign the file with out including the signatures section + output_path = tempfile.NamedTemporaryFile().name + + # call gpg + args = [ + self.required_tool_paths["gpg"], + "--detach-sign", + "--armor", + "--default-key", + keyid, + "--output", + output_path, + input_path, + ] + + with subprocess.Popen(args) as _proc: + # read file contents up to limit + with open(input_path, "wb") as input_pipe: + chunk_size = 1024 + n_chunks = int(limit / chunk_size) + with open(filename, "rb") as input_file: + for chunk_id in range(n_chunks): + input_pipe.write(input_file.read(chunk_size)) + + final_chunk_size = limit - (n_chunks * chunk_size) + if final_chunk_size != 0: + input_pipe.write(input_file.read(final_chunk_size)) + + input_pipe.close() + + # read output + with open(output_path, "rb") as output: + signature = output.read().decode() + + os.unlink(output_path) + os.unlink(input_path) + return signature diff --git a/appimagebuilder/modules/setup/executables_wrapper.py b/appimagebuilder/modules/setup/executables_wrapper.py index 5e3682ad..dfad20ec 100644 --- a/appimagebuilder/modules/setup/executables_wrapper.py +++ b/appimagebuilder/modules/setup/executables_wrapper.py @@ -30,10 +30,10 @@ class ExecutablesWrapper: EXPORTED_FILES_PREFIX = "/tmp/appimage-" def __init__( - self, - appdir_path: str, - binaries_resolver: AppRunBinariesResolver, - env: Environment, + self, + appdir_path: str, + binaries_resolver: AppRunBinariesResolver, + env: Environment, ): self.appdir_path = Path(appdir_path) self.binaries_resolver = binaries_resolver @@ -131,7 +131,7 @@ def _write_rel_shebang(self, executable, local_env_path, output): output.write(b"#!%s" % local_env_path.encode()) shebang_main = executable.shebang[0] if shebang_main.startswith("/usr/bin/env") or shebang_main.startswith( - self.EXPORTED_FILES_PREFIX + self.EXPORTED_FILES_PREFIX ): args_start = 2 else: diff --git a/appimagebuilder/modules/setup/generator.py b/appimagebuilder/modules/setup/generator.py index 024a0332..6b3b3627 100644 --- a/appimagebuilder/modules/setup/generator.py +++ b/appimagebuilder/modules/setup/generator.py @@ -187,9 +187,9 @@ def parse_env_input(self, user_env_input): v = v.replace("${APPDIR}", self.appdir_path.__str__()) if ( - k == "PATH" - or k == "APPDIR_LIBRARY_PATH" - or k == "LIBC_LIBRARY_PATH" + k == "PATH" + or k == "APPDIR_LIBRARY_PATH" + or k == "LIBC_LIBRARY_PATH" ): v = v.split(":") diff --git a/appimagebuilder/recipe/schema.py b/appimagebuilder/recipe/schema.py index 2d198c91..df49a8dc 100644 --- a/appimagebuilder/recipe/schema.py +++ b/appimagebuilder/recipe/schema.py @@ -92,7 +92,11 @@ def __init__(self): "arch": str, Optional("update-information"): str, Optional("sign-key"): str, + Optional("compression-method"): Or( + "gzip", "lzma", "lzo", "lz4", "xz", "zstd" + ), Optional("file_name"): str, + Optional("format"): Or(2, 3), } ) diff --git a/appimagebuilder/utils/file_utils.py b/appimagebuilder/utils/file_utils.py index 9389a5d1..b1144937 100644 --- a/appimagebuilder/utils/file_utils.py +++ b/appimagebuilder/utils/file_utils.py @@ -10,6 +10,7 @@ # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. import os +import shutil import stat @@ -24,3 +25,14 @@ def set_permissions_rx_all(path): | stat.S_IXOTH | stat.S_IWUSR, ) + + +def extend_file(base_filename, extension_filename, output_filename): + shutil.copyfile(base_filename, output_filename) + + with open(output_filename, "r+b") as base_fd: + # seek until the end of the base file + base_fd.seek(0, 2) + + with open(extension_filename, "rb") as extension_fd: + shutil.copyfileobj(extension_fd, base_fd) diff --git a/examples/kcalc/AppImageBuilder.yml b/examples/kcalc/AppImageBuilder.yml index 889974c8..69ff0bb7 100644 --- a/examples/kcalc/AppImageBuilder.yml +++ b/examples/kcalc/AppImageBuilder.yml @@ -70,6 +70,4 @@ AppDir: AppImage: - update-information: None - sign-key: None arch: x86_64 diff --git a/setup.py b/setup.py index 94b01d44..fe9ca961 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ "emrichen", "ruamel.yaml", "roam", + "bson", ], python_requires=">=3.6", package_data={"": []},