From a84536cf7d3b2f75f3ac5f4774613541d033bc5b Mon Sep 17 00:00:00 2001 From: Divyanshi Kaushik Date: Wed, 12 Nov 2025 00:12:35 +0530 Subject: [PATCH 1/4] KERNEL UPGRADE --- .../build_and_install_kernel.py | 291 ++++++++++++++++++ scripts/dev/upstream_merge/json_config.py | 42 +++ .../dev/upstream_merge/kernel_build_conf.json | 22 ++ .../dev/upstream_merge/utils/git_commands.py | 117 ++++++- scripts/dev/upstream_merge/utils/git_repo.py | 5 + .../upstream_merge/utils/shell_commands.py | 9 +- scripts/dev/upstream_merge/utils/toolchain.py | 83 +++++ scripts/kernel_build_conf.json | 19 ++ 8 files changed, 583 insertions(+), 5 deletions(-) create mode 100755 scripts/dev/upstream_merge/build_and_install_kernel.py create mode 100644 scripts/dev/upstream_merge/kernel_build_conf.json create mode 100644 scripts/dev/upstream_merge/utils/toolchain.py create mode 100644 scripts/kernel_build_conf.json diff --git a/scripts/dev/upstream_merge/build_and_install_kernel.py b/scripts/dev/upstream_merge/build_and_install_kernel.py new file mode 100755 index 000000000..76889bafb --- /dev/null +++ b/scripts/dev/upstream_merge/build_and_install_kernel.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Kernel build, merge (latest stable RT tag), deployment, and notification script. +Configuration via kernel_build_conf.json. +""" + +import os +import sys +import tempfile +import shutil +import re +import argparse + +script_dir = os.path.dirname(os.path.abspath(__file__)) +upstream_merge_dir = os.path.join(script_dir, 'dev', 'upstream_merge') +sys.path.append(upstream_merge_dir) +from utils.shell_commands import run_command, execute_and_stream_cmd_output +from utils.git_commands import send_email, git_fetch, git_remote, git_checkout, git_merge, git_reset, git_clean, git_merge_abort, git_tag, git_status, git_diff +from utils.git_repo import GitRepo +from utils.toolchain import detect_or_build_cross_compile +from json_config import JsonConfig + +config = None + +def parse_args(): + parser = argparse.ArgumentParser( + description="Automated kernel build, merge, and deployment script" + ) + parser.add_argument( + "-c", "--config", + type=str, + help="Path to configuration file", + default="scripts/kernel_build_conf.json", + ) + parser.add_argument("--skip-merge", action="store_true", help="Skip upstream merge step") + parser.add_argument("--skip-packages", action="store_true", help="Skip package installation check") + parser.add_argument("--dry-run", action="store_true", help="Show what would be done without executing") + parser.add_argument("--work-dir", type=str, help="Override working directory for kernel source", default=None) + return parser.parse_args() + +# === Utility Functions === +def send_email_report(subject, body): + tmpfile = tempfile.mktemp() + with open(tmpfile, "w") as f: + f.write(f"From: {config.email_from}\nTo: {config.email_to}\nSubject: {subject}\n\n{body}\n") + try: + send_email(to_address=config.email_to, subject=subject, file=tmpfile) + finally: + os.remove(tmpfile) + +# Format merge conflict notification with limited file listing to reduce email size +def _format_conflict_email(latest_tag, conflicts, git_status_output, target_branch): + conflict_list = [c for c in conflicts.splitlines() if c.strip()] + max_files = 30 + shown_files = conflict_list[:max_files] + truncated_note = "\n... (truncated)" if len(conflict_list) > max_files else "" + # Keep only lines starting with 'U' (unmerged) from status, limit to 20 + status_lines = [l for l in git_status_output.splitlines() if l.strip().startswith('U')] + status_lines = status_lines[:20] + status_block = "\n\nGit status (unmerged entries only):\n" + "\n".join(status_lines) if status_lines else "" + body = ( + f"Merge conflict while merging {latest_tag} into {target_branch}\n" + f"Total conflicting files: {len(conflict_list)}\n" + "Conflicting files:\n" + "\n".join(shown_files) + truncated_note + status_block + ) + return body + +# === Step 0: Check Required Packages === +def check_required_packages(): + print("[INFO] Checking required Ubuntu packages...") + packages = config.required_packages + if not packages: + print("[INFO] No required packages specified in configuration.") + return + missing = [] + for pkg in packages: + status, _ = run_command(f"dpkg -s {pkg}") + if status != 0: + missing.append(pkg) + if missing: + print(f"[INFO] Installing missing packages: {', '.join(missing)}") + status, output = run_command(f"sudo apt-get update && sudo apt-get install -y {' '.join(missing)}") + if status != 0: + raise RuntimeError(f"[ERROR] Package install failed: {output}") + else: + print("[INFO] All required packages are already installed.") + +# === Step 1: Detect or Build Toolchain === +# Uses shared toolchain detection utility from utils.toolchain module. + +# === Step 1.5: Ensure Kernel Source Repository === +# Clone kernel repository if missing or not a valid git directory. + +def ensure_kernel_source_repository(): + print("[INFO] Ensuring kernel source repository exists...") + if not config.kernel_src_dir: + raise RuntimeError("[ERROR] No kernel_src_dir configured in kernel_build section") + if not config.repo_url: + raise RuntimeError("[ERROR] No repo_url configured for kernel repository") + # Convert to absolute path to prevent issues after directory changes + config.kernel_src_dir = os.path.abspath(config.kernel_src_dir) + kernel_src_dir = config.kernel_src_dir + parent_dir = os.path.dirname(kernel_src_dir) + if not os.path.exists(parent_dir): + print(f"[INFO] Creating parent directory: {parent_dir}") + os.makedirs(parent_dir, exist_ok=True) + if os.path.exists(kernel_src_dir): + if os.path.isdir(os.path.join(kernel_src_dir, '.git')): + print(f"[INFO] Kernel source repository present: {kernel_src_dir}") + return + print(f"[WARNING] {kernel_src_dir} exists but is not a git repository; replacing...") + shutil.rmtree(kernel_src_dir) + status, output = run_command(f"git clone {config.repo_url} {kernel_src_dir}") + if status != 0: + raise RuntimeError(f"[ERROR] Kernel repo clone failed: {output}") + print(f"[INFO] Cloned kernel repository to: {kernel_src_dir}") + +# === Step 2: Robust Upstream Merge with Conflict Handling === +def run_upstream_merge_script(): + print("[INFO] Running upstream merge script...") + ensure_kernel_source_repository() + if not os.path.exists(config.kernel_src_dir): + raise RuntimeError(f"[ERROR] Kernel source directory {config.kernel_src_dir} missing after clone") + original_cwd = os.getcwd() + kernel_parent_dir = os.path.dirname(config.kernel_src_dir) + os.chdir(kernel_parent_dir) + try: + kernel_version = config.target_branch.split('/')[-1] if config.target_branch else "6.12" + git_obj = GitRepo( + local_repo=os.path.basename(config.kernel_src_dir), + upstream_repo_url=config.stable_rt_remote, + upstream_branch=f"linux-{kernel_version}.y-rt", + local_base_branch=config.target_branch, + upstream_repo_name="stable-rt", + fork_name="origin", + fork_url=config.repo_url + ) + os.chdir(config.kernel_src_dir) + try: + git_merge_abort() + except: + pass + git_reset(hard=True) + git_clean(force=True, directories=True, ignored_files=True) + status, _ = git_fetch() + if status != 0: + raise RuntimeError("Failed to fetch remotes") + status, _ = run_command(f"git checkout -B {config.target_branch} origin/{config.target_branch}") + if status != 0: + raise RuntimeError("Failed to checkout target branch") + status, remotes = git_remote() + if status == 0 and "stable-rt" not in remotes: + git_obj.add_remote("stable-rt", config.stable_rt_remote) + git_fetch("stable-rt", "--tags") + status, tags = git_tag(list_pattern=f"v{kernel_version}.*-rt*") + if status != 0 or not tags: + print(f"[WARNING] No v{kernel_version}-rt tags; skipping merge") + return + clean_tags = [t for t in tags.splitlines() if re.match(rf'^v{re.escape(kernel_version)}\.\d+(?:\.\d+)?-rt\d+$', t)] + if not clean_tags: + print(f"[WARNING] No clean v{kernel_version}-rt release tags; skipping merge") + return + latest_tag = sorted(clean_tags, key=lambda t: list(map(int, re.findall(r'\d+', t))))[-1] + print(f"[INFO] Latest RT tag: {latest_tag}") + merge_result = git_obj.merge_branch(latest_tag, f"Merge latest upstream {latest_tag}") + if merge_result[0] != 0: + status, conflicts = git_diff(name_only=True, diff_filter="U") + if status == 0 and conflicts: + status, git_status_output = git_status() + concise_body = _format_conflict_email(latest_tag, conflicts, git_status_output, config.target_branch) + send_email_report( + f"Merge conflict report: {config.target_branch}", + concise_body + ) + raise RuntimeError("[ERROR] Merge conflicts detected; email sent") + else: + print("[INFO] Merge successful") + finally: + os.chdir(original_cwd) + +# === Step 3: Build and Deploy Kernel === +def build_and_deploy_kernel(args): + try: + if not args.skip_packages: + check_required_packages() + CROSS_COMPILE = detect_or_build_cross_compile(config, script_dir) + env = os.environ.copy() + if config.arch: + env["ARCH"] = config.arch + env["CROSS_COMPILE"] = CROSS_COMPILE + if not args.skip_merge: + run_upstream_merge_script() + else: + ensure_kernel_source_repository() + kernel_src_dir = config.kernel_src_dir + print("[INFO] Cleaning previous builds...") + if not args.dry_run: + status, output = run_command("make mrproper", cwd=kernel_src_dir, env=env) + if status != 0: + raise RuntimeError(output) + kernel_config_target = config.kernel_config or "defconfig" + print(f"[INFO] Creating kernel configuration: {kernel_config_target}...") + if not args.dry_run: + status, output = run_command(f"make {kernel_config_target}", cwd=kernel_src_dir, env=env) + if status != 0: + raise RuntimeError(output) + make_jobs = config.make_jobs or "$(nproc)" + if make_jobs == "$(nproc)": + make_jobs = str(os.cpu_count()) + print(f"[INFO] Building kernel and modules with {make_jobs} jobs...") + if not args.dry_run: + status, output = run_command(f"make -j{make_jobs} bzImage modules", cwd=kernel_src_dir, env=env) + if status != 0: + raise RuntimeError(output) + temp_modules_dir = config.temp_modules_dir or os.path.join(kernel_src_dir, "tmp-modules") + print(f"[INFO] Staging modules in: {temp_modules_dir}...") + if not args.dry_run: + if os.path.exists(temp_modules_dir): + shutil.rmtree(temp_modules_dir) + os.makedirs(temp_modules_dir, exist_ok=True) + status, output = run_command(f"make modules_install INSTALL_MOD_PATH={temp_modules_dir}", cwd=kernel_src_dir, env=env) + if status != 0: + raise RuntimeError(output) + kernel_version = run_command("make -s kernelrelease", cwd=kernel_src_dir, env=env)[1] if not args.dry_run else "DRY-RUN-VERSION" + target_host = config.kernel_target_host or config.rt_target_IP + target_user = config.kernel_target_user or "admin" + if not target_host: + raise RuntimeError("[ERROR] No target host configured") + print(f"[INFO] Copying kernel to target {target_host}...") + if not args.dry_run: + status, output = run_command(f"scp {kernel_src_dir}/arch/x86/boot/bzImage {target_user}@{target_host}:/boot/bzImage-{kernel_version}") + if status != 0: + raise RuntimeError(output) + print("[INFO] Backing up existing kernel on target (if needed)...") + if not args.dry_run: + status, output = run_command(f"ssh {target_user}@{target_host} 'if [ -f /boot/runmode/bzImage ] && [ ! -h /boot/runmode/bzImage ]; then mv /boot/runmode/bzImage /boot/runmode/bzImage-$(uname -r); else echo Skipping backup; fi'") + if status != 0: + print(f"[WARNING] Backup step reported: {output}") + print("[INFO] Updating bootloader symlink...") + if not args.dry_run: + status, output = run_command(f"ssh {target_user}@{target_host} 'ln -sf bzImage-{kernel_version} /boot/runmode/bzImage'") + if status != 0: + raise RuntimeError(output) + print("[INFO] Copying modules to target...") + if not args.dry_run: + status, output = execute_and_stream_cmd_output(f"tar cz -C {temp_modules_dir} lib | ssh {target_user}@{target_host} tar xz -C /") + if status != 0: + raise RuntimeError(f"[ERROR] Module transfer failed: {output}") + print("[INFO] Rebooting target...") + if not args.dry_run: + status, output = execute_and_stream_cmd_output(f"ssh {target_user}@{target_host} 'reboot' || true") + if status != 0: + print(f"[WARNING] Reboot command returned non-zero: {output}") + if not args.dry_run: + send_email_report("[SUCCESS] Kernel Build+Install", f"Kernel build and deploy succeeded on {target_host}, running version {kernel_version}") + else: + print("[INFO] Dry run completed successfully") + except Exception as e: + print(f"[ERROR] {e}") + if not args.dry_run: + send_email_report("[FAILURE] Kernel Build+Install", f"Kernel build/deploy failed: {e}") + sys.exit(1) + +# === Main === +def main(): + global config + args = parse_args() + try: + config = JsonConfig(automation_conf_path=args.config, work_item_id=None) + print(f"[INFO] Loaded configuration from: {args.config}") + if args.work_dir: + work_dir = os.path.expandvars(os.path.expanduser(args.work_dir)) + config.kernel_src_dir = os.path.join(work_dir, "linux") + print(f"[INFO] Using work directory override: {args.work_dir}") + print(f"[INFO] Kernel source path: {config.kernel_src_dir}") + if getattr(config, 'kernel_src_dir', None): + # Convert to absolute path to prevent issues after directory changes + config.kernel_src_dir = os.path.abspath(config.kernel_src_dir) + target_host = config.kernel_target_host or config.rt_target_IP or "not configured" + arch = config.arch or "not configured" + branch = config.target_branch or "not configured" + print(f"[INFO] Target: {target_host}, Arch: {arch}, Branch: {branch}") + print(f"[INFO] Absolute kernel source path: {config.kernel_src_dir}") + else: + print("[WARNING] kernel_build section missing or incomplete") + except Exception as e: + print(f"[ERROR] Failed to load configuration: {e}") + sys.exit(1) + build_and_deploy_kernel(args) + +if __name__ == "__main__": + main() diff --git a/scripts/dev/upstream_merge/json_config.py b/scripts/dev/upstream_merge/json_config.py index 8f911f966..636d1ea98 100644 --- a/scripts/dev/upstream_merge/json_config.py +++ b/scripts/dev/upstream_merge/json_config.py @@ -27,3 +27,45 @@ def __init__(self, automation_conf_path, work_item_id): self.log_level = config.get("log_level") self.build_args = config.get("build_args", "") self.rt_target_IP = config.get("rt_target_IP") + + # Kernel build configuration (optional section) + kernel_config = config.get("kernel_build", {}) + # Expand environment variables in paths + self.kernel_src_dir = self._expand_path(kernel_config.get("kernel_src_dir")) + self.kernel_target_host = kernel_config.get("target_host") + self.kernel_target_user = kernel_config.get("target_user") + self.arch = kernel_config.get("arch") + self.temp_modules_dir = self._expand_path(kernel_config.get("temp_modules_dir")) + self.toolchain_prefix = self._expand_path(kernel_config.get("toolchain_prefix")) + self.merge_workdir = self._expand_path(kernel_config.get("merge_workdir")) + self.repo_url = kernel_config.get("repo_url") + self.target_branch = kernel_config.get("target_branch") + self.stable_rt_remote = kernel_config.get("stable_rt_remote") + self.nilrt_root = self._expand_path(kernel_config.get("nilrt_root")) + self.required_packages = kernel_config.get("required_packages", []) + self.make_jobs = kernel_config.get("make_jobs", "$(nproc)") + self.kernel_config = kernel_config.get("kernel_config", "defconfig") + + def _expand_path(self, path): + """Expand environment variables and user home directory in paths.""" + if path: + return os.path.expandvars(os.path.expanduser(path)) + return path + + def get_toolchain_gcc_path(self): + """Get the full path to the toolchain GCC binary.""" + if self.toolchain_prefix: + return self.toolchain_prefix + "gcc" + return None + + def get_sdk_dir(self): + """Get the SDK directory path.""" + if self.nilrt_root: + return os.path.join(self.nilrt_root, "build", "tmp-glibc", "deploy", "sdk") + return None + + def get_toolchain_build_script(self): + """Get the path to the toolchain build script.""" + if self.nilrt_root: + return os.path.join(self.nilrt_root, "scripts", "pipelines", "build.toolchain.sh") + return None diff --git a/scripts/dev/upstream_merge/kernel_build_conf.json b/scripts/dev/upstream_merge/kernel_build_conf.json new file mode 100644 index 000000000..36bf7d689 --- /dev/null +++ b/scripts/dev/upstream_merge/kernel_build_conf.json @@ -0,0 +1,22 @@ +{ + "email_from": "dkaushik@emerson.com", + "email_to": "dkaushik@emerson.com", + "kernel_build": { + "kernel_src_dir": "nilrt-kernel-build/linux", + "target_host": "10.152.8.54", + "target_user": "admin", + "arch": "x86_64", + "temp_modules_dir": "nilrt-kernel-build/linux/tmp-glibc/modules", + "toolchain_prefix": "/usr/local/oecore-x86_64/sysroots/x86_64-nilrtsdk-linux/usr/bin/x86_64-nilrt-linux/x86_64-nilrt-linux-", + "repo_url": "https://github.com/ni/linux.git", + "target_branch": "nilrt/master/6.12", + "stable_rt_remote": "https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-stable-rt.git", + "nilrt_root": "nilrt-sdk", + "required_packages": [ + "bc", "bison", "flex", "gcc", "git", + "kmod", "libelf-dev", "libssl-dev", "make", "u-boot-tools" + ], + "make_jobs": "$(nproc)", + "kernel_config": "nati_x86_64_defconfig" + } +} diff --git a/scripts/dev/upstream_merge/utils/git_commands.py b/scripts/dev/upstream_merge/utils/git_commands.py index 8155f3986..d7a0586a0 100644 --- a/scripts/dev/upstream_merge/utils/git_commands.py +++ b/scripts/dev/upstream_merge/utils/git_commands.py @@ -211,6 +211,8 @@ def git_diff( target="HEAD", compare_with=None, staged=False, + name_only=False, + diff_filter=None, capture_output=True ): """ @@ -218,12 +220,23 @@ def git_diff( :param target: Target commit or branch (default: "HEAD"). :param compare_with: Commit or branch to compare with (optional). :param staged: Whether to show staged changes. + :param name_only: Show only file names. + :param diff_filter: Filter types (e.g., 'U' for unmerged). :param capture_output: Whether to capture the command output. """ + command = "git diff" + if staged: + command += " --staged" + if name_only: + command += " --name-only" + if diff_filter: + command += f" --diff-filter={diff_filter}" + if compare_with: - command = f"git diff {target} {compare_with}" - else: - command = "git diff --staged" if staged else "git diff" + command += f" {target} {compare_with}" + elif target != "HEAD" and not staged: + command += f" {target}" + return run_command(command, capture_output) @@ -262,6 +275,104 @@ def git_pull_request( return run_command(command, capture_output) +def git_reset( + target="HEAD", + hard=False, + soft=False, + capture_output=True +): + """ + Reset the repository to a specific commit. + :param target: Target commit/branch to reset to (default: "HEAD"). + :param hard: Whether to do a hard reset (discard all changes). + :param soft: Whether to do a soft reset (keep changes staged). + :param capture_output: Whether to capture the command output. + """ + command = f"git reset" + if hard: + command += " --hard" + elif soft: + command += " --soft" + command += f" {target}" + return run_command(command, capture_output) + + +def git_clean( + force=False, + directories=False, + ignored_files=False, + capture_output=True +): + """ + Remove untracked files from the working tree. + :param force: Force removal of files. + :param directories: Remove untracked directories. + :param ignored_files: Remove ignored files as well. + :param capture_output: Whether to capture the command output. + """ + command = "git clean" + if force: + command += " -f" + if directories: + command += " -d" + if ignored_files: + command += " -x" + return run_command(command, capture_output) + + +def git_merge_abort( + capture_output=True +): + """ + Abort an ongoing merge operation. + :param capture_output: Whether to capture the command output. + """ + return run_command("git merge --abort", capture_output) + + +def git_tag( + tag_name=None, + list_pattern=None, + delete=False, + capture_output=True +): + """ + Handle git tag operations: list, create, or delete tags. + :param tag_name: Name of the tag (for create/delete operations). + :param list_pattern: Pattern to filter tags when listing. + :param delete: Whether to delete the specified tag. + :param capture_output: Whether to capture the command output. + """ + if list_pattern: + command = f"git tag -l '{list_pattern}'" + elif delete and tag_name: + command = f"git tag -d {tag_name}" + elif tag_name: + command = f"git tag {tag_name}" + else: + command = "git tag" + return run_command(command, capture_output) + + +def git_status( + short=False, + porcelain=False, + capture_output=True +): + """ + Show the working tree status. + :param short: Give the output in the short-format. + :param porcelain: Give the output in an easy-to-parse format. + :param capture_output: Whether to capture the command output. + """ + command = "git status" + if short: + command += " --short" + elif porcelain: + command += " --porcelain" + return run_command(command, capture_output) + + def send_email( to_address, subject, diff --git a/scripts/dev/upstream_merge/utils/git_repo.py b/scripts/dev/upstream_merge/utils/git_repo.py index af7e82952..18c50102d 100644 --- a/scripts/dev/upstream_merge/utils/git_repo.py +++ b/scripts/dev/upstream_merge/utils/git_repo.py @@ -16,6 +16,11 @@ git_push, git_pull_request, git_diff, + git_reset, + git_clean, + git_merge_abort, + git_tag, + git_status, ) diff --git a/scripts/dev/upstream_merge/utils/shell_commands.py b/scripts/dev/upstream_merge/utils/shell_commands.py index d8c548417..a1d644ef6 100644 --- a/scripts/dev/upstream_merge/utils/shell_commands.py +++ b/scripts/dev/upstream_merge/utils/shell_commands.py @@ -8,12 +8,14 @@ logger = logging.getLogger(__name__) -def run_command(command, capture_output=True): +def run_command(command, capture_output=True, cwd=None, env=None): """ Run a shell command and optionally capture the output. :param command: Shell command as a string. :param capture_output: Whether to capture the output. + :param cwd: Working directory for command execution. + :param env: Environment variables for command execution. :return: (return_code, output) - return code and output string (or None if not captured). """ @@ -23,7 +25,8 @@ def run_command(command, capture_output=True): try: if capture_output: result = subprocess.run( - formatted_command, capture_output=True, text=True, check=True + formatted_command, capture_output=True, text=True, check=True, + cwd=cwd, env=env ) logger.info( "Command output: %s", @@ -40,6 +43,8 @@ def run_command(command, capture_output=True): stderr=subprocess.DEVNULL, text=True, check=True, + cwd=cwd, + env=env ) logger.info( "Command output: %s", diff --git a/scripts/dev/upstream_merge/utils/toolchain.py b/scripts/dev/upstream_merge/utils/toolchain.py new file mode 100644 index 000000000..db11b8592 --- /dev/null +++ b/scripts/dev/upstream_merge/utils/toolchain.py @@ -0,0 +1,83 @@ +"""Toolchain detection/build helper: auto-detect existing repo first (ancestor walk), +ignore nilrt_root hint if already inside a clone; only clone when nothing usable exists.""" +import os +from .shell_commands import run_command +from .git_commands import git_clone + +BUILD_SCRIPT_REL = os.path.join('scripts', 'pipelines', 'build.toolchain.sh') + +def _has_build_script(root): + return root and os.path.isfile(os.path.join(root, BUILD_SCRIPT_REL)) + +def _walk_up_for_build_script(start_dir): + cur = os.path.abspath(start_dir) + while True: + if _has_build_script(cur): + return cur + parent = os.path.dirname(cur) + if parent == cur: # reached filesystem root + return None + cur = parent + +def detect_or_build_cross_compile(config, script_dir): + """Return CROSS_COMPILE prefix, building toolchain/SDK if missing. + Order: + 1. Ancestor of script_dir containing build.toolchain.sh (preferred) + 2. Provided hint (config.nilrt_root) if ancestor not found + 3. Clone into expanded hint (or ~/nilrt-sdk if empty) if still missing + """ + gcc_binary = config.get_toolchain_gcc_path() + if gcc_binary and os.path.isfile(gcc_binary): + print(f"[INFO] Found CROSS_COMPILE: {config.toolchain_prefix}") + return config.toolchain_prefix + if not config.toolchain_prefix: + raise RuntimeError('[ERROR] No toolchain_prefix configured') + + chosen_root = _walk_up_for_build_script(script_dir) + if chosen_root: + print(f"[INFO] Auto-detected existing nilrt root: {chosen_root}") + else: + hint = getattr(config, 'nilrt_root', '') or 'nilrt-sdk' + if not os.path.isabs(hint): + hint = os.path.join(os.path.expanduser('~'), hint) + if _has_build_script(hint): + chosen_root = hint + print(f"[INFO] Using hinted nilrt_root with build script: {chosen_root}") + else: + # Need to clone + if not os.path.exists(hint): + os.makedirs(hint, exist_ok=True) + print(f"[INFO] Cloning nilrt repo into {hint} (no existing build.toolchain.sh found)...") + status, _ = git_clone('https://github.com/ni/nilrt.git', hint) + if status != 0: + raise RuntimeError('[ERROR] Failed to clone nilrt repo for toolchain build') + chosen_root = hint + config.nilrt_root = chosen_root + build_script = os.path.join(chosen_root, BUILD_SCRIPT_REL) + if not os.path.isfile(build_script): + raise RuntimeError(f"[ERROR] build.toolchain.sh missing at {build_script}") + + print('[WARNING] CROSS_COMPILE not found, building toolchain...') + status, _ = run_command(f"bash {build_script}", cwd=chosen_root) + if status != 0: + raise RuntimeError('[ERROR] Toolchain build script failed') + + sdk_dir = config.get_sdk_dir() + if sdk_dir and os.path.exists(sdk_dir): + installers = [f for f in os.listdir(sdk_dir) if f.endswith('.sh')] + if installers: + installer = os.path.join(sdk_dir, installers[0]) + print('[INFO] Installing SDK to /usr/local/oecore-x86_64 ...') + status, _ = run_command(f"sudo bash {installer} -d /usr/local/oecore-x86_64 -y") + if status != 0: + raise RuntimeError('[ERROR] SDK installer failed') + else: + print(f'[WARNING] No SDK installer found in {sdk_dir}') + else: + print(f'[WARNING] SDK directory {sdk_dir} does not exist') + + gcc_binary = config.get_toolchain_gcc_path() + if gcc_binary and os.path.isfile(gcc_binary): + print(f"[INFO] Successfully built CROSS_COMPILE: {config.toolchain_prefix}") + return config.toolchain_prefix + raise RuntimeError('[ERROR] Toolchain build completed, but compiler still missing.') diff --git a/scripts/kernel_build_conf.json b/scripts/kernel_build_conf.json new file mode 100644 index 000000000..545f1598e --- /dev/null +++ b/scripts/kernel_build_conf.json @@ -0,0 +1,19 @@ +{ + "email_from": "dkaushik@emerson.com", + "email_to": "dkaushik@emerson.com", + "kernel_build": { + "kernel_src_dir": "nilrt-kernel-build/linux", + "target_host": "10.152.8.54", + "target_user": "admin", + "arch": "x86_64", + "temp_modules_dir": "nilrt-kernel-build/linux/tmp-glibc/modules", + "repo_url": "https://github.com/ni/linux.git", + "target_branch": "nilrt/master/6.12", + "stable_rt_remote": "https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-stable-rt.git", + "nilrt_root": "nilrt-sdk", + "toolchain_prefix": "/usr/local/oecore-x86_64/sysroots/x86_64-nilrtsdk-linux/usr/bin/x86_64-nilrt-linux/x86_64-nilrt-linux-", + "required_packages": ["bc", "bison", "flex", "gcc", "git", "kmod", "libelf-dev", "libssl-dev", "make", "u-boot-tools"], + "make_jobs": "$(nproc)", + "kernel_config": "nati_x86_64_defconfig" + } +} From a3db445929a04842f1d304701a0952cdf667d469 Mon Sep 17 00:00:00 2001 From: Divyanshi Kaushik Date: Wed, 12 Nov 2025 01:00:30 +0530 Subject: [PATCH 2/4] KERNEL UPGRADE --- scripts/dev/upstream_merge/build_and_install_kernel.py | 2 +- scripts/dev/upstream_merge/json_config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dev/upstream_merge/build_and_install_kernel.py b/scripts/dev/upstream_merge/build_and_install_kernel.py index 76889bafb..012b1a38f 100755 --- a/scripts/dev/upstream_merge/build_and_install_kernel.py +++ b/scripts/dev/upstream_merge/build_and_install_kernel.py @@ -265,7 +265,7 @@ def main(): global config args = parse_args() try: - config = JsonConfig(automation_conf_path=args.config, work_item_id=None) + config = JsonConfig(config_path=args.config, work_item_id=None) print(f"[INFO] Loaded configuration from: {args.config}") if args.work_dir: work_dir = os.path.expandvars(os.path.expanduser(args.work_dir)) diff --git a/scripts/dev/upstream_merge/json_config.py b/scripts/dev/upstream_merge/json_config.py index 636d1ea98..85650005c 100644 --- a/scripts/dev/upstream_merge/json_config.py +++ b/scripts/dev/upstream_merge/json_config.py @@ -9,8 +9,8 @@ class JsonConfig: """Handles loading and validating the automation_conf.json configuration file.""" - def __init__(self, automation_conf_path, work_item_id): - with open(automation_conf_path, "r", encoding="utf-8") as file: + def __init__(self, config_path, work_item_id): + with open(config_path, "r", encoding="utf-8") as file: config = json.load(file) self.work_item_id = work_item_id self.nilrt_branch = config.get("nilrt_branch") From e29899e5efc2d7ed2b705edb69c546e0b966c9d7 Mon Sep 17 00:00:00 2001 From: Divyanshi Kaushik Date: Wed, 19 Nov 2025 14:37:00 +0530 Subject: [PATCH 3/4] resolving review comments --- nilrt-kernel-build/linux | 1 + .../build_and_install_kernel.py | 139 ++++++++++-------- scripts/dev/upstream_merge/json_config.py | 1 - .../upstream_merge/utils/shell_commands.py | 6 +- scripts/dev/upstream_merge/utils/toolchain.py | 4 +- scripts/kernel_build_conf.json | 1 - 6 files changed, 86 insertions(+), 66 deletions(-) create mode 160000 nilrt-kernel-build/linux diff --git a/nilrt-kernel-build/linux b/nilrt-kernel-build/linux new file mode 160000 index 000000000..556be4f45 --- /dev/null +++ b/nilrt-kernel-build/linux @@ -0,0 +1 @@ +Subproject commit 556be4f45e0014252323bd2bb4036129898fc893 diff --git a/scripts/dev/upstream_merge/build_and_install_kernel.py b/scripts/dev/upstream_merge/build_and_install_kernel.py index 012b1a38f..b09826b83 100755 --- a/scripts/dev/upstream_merge/build_and_install_kernel.py +++ b/scripts/dev/upstream_merge/build_and_install_kernel.py @@ -90,39 +90,52 @@ def check_required_packages(): # === Step 1.5: Ensure Kernel Source Repository === # Clone kernel repository if missing or not a valid git directory. -def ensure_kernel_source_repository(): +def ensure_kernel_source_repository(args): print("[INFO] Ensuring kernel source repository exists...") + + # Use default kernel source directory if not configured if not config.kernel_src_dir: raise RuntimeError("[ERROR] No kernel_src_dir configured in kernel_build section") - if not config.repo_url: - raise RuntimeError("[ERROR] No repo_url configured for kernel repository") + + # Use NI Linux repository as default - this is the primary repo for NILRT + repo_url = "https://github.com/ni/linux.git" # Convert to absolute path to prevent issues after directory changes config.kernel_src_dir = os.path.abspath(config.kernel_src_dir) kernel_src_dir = config.kernel_src_dir parent_dir = os.path.dirname(kernel_src_dir) if not os.path.exists(parent_dir): - print(f"[INFO] Creating parent directory: {parent_dir}") - os.makedirs(parent_dir, exist_ok=True) + if not args.dry_run: + print(f"[INFO] Creating parent directory: {parent_dir}") + os.makedirs(parent_dir, exist_ok=True) + else: + print(f"[INFO] Would create parent directory: {parent_dir}") if os.path.exists(kernel_src_dir): if os.path.isdir(os.path.join(kernel_src_dir, '.git')): print(f"[INFO] Kernel source repository present: {kernel_src_dir}") return - print(f"[WARNING] {kernel_src_dir} exists but is not a git repository; replacing...") - shutil.rmtree(kernel_src_dir) - status, output = run_command(f"git clone {config.repo_url} {kernel_src_dir}") - if status != 0: - raise RuntimeError(f"[ERROR] Kernel repo clone failed: {output}") - print(f"[INFO] Cloned kernel repository to: {kernel_src_dir}") + if not args.dry_run: + print(f"[WARNING] {kernel_src_dir} exists but is not a git repository; replacing...") + shutil.rmtree(kernel_src_dir) + else: + print(f"[INFO] Would remove non-git directory: {kernel_src_dir}") + if not args.dry_run: + status, output = run_command(f"git clone {repo_url} {kernel_src_dir}") + if status != 0: + raise RuntimeError(f"[ERROR] Kernel repo clone failed: {output}") + print(f"[INFO] Cloned kernel repository to: {kernel_src_dir}") + else: + print(f"[INFO] Would clone kernel repository from {repo_url} to: {kernel_src_dir}") # === Step 2: Robust Upstream Merge with Conflict Handling === -def run_upstream_merge_script(): +def run_upstream_merge_script(args): print("[INFO] Running upstream merge script...") - ensure_kernel_source_repository() - if not os.path.exists(config.kernel_src_dir): + ensure_kernel_source_repository(args) + if not args.dry_run and not os.path.exists(config.kernel_src_dir): raise RuntimeError(f"[ERROR] Kernel source directory {config.kernel_src_dir} missing after clone") original_cwd = os.getcwd() - kernel_parent_dir = os.path.dirname(config.kernel_src_dir) - os.chdir(kernel_parent_dir) + if not args.dry_run: + kernel_parent_dir = os.path.dirname(config.kernel_src_dir) + os.chdir(kernel_parent_dir) try: kernel_version = config.target_branch.split('/')[-1] if config.target_branch else "6.12" git_obj = GitRepo( @@ -132,50 +145,58 @@ def run_upstream_merge_script(): local_base_branch=config.target_branch, upstream_repo_name="stable-rt", fork_name="origin", - fork_url=config.repo_url + fork_url="https://github.com/ni/linux.git" ) - os.chdir(config.kernel_src_dir) - try: - git_merge_abort() - except: - pass - git_reset(hard=True) - git_clean(force=True, directories=True, ignored_files=True) - status, _ = git_fetch() - if status != 0: - raise RuntimeError("Failed to fetch remotes") - status, _ = run_command(f"git checkout -B {config.target_branch} origin/{config.target_branch}") - if status != 0: - raise RuntimeError("Failed to checkout target branch") - status, remotes = git_remote() - if status == 0 and "stable-rt" not in remotes: - git_obj.add_remote("stable-rt", config.stable_rt_remote) - git_fetch("stable-rt", "--tags") - status, tags = git_tag(list_pattern=f"v{kernel_version}.*-rt*") - if status != 0 or not tags: - print(f"[WARNING] No v{kernel_version}-rt tags; skipping merge") - return - clean_tags = [t for t in tags.splitlines() if re.match(rf'^v{re.escape(kernel_version)}\.\d+(?:\.\d+)?-rt\d+$', t)] - if not clean_tags: - print(f"[WARNING] No clean v{kernel_version}-rt release tags; skipping merge") - return - latest_tag = sorted(clean_tags, key=lambda t: list(map(int, re.findall(r'\d+', t))))[-1] - print(f"[INFO] Latest RT tag: {latest_tag}") - merge_result = git_obj.merge_branch(latest_tag, f"Merge latest upstream {latest_tag}") - if merge_result[0] != 0: - status, conflicts = git_diff(name_only=True, diff_filter="U") - if status == 0 and conflicts: - status, git_status_output = git_status() - concise_body = _format_conflict_email(latest_tag, conflicts, git_status_output, config.target_branch) - send_email_report( - f"Merge conflict report: {config.target_branch}", - concise_body - ) - raise RuntimeError("[ERROR] Merge conflicts detected; email sent") + if not args.dry_run: + os.chdir(config.kernel_src_dir) + if not args.dry_run: + try: + git_merge_abort() + except: + pass + git_reset(hard=True) + git_clean(force=True, directories=True, ignored_files=True) + status, _ = git_fetch() + if status != 0: + raise RuntimeError("Failed to fetch remotes") + status, _ = run_command(f"git checkout -B {config.target_branch} origin/{config.target_branch}") + if status != 0: + raise RuntimeError("Failed to checkout target branch") + status, remotes = git_remote() + if status == 0 and "stable-rt" not in remotes: + git_obj.add_remote("stable-rt", config.stable_rt_remote) + git_fetch("stable-rt", "--tags") + else: + print("[INFO] Would reset repository and fetch latest changes") + if not args.dry_run: + status, tags = git_tag(list_pattern=f"v{kernel_version}.*-rt*") + if status != 0 or not tags: + print(f"[WARNING] No v{kernel_version}-rt tags; skipping merge") + return + clean_tags = [t for t in tags.splitlines() if re.match(rf'^v{re.escape(kernel_version)}\.\d+(?:\.\d+)?-rt\d+$', t)] + if not clean_tags: + print(f"[WARNING] No clean v{kernel_version}-rt release tags; skipping merge") + return + latest_tag = sorted(clean_tags, key=lambda t: list(map(int, re.findall(r'\d+', t))))[-1] + print(f"[INFO] Latest RT tag: {latest_tag}") + merge_result = git_obj.merge_branch(latest_tag, f"Merge latest upstream {latest_tag}") + if merge_result[0] != 0: + status, conflicts = git_diff(name_only=True, diff_filter="U") + if status == 0 and conflicts: + status, git_status_output = git_status() + concise_body = _format_conflict_email(latest_tag, conflicts, git_status_output, config.target_branch) + send_email_report( + f"Merge conflict report: {config.target_branch}", + concise_body + ) + raise RuntimeError("[ERROR] Merge conflicts detected; email sent") + else: + print("[INFO] Merge successful") else: - print("[INFO] Merge successful") + print(f"[INFO] Would fetch and merge latest RT tag for kernel version {kernel_version}") finally: - os.chdir(original_cwd) + if not args.dry_run: + os.chdir(original_cwd) # === Step 3: Build and Deploy Kernel === def build_and_deploy_kernel(args): @@ -188,9 +209,9 @@ def build_and_deploy_kernel(args): env["ARCH"] = config.arch env["CROSS_COMPILE"] = CROSS_COMPILE if not args.skip_merge: - run_upstream_merge_script() + run_upstream_merge_script(args) else: - ensure_kernel_source_repository() + ensure_kernel_source_repository(args) kernel_src_dir = config.kernel_src_dir print("[INFO] Cleaning previous builds...") if not args.dry_run: diff --git a/scripts/dev/upstream_merge/json_config.py b/scripts/dev/upstream_merge/json_config.py index 85650005c..633015bbf 100644 --- a/scripts/dev/upstream_merge/json_config.py +++ b/scripts/dev/upstream_merge/json_config.py @@ -38,7 +38,6 @@ def __init__(self, config_path, work_item_id): self.temp_modules_dir = self._expand_path(kernel_config.get("temp_modules_dir")) self.toolchain_prefix = self._expand_path(kernel_config.get("toolchain_prefix")) self.merge_workdir = self._expand_path(kernel_config.get("merge_workdir")) - self.repo_url = kernel_config.get("repo_url") self.target_branch = kernel_config.get("target_branch") self.stable_rt_remote = kernel_config.get("stable_rt_remote") self.nilrt_root = self._expand_path(kernel_config.get("nilrt_root")) diff --git a/scripts/dev/upstream_merge/utils/shell_commands.py b/scripts/dev/upstream_merge/utils/shell_commands.py index a1d644ef6..5bb42eb38 100644 --- a/scripts/dev/upstream_merge/utils/shell_commands.py +++ b/scripts/dev/upstream_merge/utils/shell_commands.py @@ -21,11 +21,10 @@ def run_command(command, capture_output=True, cwd=None, env=None): """ print(command) logger.info("Running command: %s", command) - formatted_command = shlex.split(command) try: if capture_output: result = subprocess.run( - formatted_command, capture_output=True, text=True, check=True, + command, shell=True, capture_output=True, text=True, check=True, cwd=cwd, env=env ) logger.info( @@ -38,7 +37,8 @@ def run_command(command, capture_output=True, cwd=None, env=None): ) result = subprocess.run( - formatted_command, + command, + shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True, diff --git a/scripts/dev/upstream_merge/utils/toolchain.py b/scripts/dev/upstream_merge/utils/toolchain.py index db11b8592..9857492b7 100644 --- a/scripts/dev/upstream_merge/utils/toolchain.py +++ b/scripts/dev/upstream_merge/utils/toolchain.py @@ -1,4 +1,4 @@ -"""Toolchain detection/build helper: auto-detect existing repo first (ancestor walk), + """Toolchain detection/build helper: auto-detect existing repo first (ancestor walk), ignore nilrt_root hint if already inside a clone; only clone when nothing usable exists.""" import os from .shell_commands import run_command @@ -80,4 +80,4 @@ def detect_or_build_cross_compile(config, script_dir): if gcc_binary and os.path.isfile(gcc_binary): print(f"[INFO] Successfully built CROSS_COMPILE: {config.toolchain_prefix}") return config.toolchain_prefix - raise RuntimeError('[ERROR] Toolchain build completed, but compiler still missing.') + raise RuntimeError(f'[ERROR] Toolchain build completed, but compiler still missing at expected path: {gcc_binary}') diff --git a/scripts/kernel_build_conf.json b/scripts/kernel_build_conf.json index 545f1598e..28227bdea 100644 --- a/scripts/kernel_build_conf.json +++ b/scripts/kernel_build_conf.json @@ -7,7 +7,6 @@ "target_user": "admin", "arch": "x86_64", "temp_modules_dir": "nilrt-kernel-build/linux/tmp-glibc/modules", - "repo_url": "https://github.com/ni/linux.git", "target_branch": "nilrt/master/6.12", "stable_rt_remote": "https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-stable-rt.git", "nilrt_root": "nilrt-sdk", From 05f9272119a8d88895c82fbd58c55548e10dddfb Mon Sep 17 00:00:00 2001 From: Divyanshi Kaushik Date: Wed, 19 Nov 2025 15:08:10 +0530 Subject: [PATCH 4/4] indentation issue resolved. --- scripts/dev/upstream_merge/utils/toolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/upstream_merge/utils/toolchain.py b/scripts/dev/upstream_merge/utils/toolchain.py index 9857492b7..7dbcc7278 100644 --- a/scripts/dev/upstream_merge/utils/toolchain.py +++ b/scripts/dev/upstream_merge/utils/toolchain.py @@ -1,4 +1,4 @@ - """Toolchain detection/build helper: auto-detect existing repo first (ancestor walk), +"""Toolchain detection/build helper: auto-detect existing repo first (ancestor walk), ignore nilrt_root hint if already inside a clone; only clone when nothing usable exists.""" import os from .shell_commands import run_command