coredump: Only sort VMAs when core_sort_vma sysctl is set

JIRA: https://issues.redhat.com/browse/RHEL-96077

commit 39ec9ea
Author: Kees Cook <kees@kernel.org>
Date:   Wed Feb 19 11:53:16 2025 -0800

    coredump: Only sort VMAs when core_sort_vma sysctl is set

    The sorting of VMAs by size in commit 7d442a3 ("binfmt_elf: Dump
    smaller VMAs first in ELF cores") breaks elfutils[1]. Instead, sort
    based on the setting of the new sysctl, core_sort_vma, which defaults
    to 0, no sorting.

    Reported-by: Michael Stapelberg <michael@stapelberg.ch>
    Closes: https://lore.kernel.org/all/20250218085407.61126-1-michael@stapelberg.de/ [1]
    Fixes: 7d442a3 ("binfmt_elf: Dump smaller VMAs first in ELF cores")
    Signed-off-by: Kees Cook <kees@kernel.org>

Signed-off-by: Herton R. Krzesinski <herton@redhat.com>

Author: Herton R. Krzesinski

Documentation/admin-guide/sysctl/kernel.rst

@@ -212,6 +212,17 @@ pid>/``).
 This value defaults to 0.
 
 
+core_sort_vma
+=============
+
+The default coredump writes VMAs in address order. By setting
+``core_sort_vma`` to 1, VMAs will be written from smallest size
+to largest size. This is known to break at least elfutils, but
+can be handy when dealing with very large (and truncated)
+coredumps where the more useful debugging details are included
+in the smaller VMAs.
+
+
 core_uses_pid
 =============
 

fs/coredump.c

@@ -63,6 +63,7 @@ static void free_vma_snapshot(struct coredump_params *cprm);
 
 static int core_uses_pid;
 static unsigned int core_pipe_limit;
+static unsigned int core_sort_vma;
 static char core_pattern[CORENAME_MAX_SIZE] = "core";
 static int core_name_size = CORENAME_MAX_SIZE;
 unsigned int core_file_note_size_limit = CORE_FILE_NOTE_SIZE_DEFAULT;
@@ -1025,6 +1026,15 @@ static struct ctl_table coredump_sysctls[] = {
 		.extra1		= (unsigned int *)&core_file_note_size_min,
 		.extra2		= (unsigned int *)&core_file_note_size_max,
 	},
+	{
+		.procname	= "core_sort_vma",
+		.data		= &core_sort_vma,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_douintvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
 };
 
 static int __init init_fs_coredump_sysctls(void)
@@ -1255,8 +1265,9 @@ static bool dump_vma_snapshot(struct coredump_params *cprm)
 		cprm->vma_data_size += m->dump_size;
 	}
 
-	sort(cprm->vma_meta, cprm->vma_count, sizeof(*cprm->vma_meta),
-		cmp_vma_size, NULL);
+	if (core_sort_vma)
+		sort(cprm->vma_meta, cprm->vma_count, sizeof(*cprm->vma_meta),
+		     cmp_vma_size, NULL);
 
 	return true;
 }