block: load root filesystem from disk

* load root filesystem from the disk

* update mkimage to copy all the hidden files aswell

* remove initramfs code in the kernel

* keep track of the unused items in the cache so we can reuse them to
avoid more memory allocations and disk i/o where applicable

Signed-off-by: Andy-Python-Programmer <andypythonappdeveloper@gmail.com>

Author: Andy-Python-Programmer

aero.py

@@ -25,6 +25,7 @@
 import subprocess
 import sys
 import tarfile
+import time
 
 from typing import List
 
@@ -76,9 +77,6 @@ def log_error(msg):
 
 MODULE_PATH=boot:///term_background.bmp
 MODULE_CMDLINE=background
-
-MODULE_PATH=boot:///initramfs.cpio
-MODULE_CMDLINE=initramfs
 """
 
 
@@ -437,47 +435,6 @@ def prepare_iso(args, kernel_bin, user_bins):
     shutil.copy(os.path.join(limine_path, 'BOOTAA64.EFI'), efi_boot)
     shutil.copy(os.path.join(limine_path, 'BOOTX64.EFI'), efi_boot)
 
-    sysroot_dir = os.path.join(SYSROOT_DIR, 'system-root')
-    shutil.copytree(BASE_FILES_DIR, sysroot_dir, dirs_exist_ok=True)
-
-    # dynamic linker (ld.so)
-    mlibc = os.path.join(get_userland_package(), "mlibc")
-    # gcc libraries required for rust programs
-    gcc = os.path.join(get_userland_package(), "gcc")
-
-    # FIXME
-    if "host-rust-prebuilt" in str(mlibc):
-        shutil.copytree(mlibc, sysroot_dir, dirs_exist_ok=True)
-        shutil.copytree(gcc, sysroot_dir, dirs_exist_ok=True)
-
-    for file in user_bins:
-        bin_name = os.path.basename(file)
-        shutil.copy(file, os.path.join(sysroot_dir, "usr", "bin", bin_name))
-
-    def find(path) -> List[str]:
-        _, find_output, _ = run_command(['find', '.', '-type', 'f'],
-                                        cwd=path,
-                                        stdout=subprocess.PIPE)
-
-        files_without_dot = filter(
-            lambda x: x != '.', find_output.decode('utf-8').splitlines())
-        files_without_prefix = map(
-            lambda x: remove_prefix(x, './'), files_without_dot)
-        files = list(files_without_prefix)
-
-        files.append("usr/lib/libiconv.so.2")
-        return files
-
-    files = find(sysroot_dir)
-
-    with open(os.path.join(iso_root, 'initramfs.cpio'), 'wb') as initramfs:
-        cpio_input = '\n'.join(files)
-        code, _, _ = run_command(['cpio', '-o', '-v'],
-                                 cwd=sysroot_dir,
-                                 stdout=initramfs,
-                                 stderr=subprocess.PIPE,
-                                 input=cpio_input.encode('utf-8'))
-
     with open(os.path.join(iso_root, 'limine.cfg'), 'w') as limine_cfg:
         limine_cfg.write(LIMINE_TEMPLATE)
 
@@ -641,6 +598,7 @@ def is_kvm_supported() -> bool:
 
 
 def main():
+    t0 = time.time()
     args = parse_args()
 
     # arch-aero_os
@@ -699,6 +657,8 @@ def main():
         kernel_bin = kernel_bin[0]
         iso_path = prepare_iso(args, kernel_bin, user_bins)
 
+        t1 = time.time()
+        log_info(f"build completed in {t1 - t0:.2f} seconds")
         if not args.no_run:
             run_in_emulator(build_info, iso_path)
 

src/Cargo.lock

@@ -40,9 +40,10 @@ lai = { git = "https://github.com/aero-os/lai-rs" }
 uapi = { path = "../uapi" }
 cpio_reader = { git = "https://github.com/Andy-Python-Programmer/cpio_reader" }
 static_assertions = "1.1.0"
+lru = "0.8.0"
+bytemuck = "1.12.1"
 
 limine = { git = "https://github.com/limine-bootloader/limine-rs" }
-
 cfg-if = "1.0"
 
 # X86_64 specific dependencies:

src/aero_kernel/Cargo.toml

@@ -194,13 +194,6 @@ extern "C" fn arch_aero_main() -> ! {
 
     syscall::init();
 
-    crate::INITRD_MODULE.call_once(|| {
-        modules
-            .iter()
-            .find(|e| e.cmdline.to_str().unwrap().to_str().unwrap() == "initramfs")
-            .unwrap()
-    });
-
     let boot_time = BOOT_TIME.get_response().get().unwrap();
     time::EPOCH.store(boot_time.boot_time as _, Ordering::SeqCst);
 

src/aero_kernel/src/arch/x86_64/mod.rs

@@ -262,7 +262,7 @@ impl<'a> Namespace<'a> {
             read_cmd.data_ptr.prp1 = buffer.addr().as_u64();
         }
 
-        self.controller.io_queue.lock().submit_command(read_cmd);
+        self.controller.io_queue.lock_irq().submit_command(read_cmd);
 
         // SAFETY: The buffer is initialized above.
         let buffer = unsafe { buffer.assume_init() };

src/aero_kernel/src/drivers/block/nvme/mod.rs

@@ -17,6 +17,8 @@
  * along with Aero. If not, see <https://www.gnu.org/licenses/>.
  */
 
+use crate::fs::block::BlockDeviceInterface;
+
 use super::BlockDevice;
 use core::mem::MaybeUninit;
 
@@ -126,7 +128,6 @@ impl Gpt {
         let mut header = Box::<GptTableHeader>::new_uninit();
 
         controller
-            .device()
             .read_block(1, header.as_bytes_mut())
             .expect("gpt: failed to read first sector");
 
@@ -143,7 +144,6 @@ impl Gpt {
         let mut entry_list = Box::<[GptEntry]>::new_uninit_slice(header.num_entries as usize);
 
         controller
-            .device()
             .read_block(
                 header.starting_lba as _,
                 MaybeUninit::slice_as_bytes_mut(&mut entry_list),

src/aero_kernel/src/fs/block/gpt.rs

@@ -29,19 +29,48 @@ use alloc::sync::{Arc, Weak};
 use alloc::vec::Vec;
 
 use crate::fs::devfs::install_device;
-use crate::fs::{Path, Result, MOUNT_MANAGER};
+use crate::fs::{FileSystem, Result};
 
 use crate::fs::ext2::Ext2;
 use crate::utils::sync::Mutex;
 
+use super::cache::{Cache, Cacheable};
 use super::devfs::{alloc_device_marker, Device};
 use super::inode::INodeInterface;
 
+type CachedBlockKey = (usize, usize); // (block device pointer, block)
+
+struct CachedBlock {
+    device: Weak<dyn CachedAccess>,
+    block: usize,
+    buffer: Box<[u8]>,
+}
+
+impl CachedBlock {
+    fn make_key(device: Weak<dyn CachedAccess>, block: usize) -> CachedBlockKey {
+        (device.as_ptr() as *const u8 as usize, block)
+    }
+}
+
+impl Cacheable<CachedBlockKey> for CachedBlock {
+    fn cache_key(&self) -> CachedBlockKey {
+        Self::make_key(self.device.clone(), self.block)
+    }
+}
+
+lazy_static::lazy_static! {
+    static ref BLOCK_CACHE: Arc<Cache<CachedBlockKey, CachedBlock>> = Cache::new();
+}
+
 pub trait BlockDeviceInterface: Send + Sync {
     fn block_size(&self) -> usize;
 
     fn read_block(&self, sector: usize, dest: &mut [MaybeUninit<u8>]) -> Option<usize>;
     fn write_block(&self, sector: usize, buf: &[u8]) -> Option<usize>;
+}
+
+pub trait CachedAccess: BlockDeviceInterface {
+    fn sref(&self) -> Weak<dyn CachedAccess>;
 
     fn read(&self, offset: usize, dest: &mut [MaybeUninit<u8>]) -> Option<usize> {
         let mut progress = 0;
@@ -57,10 +86,26 @@ pub trait BlockDeviceInterface: Send + Sync {
                 chunk = block_size - loc;
             }
 
-            let mut buffer = Box::<[u8]>::new_uninit_slice(block_size);
-            self.read_block(block, MaybeUninit::slice_as_bytes_mut(&mut buffer))?;
+            let key = CachedBlock::make_key(self.sref(), block);
+
+            if let Some(cached) = BLOCK_CACHE.get(key) {
+                MaybeUninit::write_slice(
+                    &mut dest[progress..(progress + chunk)],
+                    &cached.buffer[loc..loc + chunk],
+                );
+            } else {
+                let mut buffer = Box::<[u8]>::new_uninit_slice(block_size);
+
+                self.read_block(block, MaybeUninit::slice_as_bytes_mut(&mut buffer))?;
+                dest[progress..(progress + chunk)].copy_from_slice(&buffer[loc..loc + chunk]);
+
+                BLOCK_CACHE.make_item_cached(CachedBlock {
+                    device: self.sref(),
+                    block,
+                    buffer: unsafe { buffer.assume_init() },
+                });
+            }
 
-            dest[progress..(progress + chunk)].copy_from_slice(&buffer[loc..loc + chunk]);
             progress += chunk;
         }
 
@@ -85,25 +130,41 @@ pub struct BlockDevice {
     id: usize,
     name: String,
     dev: Arc<dyn BlockDeviceInterface>,
-    self_ref: Weak<BlockDevice>,
+    sref: Weak<BlockDevice>,
 }
 
 impl BlockDevice {
     pub fn new(name: String, imp: Arc<dyn BlockDeviceInterface>) -> Arc<BlockDevice> {
-        Arc::new_cyclic(|me| BlockDevice {
+        Arc::new_cyclic(|sref| BlockDevice {
             id: alloc_device_marker(),
             name,
             dev: imp,
-            self_ref: me.clone(),
+            sref: sref.clone(),
         })
     }
 
     pub fn name(&self) -> String {
         self.name.clone()
     }
+}
 
-    pub fn device(&self) -> Arc<dyn BlockDeviceInterface> {
-        self.dev.clone()
+impl BlockDeviceInterface for BlockDevice {
+    fn block_size(&self) -> usize {
+        self.dev.block_size()
+    }
+
+    fn read_block(&self, sector: usize, dest: &mut [MaybeUninit<u8>]) -> Option<usize> {
+        self.dev.read_block(sector, dest)
+    }
+
+    fn write_block(&self, sector: usize, buf: &[u8]) -> Option<usize> {
+        self.dev.write_block(sector, buf)
+    }
+}
+
+impl CachedAccess for BlockDevice {
+    fn sref(&self) -> Weak<dyn CachedAccess> {
+        self.sref.clone()
     }
 }
 
@@ -119,23 +180,27 @@ impl Device for BlockDevice {
     }
 
     fn inode(&self) -> Arc<dyn INodeInterface> {
-        self.self_ref.upgrade().unwrap().clone()
+        self.sref.upgrade().unwrap().clone()
     }
 }
 
 struct PartitionBlockDevice {
+    sref: Weak<Self>,
+
     offset: usize, // offset in sectors
     size: usize,   // capacity in sectors
     device: Arc<dyn BlockDeviceInterface>,
 }
 
 impl PartitionBlockDevice {
-    fn new(offset: usize, size: usize, device: Arc<dyn BlockDeviceInterface>) -> Self {
-        Self {
+    fn new(offset: usize, size: usize, device: Arc<dyn BlockDeviceInterface>) -> Arc<Self> {
+        Arc::new_cyclic(|sref| Self {
+            sref: sref.clone(),
+
             offset,
             size,
             device,
-        }
+        })
     }
 }
 
@@ -161,6 +226,12 @@ impl BlockDeviceInterface for PartitionBlockDevice {
     }
 }
 
+impl CachedAccess for PartitionBlockDevice {
+    fn sref(&self) -> Weak<dyn CachedAccess> {
+        self.sref.clone()
+    }
+}
+
 pub fn launch() -> Result<()> {
     let mut blocks_copy = Vec::<Arc<BlockDevice>>::new();
 
@@ -189,19 +260,27 @@ pub fn launch() -> Result<()> {
                 );
 
                 let name = alloc::format!("{}p{}", block.name(), i);
-                let partition_device = PartitionBlockDevice::new(start, size, block.device());
-                let device = BlockDevice::new(name, Arc::new(partition_device));
+                let partition_device = PartitionBlockDevice::new(start, size, block.clone());
+                let device = BlockDevice::new(name, partition_device);
 
                 install_block_device(device.clone())?;
 
                 // Check what filesystem is on this partition and mount it.
                 if let Some(ext2) = Ext2::new(device.clone()) {
                     log::info!("gpt: found ext2 filesystem on {}!", device.name());
-                    MOUNT_MANAGER.mount(super::lookup_path(Path::new("/mnt"))?, ext2)?;
+
+                    super::ROOT_FS.call_once(|| ext2.clone());
+                    super::ROOT_DIR.call_once(|| ext2.root_dir().clone());
                 }
             }
         }
     }
 
+    super::devfs::init()?;
+    log::info!("installed devfs");
+
+    super::procfs::init()?;
+    log::info!("installed procfs");
+
     Ok(())
 }