From 19f9ef3351e71dc50ed7cae6de13f13120043634 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 2 Sep 2025 22:45:55 +0800 Subject: [PATCH 001/153] vfs: rework the vfs subsystem with async Inode and superblock rework: Remove old Inode trait as it used to undertake too much responsibility. The new method use three new traits: InodeOps is used to acquire generic inode attributes. InodeFileOps and InodeDirOps handle file and directory requests respectively. All the three have async fn trait methods and don't need to be ?Sized. Then, we implement Inode, InodeFile and InodeDir for the implementors of the three "Ops" traits, erasing their actual type and provide generic dyn interface by wrapping the futures in boxes. We should provide an io worker? or some io context with an allocator for futures to reduce the overhead of io requests, or come up with some better ideas? For inode usage, we introduce InodeRef and InodeUse. InodeRef is a simple wrapper for Weak and InodeUse for Arc. This helps us use them better as we can't define impls for Arc's as they are foreign types. We also provide some more helper methods for them. After the change, we don't impose ANY structural restriction except for the spinlock wrapped InodeInfo. The InodeInfo struct design might need rethinking but the current implementation seems to be fine aside from unnecessary locks when we try to access some of its fields but this shouldn't be a VERY big problem and very urgent... Similar changes are also made to superblock traits and types. But for the superblock objects, we use a SuperBlockComplex struct to store common fields such as whether the superblock is read only, their device id and so on. Also the structs have a superblock rwsem inside. But we haven't decided how to use that (such as whether we should acquire the lock and pass it to the inode methods) and even whether they should exist and be there. This will need further thinking so we put this off for now... Filesystem rework: Rework tmpfs, fatfs and procfs with the new technology mentioned above, leaving the old ext4 unchanged. The current implementation of ext4 uses some "random" library from the "camp". Its code hasn't been fully reviewed for time reasons but seems to be rather "problematic"... We might rewrite the whole module later and the page cache subsystem requires fully reworking as well. So we put this work off as well. Block device and other parts rework: Wraps PageCacheBackend, MountCreator and BlockRequestQueue with async_trait to provide dyn compatible async functions. Dentry walking functions are also put to the heap since they are recursive functions... This has similar problems to the inode traits, ugly solution. Further optimization is required. Signed-off-by: greatbridf --- .vscode/tasks.json | 2 +- Cargo.lock | 98 +++++ Cargo.toml | 2 + src/driver/ahci/mod.rs | 63 +-- src/driver/ahci/port.rs | 48 ++- src/driver/ahci/slot.rs | 19 +- src/driver/e1000e.rs | 8 +- src/driver/serial.rs | 6 +- src/driver/virtio/riscv64.rs | 7 +- src/driver/virtio/virtio_blk.rs | 5 +- src/fs/ext4.rs | 35 +- src/fs/fat32.rs | 567 ++++++++++++++++----------- src/fs/fat32/dir.rs | 214 +++++------ src/fs/fat32/file.rs | 46 +-- src/fs/mod.rs | 3 +- src/fs/procfs.rs | 437 +++++++++------------ src/fs/shm.rs | 146 ------- src/fs/tmpfs.rs | 613 ------------------------------ src/fs/tmpfs/dir.rs | 415 ++++++++++++++++++++ src/fs/tmpfs/file.rs | 298 +++++++++++++++ src/fs/tmpfs/mod.rs | 73 ++++ src/kernel/block.rs | 38 +- src/kernel/block/mbr.rs | 2 +- src/kernel/chardev.rs | 15 +- src/kernel/mem.rs | 2 +- src/kernel/mem/mm_list/mapping.rs | 15 +- src/kernel/mem/page_cache.rs | 55 ++- src/kernel/pcie/driver.rs | 9 +- src/kernel/syscall/file_rw.rs | 94 +++-- src/kernel/syscall/mm.rs | 122 +----- src/kernel/syscall/procops.rs | 20 +- src/kernel/task/loader/elf.rs | 34 +- src/kernel/task/loader/mod.rs | 13 +- src/kernel/timer.rs | 2 + src/kernel/vfs/dentry.rs | 295 +++++++------- src/kernel/vfs/dentry/dcache.rs | 29 +- src/kernel/vfs/file/inode_file.rs | 139 ++++--- src/kernel/vfs/filearray.rs | 23 +- src/kernel/vfs/inode.rs | 494 ------------------------ src/kernel/vfs/inode/ino.rs | 31 ++ src/kernel/vfs/inode/inode.rs | 389 +++++++++++++++++++ src/kernel/vfs/inode/mod.rs | 10 + src/kernel/vfs/inode/ops.rs | 18 + src/kernel/vfs/inode/statx.rs | 97 +++++ src/kernel/vfs/mod.rs | 22 +- src/kernel/vfs/mount.rs | 32 +- src/kernel/vfs/superblock.rs | 127 +++++++ src/kernel/vfs/types/device_id.rs | 36 ++ src/kernel/vfs/types/mod.rs | 5 + src/kernel/vfs/types/mode.rs | 169 ++++++++ src/kernel/vfs/vfs.rs | 10 - src/lib.rs | 33 +- src/prelude.rs | 30 +- 53 files changed, 2986 insertions(+), 2529 deletions(-) delete mode 100644 src/fs/shm.rs delete mode 100644 src/fs/tmpfs.rs create mode 100644 src/fs/tmpfs/dir.rs create mode 100644 src/fs/tmpfs/file.rs create mode 100644 src/fs/tmpfs/mod.rs delete mode 100644 src/kernel/vfs/inode.rs create mode 100644 src/kernel/vfs/inode/ino.rs create mode 100644 src/kernel/vfs/inode/inode.rs create mode 100644 src/kernel/vfs/inode/mod.rs create mode 100644 src/kernel/vfs/inode/ops.rs create mode 100644 src/kernel/vfs/inode/statx.rs create mode 100644 src/kernel/vfs/superblock.rs create mode 100644 src/kernel/vfs/types/device_id.rs create mode 100644 src/kernel/vfs/types/mod.rs create mode 100644 src/kernel/vfs/types/mode.rs delete mode 100644 src/kernel/vfs/vfs.rs diff --git a/.vscode/tasks.json b/.vscode/tasks.json index e7a54791..a85ea0cf 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,7 +6,7 @@ { "label": "debug run riscv64", "type": "shell", - "command": "make srun ARCH=riscv64", + "command": "make srun ARCH=riscv64 IMG=/Volumes/oscomp/sdcard-rv.img", "isBackground": true, "problemMatcher": [ { diff --git a/Cargo.lock b/Cargo.lock index 59242bbc..32868677 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,17 @@ dependencies = [ "log", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic_unique_refcell" version = "0.1.0" @@ -144,6 +155,7 @@ dependencies = [ "acpi", "align_ext", "another_ext4", + "async-trait", "atomic_unique_refcell", "bitflags", "buddy_allocator", @@ -155,6 +167,7 @@ dependencies = [ "eonix_preempt", "eonix_runtime", "eonix_sync", + "futures", "intrusive-collections 0.9.8", "intrusive_list", "itertools", @@ -265,6 +278,79 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784a4df722dc6267a04af36895398f59d21d07dce47232adf31ec0ff2fa45e67" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "gimli" version = "0.32.0" @@ -332,6 +418,18 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pointers" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e70d8c65..bc7e7b0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,8 @@ another_ext4 = { git = "https://github.com/SMS-Derfflinger/another_ext4", branch stalloc = { version = "0.6.1", default-features = false, features = [ "allocator-api", ] } +async-trait = "0.1.89" +futures = { version = "0.3.31", features = ["alloc", "async-await"], default-features = false } [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index c3b1cfa0..ab405829 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -2,15 +2,16 @@ use crate::{ fs::procfs, io::Buffer as _, kernel::{ - block::{make_device, BlockDevice}, + block::BlockDevice, constants::{EINVAL, EIO}, interrupt::register_irq_handler, pcie::{self, Header, PCIDevice, PCIDriver, PciError}, - task::block_on, + vfs::types::DeviceId, }, prelude::*, }; use alloc::{format, sync::Arc}; +use async_trait::async_trait; use control::AdapterControl; use defs::*; use eonix_mm::address::{AddrOps as _, PAddr}; @@ -108,7 +109,30 @@ impl Device<'_> { } impl Device<'static> { - fn probe_ports(&self) -> KResult<()> { + async fn probe_port(&self, port: Arc>) -> KResult<()> { + port.init().await?; + + { + let port = port.clone(); + let name = format!("ahci-p{}-stats", port.nport); + procfs::populate_root(name.into_bytes().into(), move |buffer| { + port.print_stats(&mut buffer.get_writer()) + }) + .await; + } + + let port = BlockDevice::register_disk( + DeviceId::new(8, port.nport as u16 * 16), + 2147483647, // TODO: get size from device + port, + )?; + + port.partprobe().await?; + + Ok(()) + } + + async fn probe_ports(&self) -> KResult<()> { for nport in self.control.implemented_ports() { let port = Arc::new(AdapterPort::new(self.control_base, nport)); if !port.status_ok() { @@ -116,27 +140,7 @@ impl Device<'static> { } self.ports.lock_irq()[nport as usize] = Some(port.clone()); - if let Err(e) = (|| -> KResult<()> { - port.init()?; - - { - let port = port.clone(); - let name = format!("ahci-p{}-stats", port.nport); - procfs::populate_root(name.into_bytes().into(), move |buffer| { - port.print_stats(&mut buffer.get_writer()) - })?; - } - - let port = BlockDevice::register_disk( - make_device(8, nport * 16), - 2147483647, // TODO: get size from device - port, - )?; - - block_on(port.partprobe())?; - - Ok(()) - })() { + if let Err(e) = self.probe_port(port).await { self.ports.lock_irq()[nport as usize] = None; println_warn!("probe port {nport} failed with {e}"); } @@ -154,6 +158,7 @@ impl AHCIDriver { } } +#[async_trait] impl PCIDriver for AHCIDriver { fn vendor_id(&self) -> u16 { VENDOR_INTEL @@ -163,7 +168,7 @@ impl PCIDriver for AHCIDriver { DEVICE_AHCI } - fn handle_device(&self, pcidev: Arc>) -> Result<(), PciError> { + async fn handle_device(&self, pcidev: Arc>) -> Result<(), PciError> { let Header::Endpoint(header) = pcidev.header else { Err(EINVAL)? }; @@ -200,7 +205,7 @@ impl PCIDriver for AHCIDriver { let device_irq = device.clone(); register_irq_handler(irqno as i32, move || device_irq.handle_interrupt())?; - device.probe_ports()?; + device.probe_ports().await?; self.devices.lock().push(device); @@ -208,6 +213,8 @@ impl PCIDriver for AHCIDriver { } } -pub fn register_ahci_driver() { - pcie::register_driver(AHCIDriver::new()).expect("Register ahci driver failed"); +pub async fn register_ahci_driver() { + pcie::register_driver(AHCIDriver::new()) + .await + .expect("Register ahci driver failed"); } diff --git a/src/driver/ahci/port.rs b/src/driver/ahci/port.rs index f558f6e1..77286ec5 100644 --- a/src/driver/ahci/port.rs +++ b/src/driver/ahci/port.rs @@ -9,9 +9,9 @@ use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::{EINVAL, EIO}; use crate::kernel::mem::paging::Page; use crate::kernel::mem::AsMemoryBlock as _; -use crate::kernel::task::block_on; use crate::prelude::*; use alloc::collections::vec_deque::VecDeque; +use async_trait::async_trait; use core::pin::pin; use eonix_mm::address::{Addr as _, PAddr}; use eonix_sync::{SpinIrq as _, WaitList}; @@ -145,18 +145,25 @@ impl AdapterPort<'_> { self.sata_status().read_once() & 0xf == 0x3 } - fn get_free_slot(&self) -> u32 { + async fn get_free_slot(&self) -> u32 { loop { - let mut free_list = self.free_list.lock_irq(); - let free_slot = free_list.free.pop_front(); - if let Some(slot) = free_slot { - return slot; - } let mut wait = pin!(self.free_list_wait.prepare_to_wait()); - wait.as_mut().add_to_wait_list(); - drop(free_list); - block_on(wait); + { + let mut free_list = self.free_list.lock_irq(); + + if let Some(slot) = free_list.free.pop_front() { + return slot; + } + + wait.as_mut().add_to_wait_list(); + + if let Some(slot) = free_list.free.pop_front() { + return slot; + } + } + + wait.await; } } @@ -204,11 +211,11 @@ impl AdapterPort<'_> { Ok(()) } - fn send_command(&self, cmd: &impl Command) -> KResult<()> { + async fn send_command(&self, cmd: &impl Command) -> KResult<()> { let mut cmdtable = CommandTable::new(); cmdtable.setup(cmd); - let slot_index = self.get_free_slot(); + let slot_index = self.get_free_slot().await; let slot = &self.slots[slot_index as usize]; slot.prepare_command(&cmdtable, cmd.write()); @@ -222,7 +229,7 @@ impl AdapterPort<'_> { self.stats.inc_cmd_sent(); - if let Err(_) = block_on(slot.wait_finish()) { + if let Err(_) = slot.wait_finish().await { self.stats.inc_cmd_error(); return Err(EIO); }; @@ -231,16 +238,16 @@ impl AdapterPort<'_> { Ok(()) } - fn identify(&self) -> KResult<()> { + async fn identify(&self) -> KResult<()> { let cmd = IdentifyCommand::new(); // TODO: check returned data - self.send_command(&cmd)?; + self.send_command(&cmd).await?; Ok(()) } - pub fn init(&self) -> KResult<()> { + pub async fn init(&self) -> KResult<()> { self.stop_command()?; self.command_list_base() @@ -251,7 +258,7 @@ impl AdapterPort<'_> { self.start_command()?; - match self.identify() { + match self.identify().await { Err(err) => { self.stop_command()?; Err(err) @@ -269,12 +276,13 @@ impl AdapterPort<'_> { } } +#[async_trait] impl BlockRequestQueue for AdapterPort<'_> { fn max_request_pages(&self) -> u64 { 1024 } - fn submit(&self, req: BlockDeviceRequest) -> KResult<()> { + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()> { match req { BlockDeviceRequest::Read { sector, @@ -287,7 +295,7 @@ impl BlockRequestQueue for AdapterPort<'_> { let command = ReadLBACommand::new(buffer, sector, count as u16)?; - self.send_command(&command) + self.send_command(&command).await } BlockDeviceRequest::Write { sector, @@ -300,7 +308,7 @@ impl BlockRequestQueue for AdapterPort<'_> { let command = WriteLBACommand::new(buffer, sector, count as u16)?; - self.send_command(&command) + self.send_command(&command).await } } } diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 2198c457..60a66de3 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -71,19 +71,20 @@ impl<'a> CommandSlot<'a> { pub async fn wait_finish(&self) -> KResult<()> { let mut inner = loop { - let inner = self.inner.lock_irq(); - if inner.state != SlotState::Working { - break inner; - } - let mut wait = pin!(self.wait_list.prepare_to_wait()); - wait.as_mut().add_to_wait_list(); - if inner.state != SlotState::Working { - break inner; + { + let inner = self.inner.lock_irq(); + if inner.state != SlotState::Working { + break inner; + } + wait.as_mut().add_to_wait_list(); + + if inner.state != SlotState::Working { + break inner; + } } - drop(inner); wait.await; }; diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index ba31b8b1..f362f477 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -8,6 +8,7 @@ use crate::prelude::*; use alloc::boxed::Box; use alloc::sync::Arc; use alloc::vec::Vec; +use async_trait::async_trait; use core::ptr::NonNull; use eonix_hal::fence::memory_barrier; use eonix_mm::address::{Addr, PAddr}; @@ -437,6 +438,7 @@ struct Driver { dev_id: u16, } +#[async_trait] impl PCIDriver for Driver { fn vendor_id(&self) -> u16 { 0x8086 @@ -446,7 +448,7 @@ impl PCIDriver for Driver { self.dev_id } - fn handle_device(&self, device: Arc>) -> Result<(), PciError> { + async fn handle_device(&self, device: Arc>) -> Result<(), PciError> { let Header::Endpoint(header) = device.header else { Err(EINVAL)? }; @@ -473,10 +475,10 @@ impl PCIDriver for Driver { } } -pub fn register_e1000e_driver() { +pub async fn register_e1000e_driver() { let dev_ids = [0x100e, 0x10d3, 0x10ea, 0x153a]; for id in dev_ids.into_iter() { - pcie::register_driver(Driver { dev_id: id }).unwrap(); + pcie::register_driver(Driver { dev_id: id }).await.unwrap(); } } diff --git a/src/driver/serial.rs b/src/driver/serial.rs index d69965f4..b634c232 100644 --- a/src/driver/serial.rs +++ b/src/driver/serial.rs @@ -2,8 +2,8 @@ mod io; use crate::{ kernel::{ - block::make_device, console::set_console, constants::EIO, interrupt::register_irq_handler, - CharDevice, CharDeviceType, Terminal, TerminalDevice, + console::set_console, constants::EIO, interrupt::register_irq_handler, + vfs::types::DeviceId, CharDevice, CharDeviceType, Terminal, TerminalDevice, }, prelude::*, }; @@ -167,7 +167,7 @@ impl Serial { eonix_log::set_console(terminal.clone()); CharDevice::register( - make_device(4, 64 + port.id), + DeviceId::new(4, 64 + port.id as u16), port.name.clone(), CharDeviceType::Terminal(terminal), )?; diff --git a/src/driver/virtio/riscv64.rs b/src/driver/virtio/riscv64.rs index 9bdbf6ce..b33e16ac 100644 --- a/src/driver/virtio/riscv64.rs +++ b/src/driver/virtio/riscv64.rs @@ -1,8 +1,5 @@ use super::virtio_blk::HAL; -use crate::kernel::{ - block::{make_device, BlockDevice}, - task::block_on, -}; +use crate::kernel::{block::BlockDevice, task::block_on, vfs::types::DeviceId}; use alloc::{sync::Arc, vec::Vec}; use eonix_hal::arch_exported::fdt::FDT; use eonix_hal::mm::ArchPhysAccess; @@ -43,7 +40,7 @@ pub fn init() { .expect("Failed to initialize VirtIO Block device"); let block_device = BlockDevice::register_disk( - make_device(8, 16 * disk_id), + DeviceId::new(8, 16 * disk_id), 2147483647, Arc::new(Spin::new(block_device)), ) diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index 37e4fe77..86b500b6 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -7,6 +7,8 @@ use crate::{ }, prelude::KResult, }; +use alloc::boxed::Box; +use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::{ address::{Addr, PAddr, PhysAccess}, @@ -74,6 +76,7 @@ unsafe impl Hal for HAL { } } +#[async_trait] impl BlockRequestQueue for Spin> where T: Transport + Send, @@ -82,7 +85,7 @@ where 1024 } - fn submit(&self, req: BlockDeviceRequest) -> KResult<()> { + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()> { match req { BlockDeviceRequest::Write { sector, diff --git a/src/fs/ext4.rs b/src/fs/ext4.rs index 76ca4a34..121339d3 100644 --- a/src/fs/ext4.rs +++ b/src/fs/ext4.rs @@ -1,5 +1,3 @@ -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; - use crate::kernel::mem::{CachePage, CachePageStream, PageCache, PageCacheBackend}; use crate::kernel::task::block_on; use crate::kernel::timer::Ticks; @@ -31,6 +29,8 @@ use alloc::{ use another_ext4::{ Block, BlockDevice as Ext4BlockDeviceTrait, Ext4, FileType, InodeMode, PBlockId, }; +use async_trait::async_trait; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use eonix_sync::RwLock; pub struct Ext4BlockDevice { @@ -194,7 +194,7 @@ impl Ext4Fs { root_inode.inode.mtime_extra() as _, )), rwsem: RwLock::new(()), - vfs: Arc::downgrade(&ext4fs) as _, + sb: Arc::downgrade(&ext4fs) as _, }, ) }; @@ -290,7 +290,7 @@ impl Inode for FileInode { } fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let mut temp_buf = vec![0u8; buffer.total()]; @@ -334,7 +334,7 @@ impl Inode for FileInode { fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult { //let _lock = Task::block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let mut temp_buf = vec![0u8; 4096]; @@ -363,7 +363,7 @@ impl Inode for FileInode { fn chmod(&self, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let old_mode = self.mode.load(); let new_mode = old_mode.perm(mode.bits()); @@ -428,7 +428,7 @@ impl DirInode { impl Inode for DirInode { fn lookup(&self, dentry: &Arc) -> KResult>> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let name = dentry.get_name(); @@ -477,7 +477,7 @@ impl Inode for DirInode { ctime: Spin::new(Instant::new(attr.ctime as _, 0)), mtime: Spin::new(Instant::new(attr.mtime as _, 0)), rwsem: RwLock::new(()), - vfs: self.vfs.clone(), + sb: self.sb.clone(), }, ); @@ -489,7 +489,7 @@ impl Inode for DirInode { offset: usize, callback: &mut dyn FnMut(&[u8], Ino) -> KResult>, ) -> KResult { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let entries = ext4fs @@ -519,7 +519,7 @@ impl Inode for DirInode { fn creat(&self, at: &Arc, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let name = at.get_name(); @@ -534,7 +534,7 @@ impl Inode for DirInode { ) .unwrap(); - let file = FileInode::new(new_ino as u64, self.vfs.clone(), mode); + let file = FileInode::new(new_ino as u64, self.sb.clone(), mode); let now = Instant::now(); self.update_child_time(file.as_ref(), now); self.link_file(); @@ -547,7 +547,7 @@ impl Inode for DirInode { fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let name = at.get_name(); @@ -562,7 +562,7 @@ impl Inode for DirInode { ) .unwrap(); - let new_dir = DirInode::new(new_ino as u64, self.vfs.clone(), mode); + let new_dir = DirInode::new(new_ino as u64, self.sb.clone(), mode); let now = Instant::now(); self.update_child_time(new_dir.as_ref(), now); self.link_dir(); @@ -575,7 +575,7 @@ impl Inode for DirInode { fn unlink(&self, at: &Arc) -> KResult<()> { let _dir_lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let file = at.get_inode()?; @@ -602,7 +602,7 @@ impl Inode for DirInode { fn chmod(&self, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let old_mode = self.mode.load(); let new_mode = old_mode.perm(mode.bits()); @@ -638,7 +638,7 @@ impl Inode for DirInode { // TODO: may need another lock let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::().unwrap(); let old_file = old_dentry.get_inode()?; @@ -698,6 +698,7 @@ impl From for Mode { struct Ext4MountCreator; +#[async_trait] impl MountCreator for Ext4MountCreator { fn check_signature(&self, mut first_block: &[u8]) -> KResult { match first_block.split_off(1080..) { @@ -707,7 +708,7 @@ impl MountCreator for Ext4MountCreator { } } - fn create_mount(&self, source: &str, _flags: u64, mp: &Arc) -> KResult { + async fn create_mount(&self, source: &str, _flags: u64, mp: &Arc) -> KResult { let source = source.as_bytes(); let path = Path::new(source)?; diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 12eabdd5..1104337d 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,41 +1,72 @@ mod dir; mod file; -use crate::io::Stream; -use crate::kernel::constants::EIO; +use core::future::Future; +use core::ops::Deref; + +use alloc::sync::{Arc, Weak}; +use async_trait::async_trait; +use dir::{as_raw_dirents, ParseDirent}; +use eonix_sync::RwLock; +use itertools::Itertools; + +use crate::kernel::constants::{EINVAL, EIO}; use crate::kernel::mem::{AsMemoryBlock, CachePageStream}; -use crate::kernel::task::block_on; -use crate::kernel::vfs::inode::{Mode, WriteOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::types::{DeviceId, Format, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::prelude::*; use crate::{ io::{Buffer, ByteBuffer, UninitBuffer}, kernel::{ - block::{make_device, BlockDevice, BlockDeviceRequest}, + block::{BlockDevice, BlockDeviceRequest}, mem::{ paging::Page, - {CachePage, PageCache, PageCacheBackend}, + {CachePage, PageCache, PageCacheBackendOps}, }, vfs::{ dentry::Dentry, - inode::{define_struct_inode, Ino, Inode, InodeData}, + inode::{Ino, Inode}, mount::{register_filesystem, Mount, MountCreator}, - vfs::Vfs, - DevId, }, }, - prelude::*, KResult, }; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, - vec::Vec, -}; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use dir::Dirs as _; -use eonix_sync::RwLock; -use file::ClusterRead; -type ClusterNo = u32; +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Cluster(u32); + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct RawCluster(pub u32); + +impl RawCluster { + const START: u32 = 2; + const EOC: u32 = 0x0FFF_FFF8; + const INVL: u32 = 0xF000_0000; + + fn parse(self) -> Option { + match self.0 { + ..Self::START | Self::EOC..Self::INVL => None, + Self::INVL.. => { + unreachable!("invalid cluster number: RawCluster({:#08x})", self.0) + } + no => Some(Cluster(no)), + } + } +} + +impl Cluster { + pub fn as_ino(self) -> Ino { + Ino::new(self.0 as _) + } + + fn normalized(self) -> Self { + Self(self.0 - 2) + } +} const SECTOR_SIZE: usize = 512; @@ -59,7 +90,7 @@ struct Bootsector { sectors_per_fat: u32, flags: u16, fat_version: u16, - root_cluster: ClusterNo, + root_cluster: RawCluster, fsinfo_sector: u16, backup_bootsector: u16, _reserved: [u8; 12], @@ -73,222 +104,203 @@ struct Bootsector { mbr_signature: u16, } -impl_any!(FatFs); /// # Lock order /// 2. FatTable /// 3. Inodes /// struct FatFs { sectors_per_cluster: u8, - rootdir_cluster: ClusterNo, - data_start: u64, - volume_label: [u8; 11], + data_start_sector: u64, + _rootdir_cluster: Cluster, + _volume_label: Box, device: Arc, - fat: RwLock>, - weak: Weak, - icache: BTreeMap, + fat: RwLock>, } -impl Vfs for FatFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - self.device.devid() - } - - fn is_read_only(&self) -> bool { - true - } -} +impl SuperBlock for FatFs {} impl FatFs { - fn read_cluster(&self, cluster: ClusterNo, buf: &Page) -> KResult<()> { - let cluster = cluster - 2; + async fn read_cluster(&self, mut cluster: Cluster, buf: &Page) -> KResult<()> { + cluster = cluster.normalized(); let rq = BlockDeviceRequest::Read { - sector: self.data_start as u64 + cluster as u64 * self.sectors_per_cluster as u64, + sector: self.data_start_sector as u64 + + cluster.0 as u64 * self.sectors_per_cluster as u64, count: self.sectors_per_cluster as u64, buffer: core::slice::from_ref(buf), }; - self.device.commit_request(rq)?; + self.device.commit_request(rq).await?; Ok(()) } - - fn get_or_alloc_inode(&self, ino: Ino, is_directory: bool, size: u32) -> Arc { - self.icache - .get(&ino) - .cloned() - .map(FatInode::unwrap) - .unwrap_or_else(|| { - if is_directory { - DirInode::new(ino, self.weak.clone(), size) - } else { - FileInode::new(ino, self.weak.clone(), size) - } - }) - } } impl FatFs { - pub fn create(device: DevId) -> KResult<(Arc, Arc)> { + pub async fn create(device: DeviceId) -> KResult<(SbUse, InodeUse)> { let device = BlockDevice::get(device)?; - let mut fatfs_arc = Arc::new_cyclic(|weak: &Weak| Self { - device, - sectors_per_cluster: 0, - rootdir_cluster: 0, - data_start: 0, - fat: RwLock::new(Vec::new()), - weak: weak.clone(), - icache: BTreeMap::new(), - volume_label: [0; 11], - }); - let fatfs = unsafe { Arc::get_mut_unchecked(&mut fatfs_arc) }; - - let mut info: UninitBuffer = UninitBuffer::new(); - fatfs.device.read_some(0, &mut info)?.ok_or(EIO)?; + let mut info = UninitBuffer::::new(); + device.read_some(0, &mut info).await?.ok_or(EIO)?; let info = info.assume_filled_ref()?; - fatfs.sectors_per_cluster = info.sectors_per_cluster; - fatfs.rootdir_cluster = info.root_cluster; - fatfs.data_start = - info.reserved_sectors as u64 + info.fat_copies as u64 * info.sectors_per_fat as u64; - - let fat = fatfs.fat.get_mut(); - - fat.resize( - 512 * info.sectors_per_fat as usize / core::mem::size_of::(), - 0, + let mut fat = Box::new_uninit_slice( + 512 * info.sectors_per_fat as usize / core::mem::size_of::(), ); - let mut buffer = ByteBuffer::from(fat.as_mut_slice()); - - fatfs - .device - .read_some(info.reserved_sectors as usize * 512, &mut buffer)? + device + .read_some( + info.reserved_sectors as usize * 512, + &mut ByteBuffer::from(fat.as_mut()), + ) + .await? .ok_or(EIO)?; - info.volume_label - .iter() - .take_while(|&&c| c != ' ' as u8) - .take(11) - .enumerate() - .for_each(|(idx, c)| fatfs.volume_label[idx] = *c); + let sectors_per_cluster = info.sectors_per_cluster; + let rootdir_cluster = info.root_cluster.parse().ok_or(EINVAL)?; - let root_dir_cluster_count = ClusterIterator::new(fat, fatfs.rootdir_cluster).count(); - let root_dir_size = root_dir_cluster_count as u32 * info.sectors_per_cluster as u32 * 512; + let data_start_sector = + info.reserved_sectors as u64 + info.fat_copies as u64 * info.sectors_per_fat as u64; + + let volume_label = { + let end = info + .volume_label + .iter() + .position(|&c| c == b' ') + .unwrap_or(info.volume_label.len()); + + String::from_utf8_lossy(&info.volume_label[..end]) + .into_owned() + .into_boxed_str() + }; - let root_inode = DirInode::new( - (info.root_cluster & !0xF000_0000) as Ino, - fatfs.weak.clone(), - root_dir_size, + let fat = unsafe { fat.assume_init() }; + + let rootdir_cluster_count = ClusterIterator::new(fat.as_ref(), rootdir_cluster).count(); + let rootdir_size = rootdir_cluster_count as u32 * sectors_per_cluster as u32 * 512; + + let fatfs = SbUse::new( + SuperBlockInfo { + io_blksize: 4096, + device_id: device.devid(), + read_only: true, + }, + Self { + device, + sectors_per_cluster, + _rootdir_cluster: rootdir_cluster, + data_start_sector, + fat: RwLock::new(fat), + _volume_label: volume_label, + }, ); - Ok((fatfs_arc, root_inode)) + let sbref = SbRef::from(&fatfs); + Ok((fatfs, DirInode::new(rootdir_cluster, sbref, rootdir_size))) } } -struct ClusterIterator<'fat> { - fat: &'fat [ClusterNo], - cur: ClusterNo, +struct ClusterIterator<'a> { + fat: &'a [RawCluster], + cur: Option, } -impl<'fat> ClusterIterator<'fat> { - fn new(fat: &'fat [ClusterNo], start: ClusterNo) -> Self { - Self { fat, cur: start } +impl<'a> ClusterIterator<'a> { + fn new(fat: &'a [RawCluster], start: Cluster) -> Self { + Self { + fat, + cur: Some(start), + } } } impl<'fat> Iterator for ClusterIterator<'fat> { - type Item = ClusterNo; + type Item = Cluster; fn next(&mut self) -> Option { - const EOC: ClusterNo = 0x0FFF_FFF8; - const INVL: ClusterNo = 0xF000_0000; - - match self.cur { - ..2 | EOC..INVL => None, - INVL.. => unreachable!("Invalid cluster number: {}", self.cur), - next => { - self.cur = self.fat[next as usize] & !INVL; - Some(next) - } - } + self.cur.inspect(|&Cluster(no)| { + self.cur = self.fat[no as usize].parse(); + }) } } -#[allow(dead_code)] -#[derive(Clone)] -enum FatInode { - File(Arc), - Dir(Arc), +struct FileInode { + cluster: Cluster, + info: Spin, + sb: SbRef, + page_cache: PageCache, } -impl FatInode { - fn unwrap(self) -> Arc { - match self { - FatInode::File(inode) => inode, - FatInode::Dir(inode) => inode, - } +impl FileInode { + fn new(cluster: Cluster, sb: SbRef, size: u32) -> InodeUse { + InodeUse::new_cyclic(|weak: &Weak| Self { + cluster, + info: Spin::new(InodeInfo { + size: size as u64, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + sb, + page_cache: PageCache::new(weak.clone()), + }) } } -define_struct_inode! { - struct FileInode { - page_cache: PageCache, +impl InodeOps for FileInode { + type SuperBlock = FatFs; + + fn ino(&self) -> Ino { + self.cluster.as_ino() } -} -impl FileInode { - fn new(ino: Ino, weak: Weak, size: u32) -> Arc { - let inode = Arc::new_cyclic(|weak_self: &Weak| Self { - idata: InodeData::new(ino, weak), - page_cache: PageCache::new(weak_self.clone()), - }); + fn format(&self) -> Format { + Format::REG + } - // Safety: We are initializing the inode - inode.nlink.store(1, Ordering::Relaxed); - inode.mode.store(Mode::REG.perm(0o777)); - inode.size.store(size as u64, Ordering::Relaxed); + fn info(&self) -> &Spin { + &self.info + } - inode + fn super_block(&self) -> &SbRef { + &self.sb } -} -impl Inode for FileInode { fn page_cache(&self) -> Option<&PageCache> { Some(&self.page_cache) } +} - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - block_on(self.page_cache.read(buffer, offset)) +impl InodeDirOps for FileInode {} +impl InodeFileOps for FileInode { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + self.page_cache.read(buffer, offset).await } - fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::().unwrap(); - let fat = block_on(vfs.fat.read()); + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + let sb = self.sb.get()?; + let fs = &sb.backend; + let fat = sb.backend.fat.read().await; - if self.size.load(Ordering::Relaxed) as usize == 0 { + if offset >= self.info.lock().size as usize { return Ok(0); } - let cluster_size = vfs.sectors_per_cluster as usize * SECTOR_SIZE; + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; assert!(cluster_size <= 0x1000, "Cluster size is too large"); let skip_clusters = offset / cluster_size; let inner_offset = offset % cluster_size; - let cluster_iter = - ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo).skip(skip_clusters); + let cluster_iter = ClusterIterator::new(fat.as_ref(), self.cluster).skip(skip_clusters); let buffer_page = Page::alloc(); for cluster in cluster_iter { - vfs.read_cluster(cluster, &buffer_page)?; + fs.read_cluster(cluster, &buffer_page).await?; let data = unsafe { // SAFETY: We are the only one holding this page. @@ -296,7 +308,7 @@ impl Inode for FileInode { }; let end = offset + data.len(); - let real_end = core::cmp::min(end, self.size.load(Ordering::Relaxed) as usize); + let real_end = end.min(self.info.lock().size as usize); let real_size = real_end - offset; if buffer.fill(&data[..real_size])?.should_stop() { @@ -306,108 +318,203 @@ impl Inode for FileInode { Ok(buffer.wrote()) } +} - fn write(&self, _stream: &mut dyn Stream, _offset: WriteOffset) -> KResult { - todo!() +impl PageCacheBackendOps for FileInode { + async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult { + self.read_direct(page, offset).await } - fn write_direct(&self, _stream: &mut dyn Stream, _offset: usize) -> KResult { + async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult { todo!() } + + fn size(&self) -> usize { + self.info.lock().size as usize + } } -impl PageCacheBackend for FileInode { - fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult { - self.read_direct(page, offset) +struct DirInode { + cluster: Cluster, + info: Spin, + sb: SbRef, + + // TODO: Use the new PageCache... + dir_pages: RwLock>, +} + +impl DirInode { + fn new(cluster: Cluster, sb: SbRef, size: u32) -> InodeUse { + InodeUse::new(Self { + cluster, + info: Spin::new(InodeInfo { + size: size as u64, + nlink: 2, // '.' and '..' + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + sb, + dir_pages: RwLock::new(Vec::new()), + }) } - fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult { - todo!() + async fn read_dir_pages(&self) -> KResult<()> { + let mut dir_pages = self.dir_pages.write().await; + if !dir_pages.is_empty() { + return Ok(()); + } + + let sb = self.sb.get()?; + let fs = &sb.backend; + let fat = fs.fat.read().await; + + let clusters = ClusterIterator::new(fat.as_ref(), self.cluster); + + for cluster in clusters { + let page = Page::alloc(); + fs.read_cluster(cluster, &page).await?; + + dir_pages.push(page); + } + + Ok(()) } - fn size(&self) -> usize { - self.size.load(Ordering::Relaxed) as usize + async fn get_dir_pages(&self) -> KResult> + use<'_>> { + { + let dir_pages = self.dir_pages.read().await; + if !dir_pages.is_empty() { + return Ok(dir_pages); + } + } + + self.read_dir_pages().await?; + + if let Some(dir_pages) = self.dir_pages.try_read() { + return Ok(dir_pages); + } + + Ok(self.dir_pages.read().await) } } -define_struct_inode! { - struct DirInode; -} +impl InodeOps for DirInode { + type SuperBlock = FatFs; -impl DirInode { - fn new(ino: Ino, weak: Weak, size: u32) -> Arc { - let inode = Arc::new(Self { - idata: InodeData::new(ino, weak), - }); + fn ino(&self) -> Ino { + self.cluster.as_ino() + } + + fn format(&self) -> Format { + Format::DIR + } - // Safety: We are initializing the inode - inode.nlink.store(2, Ordering::Relaxed); - inode.mode.store(Mode::DIR.perm(0o777)); - inode.size.store(size as u64, Ordering::Relaxed); + fn info(&self) -> &Spin { + &self.info + } - inode + fn super_block(&self) -> &SbRef { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None } } -impl Inode for DirInode { - fn lookup(&self, dentry: &Arc) -> KResult>> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::().unwrap(); - let fat = block_on(vfs.fat.read()); - - let mut entries = ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo) - .read(vfs, 0) - .dirs(); - - let entry = entries.find(|entry| { - entry - .as_ref() - .map(|entry| &entry.filename == &***dentry.name()) - .unwrap_or(true) +impl InodeFileOps for DirInode {} +impl InodeDirOps for DirInode { + async fn lookup(&self, dentry: &Arc) -> KResult>> { + let sb = self.sb.get()?; + let dir_pages = self.get_dir_pages().await?; + + let dir_data = dir_pages.iter().map(|page| { + unsafe { + // SAFETY: No one could be writing to it. + page.as_memblk().as_bytes() + } }); - match entry { - None => Ok(None), - Some(Err(err)) => Err(err), - Some(Ok(entry)) => Ok(Some(vfs.get_or_alloc_inode( - entry.cluster as Ino, - entry.is_directory, - entry.size, - ))), + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok(); + + let mut dirents = futures::stream::iter(raw_dirents); + + while let Some(result) = dirents.next_dirent().await { + let entry = result?; + + if *entry.filename != ****dentry.name() { + continue; + } + + let sbref = SbRef::from(&sb); + + if entry.is_directory { + return Ok(Some(DirInode::new(entry.cluster, sbref, entry.size) as _)); + } else { + return Ok(Some(FileInode::new(entry.cluster, sbref, entry.size) as _)); + } } + + Ok(None) } - fn do_readdir( - &self, + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult>, - ) -> KResult { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::().unwrap(); - let fat = block_on(vfs.fat.read()); - - let cluster_iter = ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo) - .read(vfs, offset) - .dirs(); - - let mut nread = 0usize; - for entry in cluster_iter { - let entry = entry?; - - vfs.get_or_alloc_inode(entry.cluster as Ino, entry.is_directory, entry.size); - if callback(&entry.filename, entry.cluster as Ino)?.is_break() { - break; + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> impl Future>> + Send + 'r { + async move { + let sb = self.sb.get()?; + let fs = &sb.backend; + let dir_pages = self.get_dir_pages().await?; + + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; + + let cluster_offset = offset / cluster_size; + let inner_offset = offset % cluster_size; + let inner_raw_dirent_offset = inner_offset / core::mem::size_of::(); + + let dir_data = dir_pages.iter().skip(cluster_offset).map(|page| { + unsafe { + // SAFETY: No one could be writing to it. + page.as_memblk().as_bytes() + } + }); + + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok() + .skip(inner_raw_dirent_offset); + + let mut dirents = futures::stream::iter(raw_dirents); + + let mut nread = 0; + while let Some(result) = dirents.next_dirent().await { + let entry = result?; + + match callback(&entry.filename, entry.cluster.as_ino()) { + Err(err) => return Ok(Err(err)), + Ok(true) => nread += entry.entry_offset as usize, + Ok(false) => break, + } } - nread += entry.entry_offset as usize; + Ok(Ok(nread)) } - - Ok(nread) } } struct FatMountCreator; +#[async_trait] impl MountCreator for FatMountCreator { fn check_signature(&self, mut first_block: &[u8]) -> KResult { match first_block.split_off(82..) { @@ -417,8 +524,8 @@ impl MountCreator for FatMountCreator { } } - fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc) -> KResult { - let (fatfs, root_inode) = FatFs::create(make_device(8, 1))?; + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc) -> KResult { + let (fatfs, root_inode) = FatFs::create(DeviceId::new(8, 1)).await?; Mount::new(mp, fatfs, root_inode) } diff --git a/src/fs/fat32/dir.rs b/src/fs/fat32/dir.rs index c4ac6c0d..8a5b6f40 100644 --- a/src/fs/fat32/dir.rs +++ b/src/fs/fat32/dir.rs @@ -1,11 +1,16 @@ -use super::file::ClusterReadIterator; +use core::pin::Pin; + +use alloc::{boxed::Box, string::String}; +use futures::{Stream, StreamExt}; +use posix_types::result::PosixError; + use crate::kernel::constants::EINVAL; use crate::prelude::*; -use alloc::{string::String, sync::Arc}; -use itertools::Itertools; + +use super::{Cluster, RawCluster}; #[repr(C, packed)] -pub(super) struct RawDirEntry { +pub struct RawDirEntry { name: [u8; 8], extension: [u8; 3], attr: u8, @@ -21,9 +26,9 @@ pub(super) struct RawDirEntry { size: u32, } -pub(super) struct FatDirectoryEntry { - pub filename: Arc<[u8]>, - pub cluster: u32, +pub struct FatDirectoryEntry { + pub filename: Box<[u8]>, + pub cluster: Cluster, pub size: u32, pub entry_offset: u32, pub is_directory: bool, @@ -79,7 +84,7 @@ impl RawDirEntry { self.attr & Self::ATTR_DIRECTORY != 0 } - fn long_filename(&self) -> Option<[u16; 13]> { + fn as_raw_long_filename(&self) -> Option<[u16; 13]> { if !self.is_long_filename() { return None; } @@ -103,137 +108,114 @@ impl RawDirEntry { } } -impl<'data, I> RawDirs<'data> for I where I: ClusterReadIterator<'data> {} -trait RawDirs<'data>: ClusterReadIterator<'data> { - fn raw_dirs(self) -> impl Iterator> + 'data - where - Self: Sized, - { - const ENTRY_SIZE: usize = size_of::(); - - self.map(|result| { - let data = result?; - if data.len() % ENTRY_SIZE != 0 { - return Err(EINVAL); - } - - Ok(unsafe { - core::slice::from_raw_parts( - data.as_ptr() as *const RawDirEntry, - data.len() / ENTRY_SIZE, - ) - }) - }) - .flatten_ok() +pub fn as_raw_dirents(data: &[u8]) -> KResult<&[RawDirEntry]> { + let len = data.len(); + if len % size_of::() != 0 { + return Err(EINVAL); } -} - -pub(super) trait Dirs<'data>: ClusterReadIterator<'data> { - fn dirs(self) -> impl Iterator> + 'data - where - Self: Sized; -} -impl<'data, I> Dirs<'data> for I -where - I: ClusterReadIterator<'data>, -{ - fn dirs(self) -> impl Iterator> + 'data - where - Self: Sized, - { - self.raw_dirs().real_dirs() + unsafe { + Ok(core::slice::from_raw_parts( + data.as_ptr() as *const RawDirEntry, + len / size_of::(), + )) } } -trait RealDirs<'data>: Iterator> + 'data { - fn real_dirs(self) -> DirsIter<'data, Self> - where - Self: Sized; +pub trait ParseDirent { + async fn next_dirent(&mut self) -> Option>; } -impl<'data, I> RealDirs<'data> for I +impl<'a, T> ParseDirent for T where - I: Iterator> + 'data, + T: Stream>, { - fn real_dirs(self) -> DirsIter<'data, Self> - where - Self: Sized, - { - DirsIter { iter: self } - } -} + async fn next_dirent(&mut self) -> Option> { + let mut me = unsafe { Pin::new_unchecked(self) }; + + // The long filename entries are stored in reverse order. + // So we reverse all filename segments and then reverse the whole string at the end. + let mut filename_rev = String::new(); + + let mut is_lfn = false; + let mut nr_entry_scanned = 0; + let mut cur_entry; + + loop { + match me.as_mut().next().await { + Some(Err(err)) => return Some(Err(err)), + Some(Ok(ent)) => { + cur_entry = ent; + nr_entry_scanned += 1; + } + None => { + if is_lfn { + // Unterminated long filename entries are invalid. + return Some(Err(PosixError::EINVAL.into())); + } else { + return None; + } + } + }; -pub(super) struct DirsIter<'data, I> -where - I: Iterator> + 'data, -{ - iter: I, -} + if !cur_entry.is_invalid() { + break; + } -impl<'data, I> Iterator for DirsIter<'data, I> -where - I: Iterator> + 'data, -{ - type Item = KResult; - - fn next(&mut self) -> Option { - let mut filename = String::new(); - let mut entry_offset = 0; - let entry = loop { - let entry = match self.iter.next()? { - Ok(entry) => entry, - Err(err) => return Some(Err(err)), - }; - entry_offset += 1; - - let long_filename = entry.long_filename(); - if entry.is_invalid() { - if let Some(long_filename) = long_filename { - let long_filename = long_filename - .iter() - .position(|&ch| ch == 0) - .map(|pos| &long_filename[..pos]) - .unwrap_or(&long_filename); - - filename.extend( - long_filename - .into_iter() - .map(|&ch| char::from_u32(ch as u32).unwrap_or('?')) - .rev(), - ); - } + let Some(raw_long_filename) = cur_entry.as_raw_long_filename() else { continue; - } - break entry; + }; + + // We are processing a long filename entry. + is_lfn = true; + + let real_len = raw_long_filename + .iter() + .position(|&ch| ch == 0) + .unwrap_or(raw_long_filename.len()); + + let name_codes_rev = raw_long_filename.into_iter().take(real_len).rev(); + let name_chars_rev = char::decode_utf16(name_codes_rev).map(|r| r.unwrap_or('?')); + + filename_rev.extend(name_chars_rev); + } + + // From now on, `entry` represents a valid directory entry. + + let raw_cluster = + RawCluster(cur_entry.cluster_low as u32 | ((cur_entry.cluster_high as u32) << 16)); + + let Some(cluster) = raw_cluster.parse() else { + return Some(Err(PosixError::EINVAL.into())); }; - let filename: Arc<[u8]> = if filename.is_empty() { - let mut filename = entry.filename().to_vec(); - let extension = entry.extension(); + let filename; + + if filename_rev.is_empty() { + let mut name = cur_entry.filename().to_vec(); + let extension = cur_entry.extension(); if !extension.is_empty() { - filename.push(b'.'); - filename.extend_from_slice(extension); + name.push(b'.'); + name.extend_from_slice(extension); } - if entry.is_filename_lowercase() { - filename.make_ascii_lowercase(); + if cur_entry.is_filename_lowercase() { + name.make_ascii_lowercase(); } - filename.into() + filename = name.into_boxed_slice(); } else { - let mut bytes = filename.into_bytes(); - bytes.reverse(); - - bytes.into() - }; + let mut name = filename_rev.into_bytes(); + name.reverse(); + filename = name.into_boxed_slice(); + } Some(Ok(FatDirectoryEntry { - size: entry.size, - entry_offset, + size: cur_entry.size, + entry_offset: nr_entry_scanned * size_of::() as u32, filename, - cluster: entry.cluster_low as u32 | (((entry.cluster_high & !0xF000) as u32) << 16), - is_directory: entry.is_directory(), + cluster, + is_directory: cur_entry.is_directory(), })) } } diff --git a/src/fs/fat32/file.rs b/src/fs/fat32/file.rs index db16df50..2df69728 100644 --- a/src/fs/fat32/file.rs +++ b/src/fs/fat32/file.rs @@ -1,40 +1,24 @@ -use super::{ClusterIterator, FatFs}; -use crate::{ - kernel::mem::{AsMemoryBlock as _, Page}, - KResult, -}; - -pub trait ClusterReadIterator<'data>: Iterator> + 'data {} -impl<'a, I> ClusterReadIterator<'a> for I where I: Iterator> + 'a {} +use futures::Stream; -pub(super) trait ClusterRead<'data> { - fn read<'vfs>(self, vfs: &'vfs FatFs, offset: usize) -> impl ClusterReadIterator<'data> - where - Self: Sized, - 'vfs: 'data; -} +use crate::{kernel::mem::Page, prelude::KResult}; -impl<'data, 'fat: 'data> ClusterRead<'data> for ClusterIterator<'fat> { - fn read<'vfs: 'data>(self, vfs: &'vfs FatFs, offset: usize) -> impl ClusterReadIterator<'data> { - const SECTOR_SIZE: usize = 512; +use super::{ClusterIterator, FatFs}; - let cluster_size = vfs.sectors_per_cluster as usize * SECTOR_SIZE; - assert!(cluster_size <= 0x1000, "Cluster size is too large"); +pub trait ReadClusters { + fn read_clusters(self, fs: &FatFs) -> impl Stream> + Send; +} - let skip_clusters = offset / cluster_size; - let mut inner_offset = offset % cluster_size; +impl ReadClusters for ClusterIterator<'_> { + fn read_clusters(self, fs: &FatFs) -> impl Stream> + Send { + futures::stream::unfold(self, move |mut me| async { + let cluster = me.next()?; + let page = Page::alloc(); - // TODO: Use block cache. - let buffer_page = Page::alloc(); + if let Err(err) = fs.read_cluster(cluster, &page).await { + return Some((Err(err), me)); + } - self.skip(skip_clusters).map(move |cluster| { - vfs.read_cluster(cluster, &buffer_page)?; - let data = unsafe { - // SAFETY: No one could be writing to it. - &buffer_page.as_memblk().as_bytes()[inner_offset..] - }; - inner_offset = 0; - Ok(data) + Some((Ok(page), me)) }) } } diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 5d9285ec..c59ee801 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,5 +1,4 @@ +// pub mod ext4; pub mod fat32; pub mod procfs; -pub mod shm; pub mod tmpfs; -pub mod ext4; diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 2ed24613..57b881df 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -1,325 +1,264 @@ -use crate::kernel::constants::{EACCES, ENOTDIR}; -use crate::kernel::task::block_on; +use crate::kernel::constants::{EACCES, EISDIR, ENOTDIR}; use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{AtomicMode, Mode}; +use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::types::{DeviceId, Format, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; use crate::{ io::Buffer, kernel::{ mem::paging::PageBuffer, vfs::{ dentry::Dentry, - inode::{define_struct_inode, AtomicIno, Ino, Inode, InodeData}, + inode::{Ino, Inode}, mount::{dump_mounts, register_filesystem, Mount, MountCreator}, - vfs::Vfs, - DevId, }, }, prelude::*, }; -use alloc::sync::{Arc, Weak}; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use eonix_sync::{AsProof as _, AsProofMut as _, LazyLock, Locked}; -use itertools::Itertools; - -#[allow(dead_code)] -pub trait ProcFsFile: Send + Sync { - fn can_read(&self) -> bool { - false - } - - fn can_write(&self) -> bool { - false - } +use alloc::sync::Arc; +use async_trait::async_trait; +use core::future::Future; +use core::sync::atomic::{AtomicU64, Ordering}; +use eonix_sync::{LazyLock, RwLock}; + +struct Node { + ino: Ino, + sb: SbRef, + info: Spin, + kind: NodeKind, +} - fn read(&self, _buffer: &mut PageBuffer) -> KResult { - Err(EACCES) - } +enum NodeKind { + File(FileInode), + Dir(DirInode), +} - fn write(&self, _buffer: &[u8]) -> KResult { - Err(EACCES) - } +struct FileInode { + read: Option KResult<()> + Send + Sync>>, + write: Option<()>, } -pub enum ProcFsNode { - File(Arc), - Dir(Arc), +struct DirInode { + entries: RwLock, InodeUse)>>, } -impl ProcFsNode { - fn unwrap(&self) -> Arc { - match self { - ProcFsNode::File(inode) => inode.clone(), - ProcFsNode::Dir(inode) => inode.clone(), - } - } +impl InodeOps for Node { + type SuperBlock = ProcFs; fn ino(&self) -> Ino { - match self { - ProcFsNode::File(inode) => inode.ino, - ProcFsNode::Dir(inode) => inode.ino, - } + self.ino } -} -define_struct_inode! { - pub struct FileInode { - file: Box, - } -} - -impl FileInode { - pub fn new(ino: Ino, vfs: Weak, file: Box) -> Arc { - let mut mode = Mode::REG; - if file.can_read() { - mode.set_perm(0o444); - } - if file.can_write() { - mode.set_perm(0o222); + fn format(&self) -> Format { + match &self.kind { + NodeKind::File(_) => Format::REG, + NodeKind::Dir(_) => Format::DIR, } + } - let mut inode = Self { - idata: InodeData::new(ino, vfs), - file, - }; + fn info(&self) -> &Spin { + &self.info + } - inode.idata.mode.store(mode); - inode.idata.nlink.store(1, Ordering::Relaxed); - *inode.ctime.get_mut() = Instant::now(); - *inode.mtime.get_mut() = Instant::now(); - *inode.atime.get_mut() = Instant::now(); + fn super_block(&self) -> &SbRef { + &self.sb + } - Arc::new(inode) + fn page_cache(&self) -> Option<&crate::kernel::mem::PageCache> { + None } } -impl Inode for FileInode { - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - if !self.file.can_read() { +impl InodeFileOps for Node { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + let NodeKind::File(file_inode) = &self.kind else { + return Err(EISDIR); + }; + + let Some(read_fn) = &file_inode.read else { return Err(EACCES); - } + }; let mut page_buffer = PageBuffer::new(); - self.file.read(&mut page_buffer)?; + read_fn(&mut page_buffer)?; - let data = page_buffer - .data() - .split_at_checked(offset) - .map(|(_, data)| data); + let Some((_, data)) = page_buffer.data().split_at_checked(offset) else { + return Ok(0); + }; - match data { - None => Ok(0), - Some(data) => Ok(buffer.fill(data)?.allow_partial()), - } + Ok(buffer.fill(data)?.allow_partial()) } } -define_struct_inode! { - pub struct DirInode { - entries: Locked, ProcFsNode)>, ()>, - } -} +impl InodeDirOps for Node { + async fn lookup(&self, dentry: &Arc) -> KResult>> { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; -impl DirInode { - pub fn new(ino: Ino, vfs: Weak) -> Arc { - Self::new_locked(ino, vfs, |inode, rwsem| unsafe { - addr_of_mut_field!(inode, entries).write(Locked::new(vec![], rwsem)); - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(Mode::DIR.perm(0o755))); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} + let entries = dir.entries.read().await; + + let dent_name = dentry.name(); + for (name, node) in entries.iter() { + if *name == ***dent_name { + return Ok(Some(node.clone() as _)); + } + } -impl Inode for DirInode { - fn lookup(&self, dentry: &Arc) -> KResult>> { - let lock = block_on(self.rwsem.read()); - Ok(self - .entries - .access(lock.prove()) - .iter() - .find_map(|(name, node)| (name == &***dentry.name()).then(|| node.unwrap()))) + Ok(None) } - fn do_readdir( - &self, + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult>, - ) -> KResult { - let lock = block_on(self.rwsem.read()); - self.entries - .access(lock.prove()) - .iter() - .skip(offset) - .map(|(name, node)| callback(name.as_ref(), node.ino())) - .take_while(|result| result.map_or(true, |flow| flow.is_continue())) - .take_while_inclusive(|result| result.is_ok()) - .fold_ok(0, |acc, _| acc + 1) + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> impl Future>> + Send + 'r { + Box::pin(async move { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; + + let entries = dir.entries.read().await; + + let mut count = 0; + for (name, node) in entries.iter().skip(offset) { + match callback(name.as_ref(), node.ino) { + Err(err) => return Ok(Err(err)), + Ok(true) => count += 1, + Ok(false) => break, + } + } + + Ok(Ok(count)) + }) } } -impl_any!(ProcFs); -pub struct ProcFs { - root_node: Arc, - next_ino: AtomicIno, -} - -impl Vfs for ProcFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - 10 +impl Node { + pub fn new_file( + ino: Ino, + sb: SbRef, + read: impl Fn(&mut PageBuffer) -> KResult<()> + Send + Sync + 'static, + ) -> InodeUse { + InodeUse::new(Self { + ino, + sb, + info: Spin::new(InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o444), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + kind: NodeKind::File(FileInode::new(Box::new(read))), + }) } - fn is_read_only(&self) -> bool { - false + fn new_dir(ino: Ino, sb: SbRef) -> InodeUse { + InodeUse::new(Self { + ino, + sb, + info: Spin::new(InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o755), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }), + kind: NodeKind::Dir(DirInode::new()), + }) } } -static GLOBAL_PROCFS: LazyLock> = LazyLock::new(|| { - Arc::new_cyclic(|weak: &Weak| ProcFs { - root_node: DirInode::new(0, weak.clone()), - next_ino: AtomicIno::new(1), - }) -}); - -struct ProcFsMountCreator; - -#[allow(dead_code)] -impl ProcFsMountCreator { - pub fn get() -> Arc { - GLOBAL_PROCFS.clone() - } - - pub fn get_weak() -> Weak { - Arc::downgrade(&GLOBAL_PROCFS) +impl FileInode { + fn new(read: Box KResult<()> + Send + Sync>) -> Self { + Self { + read: Some(read), + write: None, + } } } -impl MountCreator for ProcFsMountCreator { - fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc) -> KResult { - let vfs = ProcFsMountCreator::get(); - let root_inode = vfs.root_node.clone(); - Mount::new(mp, vfs, root_inode) - } - - fn check_signature(&self, _: &[u8]) -> KResult { - Ok(true) +impl DirInode { + pub fn new() -> Self { + Self { + entries: RwLock::new(vec![]), + } } } -pub fn root() -> ProcFsNode { - let vfs = ProcFsMountCreator::get(); - let root = vfs.root_node.clone(); - - ProcFsNode::Dir(root) +pub struct ProcFs { + root: InodeUse, + next_ino: AtomicU64, } -pub fn creat( - parent: &ProcFsNode, - name: Arc<[u8]>, - file: Box, -) -> KResult { - let parent = match parent { - ProcFsNode::File(_) => return Err(ENOTDIR), - ProcFsNode::Dir(parent) => parent, - }; - - let fs = ProcFsMountCreator::get(); - let ino = fs.next_ino.fetch_add(1, Ordering::Relaxed); - - let inode = FileInode::new(ino, Arc::downgrade(&fs), file); - - { - let lock = block_on(parent.idata.rwsem.write()); - parent - .entries - .access_mut(lock.prove_mut()) - .push((name, ProcFsNode::File(inode.clone()))); +impl SuperBlock for ProcFs {} +impl ProcFs { + fn assign_ino(&self) -> Ino { + Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) } - - Ok(ProcFsNode::File(inode)) } -#[allow(dead_code)] -pub fn mkdir(parent: &ProcFsNode, name: &[u8]) -> KResult { - let parent = match parent { - ProcFsNode::File(_) => return Err(ENOTDIR), - ProcFsNode::Dir(parent) => parent, - }; - - let fs = ProcFsMountCreator::get(); - let ino = fs.next_ino.fetch_add(1, Ordering::Relaxed); - - let inode = DirInode::new(ino, Arc::downgrade(&fs)); +static GLOBAL_PROCFS: LazyLock> = LazyLock::new(|| { + SbUse::new_cyclic( + SuperBlockInfo { + io_blksize: 4096, + device_id: DeviceId::new(0, 10), + read_only: false, + }, + |sbref| ProcFs { + root: Node::new_dir(Ino::new(0), sbref), + next_ino: AtomicU64::new(1), + }, + ) +}); - parent - .entries - .access_mut(block_on(inode.rwsem.write()).prove_mut()) - .push((Arc::from(name), ProcFsNode::Dir(inode.clone()))); +struct ProcFsMountCreator; - Ok(ProcFsNode::Dir(inode)) -} +#[async_trait] +impl MountCreator for ProcFsMountCreator { + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc) -> KResult { + let fs = GLOBAL_PROCFS.clone(); + let root_inode = fs.backend.root.clone(); -struct DumpMountsFile; -impl ProcFsFile for DumpMountsFile { - fn can_read(&self) -> bool { - true + Mount::new(mp, fs, root_inode) } - fn read(&self, buffer: &mut PageBuffer) -> KResult { - dump_mounts(&mut buffer.get_writer()); - - Ok(buffer.data().len()) + fn check_signature(&self, _: &[u8]) -> KResult { + Ok(true) } } -pub fn init() { - register_filesystem("procfs", Arc::new(ProcFsMountCreator)).unwrap(); - - creat( - &root(), - Arc::from(b"mounts".as_slice()), - Box::new(DumpMountsFile), - ) - .unwrap(); -} - -pub struct GenericProcFsFile +pub async fn populate_root(name: Arc<[u8]>, read_fn: F) where - ReadFn: Send + Sync + Fn(&mut PageBuffer) -> KResult<()>, + F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, { - read_fn: Option, -} + let procfs = &GLOBAL_PROCFS.backend; + let root = &procfs.root; -impl ProcFsFile for GenericProcFsFile -where - ReadFn: Send + Sync + Fn(&mut PageBuffer) -> KResult<()>, -{ - fn can_read(&self) -> bool { - self.read_fn.is_some() - } + let NodeKind::Dir(root) = &root.kind else { + unreachable!(); + }; - fn read(&self, buffer: &mut PageBuffer) -> KResult { - self.read_fn.as_ref().ok_or(EACCES)?(buffer).map(|_| buffer.data().len()) - } + let mut entries = root.entries.write().await; + entries.push(( + name.clone(), + Node::new_file(procfs.assign_ino(), SbRef::from(&GLOBAL_PROCFS), read_fn), + )); } -pub fn populate_root(name: Arc<[u8]>, read_fn: F) -> KResult<()> -where - F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, -{ - let root = root(); - - creat( - &root, - name, - Box::new(GenericProcFsFile { - read_fn: Some(read_fn), - }), - ) - .map(|_| ()) +pub async fn init() { + register_filesystem("procfs", Arc::new(ProcFsMountCreator)).unwrap(); + + populate_root(Arc::from(b"mounts".as_slice()), |buffer| { + dump_mounts(&mut buffer.get_writer()); + Ok(()) + }) + .await; } diff --git a/src/fs/shm.rs b/src/fs/shm.rs deleted file mode 100644 index 09d36141..00000000 --- a/src/fs/shm.rs +++ /dev/null @@ -1,146 +0,0 @@ -use core::sync::atomic::{AtomicU32, Ordering}; - -use alloc::{collections::btree_map::BTreeMap, sync::Arc}; -use bitflags::bitflags; -use eonix_sync::{LazyLock, Mutex}; - -use crate::{ - fs::tmpfs::{DirectoryInode, FileInode, TmpFs}, - kernel::{constants::ENOSPC, vfs::inode::Mode}, - prelude::KResult, -}; - -bitflags! { - #[derive(Debug, Clone, Copy)] - pub struct ShmFlags: u32 { - /// Create a new segment. If this flag is not used, then shmget() will - /// find the segment associated with key and check to see if the user - /// has permission to access the segment. - const IPC_CREAT = 0o1000; - /// This flag is used with IPC_CREAT to ensure that this call creates - /// the segment. If the segment already exists, the call fails. - const IPC_EXCL = 0o2000; - - /// Attach the segment for read-only access.If this flag is not specified, - /// the segment is attached for read and write access, and the process - /// must have read and write permission for the segment. - const SHM_RDONLY = 0o10000; - /// round attach address to SHMLBA boundary - const SHM_RND = 0o20000; - /// Allow the contents of the segment to be executed. - const SHM_EXEC = 0o100000; - } -} - -pub const IPC_PRIVATE: usize = 0; - -pub struct ShmManager { - tmpfs: Arc, - root: Arc, - areas: BTreeMap, -} - -#[repr(C)] -#[derive(Default, Clone, Copy, Debug)] -pub struct IpcPerm { - key: i32, - uid: u32, - gid: u32, - cuid: u32, - cgid: u32, - mode: u16, - seq: u16, -} - -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct ShmIdDs { - // Ownership and permissions - pub shm_perm: IpcPerm, - // Size of segment (bytes). In our system, this must be aligned - pub shm_segsz: usize, - // Last attach time - pub shm_atime: usize, - // Last detach time - pub shm_dtime: usize, - // Creation time/time of last modification via shmctl() - pub shm_ctime: usize, - // PID of creator - pub shm_cpid: usize, - // PID of last shmat(2)/shmdt(2) - pub shm_lpid: usize, - // No. of current attaches - pub shm_nattch: usize, -} - -impl ShmIdDs { - fn new(size: usize, pid: u32) -> Self { - Self { - shm_perm: IpcPerm::default(), - shm_segsz: size, - shm_atime: 0, - shm_dtime: 0, - shm_ctime: 0, // Should set instant now - shm_cpid: pid as usize, - shm_lpid: 0, - shm_nattch: 0, - } - } -} - -#[derive(Debug)] -pub struct ShmArea { - pub area: Arc, - pub shmid_ds: ShmIdDs, -} - -// A big lock here to protect the shared memory area. -// Can be improved with finer-grained locking? -pub static SHM_MANAGER: LazyLock> = - LazyLock::new(|| Mutex::new(ShmManager::new())); - -impl ShmManager { - fn new() -> Self { - let (tmpfs, root) = TmpFs::create(false).expect("should create shm_area successfully"); - Self { - tmpfs, - root, - areas: BTreeMap::new(), - } - } - - pub fn create_shared_area(&self, size: usize, pid: u32, mode: Mode) -> ShmArea { - let ino = self.tmpfs.assign_ino(); - let vfs = Arc::downgrade(&self.tmpfs); - ShmArea { - area: FileInode::new(ino, vfs, size, mode), - shmid_ds: ShmIdDs::new(size, pid), - } - } - - pub fn get(&self, shmid: u32) -> Option<&ShmArea> { - self.areas.get(&shmid) - } - - pub fn insert(&mut self, shmid: u32, area: ShmArea) { - self.areas.insert(shmid, area); - } -} - -pub fn gen_shm_id(key: usize) -> KResult { - const SHM_MAGIC: u32 = 114514000; - - static NEXT_SHMID: AtomicU32 = AtomicU32::new(0); - - if key == IPC_PRIVATE { - let shmid = NEXT_SHMID.fetch_add(1, Ordering::Relaxed); - - if shmid >= SHM_MAGIC { - return Err(ENOSPC); - } else { - return Ok(shmid); - } - } - - (key as u32).checked_add(SHM_MAGIC).ok_or(ENOSPC) -} diff --git a/src/fs/tmpfs.rs b/src/fs/tmpfs.rs deleted file mode 100644 index 7a5bd52b..00000000 --- a/src/fs/tmpfs.rs +++ /dev/null @@ -1,613 +0,0 @@ -use crate::io::Stream; -use crate::kernel::constants::{EEXIST, EINVAL, EIO, EISDIR, ENOENT, ENOSYS, ENOTDIR}; -use crate::kernel::mem::{CachePage, CachePageStream, PageCache, PageCacheBackend}; -use crate::kernel::task::block_on; -use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::RenameData; -use crate::kernel::vfs::inode::{AtomicMode, InodeData}; -use crate::{ - io::Buffer, - kernel::vfs::{ - dentry::{dcache, Dentry}, - inode::{define_struct_inode, AtomicIno, Ino, Inode, Mode, WriteOffset}, - mount::{register_filesystem, Mount, MountCreator, MS_RDONLY}, - vfs::Vfs, - DevId, - }, - prelude::*, -}; -use alloc::sync::{Arc, Weak}; -use core::fmt::Debug; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use eonix_mm::paging::PAGE_SIZE; -use eonix_sync::{AsProof as _, AsProofMut as _, Locked, Mutex, ProofMut}; -use itertools::Itertools; - -fn acquire(vfs: &Weak) -> KResult> { - vfs.upgrade().ok_or(EIO) -} - -fn astmp(vfs: &Arc) -> &TmpFs { - vfs.as_any() - .downcast_ref::() - .expect("corrupted tmpfs data structure") -} - -define_struct_inode! { - struct NodeInode { - devid: DevId, - } -} - -impl NodeInode { - fn new(ino: Ino, vfs: Weak, mode: Mode, devid: DevId) -> Arc { - Self::new_locked(ino, vfs, |inode, _| unsafe { - addr_of_mut_field!(inode, devid).write(devid); - - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(mode)); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} - -impl Inode for NodeInode { - fn devid(&self) -> KResult { - Ok(self.devid) - } -} - -define_struct_inode! { - pub(super) struct DirectoryInode { - entries: Locked, Ino)>, ()>, - } -} - -impl DirectoryInode { - fn new(ino: Ino, vfs: Weak, mode: Mode) -> Arc { - Self::new_locked(ino, vfs, |inode, rwsem| unsafe { - addr_of_mut_field!(inode, entries) - .write(Locked::new(vec![(Arc::from(b".".as_slice()), ino)], rwsem)); - - addr_of_mut_field!(&mut *inode, size).write(1.into()); - addr_of_mut_field!(&mut *inode, mode) - .write(AtomicMode::from(Mode::DIR.perm(mode.non_format_bits()))); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); // link from `.` to itself - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } - - fn link(&self, name: Arc<[u8]>, file: &dyn Inode, dlock: ProofMut<'_, ()>) { - let now = Instant::now(); - - // SAFETY: Only `unlink` will do something based on `nlink` count - // No need to synchronize here - file.nlink.fetch_add(1, Ordering::Relaxed); - *self.ctime.lock() = now; - - // SAFETY: `rwsem` has done the synchronization - self.size.fetch_add(1, Ordering::Relaxed); - *self.mtime.lock() = now; - - self.entries.access_mut(dlock).push((name, file.ino)); - } - - fn do_unlink( - &self, - file: &Arc, - filename: &[u8], - entries: &mut Vec<(Arc<[u8]>, Ino)>, - now: Instant, - decrease_size: bool, - _dir_lock: ProofMut<()>, - _file_lock: ProofMut<()>, - ) -> KResult<()> { - // SAFETY: `file_lock` has done the synchronization - if file.mode.load().is_dir() { - return Err(EISDIR); - } - - entries.retain(|(name, ino)| *ino != file.ino || name.as_ref() != filename); - - if decrease_size { - // SAFETY: `dir_lock` has done the synchronization - self.size.fetch_sub(1, Ordering::Relaxed); - } - - *self.mtime.lock() = now; - - // The last reference to the inode is held by some dentry - // and will be released when the dentry is released - - // SAFETY: `file_lock` has done the synchronization - file.nlink.fetch_sub(1, Ordering::Relaxed); - *file.ctime.lock() = now; - - Ok(()) - } -} - -impl Inode for DirectoryInode { - fn do_readdir( - &self, - offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult>, - ) -> KResult { - let lock = block_on(self.rwsem.read()); - self.entries - .access(lock.prove()) - .iter() - .skip(offset) - .map(|(name, ino)| callback(&name, *ino)) - .take_while(|result| result.map_or(true, |flow| flow.is_continue())) - .take_while_inclusive(|result| result.is_ok()) - .fold_ok(0, |acc, _| acc + 1) - } - - fn creat(&self, at: &Arc, mode: Mode) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = FileInode::new(ino, self.vfs.clone(), 0, mode); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_reg(file) - } - - fn mknod(&self, at: &Dentry, mode: Mode, dev: DevId) -> KResult<()> { - if !mode.is_chr() && !mode.is_blk() { - return Err(EINVAL); - } - - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = NodeInode::new(ino, self.vfs.clone(), mode, dev); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_reg(file) - } - - fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = SymlinkInode::new(ino, self.vfs.clone(), target.into()); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_symlink(file) - } - - fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let newdir = DirectoryInode::new(ino, self.vfs.clone(), mode); - - self.link(at.get_name(), newdir.as_ref(), rwsem.prove_mut()); - at.save_dir(newdir) - } - - fn unlink(&self, at: &Arc) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - - let dir_lock = block_on(self.rwsem.write()); - - let file = at.get_inode()?; - let filename = at.get_name(); - let file_lock = block_on(file.rwsem.write()); - - let entries = self.entries.access_mut(dir_lock.prove_mut()); - - self.do_unlink( - &file, - &filename, - entries, - Instant::now(), - true, - dir_lock.prove_mut(), - file_lock.prove_mut(), - )?; - - // Remove the dentry from the dentry cache immediately - // so later lookup will fail with ENOENT - dcache::d_remove(at); - - Ok(()) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - let _lock = block_on(self.rwsem.write()); - - // SAFETY: `rwsem` has done the synchronization - let old = self.mode.load(); - self.mode.store(old.perm(mode.non_format_bits())); - *self.ctime.lock() = Instant::now(); - - Ok(()) - } - - fn rename(&self, rename_data: RenameData) -> KResult<()> { - let RenameData { - old_dentry, - new_dentry, - new_parent, - is_exchange, - no_replace, - vfs, - } = rename_data; - - if is_exchange { - println_warn!("TmpFs does not support exchange rename for now"); - return Err(ENOSYS); - } - - let vfs = vfs - .as_any() - .downcast_ref::() - .expect("vfs must be a TmpFs"); - - let _rename_lock = block_on(vfs.rename_lock.lock()); - - let old_file = old_dentry.get_inode()?; - let new_file = new_dentry.get_inode(); - - if no_replace && new_file.is_ok() { - return Err(EEXIST); - } - - let same_parent = Arc::as_ptr(&new_parent) == &raw const *self; - if same_parent { - // Same directory rename - // Remove from old location and add to new location - let parent_lock = block_on(self.rwsem.write()); - let entries = self.entries.access_mut(parent_lock.prove_mut()); - - fn rename_old( - old_entry: &mut (Arc<[u8]>, Ino), - old_file: &Arc, - new_dentry: &Arc, - now: Instant, - ) { - let (name, _) = old_entry; - *name = new_dentry.get_name(); - *old_file.ctime.lock() = now; - } - - let old_ino = old_file.ino; - let new_ino = new_file.as_ref().ok().map(|f| f.ino); - let old_name = old_dentry.get_name(); - let new_name = new_dentry.get_name(); - - // Find the old and new entries in the directory after we've locked the directory. - let indices = - entries - .iter() - .enumerate() - .fold([None, None], |[old, new], (idx, (name, ino))| { - if Some(*ino) == new_ino && *name == new_name { - [old, Some(idx)] - } else if *ino == old_ino && *name == old_name { - [Some(idx), new] - } else { - [old, new] - } - }); - - let (old_entry_idx, new_entry_idx) = match indices { - [None, ..] => return Err(ENOENT), - [Some(old_idx), new_idx] => (old_idx, new_idx), - }; - - let now = Instant::now(); - - if let Some(new_idx) = new_entry_idx { - // Replace existing file (i.e. rename the old and unlink the new) - let new_file = new_file.unwrap(); - let _new_file_lock = block_on(new_file.rwsem.write()); - - // SAFETY: `new_file_lock` has done the synchronization - match (new_file.mode.load(), old_file.mode.load()) { - (Mode::DIR, _) => return Err(EISDIR), - (_, Mode::DIR) => return Err(ENOTDIR), - _ => {} - } - - entries.remove(new_idx); - - // SAFETY: `parent_lock` has done the synchronization - self.size.fetch_sub(1, Ordering::Relaxed); - - // The last reference to the inode is held by some dentry - // and will be released when the dentry is released - - // SAFETY: `new_file_lock` has done the synchronization - new_file.nlink.fetch_sub(1, Ordering::Relaxed); - *new_file.ctime.lock() = now; - } - - rename_old(&mut entries[old_entry_idx], &old_file, new_dentry, now); - *self.mtime.lock() = now; - } else { - // Cross-directory rename - handle similar to same directory case - - // Get new parent directory - let new_parent_inode = new_dentry.parent().get_inode()?; - assert!(new_parent_inode.is_dir()); - let new_parent = (new_parent_inode.as_ref() as &dyn Any) - .downcast_ref::() - .expect("new parent must be a DirectoryInode"); - - let old_parent_lock = block_on(self.rwsem.write()); - let new_parent_lock = block_on(new_parent_inode.rwsem.write()); - - let old_ino = old_file.ino; - let new_ino = new_file.as_ref().ok().map(|f| f.ino); - let old_name = old_dentry.get_name(); - let new_name = new_dentry.get_name(); - - // Find the old entry in the old directory - let old_entries = self.entries.access_mut(old_parent_lock.prove_mut()); - let old_pos = old_entries - .iter() - .position(|(name, ino)| *ino == old_ino && *name == old_name) - .ok_or(ENOENT)?; - - // Find the new entry in the new directory (if it exists) - let new_entries = new_parent.entries.access_mut(new_parent_lock.prove_mut()); - let has_new = new_entries - .iter() - .position(|(name, ino)| Some(*ino) == new_ino && *name == new_name) - .is_some(); - - let now = Instant::now(); - - if has_new { - // Replace existing file (i.e. move the old and unlink the new) - let new_file = new_file.unwrap(); - let new_file_lock = block_on(new_file.rwsem.write()); - - match (old_file.mode.load(), new_file.mode.load()) { - (Mode::DIR, Mode::DIR) => {} - (Mode::DIR, _) => return Err(ENOTDIR), - (_, _) => {} - } - - // Unlink the old file that was replaced - new_parent.do_unlink( - &new_file, - &new_name, - new_entries, - now, - false, - new_parent_lock.prove_mut(), - new_file_lock.prove_mut(), - )?; - } else { - new_parent.size.fetch_add(1, Ordering::Relaxed); - } - - // Remove from old directory - old_entries.remove(old_pos); - - // Add new entry - new_entries.push((new_name, old_ino)); - - self.size.fetch_sub(1, Ordering::Relaxed); - *self.mtime.lock() = now; - *old_file.ctime.lock() = now; - } - - block_on(dcache::d_exchange(old_dentry, new_dentry)); - - Ok(()) - } -} - -define_struct_inode! { - struct SymlinkInode { - target: Arc<[u8]>, - } -} - -impl SymlinkInode { - fn new(ino: Ino, vfs: Weak, target: Arc<[u8]>) -> Arc { - Self::new_locked(ino, vfs, |inode, _| unsafe { - let len = target.len(); - addr_of_mut_field!(inode, target).write(target); - - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(Mode::LNK.perm(0o777))); - addr_of_mut_field!(&mut *inode, size).write((len as u64).into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} - -impl Inode for SymlinkInode { - fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { - buffer - .fill(self.target.as_ref()) - .map(|result| result.allow_partial()) - } - - fn chmod(&self, _: Mode) -> KResult<()> { - Ok(()) - } -} - -define_struct_inode! { - pub struct FileInode { - pages: PageCache, - } -} - -impl Debug for FileInode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "FileInode({:?})", self.idata) - } -} - -impl FileInode { - pub fn new(ino: Ino, vfs: Weak, size: usize, mode: Mode) -> Arc { - let inode = Arc::new_cyclic(|weak_self: &Weak| FileInode { - idata: InodeData::new(ino, vfs), - pages: PageCache::new(weak_self.clone()), - }); - - inode.mode.store(Mode::REG.perm(mode.non_format_bits())); - inode.nlink.store(1, Ordering::Relaxed); - inode.size.store(size as u64, Ordering::Relaxed); - inode - } -} - -impl PageCacheBackend for FileInode { - fn read_page(&self, _cache_page: &mut CachePage, _offset: usize) -> KResult { - Ok(PAGE_SIZE) - } - - fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult { - Ok(PAGE_SIZE) - } - - fn size(&self) -> usize { - self.size.load(Ordering::Relaxed) as usize - } -} - -impl Inode for FileInode { - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.pages) - } - - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - let _lock = block_on(self.rwsem.write()); - block_on(self.pages.read(buffer, offset)) - } - - fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult { - // TODO: We don't need that strong guarantee, find some way to avoid locks - let _lock = block_on(self.rwsem.write()); - - let mut store_new_end = None; - let offset = match offset { - WriteOffset::Position(offset) => offset, - WriteOffset::End(end) => { - store_new_end = Some(end); - - // SAFETY: `lock` has done the synchronization - self.size.load(Ordering::Relaxed) as usize - } - }; - - let wrote = block_on(self.pages.write(stream, offset))?; - let cursor_end = offset + wrote; - - if let Some(store_end) = store_new_end { - *store_end = cursor_end; - } - - // SAFETY: `lock` has done the synchronization - *self.mtime.lock() = Instant::now(); - self.size.store(cursor_end as u64, Ordering::Relaxed); - - Ok(wrote) - } - - fn truncate(&self, length: usize) -> KResult<()> { - let _lock = block_on(self.rwsem.write()); - block_on(self.pages.resize(length))?; - self.size.store(length as u64, Ordering::Relaxed); - *self.mtime.lock() = Instant::now(); - Ok(()) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - let _lock = block_on(self.rwsem.write()); - - // SAFETY: `rwsem` has done the synchronization - let old = self.mode.load(); - self.mode.store(old.perm(mode.non_format_bits())); - *self.ctime.lock() = Instant::now(); - - Ok(()) - } -} - -impl_any!(TmpFs); -pub(super) struct TmpFs { - next_ino: AtomicIno, - readonly: bool, - rename_lock: Mutex<()>, -} - -impl Vfs for TmpFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - 2 - } - - fn is_read_only(&self) -> bool { - self.readonly - } -} - -impl TmpFs { - pub(super) fn assign_ino(&self) -> Ino { - self.next_ino.fetch_add(1, Ordering::AcqRel) - } - - pub fn create(readonly: bool) -> KResult<(Arc, Arc)> { - let tmpfs = Arc::new(Self { - next_ino: AtomicIno::new(1), - readonly, - rename_lock: Mutex::new(()), - }); - - let weak = Arc::downgrade(&tmpfs); - let root_dir = DirectoryInode::new(0, weak, Mode::new(0o755)); - - Ok((tmpfs, root_dir)) - } -} - -struct TmpFsMountCreator; - -impl MountCreator for TmpFsMountCreator { - fn create_mount(&self, _source: &str, flags: u64, mp: &Arc) -> KResult { - let (fs, root_inode) = TmpFs::create(flags & MS_RDONLY != 0)?; - - Mount::new(mp, fs, root_inode) - } - - fn check_signature(&self, _: &[u8]) -> KResult { - Ok(true) - } -} - -pub fn init() { - register_filesystem("tmpfs", Arc::new(TmpFsMountCreator)).unwrap(); -} diff --git a/src/fs/tmpfs/dir.rs b/src/fs/tmpfs/dir.rs new file mode 100644 index 00000000..e2be1d12 --- /dev/null +++ b/src/fs/tmpfs/dir.rs @@ -0,0 +1,415 @@ +use core::{any::Any, future::Future}; + +use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; +use eonix_log::println_warn; +use eonix_sync::{LazyLock, RwLock, Spin}; + +use crate::{ + kernel::{ + constants::{EEXIST, EINVAL, EISDIR, ENOENT, ENOSYS, ENOTDIR}, + mem::PageCache, + timer::Instant, + vfs::{ + dentry::{dcache, Dentry}, + inode::{ + Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, RenameData, + }, + types::{DeviceId, Format, Mode, Permission}, + SbRef, + }, + }, + prelude::KResult, +}; + +use super::{ + file::{DeviceInode, FileInode, SymlinkInode}, + TmpFs, +}; + +pub struct DirectoryInode { + sb: SbRef, + ino: Ino, + info: Spin, + entries: RwLock, Ino)>>, +} + +impl InodeOps for DirectoryInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + Format::DIR + } + + fn info(&self) -> &Spin { + &self.info + } + + fn super_block(&self) -> &SbRef { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None + } +} + +impl DirectoryInode { + pub fn new(ino: Ino, sb: SbRef, perm: Permission) -> InodeUse { + static DOT: LazyLock> = LazyLock::new(|| Arc::from(b".".as_slice())); + + let now = Instant::now(); + + InodeUse::new(Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: 1, + nlink: 1, // link from `.` to itself + perm, + ctime: now, + mtime: now, + atime: now, + uid: 0, + gid: 0, + }), + entries: RwLock::new(vec![(DOT.clone(), ino)]), + }) + } + + fn link( + &self, + entries: &mut Vec<(Arc<[u8]>, Ino)>, + name: Arc<[u8]>, + file: &InodeUse, + ) { + let mut self_info = self.info.lock(); + let mut file_info = file.info().lock(); + + let now = Instant::now(); + + file_info.nlink += 1; + file_info.ctime = now; + + self_info.size += 1; + self_info.mtime = now; + self_info.ctime = now; + + entries.push((name, file.ino())); + } + + fn do_unlink( + &self, + file: &InodeUse, + filename: &[u8], + entries: &mut Vec<(Arc<[u8]>, Ino)>, + now: Instant, + decrease_size: bool, + self_info: &mut InodeInfo, + file_info: &mut InodeInfo, + ) -> KResult<()> { + // SAFETY: `file_lock` has done the synchronization + if file.format() == Format::DIR { + return Err(EISDIR); + } + + let file_ino = file.ino(); + entries.retain(|(name, ino)| *ino != file_ino || name.as_ref() != filename); + + if decrease_size { + self_info.size -= 1; + } + + self_info.mtime = now; + self_info.ctime = now; + + // The last reference to the inode is held by some dentry + // and will be released when the dentry is released + + file_info.nlink -= 1; + file_info.ctime = now; + + // TODO!!!: Remove the file if nlink == 1 + + Ok(()) + } +} + +impl InodeDirOps for DirectoryInode { + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> impl Future>> + Send + 'r { + Box::pin(async move { + let _sb = self.sb.get()?; + let entries = self.entries.read().await; + + let mut count = 0; + for entry in entries.iter().skip(offset) { + match for_each_entry(&entry.0, entry.1) { + Err(err) => return Ok(Err(err)), + Ok(false) => break, + Ok(true) => count += 1, + } + } + + Ok(Ok(count)) + }) + } + + async fn create(&self, at: &Arc, perm: Permission) -> KResult<()> { + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file: InodeUse = FileInode::new(ino, self.sb.clone(), 0, perm); + + self.link(&mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + if !mode.is_chr() && !mode.is_blk() { + return Err(EINVAL); + } + + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file: InodeUse = DeviceInode::new(ino, self.sb.clone(), mode, dev); + + self.link(&mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()> { + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file: InodeUse = SymlinkInode::new(ino, self.sb.clone(), target.into()); + + self.link(&mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { + let sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let new_dir: InodeUse = DirectoryInode::new(ino, self.sb.clone(), perm); + + self.link(&mut entries, at.get_name(), &new_dir); + at.fill(new_dir); + + Ok(()) + } + + async fn unlink(&self, at: &Arc) -> KResult<()> { + let _sb = self.sb.get()?; + let mut entries = self.entries.write().await; + + let file = at.get_inode()?; + let filename = at.get_name(); + + self.do_unlink( + &file, + &filename, + &mut entries, + Instant::now(), + true, + &mut self.info.lock(), + &mut file.info().lock(), + )?; + + // Remove the dentry from the dentry cache immediately + // so later lookup will fail with ENOENT + dcache::d_remove(at); + + Ok(()) + } + + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { + let sb = self.sb.get()?; + let _rename_lock = sb.backend.rename_lock.lock().await; + let mut self_entries = self.entries.write().await; + + let RenameData { + old_dentry, + new_dentry, + new_parent, + is_exchange, + no_replace, + } = rename_data; + + if is_exchange { + println_warn!("TmpFs does not support exchange rename for now"); + return Err(ENOSYS); + } + + let old_file = old_dentry.get_inode()?; + let new_file = new_dentry.inode(); + + if no_replace && new_file.is_some() { + return Err(EEXIST); + } + + if new_parent.as_raw() == &raw const *self { + // Same directory rename + // Remove from old location and add to new location + let old_ino = old_file.ino(); + let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_name = old_dentry.get_name(); + let new_name = new_dentry.get_name(); + + // Find the old and new entries in the directory after we've locked the directory. + let (mut old_ent_idx, mut new_ent_idx) = (None, None); + for (idx, (name, ino)) in self_entries.iter().enumerate() { + if *ino == old_ino && *name == old_name { + old_ent_idx = Some(idx); + } + + if Some(*ino) == new_ino && *name == new_name { + new_ent_idx = Some(idx); + } + } + + let Some(old_ent_idx) = old_ent_idx else { + return Err(ENOENT); + }; + + if Some(old_ent_idx) == new_ent_idx { + return Ok(()); + } + + let now = Instant::now(); + if let Some(new_idx) = new_ent_idx { + // Replace existing file (i.e. rename the old and unlink the new) + let new_file = new_file.unwrap(); + + match (new_file.format(), old_file.format()) { + (Format::DIR, _) => return Err(EISDIR), + (_, Format::DIR) => return Err(ENOTDIR), + _ => {} + } + + self_entries.remove(new_idx); + + self.info.lock().size -= 1; + + // The last reference to the inode is held by some dentry + // and will be released when the dentry is released + + let mut new_info = new_file.info().lock(); + + new_info.nlink -= 1; + new_info.mtime = now; + new_info.ctime = now; + } + + let (name, _) = &mut self_entries[old_ent_idx]; + *name = new_dentry.get_name(); + + let mut self_info = self.info.lock(); + self_info.mtime = now; + self_info.ctime = now; + } else { + // Cross-directory rename - handle similar to same directory case + + // Get new parent directory + let new_parent_inode = new_dentry.parent().get_inode()?; + assert_eq!(new_parent_inode.format(), Format::DIR); + + let new_parent = (&new_parent_inode as &dyn Any) + .downcast_ref::() + .expect("new parent must be a DirectoryInode"); + + let mut new_entries = new_parent.entries.write().await; + + let old_ino = old_file.ino(); + let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_name = old_dentry.get_name(); + let new_name = new_dentry.get_name(); + + // Find the old entry in the old directory + let old_pos = self_entries + .iter() + .position(|(name, ino)| *ino == old_ino && *name == old_name) + .ok_or(ENOENT)?; + + // Find the new entry in the new directory (if it exists) + let has_new = new_entries + .iter() + .position(|(name, ino)| Some(*ino) == new_ino && *name == new_name) + .is_some(); + + let now = Instant::now(); + + if has_new { + // Replace existing file (i.e. move the old and unlink the new) + let new_file = new_file.unwrap(); + + match (old_file.format(), new_file.format()) { + (Format::DIR, Format::DIR) => {} + (Format::DIR, _) => return Err(ENOTDIR), + (_, _) => {} + } + + // Unlink the old file that was replaced + new_parent.do_unlink( + &new_file, + &new_name, + &mut new_entries, + now, + false, + &mut new_parent.info.lock(), + &mut new_file.info().lock(), + )?; + } else { + new_parent.info.lock().size += 1; + new_parent.info.lock().mtime = now; + new_parent.info.lock().ctime = now; + } + + // Remove from old directory + self_entries.remove(old_pos); + + // Add new entry + new_entries.push((new_name, old_ino)); + + let mut self_info = self.info.lock(); + self_info.size -= 1; + self_info.mtime = now; + self_info.ctime = now; + } + + dcache::d_exchange(old_dentry, new_dentry).await; + Ok(()) + } +} + +impl InodeFileOps for DirectoryInode { + async fn chmod(&self, perm: Permission) -> KResult<()> { + let _sb = self.sb.get()?; + + { + let mut info = self.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); + } + + Ok(()) + } +} diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs new file mode 100644 index 00000000..624112e0 --- /dev/null +++ b/src/fs/tmpfs/file.rs @@ -0,0 +1,298 @@ +use alloc::sync::Arc; +use eonix_mm::paging::PAGE_SIZE; +use eonix_sync::{RwLock, Spin}; + +use crate::{ + io::{Buffer, Stream}, + kernel::{ + mem::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}, + timer::Instant, + vfs::{ + inode::{Ino, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, WriteOffset}, + types::{DeviceId, Format, Mode, Permission}, + SbRef, + }, + }, + prelude::KResult, +}; + +use super::TmpFs; + +pub struct FileInode { + sb: SbRef, + ino: Ino, + info: Spin, + rwsem: RwLock<()>, + pages: PageCache, +} + +impl FileInode { + pub fn new(ino: Ino, sb: SbRef, size: usize, perm: Permission) -> InodeUse { + let now = Instant::now(); + + InodeUse::new_cyclic(|weak| Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: size as _, + nlink: 1, + uid: 0, + gid: 0, + perm, + atime: now, + ctime: now, + mtime: now, + }), + rwsem: RwLock::new(()), + pages: PageCache::new(weak.clone() as _), + }) + } +} + +impl PageCacheBackendOps for FileInode { + async fn read_page(&self, _cache_page: &mut CachePage, _offset: usize) -> KResult { + Ok(PAGE_SIZE) + } + + async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult { + Ok(PAGE_SIZE) + } + + fn size(&self) -> usize { + self.info.lock().size as usize + } +} + +impl InodeOps for FileInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + Format::REG + } + + fn info(&self) -> &Spin { + &self.info + } + + fn super_block(&self) -> &SbRef { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + Some(&self.pages) + } +} + +impl InodeDirOps for FileInode {} +impl InodeFileOps for FileInode { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + let _lock = self.rwsem.read().await; + self.pages.read(buffer, offset).await + } + + async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult { + let _lock = self.rwsem.write().await; + + let mut store_new_end = None; + let offset = match offset { + WriteOffset::Position(offset) => offset, + WriteOffset::End(end) => { + store_new_end = Some(end); + + // `info.size` won't change since we are holding the write lock. + self.info.lock().size as usize + } + }; + + let wrote = self.pages.write(stream, offset).await?; + let cursor_end = offset + wrote; + + if let Some(store_end) = store_new_end { + *store_end = cursor_end; + } + + { + let now = Instant::now(); + let mut info = self.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = info.size.max(cursor_end as u64); + } + + Ok(wrote) + } + + async fn truncate(&self, length: usize) -> KResult<()> { + let _lock = self.rwsem.write().await; + + self.pages.resize(length).await?; + + { + let now = Instant::now(); + let mut info = self.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = length as u64; + } + + Ok(()) + } + + async fn chmod(&self, perm: Permission) -> KResult<()> { + let _sb = self.sb.get()?; + + { + let mut info = self.info.lock(); + + info.perm = perm; + info.ctime = Instant::now(); + } + + Ok(()) + } +} + +pub struct DeviceInode { + sb: SbRef, + ino: Ino, + info: Spin, + is_block: bool, + devid: DeviceId, +} + +impl DeviceInode { + pub fn new(ino: Ino, sb: SbRef, mode: Mode, devid: DeviceId) -> InodeUse { + let now = Instant::now(); + + InodeUse::new(Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(mode.non_format_bits()), + atime: now, + ctime: now, + mtime: now, + }), + is_block: mode.format() == Format::BLK, + devid, + }) + } +} + +impl InodeOps for DeviceInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + if self.is_block { + Format::BLK + } else { + Format::CHR + } + } + + fn info(&self) -> &Spin { + &self.info + } + + fn super_block(&self) -> &SbRef { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None + } +} + +impl InodeDirOps for DeviceInode {} +impl InodeFileOps for DeviceInode { + async fn chmod(&self, perm: Permission) -> KResult<()> { + let _sb = self.sb.get()?; + + { + let mut info = self.info.lock(); + + info.perm = perm; + info.ctime = Instant::now(); + } + + Ok(()) + } + + fn devid(&self) -> KResult { + Ok(self.devid) + } +} + +pub struct SymlinkInode { + sb: SbRef, + ino: Ino, + info: Spin, + target: Arc<[u8]>, +} + +impl SymlinkInode { + pub fn new(ino: Ino, sb: SbRef, target: Arc<[u8]>) -> InodeUse { + let now = Instant::now(); + + InodeUse::new(Self { + sb, + ino, + info: Spin::new(InodeInfo { + size: target.len() as _, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: now, + ctime: now, + mtime: now, + }), + target, + }) + } +} + +impl InodeDirOps for SymlinkInode {} +impl InodeOps for SymlinkInode { + type SuperBlock = TmpFs; + + fn ino(&self) -> Ino { + self.ino + } + + fn format(&self) -> Format { + Format::LNK + } + + fn info(&self) -> &Spin { + &self.info + } + + fn super_block(&self) -> &SbRef { + &self.sb + } + + fn page_cache(&self) -> Option<&PageCache> { + None + } +} + +impl InodeFileOps for SymlinkInode { + async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { + buffer + .fill(self.target.as_ref()) + .map(|result| result.allow_partial()) + } +} diff --git a/src/fs/tmpfs/mod.rs b/src/fs/tmpfs/mod.rs new file mode 100644 index 00000000..2bef67b6 --- /dev/null +++ b/src/fs/tmpfs/mod.rs @@ -0,0 +1,73 @@ +mod dir; +mod file; + +use crate::kernel::vfs::inode::{Ino, InodeUse}; +use crate::kernel::vfs::types::{DeviceId, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::{ + kernel::vfs::{ + dentry::Dentry, + mount::{register_filesystem, Mount, MountCreator}, + }, + prelude::*, +}; +use alloc::sync::Arc; +use async_trait::async_trait; +use core::sync::atomic::AtomicU64; +use core::sync::atomic::Ordering; +use dir::DirectoryInode; +use eonix_sync::Mutex; + +pub struct TmpFs { + next_ino: AtomicU64, + rename_lock: Mutex<()>, +} + +impl SuperBlock for TmpFs {} + +impl TmpFs { + fn assign_ino(&self) -> Ino { + Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) + } + + fn create() -> KResult<(SbUse, InodeUse)> { + let tmpfs = SbUse::new( + SuperBlockInfo { + io_blksize: 4096, + device_id: DeviceId::new(0, 2), + read_only: false, + }, + Self { + next_ino: AtomicU64::new(1), + rename_lock: Mutex::new(()), + }, + ); + + let root_dir = DirectoryInode::new( + tmpfs.backend.assign_ino(), + SbRef::from(&tmpfs), + Permission::new(0o755), + ); + + Ok((tmpfs, root_dir)) + } +} + +struct TmpFsMountCreator; + +#[async_trait] +impl MountCreator for TmpFsMountCreator { + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc) -> KResult { + let (fs, root_inode) = TmpFs::create()?; + + Mount::new(mp, fs, root_inode) + } + + fn check_signature(&self, _: &[u8]) -> KResult { + Ok(true) + } +} + +pub fn init() { + register_filesystem("tmpfs", Arc::new(TmpFsMountCreator)).unwrap(); +} diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 349e3656..3e4b65d1 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -3,7 +3,7 @@ mod mbr; use super::{ constants::ENOENT, mem::{paging::Page, AsMemoryBlock as _}, - vfs::DevId, + vfs::types::DeviceId, }; use crate::kernel::constants::{EEXIST, EINVAL}; use crate::{ @@ -14,13 +14,10 @@ use alloc::{ collections::btree_map::{BTreeMap, Entry}, sync::Arc, }; +use async_trait::async_trait; use core::cmp::Ordering; use mbr::MBRPartTable; -pub fn make_device(major: u32, minor: u32) -> DevId { - (major << 8) & 0xff00u32 | minor & 0xffu32 -} - pub struct Partition { pub lba_offset: u64, pub sector_count: u64, @@ -30,11 +27,12 @@ pub trait PartTable { fn partitions(&self) -> impl Iterator + use<'_, Self>; } +#[async_trait] pub trait BlockRequestQueue: Send + Sync { /// Maximum number of sectors that can be read in one request fn max_request_pages(&self) -> u64; - fn submit(&self, req: BlockDeviceRequest) -> KResult<()>; + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()>; } enum BlockDeviceType { @@ -42,7 +40,7 @@ enum BlockDeviceType { queue: Arc, }, Partition { - disk_dev: DevId, + disk_dev: DeviceId, lba_offset: u64, queue: Arc, }, @@ -50,7 +48,7 @@ enum BlockDeviceType { pub struct BlockDevice { /// Unique device identifier, major and minor numbers - devid: DevId, + devid: DeviceId, /// Total size of the device in sectors (512 bytes each) sector_count: u64, @@ -77,11 +75,11 @@ impl Ord for BlockDevice { } } -static BLOCK_DEVICE_LIST: Spin>> = Spin::new(BTreeMap::new()); +static BLOCK_DEVICE_LIST: Spin>> = Spin::new(BTreeMap::new()); impl BlockDevice { pub fn register_disk( - devid: DevId, + devid: DeviceId, size: u64, queue: Arc, ) -> KResult> { @@ -97,13 +95,13 @@ impl BlockDevice { } } - pub fn get(devid: DevId) -> KResult> { + pub fn get(devid: DeviceId) -> KResult> { BLOCK_DEVICE_LIST.lock().get(&devid).cloned().ok_or(ENOENT) } } impl BlockDevice { - pub fn devid(&self) -> DevId { + pub fn devid(&self) -> DeviceId { self.devid } @@ -121,7 +119,7 @@ impl BlockDevice { }; let device = Arc::new(BlockDevice { - devid: make_device(self.devid >> 8, (self.devid & 0xff) + idx as u32 + 1), + devid: DeviceId::new(self.devid.major, self.devid.minor + idx as u16 + 1), sector_count: size, dev_type: BlockDeviceType::Partition { disk_dev: self.devid, @@ -159,7 +157,7 @@ impl BlockDevice { /// - `req.sector` must be within the disk size /// - `req.buffer` must be enough to hold the data /// - pub fn commit_request(&self, mut req: BlockDeviceRequest) -> KResult<()> { + pub async fn commit_request(&self, mut req: BlockDeviceRequest<'_>) -> KResult<()> { // Verify the request parameters. match &mut req { BlockDeviceRequest::Read { sector, count, .. } => { @@ -184,7 +182,7 @@ impl BlockDevice { } } - self.queue().submit(req) + self.queue().submit(req).await } /// Read some from the block device, may involve some copy and fragmentation @@ -194,7 +192,7 @@ impl BlockDevice { /// # Arguments /// `offset` - offset in bytes /// - pub fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult { + pub async fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult { let mut sector_start = offset as u64 / 512; let mut first_sector_offset = offset as u64 % 512; let mut sector_count = (first_sector_offset + buffer.total() as u64 + 511) / 512; @@ -241,7 +239,7 @@ impl BlockDevice { buffer: &pages, }; - self.commit_request(req)?; + self.commit_request(req).await?; for page in pages.iter() { // SAFETY: We are the only owner of the page so no one could be mutating it. @@ -277,7 +275,7 @@ impl BlockDevice { /// `offset` - offset in bytes /// `data` - data to write /// - pub fn write_some(&self, offset: usize, data: &[u8]) -> KResult { + pub async fn write_some(&self, offset: usize, data: &[u8]) -> KResult { let mut sector_start = offset as u64 / 512; let mut first_sector_offset = offset as u64 % 512; let mut remaining_data = data; @@ -320,7 +318,7 @@ impl BlockDevice { count: sector_count, buffer: pages, }; - self.commit_request(read_req)?; + self.commit_request(read_req).await?; } let mut data_offset = 0; @@ -356,7 +354,7 @@ impl BlockDevice { count: sector_count, buffer: pages, }; - self.commit_request(write_req)?; + self.commit_request(write_req).await?; let bytes_written = data_offset; nwritten += bytes_written; diff --git a/src/kernel/block/mbr.rs b/src/kernel/block/mbr.rs index 74cdc36e..c5820679 100644 --- a/src/kernel/block/mbr.rs +++ b/src/kernel/block/mbr.rs @@ -31,7 +31,7 @@ pub struct MBRPartTable { impl MBRPartTable { pub async fn from_disk(disk: &BlockDevice) -> KResult { let mut mbr: UninitBuffer = UninitBuffer::new(); - disk.read_some(0, &mut mbr)?.ok_or(EIO)?; + disk.read_some(0, &mut mbr).await?.ok_or(EIO)?; let mbr = mbr.assume_init()?; if mbr.magic != [0x55, 0xaa] { diff --git a/src/kernel/chardev.rs b/src/kernel/chardev.rs index aff3271e..4e01d83a 100644 --- a/src/kernel/chardev.rs +++ b/src/kernel/chardev.rs @@ -1,10 +1,9 @@ use super::{ - block::make_device, console::get_console, constants::{EEXIST, EIO}, task::{block_on, ProcessList, Thread}, terminal::Terminal, - vfs::{DevId, File, FileType, TerminalFile}, + vfs::{types::DeviceId, File, FileType, TerminalFile}, }; use crate::{ io::{Buffer, Stream, StreamRead}, @@ -34,7 +33,7 @@ pub struct CharDevice { device: CharDeviceType, } -static CHAR_DEVICES: Spin>> = Spin::new(BTreeMap::new()); +static CHAR_DEVICES: Spin>> = Spin::new(BTreeMap::new()); impl CharDevice { pub fn read(&self, buffer: &mut dyn Buffer) -> KResult { @@ -54,11 +53,11 @@ impl CharDevice { } } - pub fn get(devid: DevId) -> Option> { + pub fn get(devid: DeviceId) -> Option> { CHAR_DEVICES.lock().get(&devid).cloned() } - pub fn register(devid: DevId, name: Arc, device: CharDeviceType) -> KResult<()> { + pub fn register(devid: DeviceId, name: Arc, device: CharDeviceType) -> KResult<()> { match CHAR_DEVICES.lock().entry(devid) { Entry::Vacant(entry) => { entry.insert(Arc::new(CharDevice { name, device })); @@ -134,19 +133,19 @@ impl VirtualCharDevice for ConsoleDevice { impl CharDevice { pub fn init() -> KResult<()> { Self::register( - make_device(1, 3), + DeviceId::new(1, 3), Arc::from("null"), CharDeviceType::Virtual(Box::new(NullDevice)), )?; Self::register( - make_device(1, 5), + DeviceId::new(1, 5), Arc::from("zero"), CharDeviceType::Virtual(Box::new(ZeroDevice)), )?; Self::register( - make_device(5, 1), + DeviceId::new(5, 1), Arc::from("console"), CharDeviceType::Virtual(Box::new(ConsoleDevice)), )?; diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index efd06824..c147306e 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -12,5 +12,5 @@ pub use access::{AsMemoryBlock, MemoryBlock, PhysAccess}; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; -pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackend}; +pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}; pub use paging::{Page, PageBuffer}; diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 662000ba..5446ae42 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,24 +1,15 @@ -use core::fmt::Debug; - -use crate::kernel::vfs::inode::Inode; -use alloc::sync::Arc; +use crate::kernel::vfs::inode::{Inode, InodeUse}; use eonix_mm::paging::PAGE_SIZE; #[derive(Debug, Clone)] pub struct FileMapping { - pub file: Arc, + pub file: InodeUse, /// Offset in the file, aligned to 4KB boundary. pub offset: usize, /// Length of the mapping. Exceeding part will be zeroed. pub length: usize, } -impl Debug for dyn Inode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "Inode()") - } -} - #[derive(Debug, Clone)] pub enum Mapping { // private anonymous memory @@ -28,7 +19,7 @@ pub enum Mapping { } impl FileMapping { - pub fn new(file: Arc, offset: usize, length: usize) -> Self { + pub fn new(file: InodeUse, offset: usize, length: usize) -> Self { assert_eq!(offset & (PAGE_SIZE - 1), 0); Self { file, diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 3ccf3255..9deb50cf 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -6,8 +6,10 @@ use crate::{ GlobalPageAlloc, }; use align_ext::AlignExt; +use alloc::boxed::Box; use alloc::{collections::btree_map::BTreeMap, sync::Weak}; -use core::mem::ManuallyDrop; +use async_trait::async_trait; +use core::{future::Future, mem::ManuallyDrop}; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::{ address::{PAddr, PhysAccess}, @@ -159,7 +161,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE))?; + .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) + .await?; pages.insert(page_id, new_page); } } @@ -205,7 +208,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE))?; + .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) + .await?; new_page }; @@ -224,7 +228,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .write_page(&mut CachePageStream::new(*page), page_id << PAGE_SIZE_BITS)?; + .write_page(&mut CachePageStream::new(*page), page_id << PAGE_SIZE_BITS) + .await?; page.clear_dirty(); } } @@ -286,7 +291,8 @@ impl PageCache { self.backend .upgrade() .unwrap() - .read_page(&mut new_page, offset_aligin)?; + .read_page(&mut new_page, offset_aligin) + .await?; pages.insert(page_id, new_page); new_page.0 } @@ -349,14 +355,47 @@ impl Stream for CachePageStream { // for fs, offset is file offset (floor algin to PAGE_SIZE) // for blkdev, offset is block idx (floor align to PAGE_SIZE / BLK_SIZE) // Oh no, this would make unnecessary cache -pub trait PageCacheBackend { - fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult; +pub trait PageCacheBackendOps: Sized { + fn read_page( + &self, + page: &mut CachePage, + offset: usize, + ) -> impl Future> + Send; + + fn write_page( + &self, + page: &mut CachePageStream, + offset: usize, + ) -> impl Future> + Send; - fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult; + fn size(&self) -> usize; +} +#[async_trait] +pub trait PageCacheBackend: Send + Sync { + async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult; + async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult; fn size(&self) -> usize; } +#[async_trait] +impl PageCacheBackend for T +where + T: PageCacheBackendOps + Send + Sync + 'static, +{ + async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult { + self.read_page(page, offset).await + } + + async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult { + self.write_page(page, offset).await + } + + fn size(&self) -> usize { + self.size() + } +} + pub trait PageCacheRawPage: RawPage { fn valid_size(&self) -> &mut usize; diff --git a/src/kernel/pcie/driver.rs b/src/kernel/pcie/driver.rs index be88b7df..eebaa896 100644 --- a/src/kernel/pcie/driver.rs +++ b/src/kernel/pcie/driver.rs @@ -4,21 +4,24 @@ use super::{ }; use crate::{kernel::constants::EEXIST, KResult}; use alloc::{ + boxed::Box, collections::btree_map::{self, BTreeMap}, sync::Arc, }; +use async_trait::async_trait; use eonix_sync::Spin; static PCIE_DRIVERS: Spin>> = Spin::new(BTreeMap::new()); +#[async_trait] pub trait PCIDriver: Send + Sync { fn vendor_id(&self) -> u16; fn device_id(&self) -> u16; - fn handle_device(&self, device: Arc>) -> Result<(), PciError>; + async fn handle_device(&self, device: Arc>) -> Result<(), PciError>; } -pub fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { +pub async fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { let index = (driver.vendor_id() as u32) << 16 | driver.device_id() as u32; let driver = Arc::new(driver); @@ -31,7 +34,7 @@ pub fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { let devices = PCIE_DEVICES.lock().get(&index).cloned(); if let Some(devices) = devices { for device in devices { - driver.handle_device(device)?; + driver.handle_device(device).await?; } }; diff --git a/src/kernel/syscall/file_rw.rs b/src/kernel/syscall/file_rw.rs index 1a48b255..db32b0e5 100644 --- a/src/kernel/syscall/file_rw.rs +++ b/src/kernel/syscall/file_rw.rs @@ -7,7 +7,7 @@ use crate::kernel::syscall::UserMut; use crate::kernel::task::Thread; use crate::kernel::timer::sleep; use crate::kernel::vfs::filearray::FD; -use crate::kernel::vfs::inode::Mode; +use crate::kernel::vfs::types::{DeviceId, Mode}; use crate::kernel::vfs::{PollEvent, SeekOption}; use crate::{ io::{Buffer, BufferFill}, @@ -41,7 +41,7 @@ impl FromSyscallArg for AtFlags { } } -fn dentry_from( +async fn dentry_from( thread: &Thread, dirfd: FD, pathname: User, @@ -52,7 +52,7 @@ fn dentry_from( match (path.as_cstr().to_bytes_with_nul()[0], dirfd) { (b'/', _) | (_, FD::AT_FDCWD) => { let path = Path::new(path.as_cstr().to_bytes())?; - Dentry::open(&thread.fs_context, path, follow_symlink) + Dentry::open(&thread.fs_context, path, follow_symlink).await } (0, dirfd) => { let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; @@ -63,7 +63,7 @@ fn dentry_from( let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; let dir_dentry = dir_file.as_path().ok_or(ENOTDIR)?; - Dentry::open_at(&thread.fs_context, dir_dentry, path, follow_symlink) + Dentry::open_at(&thread.fs_context, dir_dentry, path, follow_symlink).await } } } @@ -119,13 +119,11 @@ async fn pwrite64(fd: FD, buffer: User, count: usize, offset: usize) -> KRes } #[eonix_macros::define_syscall(SYS_OPENAT)] -async fn openat(dirfd: FD, pathname: User, flags: OpenFlags, mut mode: Mode) -> KResult { - let dentry = dentry_from(thread, dirfd, pathname, flags.follow_symlink())?; +async fn openat(dirfd: FD, pathname: User, flags: OpenFlags, mode: Mode) -> KResult { + let dentry = dentry_from(thread, dirfd, pathname, flags.follow_symlink()).await?; + let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); - - thread.files.open(&dentry, flags, mode) + thread.files.open(&dentry, flags, perm).await } #[cfg(target_arch = "x86_64")] @@ -206,7 +204,7 @@ async fn newfstatat( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; let statbuf = UserPointerMut::new(statbuf)?; @@ -247,7 +245,7 @@ async fn statx( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; dentry.statx(&mut statx, mask)?; @@ -257,12 +255,11 @@ async fn statx( } #[eonix_macros::define_syscall(SYS_MKDIRAT)] -async fn mkdirat(dirfd: FD, pathname: User, mut mode: Mode) -> KResult<()> { - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); +async fn mkdirat(dirfd: FD, pathname: User, mode: Mode) -> KResult<()> { + let dentry = dentry_from(thread, dirfd, pathname, true).await?; + let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - let dentry = dentry_from(thread, dirfd, pathname, true)?; - dentry.mkdir(mode) + dentry.mkdir(perm).await } #[cfg(target_arch = "x86_64")] @@ -274,7 +271,7 @@ async fn mkdir(pathname: User, mode: u32) -> KResult<()> { #[eonix_macros::define_syscall(SYS_FTRUNCATE64)] async fn truncate64(fd: FD, length: usize) -> KResult<()> { let file = thread.files.get(fd).ok_or(EBADF)?; - file.as_path().ok_or(EBADF)?.truncate(length) + file.as_path().ok_or(EBADF)?.truncate(length).await } #[cfg(target_arch = "x86_64")] @@ -290,7 +287,10 @@ async fn truncate(pathname: User, length: usize) -> KResult<()> { #[eonix_macros::define_syscall(SYS_UNLINKAT)] async fn unlinkat(dirfd: FD, pathname: User) -> KResult<()> { - dentry_from(thread, dirfd, pathname, false)?.unlink() + dentry_from(thread, dirfd, pathname, false) + .await? + .unlink() + .await } #[cfg(target_arch = "x86_64")] @@ -302,9 +302,9 @@ async fn unlink(pathname: User) -> KResult<()> { #[eonix_macros::define_syscall(SYS_SYMLINKAT)] async fn symlinkat(target: User, dirfd: FD, linkpath: User) -> KResult<()> { let target = UserString::new(target)?; - let dentry = dentry_from(thread, dirfd, linkpath, false)?; + let dentry = dentry_from(thread, dirfd, linkpath, false).await?; - dentry.symlink(target.as_cstr().to_bytes()) + dentry.symlink(target.as_cstr().to_bytes()).await } #[cfg(target_arch = "x86_64")] @@ -313,18 +313,36 @@ async fn symlink(target: User, linkpath: User) -> KResult<()> { sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath) } +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +struct UserDeviceId(u32); + +impl FromSyscallArg for UserDeviceId { + fn from_arg(value: usize) -> Self { + Self(value as u32) + } +} + +impl UserDeviceId { + pub fn into_devid(self) -> DeviceId { + let major = (self.0 >> 8) & 0xfff; + let minor = (self.0 & 0xff) | ((self.0 >> 12) & 0xfff00); + + // TODO: We strip off the high 4 bits of the minor ID for now... + DeviceId::new(major as u16, minor as u16) + } +} + #[eonix_macros::define_syscall(SYS_MKNODAT)] -async fn mknodat(dirfd: FD, pathname: User, mut mode: Mode, dev: u32) -> KResult<()> { +async fn mknodat(dirfd: FD, pathname: User, mut mode: Mode, dev: UserDeviceId) -> KResult<()> { if !mode.is_blk() && !mode.is_chr() { return Err(EINVAL); } - let dentry = dentry_from(thread, dirfd, pathname, true)?; - - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); + let dentry = dentry_from(thread, dirfd, pathname, true).await?; + mode.set_perm(mode.perm().mask_with(*thread.fs_context.umask.lock())); - dentry.mknod(mode, dev) + dentry.mknod(mode, dev.into_devid()).await } #[cfg(target_arch = "x86_64")] @@ -340,10 +358,10 @@ async fn readlinkat( buffer: UserMut, bufsize: usize, ) -> KResult { - let dentry = dentry_from(thread, dirfd, pathname, false)?; + let dentry = dentry_from(thread, dirfd, pathname, false).await?; let mut buffer = UserBuffer::new(buffer, bufsize)?; - dentry.readlink(&mut buffer) + dentry.readlink(&mut buffer).await } #[cfg(target_arch = "x86_64")] @@ -471,7 +489,7 @@ async fn faccessat(dirfd: FD, pathname: User, _mode: u32, flags: AtFlags) -> let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { @@ -614,12 +632,12 @@ async fn fchownat( gid: u32, flags: AtFlags, ) -> KResult<()> { - let dentry = dentry_from(thread, dirfd, pathname, !flags.no_follow())?; + let dentry = dentry_from(thread, dirfd, pathname, !flags.no_follow()).await?; if !dentry.is_valid() { return Err(ENOENT); } - dentry.chown(uid, gid) + dentry.chown(uid, gid).await } #[eonix_macros::define_syscall(SYS_FCHMODAT)] @@ -628,14 +646,14 @@ async fn fchmodat(dirfd: FD, pathname: User, mode: Mode, flags: AtFlags) -> let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { return Err(ENOENT); } - dentry.chmod(mode) + dentry.chmod(mode).await } #[eonix_macros::define_syscall(SYS_FCHMOD)] @@ -654,7 +672,7 @@ async fn utimensat( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { @@ -688,10 +706,10 @@ async fn renameat2( Err(EINVAL)?; } - let old_dentry = dentry_from(thread, old_dirfd, old_pathname, false)?; - let new_dentry = dentry_from(thread, new_dirfd, new_pathname, false)?; + let old_dentry = dentry_from(thread, old_dirfd, old_pathname, false).await?; + let new_dentry = dentry_from(thread, new_dirfd, new_pathname, false).await?; - old_dentry.rename(&new_dentry, flags) + old_dentry.rename(&new_dentry, flags).await } #[cfg(target_arch = "x86_64")] diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index c6300ac7..4cb7908c 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -1,10 +1,8 @@ use super::FromSyscallArg; -use crate::fs::shm::{gen_shm_id, ShmFlags, IPC_PRIVATE, SHM_MANAGER}; -use crate::kernel::constants::{EBADF, EEXIST, EINVAL, ENOENT}; +use crate::kernel::constants::{EBADF, EINVAL}; use crate::kernel::mem::FileMapping; use crate::kernel::task::Thread; use crate::kernel::vfs::filearray::FD; -use crate::kernel::vfs::inode::Mode; use crate::{ kernel::{ constants::{UserMmapFlags, UserMmapProtocol}, @@ -66,13 +64,7 @@ async fn do_mmap2( if !is_shared { Mapping::Anonymous } else { - // The mode is unimportant here, since we are checking prot in mm_area. - let shared_area = SHM_MANAGER.lock().await.create_shared_area( - len, - thread.process.pid, - Mode::REG.perm(0o777), - ); - Mapping::File(FileMapping::new(shared_area.area.clone(), 0, len)) + unimplemented!("mmap MAP_ANONYMOUS | MAP_SHARED"); } } else { let file = thread @@ -179,114 +171,4 @@ async fn mprotect(addr: usize, len: usize, prot: UserMmapProtocol) -> KResult<() .await } -#[eonix_macros::define_syscall(SYS_SHMGET)] -async fn shmget(key: usize, size: usize, shmflg: u32) -> KResult { - let size = size.align_up(PAGE_SIZE); - - let mut shm_manager = SHM_MANAGER.lock().await; - let shmid = gen_shm_id(key)?; - - let mode = Mode::REG.perm(shmflg); - let shmflg = ShmFlags::from_bits_truncate(shmflg); - - if key == IPC_PRIVATE { - let new_shm = shm_manager.create_shared_area(size, thread.process.pid, mode); - shm_manager.insert(shmid, new_shm); - return Ok(shmid); - } - - if let Some(_) = shm_manager.get(shmid) { - if shmflg.contains(ShmFlags::IPC_CREAT | ShmFlags::IPC_EXCL) { - return Err(EEXIST); - } - - return Ok(shmid); - } - - if shmflg.contains(ShmFlags::IPC_CREAT) { - let new_shm = shm_manager.create_shared_area(size, thread.process.pid, mode); - shm_manager.insert(shmid, new_shm); - return Ok(shmid); - } - - Err(ENOENT) -} - -#[eonix_macros::define_syscall(SYS_SHMAT)] -async fn shmat(shmid: u32, addr: usize, shmflg: u32) -> KResult { - let mm_list = &thread.process.mm_list; - let shm_manager = SHM_MANAGER.lock().await; - let shm_area = shm_manager.get(shmid).ok_or(EINVAL)?; - - // Why is this not used? - let _mode = shmflg & 0o777; - let shmflg = ShmFlags::from_bits_truncate(shmflg); - - let mut permission = Permission { - read: true, - write: true, - execute: false, - }; - - if shmflg.contains(ShmFlags::SHM_EXEC) { - permission.execute = true; - } - if shmflg.contains(ShmFlags::SHM_RDONLY) { - permission.write = false; - } - - let size = shm_area.shmid_ds.shm_segsz; - - let mapping = Mapping::File(FileMapping { - file: shm_area.area.clone(), - offset: 0, - length: size, - }); - - let addr = if addr != 0 { - if addr % PAGE_SIZE != 0 && !shmflg.contains(ShmFlags::SHM_RND) { - return Err(EINVAL); - } - let addr = VAddr::from(addr.align_down(PAGE_SIZE)); - mm_list - .mmap_fixed(addr, size, mapping, permission, true) - .await - } else { - mm_list - .mmap_hint(VAddr::NULL, size, mapping, permission, true) - .await - }?; - - thread.process.shm_areas.lock().insert(addr, size); - - Ok(addr.addr()) -} - -#[eonix_macros::define_syscall(SYS_SHMDT)] -async fn shmdt(addr: usize) -> KResult<()> { - let addr = VAddr::from(addr); - - let size = { - let mut shm_areas = thread.process.shm_areas.lock(); - let size = *shm_areas.get(&addr).ok_or(EINVAL)?; - shm_areas.remove(&addr); - - size - }; - - thread.process.mm_list.unmap(addr, size).await -} - -#[eonix_macros::define_syscall(SYS_SHMCTL)] -async fn shmctl(_shmid: u32, _op: i32, _shmid_ds: usize) -> KResult { - // TODO - Ok(0) -} - -#[eonix_macros::define_syscall(SYS_MEMBARRIER)] -async fn membarrier(_cmd: usize, _flags: usize) -> KResult<()> { - // TODO - Ok(()) -} - pub fn keep_alive() {} diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 7dd573cc..b4d3e449 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -16,7 +16,7 @@ use crate::kernel::task::{parse_futexop, CloneArgs}; use crate::kernel::timer::sleep; use crate::kernel::user::UserString; use crate::kernel::user::{UserPointer, UserPointerMut}; -use crate::kernel::vfs::inode::Mode; +use crate::kernel::vfs::types::Permission; use crate::kernel::vfs::{self, dentry::Dentry}; use crate::path::Path; use crate::{kernel::user::UserBuffer, prelude::*}; @@ -100,10 +100,11 @@ async fn clock_nanosleep( } #[eonix_macros::define_syscall(SYS_UMASK)] -async fn umask(mask: Mode) -> KResult { - let mut umask = thread.fs_context.umask.lock(); +async fn umask(raw_new_mask: u32) -> KResult { + let new_mask = Permission::new(!raw_new_mask); + let old_mask = core::mem::replace(&mut *thread.fs_context.umask.lock(), new_mask); - Ok(core::mem::replace(&mut *umask, mask.non_format())) + Ok(!old_mask.bits()) } #[eonix_macros::define_syscall(SYS_GETCWD)] @@ -124,7 +125,7 @@ async fn chdir(path: User) -> KResult<()> { let path = UserString::new(path)?; let path = Path::new(path.as_cstr().to_bytes())?; - let dentry = Dentry::open(&thread.fs_context, path, true)?; + let dentry = Dentry::open(&thread.fs_context, path, true).await?; if !dentry.is_valid() { return Err(ENOENT); } @@ -159,7 +160,8 @@ async fn mount(source: User, target: User, fstype: User, flags: usiz &thread.fs_context, Path::new(target.as_cstr().to_bytes())?, true, - )?; + ) + .await?; if !mountpoint.is_valid() { return Err(ENOENT); @@ -172,6 +174,7 @@ async fn mount(source: User, target: User, fstype: User, flags: usiz fstype.as_cstr().to_str().map_err(|_| EINVAL)?, flags as u64, ) + .await } fn get_strings(mut ptr_strings: UserPointer<'_, PtrT>) -> KResult> { @@ -199,14 +202,15 @@ async fn execve(exec: User, argv: User, envp: User) -> KResult Elf { Err(ENOEXEC) } - fn parse(elf_file: Arc) -> KResult { + async fn parse(elf_file: Arc) -> KResult { let mut elf_header = UninitBuffer::>::new(); - elf_file.read(&mut elf_header, 0)?; + elf_file.read(&mut elf_header, 0).await?; let elf_header = elf_header.assume_init().map_err(|_| ENOEXEC)?; @@ -203,10 +203,12 @@ impl Elf { let ph_count = elf_header.pt2.ph_count; let mut program_headers = vec![E::Ph::default(); ph_count as usize]; - elf_file.read( - &mut ByteBuffer::from(program_headers.as_mut_slice()), - ph_offset.into_usize(), - )?; + elf_file + .read( + &mut ByteBuffer::from(program_headers.as_mut_slice()), + ph_offset.into_usize(), + ) + .await?; Ok(Self { file: elf_file, @@ -390,12 +392,13 @@ impl Elf { } async fn load_ldso(&self, mm_list: &MMList) -> KResult> { - let ldso_path = self.ldso_path()?; + let ldso_path = self.ldso_path().await?; if let Some(ldso_path) = ldso_path { let fs_context = FsContext::global(); - let ldso_file = Dentry::open(fs_context, Path::new(ldso_path.as_bytes())?, true)?; - let ldso_elf = Elf::::parse(ldso_file)?; + let ldso_file = + Dentry::open(fs_context, Path::new(ldso_path.as_bytes())?, true).await?; + let ldso_elf = Elf::::parse(ldso_file).await?; let base = VAddr::from(E::LDSO_BASE_ADDR); @@ -420,7 +423,7 @@ impl Elf { mm_list.map_vdso().await } - fn ldso_path(&self) -> KResult> { + async fn ldso_path(&self) -> KResult> { for program_header in &self.program_headers { let type_ = program_header.type_().map_err(|_| ENOEXEC)?; @@ -430,7 +433,8 @@ impl Elf { let mut ldso_vec = vec![0u8; file_size - 1]; // -1 due to '\0' self.file - .read(&mut ByteBuffer::from(ldso_vec.as_mut_slice()), file_offset)?; + .read(&mut ByteBuffer::from(ldso_vec.as_mut_slice()), file_offset) + .await?; let ldso_path = String::from_utf8(ldso_vec).map_err(|_| ENOEXEC)?; return Ok(Some(ldso_path)); } @@ -445,16 +449,16 @@ pub enum ELF { } impl ELF { - pub fn parse(elf_file: Arc) -> KResult { + pub async fn parse(elf_file: Arc) -> KResult { let mut header_pt1 = UninitBuffer::::new(); - elf_file.read(&mut header_pt1, 0)?; + elf_file.read(&mut header_pt1, 0).await?; let header_pt1 = header_pt1.assume_init().map_err(|_| ENOEXEC)?; assert_eq!(header_pt1.magic, ELF_MAGIC); match header_pt1.class() { - Class::ThirtyTwo => Ok(ELF::Elf32(Elf::parse(elf_file)?)), - Class::SixtyFour => Ok(ELF::Elf64(Elf::parse(elf_file)?)), + Class::ThirtyTwo => Ok(ELF::Elf32(Elf::parse(elf_file).await?)), + Class::SixtyFour => Ok(ELF::Elf64(Elf::parse(elf_file).await?)), _ => Err(ENOEXEC), } } diff --git a/src/kernel/task/loader/mod.rs b/src/kernel/task/loader/mod.rs index 4e3f4db1..7679aaf4 100644 --- a/src/kernel/task/loader/mod.rs +++ b/src/kernel/task/loader/mod.rs @@ -33,7 +33,7 @@ pub struct ProgramLoader { } impl ProgramLoader { - pub fn parse( + pub async fn parse( fs_context: &FsContext, mut exec_path: CString, mut file: Arc, @@ -49,12 +49,15 @@ impl ProgramLoader { } let mut magic = [0; 4]; - file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0)?; + file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0) + .await?; match magic { [b'#', b'!', ..] => { let mut interpreter_line = [0; 256]; - let nread = file.read(&mut ByteBuffer::new(&mut interpreter_line), 0)?; + let nread = file + .read(&mut ByteBuffer::new(&mut interpreter_line), 0) + .await?; // There is a tiny time gap between reading the magic number and // reading the interpreter line, so we need to check if the line @@ -77,7 +80,7 @@ impl ProgramLoader { } let path = Path::new(interpreter_name.as_bytes())?; - file = Dentry::open(fs_context, path, true)?; + file = Dentry::open(fs_context, path, true).await?; args.insert(0, interpreter_name.clone()); if let Some(arg) = interpreter_arg { @@ -92,7 +95,7 @@ impl ProgramLoader { exec_path = interpreter_name; } - ELF_MAGIC => break ELF::parse(file)?, + ELF_MAGIC => break ELF::parse(file).await?, _ => return Err(ENOEXEC), } diff --git a/src/kernel/timer.rs b/src/kernel/timer.rs index 9b6a3ff2..1dbb1382 100644 --- a/src/kernel/timer.rs +++ b/src/kernel/timer.rs @@ -76,6 +76,8 @@ impl Ticks { } impl Instant { + pub const UNIX_EPOCH: Self = Self::default(); + pub const fn default() -> Self { Instant { secs_since_epoch: 0, diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index 8bcd9f8a..5ac4e407 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -1,8 +1,9 @@ pub mod dcache; use super::{ - inode::{Ino, Inode, Mode, RenameData, WriteOffset}, - DevId, FsContext, + inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, + types::{DeviceId, Format, Mode, Permission}, + FsContext, }; use crate::{ hash::KernelHasher, @@ -14,22 +15,31 @@ use crate::{ }; use crate::{ io::Stream, - kernel::constants::{EEXIST, EINVAL, EIO, EISDIR, ELOOP, ENOENT, ENOTDIR, EPERM, ERANGE}, + kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, ENOTDIR, EPERM, ERANGE}, }; -use alloc::sync::{Arc, Weak}; +use alloc::sync::Arc; use core::{ fmt, + future::Future, hash::{BuildHasher, BuildHasherDefault, Hasher}, - ops::ControlFlow, + pin::Pin, sync::atomic::{AtomicPtr, AtomicU64, Ordering}, }; use eonix_sync::LazyLock; use pointers::BorrowedArc; use posix_types::{namei::RenameFlags, open::OpenFlags, result::PosixError, stat::StatX}; +#[derive(PartialEq, Eq)] +enum DentryKind { + Regular, + Directory, + Symlink, + Mountpoint, +} + struct DentryData { - inode: Arc, - flags: u64, + inode: InodeUse, + kind: DentryKind, } /// # Safety @@ -79,12 +89,6 @@ impl fmt::Debug for Dentry { } } -const D_DIRECTORY: u64 = 1; -#[allow(dead_code)] -const D_MOUNTPOINT: u64 = 2; -const D_SYMLINK: u64 = 4; -const D_REGULAR: u64 = 8; - impl RCUNode for Dentry { fn rcu_prev(&self) -> &AtomicPtr { &self.prev @@ -116,11 +120,11 @@ impl Dentry { self.hash.store(hash, Ordering::Relaxed); } - fn find(self: &Arc, name: &[u8]) -> KResult> { + async fn find(self: &Arc, name: &[u8]) -> KResult> { let data = self.data.load(); let data = data.as_ref().ok_or(ENOENT)?; - if data.flags & D_DIRECTORY == 0 { + if data.kind != DentryKind::Directory { return Err(ENOTDIR); } @@ -141,7 +145,7 @@ impl Dentry { return Ok(found); } - dcache::d_try_revalidate(&dentry); + let _ = dcache::d_try_revalidate(&dentry).await; dcache::d_add(dentry.clone()); Ok(dentry) @@ -192,8 +196,8 @@ impl Dentry { .map_or(core::ptr::null(), |parent| Arc::as_ptr(&parent)) } - fn save_data(&self, inode: Arc, flags: u64) -> KResult<()> { - let new = DentryData { inode, flags }; + fn save(&self, inode: InodeUse, kind: DentryKind) { + let new = DentryData { inode, kind }; // TODO!!!: We don't actually need to use `RCUPointer` here // Safety: this function may only be called from `create`-like functions which requires the @@ -201,41 +205,35 @@ impl Dentry { // can't get a reference to the old data. let old = unsafe { self.data.swap(Some(Arc::new(new))) }; assert!(old.is_none()); - - Ok(()) } - pub fn save_reg(&self, file: Arc) -> KResult<()> { - self.save_data(file, D_REGULAR) - } - - pub fn save_symlink(&self, link: Arc) -> KResult<()> { - self.save_data(link, D_SYMLINK) + pub fn fill(&self, file: InodeUse) { + match file.format() { + Format::REG | Format::BLK | Format::CHR => self.save(file, DentryKind::Regular), + Format::DIR => self.save(file, DentryKind::Directory), + Format::LNK => self.save(file, DentryKind::Symlink), + } } - pub fn save_dir(&self, dir: Arc) -> KResult<()> { - self.save_data(dir, D_DIRECTORY) + pub fn inode(&self) -> Option> { + self.data.load().as_ref().map(|data| data.inode.clone()) } - pub fn get_inode(&self) -> KResult> { - self.data - .load() - .as_ref() - .ok_or(ENOENT) - .map(|data| data.inode.clone()) + pub fn get_inode(&self) -> KResult> { + self.inode().ok_or(ENOENT) } pub fn is_directory(&self) -> bool { let data = self.data.load(); data.as_ref() - .map_or(false, |data| data.flags & D_DIRECTORY != 0) + .map_or(false, |data| data.kind == DentryKind::Directory) } pub fn is_valid(&self) -> bool { self.data.load().is_some() } - pub fn open_check(self: &Arc, flags: OpenFlags, mode: Mode) -> KResult<()> { + pub async fn open_check(self: &Arc, flags: OpenFlags, perm: Permission) -> KResult<()> { let data = self.data.load(); if data.is_some() { @@ -250,7 +248,7 @@ impl Dentry { } let parent = self.parent().get_inode()?; - parent.creat(self, mode) + parent.create(self, perm).await } } } @@ -260,110 +258,120 @@ impl Dentry { context: &FsContext, dentry: Arc, nrecur: u32, - ) -> KResult> { - if nrecur >= 16 { - return Err(ELOOP); - } + ) -> Pin>> + use<'_>>> { + Box::pin(async move { + if nrecur >= 16 { + return Err(ELOOP); + } - let data = dentry.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; + let data = dentry.data.load(); + let data = data.as_ref().ok_or(ENOENT)?; - match data.flags { - flags if flags & D_REGULAR != 0 => Err(ENOTDIR), - flags if flags & D_DIRECTORY != 0 => Ok(dentry), - flags if flags & D_SYMLINK != 0 => { - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); + match data.kind { + DentryKind::Regular => Err(ENOTDIR), + DentryKind::Directory => Ok(dentry), + DentryKind::Symlink => { + let mut buffer = [0u8; 256]; + let mut buffer = ByteBuffer::new(&mut buffer); - data.inode.readlink(&mut buffer)?; - let path = Path::new(buffer.data())?; + data.inode.readlink(&mut buffer).await?; + let path = Path::new(buffer.data())?; - let dentry = - Self::open_recursive(context, &dentry.parent(), path, true, nrecur + 1)?; + let dentry = + Self::open_recursive(context, &dentry.parent(), path, true, nrecur + 1) + .await?; - Self::resolve_directory(context, dentry, nrecur + 1) + Self::resolve_directory(context, dentry, nrecur + 1).await + } + _ => panic!("Invalid dentry flags"), } - _ => panic!("Invalid dentry flags"), - } + }) } - pub fn open_recursive( - context: &FsContext, - cwd: &Arc, - path: Path, + pub fn open_recursive<'r, 'a: 'r, 'b: 'r, 'c: 'r>( + context: &'a FsContext, + cwd: &'b Arc, + path: Path<'c>, follow: bool, nrecur: u32, - ) -> KResult> { - // too many recursive search layers will cause stack overflow - // so we use 16 for now - if nrecur >= 16 { - return Err(ELOOP); - } + ) -> Pin>> + 'r>> { + Box::pin(async move { + // too many recursive search layers will cause stack overflow + // so we use 16 for now + if nrecur >= 16 { + return Err(ELOOP); + } - let mut cwd = if path.is_absolute() { - context.fsroot.clone() - } else { - cwd.clone() - }; + let mut cwd = if path.is_absolute() { + context.fsroot.clone() + } else { + cwd.clone() + }; - for item in path.iter() { - if let PathComponent::TrailingEmpty = item { - if cwd.data.load().as_ref().is_none() { - return Ok(cwd); + for item in path.iter() { + if let PathComponent::TrailingEmpty = item { + if cwd.data.load().as_ref().is_none() { + return Ok(cwd); + } } - } - cwd = Self::resolve_directory(context, cwd, nrecur)?; + cwd = Self::resolve_directory(context, cwd, nrecur).await?; - match item { - PathComponent::TrailingEmpty | PathComponent::Current => {} // pass - PathComponent::Parent => { - if !cwd.hash_eq(&context.fsroot) { - let parent = cwd.parent().clone(); - cwd = Self::resolve_directory(context, parent, nrecur)?; + match item { + PathComponent::TrailingEmpty | PathComponent::Current => {} // pass + PathComponent::Parent => { + if !cwd.hash_eq(&context.fsroot) { + let parent = cwd.parent().clone(); + cwd = Self::resolve_directory(context, parent, nrecur).await?; + } + continue; + } + PathComponent::Name(name) => { + cwd = cwd.find(name).await?; } - continue; - } - PathComponent::Name(name) => { - cwd = cwd.find(name)?; } } - } - if follow { - let data = cwd.data.load(); + if follow { + let data = cwd.data.load(); - if let Some(data) = data.as_ref() { - if data.flags & D_SYMLINK != 0 { - let data = cwd.data.load(); - let data = data.as_ref().unwrap(); - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); + if let Some(data) = data.as_ref() { + if data.kind == DentryKind::Symlink { + let data = cwd.data.load(); + let data = data.as_ref().unwrap(); + let mut buffer = [0u8; 256]; + let mut buffer = ByteBuffer::new(&mut buffer); - data.inode.readlink(&mut buffer)?; - let path = Path::new(buffer.data())?; + data.inode.readlink(&mut buffer).await?; + let path = Path::new(buffer.data())?; - let parent = cwd.parent().clone(); - cwd = Self::open_recursive(context, &parent, path, true, nrecur + 1)?; + let parent = cwd.parent().clone(); + cwd = + Self::open_recursive(context, &parent, path, true, nrecur + 1).await?; + } } } - } - Ok(cwd) + Ok(cwd) + }) } - pub fn open(context: &FsContext, path: Path, follow_symlinks: bool) -> KResult> { + pub async fn open( + context: &FsContext, + path: Path<'_>, + follow_symlinks: bool, + ) -> KResult> { let cwd = context.cwd.lock().clone(); - Dentry::open_recursive(context, &cwd, path, follow_symlinks, 0) + Dentry::open_recursive(context, &cwd, path, follow_symlinks, 0).await } - pub fn open_at( + pub async fn open_at( context: &FsContext, at: &Arc, - path: Path, + path: Path<'_>, follow_symlinks: bool, ) -> KResult> { - Dentry::open_recursive(context, at, path, follow_symlinks, 0) + Dentry::open_recursive(context, at, path, follow_symlinks, 0).await } pub fn get_path( @@ -405,18 +413,18 @@ impl Dentry { } impl Dentry { - pub fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + pub async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.mode.load().format() { - Mode::DIR => Err(EISDIR), - Mode::REG => inode.read(buffer, offset), - Mode::BLK => { + match inode.format() { + Format::DIR => Err(EISDIR), + Format::REG => inode.read(buffer, offset).await, + Format::BLK => { let device = BlockDevice::get(inode.devid()?)?; - Ok(device.read_some(offset, buffer)?.allow_partial()) + Ok(device.read_some(offset, buffer).await?.allow_partial()) } - Mode::CHR => { + Format::CHR => { let device = CharDevice::get(inode.devid()?).ok_or(EPERM)?; device.read(buffer) } @@ -424,32 +432,32 @@ impl Dentry { } } - pub fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult { + pub async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.mode.load().format() { - Mode::DIR => Err(EISDIR), - Mode::REG => inode.write(stream, offset), - Mode::BLK => Err(EINVAL), // TODO - Mode::CHR => CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream), + match inode.format() { + Format::DIR => Err(EISDIR), + Format::REG => inode.write(stream, offset).await, + Format::BLK => Err(EINVAL), // TODO + Format::CHR => CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream), _ => Err(EINVAL), } } - pub fn readdir(&self, offset: usize, mut callback: F) -> KResult + pub async fn readdir(&self, offset: usize, mut for_each_entry: F) -> KResult> where - F: FnMut(&[u8], Ino) -> KResult>, + F: FnMut(&[u8], Ino) -> KResult + Send, { let dir = self.get_inode()?; - dir.do_readdir(offset, &mut callback) + dir.readdir(offset, &mut for_each_entry).await } - pub fn mkdir(&self, mode: Mode) -> KResult<()> { + pub async fn mkdir(&self, perm: Permission) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.mkdir(self, mode) + dir.mkdir(self, perm).await } } @@ -457,50 +465,50 @@ impl Dentry { self.get_inode()?.statx(stat, mask) } - pub fn truncate(&self, size: usize) -> KResult<()> { - self.get_inode()?.truncate(size) + pub async fn truncate(&self, size: usize) -> KResult<()> { + self.get_inode()?.truncate(size).await } - pub fn unlink(self: &Arc) -> KResult<()> { + pub async fn unlink(self: &Arc) -> KResult<()> { if self.get_inode().is_err() { Err(ENOENT) } else { let dir = self.parent().get_inode()?; - dir.unlink(self) + dir.unlink(self).await } } - pub fn symlink(self: &Arc, link: &[u8]) -> KResult<()> { + pub async fn symlink(self: &Arc, link: &[u8]) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.symlink(self, link) + dir.symlink(self, link).await } } - pub fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { - self.get_inode()?.readlink(buffer) + pub async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { + self.get_inode()?.readlink(buffer).await } - pub fn mknod(&self, mode: Mode, devid: DevId) -> KResult<()> { + pub async fn mknod(&self, mode: Mode, devid: DeviceId) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.mknod(self, mode, devid) + dir.mknod(self, mode, devid).await } } - pub fn chmod(&self, mode: Mode) -> KResult<()> { - self.get_inode()?.chmod(mode) + pub async fn chmod(&self, mode: Mode) -> KResult<()> { + self.get_inode()?.chmod(mode).await } - pub fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - self.get_inode()?.chown(uid, gid) + pub async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { + self.get_inode()?.chown(uid, gid).await } - pub fn rename(self: &Arc, new: &Arc, flags: RenameFlags) -> KResult<()> { + pub async fn rename(self: &Arc, new: &Arc, flags: RenameFlags) -> KResult<()> { if Arc::ptr_eq(self, new) { return Ok(()); } @@ -509,22 +517,19 @@ impl Dentry { let new_parent = new.parent().get_inode()?; // If the two dentries are not in the same filesystem, return EXDEV. - if !Weak::ptr_eq(&old_parent.vfs, &new_parent.vfs) { + if old_parent.sbref().eq(&new_parent.sbref()) { Err(PosixError::EXDEV)?; } - let vfs = old_parent.vfs.upgrade().ok_or(EIO)?; - let rename_data = RenameData { old_dentry: self, new_dentry: new, new_parent, - vfs, is_exchange: flags.contains(RenameFlags::RENAME_EXCHANGE), no_replace: flags.contains(RenameFlags::RENAME_NOREPLACE), }; // Delegate to the parent directory's rename implementation - old_parent.rename(rename_data) + old_parent.rename(rename_data).await } } diff --git a/src/kernel/vfs/dentry/dcache.rs b/src/kernel/vfs/dentry/dcache.rs index 188a1cfc..e2491235 100644 --- a/src/kernel/vfs/dentry/dcache.rs +++ b/src/kernel/vfs/dentry/dcache.rs @@ -1,7 +1,5 @@ -use super::{Dentry, Inode}; +use super::Dentry; use crate::kernel::constants::ENOENT; -use crate::kernel::task::block_on; -use crate::kernel::vfs::inode::Mode; use crate::rcu::RCUPointer; use crate::{ prelude::*, @@ -41,27 +39,14 @@ pub fn d_find_fast(dentry: &Dentry) -> Option> { /// Call `lookup()` on the parent inode to try find if the dentry points to a valid inode /// /// Silently fail without any side effects -pub fn d_try_revalidate(dentry: &Arc) { - let _lock = block_on(D_EXCHANGE_LOCK.lock()); - - (|| -> KResult<()> { - let parent = dentry.parent().get_inode()?; - let inode = parent.lookup(dentry)?.ok_or(ENOENT)?; +pub async fn d_try_revalidate(dentry: &Arc) -> KResult<()> { + let _lock = D_EXCHANGE_LOCK.lock().await; - d_save(dentry, inode) - })() - .unwrap_or_default(); -} + let parent = dentry.parent().get_inode()?; + let inode = parent.lookup(dentry).await?.ok_or(ENOENT)?; -/// Save the inode to the dentry. -/// -/// Dentry flags will be determined by the inode's mode. -pub fn d_save(dentry: &Arc, inode: Arc) -> KResult<()> { - match inode.mode.load().format() { - Mode::DIR => dentry.save_dir(inode), - Mode::LNK => dentry.save_symlink(inode), - _ => dentry.save_reg(inode), - } + dentry.fill(inode); + Ok(()) } /// Replace the old dentry with the new one in the dcache diff --git a/src/kernel/vfs/file/inode_file.rs b/src/kernel/vfs/file/inode_file.rs index 6386ba92..96526ee9 100644 --- a/src/kernel/vfs/file/inode_file.rs +++ b/src/kernel/vfs/file/inode_file.rs @@ -5,13 +5,13 @@ use crate::{ constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}, vfs::{ dentry::Dentry, - inode::{Inode, Mode, WriteOffset}, + inode::{Inode, InodeUse, WriteOffset}, + types::Format, }, }, prelude::KResult, }; use alloc::sync::Arc; -use core::{ops::ControlFlow, sync::atomic::Ordering}; use eonix_sync::Mutex; use posix_types::{ getdent::{UserDirent, UserDirent64}, @@ -25,7 +25,7 @@ pub struct InodeFile { pub a: bool, /// Only a few modes those won't possibly change are cached here to speed up file operations. /// Specifically, `S_IFMT` masked bits. - pub mode: Mode, + pub format: Format, cursor: Mutex, dentry: Arc, } @@ -34,12 +34,7 @@ impl InodeFile { pub fn new(dentry: Arc, flags: OpenFlags) -> File { // SAFETY: `dentry` used to create `InodeFile` is valid. // SAFETY: `mode` should never change with respect to the `S_IFMT` fields. - let cached_mode = dentry - .get_inode() - .expect("`dentry` is invalid") - .mode - .load() - .format(); + let format = dentry.inode().expect("dentry should be invalid").format(); let (r, w, a) = flags.as_rwa(); @@ -50,15 +45,15 @@ impl InodeFile { r, w, a, - mode: cached_mode, + format, cursor: Mutex::new(0), }), ) } pub fn sendfile_check(&self) -> KResult<()> { - match self.mode { - Mode::REG | Mode::BLK => Ok(()), + match self.format { + Format::REG | Format::BLK => Ok(()), _ => Err(EBADF), } } @@ -70,21 +65,19 @@ impl InodeFile { let mut cursor = self.cursor.lock().await; - if self.a { - let nwrote = self.dentry.write(stream, WriteOffset::End(&mut cursor))?; + let (offset, update_offset) = match (self.a, offset) { + (true, _) => (WriteOffset::End(&mut cursor), None), + (false, Some(offset)) => (WriteOffset::Position(offset), None), + (false, None) => (WriteOffset::Position(*cursor), Some(&mut *cursor)), + }; - Ok(nwrote) - } else { - let nwrote = if let Some(offset) = offset { - self.dentry.write(stream, WriteOffset::Position(offset))? - } else { - let nwrote = self.dentry.write(stream, WriteOffset::Position(*cursor))?; - *cursor += nwrote; - nwrote - }; - - Ok(nwrote) + let nr_write = self.dentry.write(stream, offset).await?; + + if let Some(update_offset) = update_offset { + *update_offset += nr_write; } + + Ok(nr_write) } pub async fn read(&self, buffer: &mut dyn Buffer, offset: Option) -> KResult { @@ -92,24 +85,20 @@ impl InodeFile { return Err(EBADF); } - let nread = if let Some(offset) = offset { - let nread = self.dentry.read(buffer, offset)?; - nread - } else { - let mut cursor = self.cursor.lock().await; - - let nread = self.dentry.read(buffer, *cursor)?; + if let Some(offset) = offset { + return Ok(self.dentry.read(buffer, offset).await?); + } - *cursor += nread; - nread - }; + let mut cursor = self.cursor.lock().await; + let nread = self.dentry.read(buffer, *cursor).await?; + *cursor += nread; Ok(nread) } } impl File { - pub fn get_inode(&self) -> KResult>> { + pub fn get_inode(&self) -> KResult>> { if let FileType::Inode(inode_file) = &**self { Ok(Some(inode_file.dentry.get_inode()?)) } else { @@ -124,27 +113,30 @@ impl File { let mut cursor = inode_file.cursor.lock().await; - let nread = inode_file.dentry.readdir(*cursor, |filename, ino| { - // + 1 for filename length padding '\0', + 1 for d_type. - let real_record_len = core::mem::size_of::() + filename.len() + 2; + let nread = inode_file + .dentry + .readdir(*cursor, |filename, ino| { + // + 1 for filename length padding '\0', + 1 for d_type. + let real_record_len = core::mem::size_of::() + filename.len() + 2; - if buffer.available() < real_record_len { - return Ok(ControlFlow::Break(())); - } + if buffer.available() < real_record_len { + return Ok(false); + } - let record = UserDirent { - d_ino: ino as u32, - d_off: 0, - d_reclen: real_record_len as u16, - d_name: [0; 0], - }; + let record = UserDirent { + d_ino: ino.as_raw() as u32, + d_off: 0, + d_reclen: real_record_len as u16, + d_name: [0; 0], + }; - buffer.copy(&record)?.ok_or(EFAULT)?; - buffer.fill(filename)?.ok_or(EFAULT)?; - buffer.fill(&[0, 0])?.ok_or(EFAULT)?; + buffer.copy(&record)?.ok_or(EFAULT)?; + buffer.fill(filename)?.ok_or(EFAULT)?; + buffer.fill(&[0, 0])?.ok_or(EFAULT)?; - Ok(ControlFlow::Continue(())) - })?; + Ok(true) + }) + .await??; *cursor += nread; Ok(()) @@ -157,28 +149,31 @@ impl File { let mut cursor = inode_file.cursor.lock().await; - let nread = inode_file.dentry.readdir(*cursor, |filename, ino| { - // Filename length + 1 for padding '\0' - let real_record_len = core::mem::size_of::() + filename.len() + 1; + let nread = inode_file + .dentry + .readdir(*cursor, |filename, ino| { + // Filename length + 1 for padding '\0' + let real_record_len = core::mem::size_of::() + filename.len() + 1; - if buffer.available() < real_record_len { - return Ok(ControlFlow::Break(())); - } + if buffer.available() < real_record_len { + return Ok(false); + } - let record = UserDirent64 { - d_ino: ino, - d_off: 0, - d_reclen: real_record_len as u16, - d_type: 0, - d_name: [0; 0], - }; + let record = UserDirent64 { + d_ino: ino.as_raw(), + d_off: 0, + d_reclen: real_record_len as u16, + d_type: 0, + d_name: [0; 0], + }; - buffer.copy(&record)?.ok_or(EFAULT)?; - buffer.fill(filename)?.ok_or(EFAULT)?; - buffer.fill(&[0])?.ok_or(EFAULT)?; + buffer.copy(&record)?.ok_or(EFAULT)?; + buffer.fill(filename)?.ok_or(EFAULT)?; + buffer.fill(&[0])?.ok_or(EFAULT)?; - Ok(ControlFlow::Continue(())) - })?; + Ok(true) + }) + .await??; *cursor += nread; Ok(()) @@ -196,7 +191,7 @@ impl File { SeekOption::Set(n) => n, SeekOption::End(off) => { let inode = inode_file.dentry.get_inode()?; - let size = inode.size.load(Ordering::Relaxed) as usize; + let size = inode.info().lock().size as usize; size.checked_add_signed(off).ok_or(EOVERFLOW)? } }; diff --git a/src/kernel/vfs/filearray.rs b/src/kernel/vfs/filearray.rs index b457a425..1862a3e1 100644 --- a/src/kernel/vfs/filearray.rs +++ b/src/kernel/vfs/filearray.rs @@ -1,6 +1,6 @@ use super::{ file::{File, InodeFile, Pipe}, - inode::Mode, + types::{Format, Permission}, Spin, TerminalFile, }; use crate::kernel::{ @@ -280,26 +280,31 @@ impl FileArray { Ok((read_fd, write_fd)) } - pub fn open(&self, dentry: &Arc, flags: OpenFlags, mode: Mode) -> KResult { - dentry.open_check(flags, mode)?; + pub async fn open( + &self, + dentry: &Arc, + flags: OpenFlags, + perm: Permission, + ) -> KResult { + dentry.open_check(flags, perm).await?; let fdflag = flags.as_fd_flags(); let inode = dentry.get_inode()?; - let file_format = inode.mode.load().format(); + let file_format = inode.format(); match (flags.directory(), file_format, flags.write()) { - (true, Mode::DIR, _) => {} + (true, Format::DIR, _) => {} (true, _, _) => return Err(ENOTDIR), - (false, Mode::DIR, true) => return Err(EISDIR), + (false, Format::DIR, true) => return Err(EISDIR), _ => {} } - if flags.truncate() && flags.write() && file_format.is_reg() { - inode.truncate(0)?; + if flags.truncate() && flags.write() && file_format == Format::REG { + inode.truncate(0).await?; } - let file = if file_format.is_chr() { + let file = if file_format == Format::CHR { let device = CharDevice::get(inode.devid()?).ok_or(ENXIO)?; device.open(flags)? } else { diff --git a/src/kernel/vfs/inode.rs b/src/kernel/vfs/inode.rs deleted file mode 100644 index 52529f84..00000000 --- a/src/kernel/vfs/inode.rs +++ /dev/null @@ -1,494 +0,0 @@ -use super::{dentry::Dentry, vfs::Vfs, DevId}; -use crate::io::Stream; -use crate::kernel::constants::{ - EINVAL, EISDIR, ENOTDIR, EPERM, STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, - STATX_MODE, STATX_MTIME, STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, S_IFBLK, S_IFCHR, - S_IFDIR, S_IFLNK, S_IFMT, S_IFREG, -}; -use crate::kernel::mem::PageCache; -use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; -use crate::kernel::task::block_on; -use crate::kernel::timer::Instant; -use crate::{io::Buffer, prelude::*}; -use alloc::sync::{Arc, Weak}; -use core::{ - mem::MaybeUninit, - ops::ControlFlow, - ptr::addr_of_mut, - sync::atomic::{AtomicU32, AtomicU64, Ordering}, -}; -use eonix_sync::RwLock; -use posix_types::stat::StatX; - -pub type Ino = u64; -pub type AtomicIno = AtomicU64; -#[allow(dead_code)] -pub type ISize = u64; -pub type AtomicISize = AtomicU64; -#[allow(dead_code)] -pub type Nlink = u64; -pub type AtomicNlink = AtomicU64; -#[allow(dead_code)] -pub type Uid = u32; -pub type AtomicUid = AtomicU32; -#[allow(dead_code)] -pub type Gid = u32; -pub type AtomicGid = AtomicU32; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub struct Mode(u32); - -pub struct AtomicMode(AtomicU32); - -#[derive(Debug)] -pub struct InodeData { - pub ino: Ino, - pub size: AtomicISize, - pub nlink: AtomicNlink, - - pub uid: AtomicUid, - pub gid: AtomicGid, - pub mode: AtomicMode, - - pub atime: Spin, - pub ctime: Spin, - pub mtime: Spin, - - pub rwsem: RwLock<()>, - - pub vfs: Weak, -} - -impl InodeData { - pub fn new(ino: Ino, vfs: Weak) -> Self { - Self { - ino, - vfs, - atime: Spin::new(Instant::now()), - ctime: Spin::new(Instant::now()), - mtime: Spin::new(Instant::now()), - rwsem: RwLock::new(()), - size: AtomicU64::new(0), - nlink: AtomicNlink::new(0), - uid: AtomicUid::new(0), - gid: AtomicGid::new(0), - mode: AtomicMode::new(0), - } - } -} - -#[allow(dead_code)] -pub trait InodeInner: - Send + Sync + core::ops::Deref + core::ops::DerefMut -{ - fn data(&self) -> &InodeData; - fn data_mut(&mut self) -> &mut InodeData; -} - -pub enum WriteOffset<'end> { - Position(usize), - End(&'end mut usize), -} - -pub struct RenameData<'a, 'b> { - pub old_dentry: &'a Arc, - pub new_dentry: &'b Arc, - pub new_parent: Arc, - pub vfs: Arc, - pub is_exchange: bool, - pub no_replace: bool, -} - -#[allow(unused_variables)] -pub trait Inode: Send + Sync + InodeInner + Any { - fn is_dir(&self) -> bool { - self.mode.load().is_dir() - } - - fn lookup(&self, dentry: &Arc) -> KResult>> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn creat(&self, at: &Arc, mode: Mode) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn mknod(&self, at: &Dentry, mode: Mode, dev: DevId) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn unlink(&self, at: &Arc) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn devid(&self) -> KResult { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn truncate(&self, length: usize) -> KResult<()> { - Err(if self.is_dir() { EISDIR } else { EPERM }) - } - - fn rename(&self, rename_data: RenameData) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn do_readdir( - &self, - offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult>, - ) -> KResult { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - Err(EPERM) - } - - fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - Err(EPERM) - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } - - fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { - // Safety: ffi should have checked reference - let vfs = self.vfs.upgrade().expect("Vfs is dropped"); - - let size = self.size.load(Ordering::Relaxed); - let mode = self.mode.load(); - - if mask & STATX_NLINK != 0 { - stat.stx_nlink = self.nlink.load(Ordering::Acquire) as _; - stat.stx_mask |= STATX_NLINK; - } - - if mask & STATX_ATIME != 0 { - let atime = *self.atime.lock(); - stat.stx_atime = atime.into(); - stat.stx_mask |= STATX_ATIME; - } - - if mask & STATX_MTIME != 0 { - let mtime = *self.mtime.lock(); - stat.stx_mtime = mtime.into(); - stat.stx_mask |= STATX_MTIME; - } - - if mask & STATX_CTIME != 0 { - let ctime = *self.ctime.lock(); - stat.stx_ctime = ctime.into(); - stat.stx_mask |= STATX_CTIME; - } - - if mask & STATX_SIZE != 0 { - stat.stx_size = self.size.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_SIZE; - } - - stat.stx_mode = 0; - if mask & STATX_MODE != 0 { - stat.stx_mode |= mode.non_format_bits() as u16; - stat.stx_mask |= STATX_MODE; - } - - if mask & STATX_TYPE != 0 { - stat.stx_mode |= mode.format_bits() as u16; - if mode.is_blk() || mode.is_chr() { - let devid = self.devid(); - stat.stx_rdev_major = (devid? >> 8) & 0xff; - stat.stx_rdev_minor = devid? & 0xff; - } - stat.stx_mask |= STATX_TYPE; - } - - if mask & STATX_INO != 0 { - stat.stx_ino = self.ino as _; - stat.stx_mask |= STATX_INO; - } - - if mask & STATX_BLOCKS != 0 { - stat.stx_blocks = (size + 512 - 1) / 512; - stat.stx_blksize = vfs.io_blksize() as _; - stat.stx_mask |= STATX_BLOCKS; - } - - if mask & STATX_UID != 0 { - stat.stx_uid = self.uid.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_UID; - } - - if mask & STATX_GID != 0 { - stat.stx_gid = self.gid.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_GID; - } - - let fsdev = vfs.fs_devid(); - stat.stx_dev_major = (fsdev >> 8) & 0xff; - stat.stx_dev_minor = fsdev & 0xff; - - // TODO: support more attributes - stat.stx_attributes_mask = 0; - - Ok(()) - } - - fn new_locked(ino: Ino, vfs: Weak, f: F) -> Arc - where - Self: Sized, - F: FnOnce(*mut Self, &()), - { - let mut uninit = Arc::::new_uninit(); - - let uninit_mut = Arc::get_mut(&mut uninit).unwrap(); - - // Safety: `idata` is owned by `uninit` - let idata = unsafe { - addr_of_mut!(*(*uninit_mut.as_mut_ptr()).data_mut()) - .cast::>() - .as_mut() - .unwrap() - }; - - idata.write(InodeData::new(ino, vfs)); - - f( - uninit_mut.as_mut_ptr(), - // SAFETY: `idata` is initialized and we will never move the lock. - &block_on(unsafe { idata.assume_init_ref() }.rwsem.read()), - ); - - // Safety: `uninit` is initialized - unsafe { uninit.assume_init() } - } -} - -// TODO: define multiple inode structs a time -macro_rules! define_struct_inode { - ($v:vis struct $inode_t:ident;) => { - $v struct $inode_t { - /// Do not use this directly - idata: $crate::kernel::vfs::inode::InodeData, - } - - impl core::ops::Deref for $inode_t { - type Target = $crate::kernel::vfs::inode::InodeData; - - fn deref(&self) -> &Self::Target { - &self.idata - } - } - - impl core::ops::DerefMut for $inode_t { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.idata - } - } - - impl $crate::kernel::vfs::inode::InodeInner for $inode_t { - fn data(&self) -> &$crate::kernel::vfs::inode::InodeData { - &self.idata - } - - fn data_mut(&mut self) -> &mut $crate::kernel::vfs::inode::InodeData { - &mut self.idata - } - } - }; - ($v:vis struct $inode_t:ident { $($vis:vis $name:ident: $type:ty,)* }) => { - $v struct $inode_t { - /// Do not use this directly - idata: $crate::kernel::vfs::inode::InodeData, - $($vis $name: $type,)* - } - - impl core::ops::Deref for $inode_t { - type Target = $crate::kernel::vfs::inode::InodeData; - - fn deref(&self) -> &Self::Target { - &self.idata - } - } - - impl core::ops::DerefMut for $inode_t { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.idata - } - } - - impl $crate::kernel::vfs::inode::InodeInner for $inode_t { - fn data(&self) -> &$crate::kernel::vfs::inode::InodeData { - &self.idata - } - - fn data_mut(&mut self) -> &mut $crate::kernel::vfs::inode::InodeData { - &mut self.idata - } - } - }; -} - -pub(crate) use define_struct_inode; - -impl Mode { - pub const REG: Self = Self(S_IFREG); - pub const DIR: Self = Self(S_IFDIR); - pub const LNK: Self = Self(S_IFLNK); - pub const BLK: Self = Self(S_IFBLK); - pub const CHR: Self = Self(S_IFCHR); - - pub const fn new(bits: u32) -> Self { - Self(bits) - } - - pub const fn is_blk(&self) -> bool { - (self.0 & S_IFMT) == S_IFBLK - } - - pub const fn is_chr(&self) -> bool { - (self.0 & S_IFMT) == S_IFCHR - } - - pub const fn is_reg(&self) -> bool { - (self.0 & S_IFMT) == S_IFREG - } - - pub const fn is_dir(&self) -> bool { - (self.0 & S_IFMT) == S_IFDIR - } - - pub const fn is_lnk(&self) -> bool { - (self.0 & S_IFMT) == S_IFLNK - } - - pub const fn bits(&self) -> u32 { - self.0 - } - - pub const fn format_bits(&self) -> u32 { - self.0 & S_IFMT - } - - pub const fn format(&self) -> Self { - Self::new(self.format_bits()) - } - - pub const fn non_format_bits(&self) -> u32 { - self.0 & !S_IFMT - } - - pub const fn non_format(&self) -> Self { - Self::new(self.non_format_bits()) - } - - pub const fn perm(self, perm: u32) -> Self { - Self::new((self.0 & !0o777) | (perm & 0o777)) - } - - pub const fn set_perm(&mut self, perm: u32) { - *self = self.perm(perm); - } - - pub const fn mask_perm(&mut self, perm_mask: u32) { - let perm_mask = perm_mask & 0o777; - let self_perm = self.non_format_bits() & 0o777; - - *self = self.perm(self_perm & perm_mask); - } -} - -impl AtomicMode { - pub const fn new(bits: u32) -> Self { - Self(AtomicU32::new(bits)) - } - - pub const fn from(mode: Mode) -> Self { - Self::new(mode.0) - } - - pub fn load(&self) -> Mode { - Mode(self.0.load(Ordering::Relaxed)) - } - - pub fn store(&self, mode: Mode) { - self.0.store(mode.0, Ordering::Relaxed); - } -} - -impl core::fmt::Debug for AtomicMode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("AtomicMode") - .field("bits", &self.load().0) - .finish() - } -} - -impl core::fmt::Debug for Mode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let format_name = match self.format() { - Mode::REG => "REG", - Mode::DIR => "DIR", - Mode::LNK => "LNK", - Mode::BLK => "BLK", - Mode::CHR => "CHR", - _ => "UNK", - }; - - match self.non_format_bits() & !0o777 { - 0 => write!( - f, - "Mode({format_name}, {perm:#o})", - perm = self.non_format_bits() - )?, - rem => write!( - f, - "Mode({format_name}, {perm:#o}, rem={rem:#x})", - perm = self.non_format_bits() & 0o777 - )?, - } - - Ok(()) - } -} - -impl FromSyscallArg for Mode { - fn from_arg(value: usize) -> Self { - Mode::new(value as u32) - } -} - -impl SyscallRetVal for Mode { - fn into_retval(self) -> Option { - Some(self.bits() as usize) - } -} diff --git a/src/kernel/vfs/inode/ino.rs b/src/kernel/vfs/inode/ino.rs new file mode 100644 index 00000000..b5ee7ac0 --- /dev/null +++ b/src/kernel/vfs/inode/ino.rs @@ -0,0 +1,31 @@ +use core::{ + fmt::{Debug, Display, Formatter}, + sync::atomic::AtomicU64, +}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Ino(u64); + +pub struct AtomicIno(AtomicU64); + +impl Ino { + pub const fn new(ino: u64) -> Self { + Self(ino) + } + + pub const fn as_raw(self) -> u64 { + self.0 + } +} + +impl Debug for Ino { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "Ino({})", self.0) + } +} + +impl Display for Ino { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs new file mode 100644 index 00000000..786a31fe --- /dev/null +++ b/src/kernel/vfs/inode/inode.rs @@ -0,0 +1,389 @@ +use alloc::boxed::Box; +use core::{ + any::Any, + future::Future, + marker::Unsize, + ops::{CoerceUnsized, Deref}, + pin::Pin, +}; +use eonix_sync::Spin; + +use alloc::sync::{Arc, Weak}; +use async_trait::async_trait; + +use crate::{ + io::{Buffer, Stream}, + kernel::{ + constants::{EINVAL, EPERM}, + mem::PageCache, + timer::Instant, + vfs::{ + dentry::Dentry, + types::{DeviceId, Format, Mode, Permission}, + SbRef, SbUse, SuperBlock, + }, + }, + prelude::KResult, +}; + +use super::{Ino, RenameData, WriteOffset}; + +pub trait InodeOps: Sized + Send + Sync + 'static { + type SuperBlock: SuperBlock + Sized; + + fn ino(&self) -> Ino; + fn format(&self) -> Format; + fn info(&self) -> &Spin; + + fn super_block(&self) -> &SbRef; + + fn page_cache(&self) -> Option<&PageCache>; +} + +#[allow(unused_variables)] +pub trait InodeDirOps: InodeOps { + fn lookup( + &self, + dentry: &Arc, + ) -> impl Future>>> + Send { + async { Err(EPERM) } + } + + /// Read directory entries and call the given closure for each entry. + /// + /// # Returns + /// - Ok(count): The number of entries read. + /// - Ok(Err(err)): Some error occurred while calling the given closure. + /// - Err(err): An error occurred while reading the directory. + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> impl Future>> + Send + 'r { + async { Err(EPERM) } + } + + fn create( + &self, + at: &Arc, + mode: Permission, + ) -> impl Future> + Send { + async { Err(EPERM) } + } + + fn mkdir(&self, at: &Dentry, mode: Permission) -> impl Future> + Send { + async { Err(EPERM) } + } + + fn mknod( + &self, + at: &Dentry, + mode: Mode, + dev: DeviceId, + ) -> impl Future> + Send { + async { Err(EPERM) } + } + + fn unlink(&self, at: &Arc) -> impl Future> + Send { + async { Err(EPERM) } + } + + fn symlink(&self, at: &Arc, target: &[u8]) -> impl Future> + Send { + async { Err(EPERM) } + } + + fn rename(&self, rename_data: RenameData<'_, '_>) -> impl Future> + Send { + async { Err(EPERM) } + } +} + +#[allow(unused_variables)] +pub trait InodeFileOps: InodeOps { + fn read( + &self, + buffer: &mut dyn Buffer, + offset: usize, + ) -> impl Future> + Send { + async { Err(EINVAL) } + } + + fn read_direct( + &self, + buffer: &mut dyn Buffer, + offset: usize, + ) -> impl Future> + Send { + async { Err(EINVAL) } + } + + fn write( + &self, + stream: &mut dyn Stream, + offset: WriteOffset<'_>, + ) -> impl Future> + Send { + async { Err(EINVAL) } + } + + fn write_direct( + &self, + stream: &mut dyn Stream, + offset: usize, + ) -> impl Future> + Send { + async { Err(EINVAL) } + } + + fn devid(&self) -> KResult { + Err(EINVAL) + } + + fn readlink(&self, buffer: &mut dyn Buffer) -> impl Future> + Send { + async { Err(EINVAL) } + } + + fn truncate(&self, length: usize) -> impl Future> + Send { + async { Err(EPERM) } + } + + fn chmod(&self, perm: Permission) -> impl Future> + Send { + async { Err(EPERM) } + } + + fn chown(&self, uid: u32, gid: u32) -> impl Future> + Send { + async { Err(EPERM) } + } +} + +#[async_trait] +pub trait InodeDir { + async fn lookup(&self, dentry: &Arc) -> KResult>>; + async fn create(&self, at: &Arc, perm: Permission) -> KResult<()>; + async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()>; + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()>; + async fn unlink(&self, at: &Arc) -> KResult<()>; + async fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()>; + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()>; + + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> Pin>> + Send + 'r>>; +} + +#[async_trait] +pub trait InodeFile { + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult; + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult; + async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult; + async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult; + fn devid(&self) -> KResult; + async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult; + async fn truncate(&self, length: usize) -> KResult<()>; + async fn chmod(&self, mode: Mode) -> KResult<()>; + async fn chown(&self, uid: u32, gid: u32) -> KResult<()>; +} + +pub trait Inode: InodeFile + InodeDir + Any + Send + Sync + 'static { + fn ino(&self) -> Ino; + fn format(&self) -> Format; + fn info(&self) -> &Spin; + + // TODO: This might should be removed... Temporary workaround for now. + fn page_cache(&self) -> Option<&PageCache>; + + fn sbref(&self) -> SbRef; + fn sbget(&self) -> KResult>; +} + +#[async_trait] +impl InodeFile for T +where + T: InodeFileOps, +{ + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + self.read(buffer, offset).await + } + + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + self.read_direct(buffer, offset).await + } + + async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult { + self.write(stream, offset).await + } + + async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult { + self.write_direct(stream, offset).await + } + + fn devid(&self) -> KResult { + self.devid() + } + + async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { + self.readlink(buffer).await + } + + async fn truncate(&self, length: usize) -> KResult<()> { + self.truncate(length).await + } + + async fn chmod(&self, mode: Mode) -> KResult<()> { + self.chmod(Permission::new(mode.non_format_bits())).await + } + + async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { + self.chown(uid, gid).await + } +} + +#[async_trait] +impl InodeDir for T +where + T: InodeDirOps, +{ + async fn lookup(&self, dentry: &Arc) -> KResult>> { + self.lookup(dentry).await + } + + async fn create(&self, at: &Arc, perm: Permission) -> KResult<()> { + self.create(at, perm).await + } + + async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { + self.mkdir(at, perm).await + } + + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + self.mknod(at, mode, dev).await + } + + async fn unlink(&self, at: &Arc) -> KResult<()> { + self.unlink(at).await + } + + async fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()> { + self.symlink(at, target).await + } + + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { + self.rename(rename_data).await + } + + fn readdir<'r, 'a: 'r, 'b: 'r>( + &'a self, + offset: usize, + callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> Pin>> + Send + 'r>> { + Box::pin(self.readdir(offset, callback)) + } +} + +impl Inode for T +where + T: InodeOps + InodeFile + InodeDir, +{ + fn ino(&self) -> Ino { + self.ino() + } + + fn format(&self) -> Format { + self.format() + } + + fn info(&self) -> &Spin { + self.info() + } + + fn page_cache(&self) -> Option<&PageCache> { + self.page_cache() + } + + fn sbref(&self) -> SbRef { + self.super_block().clone() + } + + fn sbget(&self) -> KResult> { + self.super_block().get().map(|sb| sb as _) + } +} + +#[derive(Debug, Clone)] +pub struct InodeInfo { + pub size: u64, + pub nlink: u64, + + pub uid: u32, + pub gid: u32, + pub perm: Permission, + + pub atime: Instant, + pub ctime: Instant, + pub mtime: Instant, +} + +#[derive(Clone)] +pub struct InodeRef(Weak) +where + I: Inode + ?Sized; + +pub struct InodeUse(Arc) +where + I: Inode + ?Sized; + +impl InodeUse +where + I: Inode, +{ + pub fn new(inode: I) -> Self { + Self(Arc::new(inode)) + } + + pub fn new_cyclic(inode_func: impl FnOnce(&Weak) -> I) -> Self { + Self(Arc::new_cyclic(inode_func)) + } +} + +impl InodeUse +where + I: Inode + ?Sized, +{ + pub fn as_raw(&self) -> *const I { + Arc::as_ptr(&self.0) + } +} + +impl CoerceUnsized> for InodeUse +where + T: Inode + Unsize + ?Sized, + U: Inode + ?Sized, +{ +} + +impl Clone for InodeUse +where + I: Inode + ?Sized, +{ + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl core::fmt::Debug for InodeUse +where + I: Inode + ?Sized, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "InodeUse(ino={})", self.ino()) + } +} + +impl Deref for InodeUse +where + I: Inode + ?Sized, +{ + type Target = I; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} diff --git a/src/kernel/vfs/inode/mod.rs b/src/kernel/vfs/inode/mod.rs new file mode 100644 index 00000000..786d30fa --- /dev/null +++ b/src/kernel/vfs/inode/mod.rs @@ -0,0 +1,10 @@ +mod ino; +mod inode; +mod ops; +mod statx; + +pub use ino::Ino; +pub use inode::{ + Inode, InodeDir, InodeDirOps, InodeFile, InodeFileOps, InodeInfo, InodeOps, InodeRef, InodeUse, +}; +pub use ops::{RenameData, WriteOffset}; diff --git a/src/kernel/vfs/inode/ops.rs b/src/kernel/vfs/inode/ops.rs new file mode 100644 index 00000000..baab1a80 --- /dev/null +++ b/src/kernel/vfs/inode/ops.rs @@ -0,0 +1,18 @@ +use alloc::sync::Arc; + +use crate::kernel::vfs::dentry::Dentry; + +use super::{inode::InodeUse, Inode}; + +pub enum WriteOffset<'end> { + Position(usize), + End(&'end mut usize), +} + +pub struct RenameData<'a, 'b> { + pub old_dentry: &'a Arc, + pub new_dentry: &'b Arc, + pub new_parent: InodeUse, + pub is_exchange: bool, + pub no_replace: bool, +} diff --git a/src/kernel/vfs/inode/statx.rs b/src/kernel/vfs/inode/statx.rs new file mode 100644 index 00000000..a85ef3af --- /dev/null +++ b/src/kernel/vfs/inode/statx.rs @@ -0,0 +1,97 @@ +use posix_types::stat::StatX; + +use crate::{ + kernel::{ + constants::{ + STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, STATX_MODE, STATX_MTIME, + STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, + }, + vfs::types::Format, + }, + prelude::KResult, +}; + +use super::{inode::InodeUse, Inode}; + +impl InodeUse +where + I: Inode + ?Sized, +{ + pub fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { + let sb = self.sbget()?; + let info = self.info().lock(); + + if mask & STATX_NLINK != 0 { + stat.stx_nlink = info.nlink as _; + stat.stx_mask |= STATX_NLINK; + } + + if mask & STATX_ATIME != 0 { + stat.stx_atime = info.atime.into(); + stat.stx_mask |= STATX_ATIME; + } + + if mask & STATX_MTIME != 0 { + stat.stx_mtime = info.mtime.into(); + stat.stx_mask |= STATX_MTIME; + } + + if mask & STATX_CTIME != 0 { + stat.stx_ctime = info.ctime.into(); + stat.stx_mask |= STATX_CTIME; + } + + if mask & STATX_SIZE != 0 { + stat.stx_size = info.size as _; + stat.stx_mask |= STATX_SIZE; + } + + stat.stx_mode = 0; + if mask & STATX_MODE != 0 { + stat.stx_mode |= info.perm.bits() as u16; + stat.stx_mask |= STATX_MODE; + } + + if mask & STATX_TYPE != 0 { + let format = self.format(); + + stat.stx_mode |= format.as_raw() as u16; + if let Format::BLK | Format::CHR = format { + let devid = self.devid()?; + stat.stx_rdev_major = devid.major as _; + stat.stx_rdev_minor = devid.minor as _; + } + stat.stx_mask |= STATX_TYPE; + } + + if mask & STATX_INO != 0 { + stat.stx_ino = self.ino().as_raw(); + stat.stx_mask |= STATX_INO; + } + + if mask & STATX_BLOCKS != 0 { + stat.stx_blocks = (info.size + 512 - 1) / 512; + stat.stx_blksize = sb.info.io_blksize as _; + stat.stx_mask |= STATX_BLOCKS; + } + + if mask & STATX_UID != 0 { + stat.stx_uid = info.uid; + stat.stx_mask |= STATX_UID; + } + + if mask & STATX_GID != 0 { + stat.stx_gid = info.gid; + stat.stx_mask |= STATX_GID; + } + + let fsdev = sb.info.device_id; + stat.stx_dev_major = fsdev.major as _; + stat.stx_dev_minor = fsdev.minor as _; + + // TODO: support more attributes + stat.stx_attributes_mask = 0; + + Ok(()) + } +} diff --git a/src/kernel/vfs/mod.rs b/src/kernel/vfs/mod.rs index f62cb9b9..5b8eca5a 100644 --- a/src/kernel/vfs/mod.rs +++ b/src/kernel/vfs/mod.rs @@ -1,31 +1,31 @@ -use crate::prelude::*; -use alloc::sync::Arc; -use dentry::Dentry; -use eonix_sync::LazyLock; -use inode::Mode; - pub mod dentry; mod file; pub mod filearray; pub mod inode; pub mod mount; -pub mod vfs; +mod superblock; +pub mod types; -pub use file::{File, FileType, PollEvent, SeekOption, TerminalFile}; +use crate::prelude::*; +use alloc::sync::Arc; +use dentry::Dentry; +use eonix_sync::LazyLock; +use types::Permission; -pub type DevId = u32; +pub use file::{File, FileType, PollEvent, SeekOption, TerminalFile}; +pub use superblock::{SbRef, SbUse, SuperBlock, SuperBlockInfo, SuperBlockLock}; pub struct FsContext { pub fsroot: Arc, pub cwd: Spin>, - pub umask: Spin, + pub umask: Spin, } static GLOBAL_FS_CONTEXT: LazyLock> = LazyLock::new(|| { Arc::new(FsContext { fsroot: Dentry::root().clone(), cwd: Spin::new(Dentry::root().clone()), - umask: Spin::new(Mode::new(0o022)), + umask: Spin::new(Permission::new(0o755)), }) }); diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 0b38e0c0..213acae9 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -1,11 +1,15 @@ use super::{ dentry::{dcache, Dentry, DROOT}, - inode::Inode, - vfs::Vfs, + inode::{Inode, InodeUse}, + SbUse, SuperBlock, +}; +use crate::kernel::{ + constants::{EEXIST, ENODEV, ENOTDIR}, + task::block_on, }; -use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; use crate::prelude::*; use alloc::{collections::btree_map::BTreeMap, string::ToString as _, sync::Arc}; +use async_trait::async_trait; use eonix_sync::LazyLock; pub const MS_RDONLY: u64 = 1 << 0; @@ -30,17 +34,21 @@ static MOUNT_CREATORS: Spin>> = Spin::new static MOUNTS: Spin, MountPointData)>> = Spin::new(vec![]); pub struct Mount { - _vfs: Arc, + sb: SbUse, root: Arc, } impl Mount { - pub fn new(mp: &Dentry, vfs: Arc, root_inode: Arc) -> KResult { + pub fn new( + mp: &Dentry, + sb: SbUse, + root_inode: InodeUse, + ) -> KResult { let root_dentry = Dentry::create(mp.parent().clone(), &mp.get_name()); - root_dentry.save_dir(root_inode)?; + root_dentry.fill(root_inode); Ok(Self { - _vfs: vfs, + sb, root: root_dentry, }) } @@ -53,9 +61,10 @@ impl Mount { unsafe impl Send for Mount {} unsafe impl Sync for Mount {} +#[async_trait] pub trait MountCreator: Send + Sync { fn check_signature(&self, first_block: &[u8]) -> KResult; - fn create_mount(&self, source: &str, flags: u64, mp: &Arc) -> KResult; + async fn create_mount(&self, source: &str, flags: u64, mp: &Arc) -> KResult; } pub fn register_filesystem(fstype: &str, creator: Arc) -> KResult<()> { @@ -77,7 +86,7 @@ struct MountPointData { flags: u64, } -pub fn do_mount( +pub async fn do_mount( mountpoint: &Arc, source: &str, mountpoint_str: &str, @@ -101,7 +110,7 @@ pub fn do_mount( let creators = { MOUNT_CREATORS.lock() }; creators.get(fstype).ok_or(ENODEV)?.clone() }; - let mount = creator.create_mount(source, flags, mountpoint)?; + let mount = creator.create_mount(source, flags, mountpoint).await?; let root_dentry = mount.root().clone(); @@ -165,8 +174,7 @@ impl Dentry { .cloned() .expect("tmpfs not registered."); - let mount = creator - .create_mount(&source, mount_flags, &DROOT) + let mount = block_on(creator.create_mount(&source, mount_flags, &DROOT)) .expect("Failed to create root mount."); let root_dentry = mount.root().clone(); diff --git a/src/kernel/vfs/superblock.rs b/src/kernel/vfs/superblock.rs new file mode 100644 index 00000000..85b28c01 --- /dev/null +++ b/src/kernel/vfs/superblock.rs @@ -0,0 +1,127 @@ +use core::{ + marker::Unsize, + ops::{CoerceUnsized, Deref}, +}; + +use alloc::sync::{Arc, Weak}; +use eonix_sync::RwLock; + +use crate::{kernel::constants::EIO, prelude::KResult}; + +use super::types::DeviceId; + +pub trait SuperBlock: Send + Sync + 'static {} + +#[derive(Debug, Clone)] +pub struct SuperBlockInfo { + pub io_blksize: u32, + pub device_id: DeviceId, + pub read_only: bool, +} + +pub struct SuperBlockLock(()); + +pub struct SuperBlockComplex +where + Backend: SuperBlock + ?Sized, +{ + pub info: SuperBlockInfo, + pub rwsem: RwLock, + pub backend: Backend, +} + +pub struct SbRef(Weak>) +where + S: SuperBlock + ?Sized; + +pub struct SbUse(Arc>) +where + S: SuperBlock + ?Sized; + +impl SbRef +where + S: SuperBlock + ?Sized, +{ + pub fn try_get(&self) -> Option> { + self.0.upgrade().map(|arc| SbUse(arc)) + } + + pub fn get(&self) -> KResult> { + self.try_get().ok_or(EIO) + } + + pub fn from(sb: &SbUse) -> Self { + SbRef(Arc::downgrade(&sb.0)) + } + + pub fn eq(&self, other: &SbRef) -> bool + where + U: SuperBlock + ?Sized, + { + core::ptr::addr_eq(self.0.as_ptr(), other.0.as_ptr()) + } +} + +impl SbUse +where + S: SuperBlock, +{ + pub fn new(info: SuperBlockInfo, backend: S) -> Self { + Self(Arc::new(SuperBlockComplex { + info, + rwsem: RwLock::new(SuperBlockLock(())), + backend, + })) + } + + pub fn new_cyclic(info: SuperBlockInfo, backend_func: impl FnOnce(SbRef) -> S) -> Self { + Self(Arc::new_cyclic(|weak| SuperBlockComplex { + info, + rwsem: RwLock::new(SuperBlockLock(())), + backend: backend_func(SbRef(weak.clone())), + })) + } +} + +impl Clone for SbRef +where + S: SuperBlock + ?Sized, +{ + fn clone(&self) -> Self { + SbRef(self.0.clone()) + } +} + +impl Clone for SbUse +where + S: SuperBlock + ?Sized, +{ + fn clone(&self) -> Self { + SbUse(self.0.clone()) + } +} + +impl CoerceUnsized> for SbRef +where + T: SuperBlock + Unsize + ?Sized, + U: SuperBlock + ?Sized, +{ +} + +impl CoerceUnsized> for SbUse +where + T: SuperBlock + Unsize + ?Sized, + U: SuperBlock + ?Sized, +{ +} + +impl Deref for SbUse +where + S: SuperBlock + ?Sized, +{ + type Target = SuperBlockComplex; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} diff --git a/src/kernel/vfs/types/device_id.rs b/src/kernel/vfs/types/device_id.rs new file mode 100644 index 00000000..cf3ea886 --- /dev/null +++ b/src/kernel/vfs/types/device_id.rs @@ -0,0 +1,36 @@ +use core::fmt::{Debug, Display, Formatter}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DeviceId { + pub major: u16, + pub minor: u16, +} + +impl DeviceId { + pub const fn new(major: u16, minor: u16) -> Self { + Self { major, minor } + } + + pub const fn from_raw(raw: u32) -> Self { + Self { + major: (raw >> 16) as u16, + minor: (raw & 0xFFFF) as u16, + } + } + + pub const fn to_raw(self) -> u32 { + ((self.major as u32) << 16) | (self.minor as u32) + } +} + +impl Debug for DeviceId { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "DeviceId({:04x}:{:04x})", self.major, self.minor) + } +} + +impl Display for DeviceId { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "{:04x}:{:04x}", self.major, self.minor) + } +} diff --git a/src/kernel/vfs/types/mod.rs b/src/kernel/vfs/types/mod.rs new file mode 100644 index 00000000..4a7505f7 --- /dev/null +++ b/src/kernel/vfs/types/mod.rs @@ -0,0 +1,5 @@ +mod device_id; +mod mode; + +pub use device_id::DeviceId; +pub use mode::{Format, Mode, Permission}; diff --git a/src/kernel/vfs/types/mode.rs b/src/kernel/vfs/types/mode.rs new file mode 100644 index 00000000..dc1b88ec --- /dev/null +++ b/src/kernel/vfs/types/mode.rs @@ -0,0 +1,169 @@ +use crate::kernel::{ + constants::{S_IFBLK, S_IFCHR, S_IFDIR, S_IFLNK, S_IFMT, S_IFREG}, + syscall::{FromSyscallArg, SyscallRetVal}, +}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Mode(u32); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Format { + REG, + DIR, + LNK, + BLK, + CHR, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Permission(u32); + +impl Mode { + pub const fn new(bits: u32) -> Self { + Self(bits) + } + + pub const fn is_blk(&self) -> bool { + (self.0 & S_IFMT) == S_IFBLK + } + + pub const fn is_chr(&self) -> bool { + (self.0 & S_IFMT) == S_IFCHR + } + + pub const fn bits(&self) -> u32 { + self.0 + } + + pub const fn format_bits(&self) -> u32 { + self.0 & S_IFMT + } + + pub const fn non_format_bits(&self) -> u32 { + self.0 & !S_IFMT + } + + pub fn format(&self) -> Format { + match self.format_bits() { + S_IFREG => Format::REG, + S_IFDIR => Format::DIR, + S_IFLNK => Format::LNK, + S_IFBLK => Format::BLK, + S_IFCHR => Format::CHR, + _ => panic!("unknown format bits: {:#o}", self.format_bits()), + } + } + + pub fn perm(&self) -> Permission { + Permission::new(self.non_format_bits()) + } + + pub const fn non_format(&self) -> Self { + Self::new(self.non_format_bits()) + } + + pub const fn set_perm(&mut self, perm: Permission) { + self.0 = self.format_bits() | perm.bits(); + } +} + +impl Format { + pub const fn as_raw(&self) -> u32 { + match self { + Self::REG => S_IFREG, + Self::DIR => S_IFDIR, + Self::LNK => S_IFLNK, + Self::BLK => S_IFBLK, + Self::CHR => S_IFCHR, + } + } +} + +impl Permission { + const RWX: [&str; 8] = ["---", "--x", "-w-", "-wx", "r--", "r-x", "rw-", "rwx"]; + + pub const fn new(perm_bits: u32) -> Self { + Self(perm_bits & 0o7777) + } + + pub const fn bits(&self) -> u32 { + self.0 + } + + pub const fn mask_with(&self, mask: Self) -> Self { + Self(self.0 & mask.0) + } +} + +impl core::fmt::Debug for Mode { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.non_format_bits() & !0o777 { + 0 => write!( + f, + "Mode({format:?}, {perm:#o})", + format = self.format(), + perm = self.non_format_bits() + )?, + rem => write!( + f, + "Mode({format:?}, {perm:#o}, rem={rem:#x})", + format = self.format(), + perm = self.non_format_bits() & 0o777 + )?, + } + + Ok(()) + } +} + +impl core::fmt::Debug for Format { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::REG => write!(f, "REG"), + Self::DIR => write!(f, "DIR"), + Self::LNK => write!(f, "LNK"), + Self::BLK => write!(f, "BLK"), + Self::CHR => write!(f, "CHR"), + } + } +} + +impl core::fmt::Debug for Permission { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let owner = self.0 >> 6 & 0o7; + let group = self.0 >> 3 & 0o7; + let other = self.0 & 0o7; + + write!( + f, + "{}{}{}", + Self::RWX[owner as usize], + Self::RWX[group as usize], + Self::RWX[other as usize] + ) + } +} + +impl FromSyscallArg for Mode { + fn from_arg(value: usize) -> Self { + Mode::new(value as u32) + } +} + +impl SyscallRetVal for Mode { + fn into_retval(self) -> Option { + Some(self.bits() as usize) + } +} + +impl FromSyscallArg for Permission { + fn from_arg(value: usize) -> Self { + Permission::new(value as u32) + } +} + +impl SyscallRetVal for Permission { + fn into_retval(self) -> Option { + Some(self.bits() as usize) + } +} diff --git a/src/kernel/vfs/vfs.rs b/src/kernel/vfs/vfs.rs deleted file mode 100644 index ee66f0b6..00000000 --- a/src/kernel/vfs/vfs.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::prelude::*; - -use super::DevId; - -#[allow(dead_code)] -pub trait Vfs: Send + Sync + AsAny { - fn io_blksize(&self) -> usize; - fn fs_devid(&self) -> DevId; - fn is_read_only(&self) -> bool; -} diff --git a/src/lib.rs b/src/lib.rs index 80d24c28..98e196f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,10 +2,12 @@ #![no_main] #![feature(allocator_api)] #![feature(c_size_t)] +#![feature(coerce_unsized)] #![feature(concat_idents)] #![feature(arbitrary_self_types)] #![feature(get_mut_unchecked)] #![feature(macro_metavar_expr)] +#![feature(unsize)] extern crate alloc; @@ -46,8 +48,8 @@ use kernel::{ task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}, vfs::{ dentry::Dentry, - inode::Mode, mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}, + types::Permission, FsContext, }, CharDevice, @@ -192,16 +194,16 @@ async fn init_process(early_kstack: PRange) { { // We might want the serial initialized as soon as possible. driver::serial::init().unwrap(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; } #[cfg(target_arch = "riscv64")] { driver::serial::init().unwrap(); driver::virtio::init_virtio_devices(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; driver::goldfish_rtc::probe(); } @@ -209,21 +211,26 @@ async fn init_process(early_kstack: PRange) { { driver::serial::init().unwrap(); driver::virtio::init_virtio_devices(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; } fs::tmpfs::init(); - fs::procfs::init(); + fs::procfs::init().await; fs::fat32::init(); - fs::ext4::init(); + // fs::ext4::init(); let load_info = { // mount fat32 /mnt directory let fs_context = FsContext::global(); - let mnt_dir = Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true).unwrap(); + let mnt_dir = Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true) + .await + .unwrap(); - mnt_dir.mkdir(Mode::new(0o755)).unwrap(); + mnt_dir + .mkdir(Permission::new(0o755)) + .await + .expect("Failed to create /mnt directory"); do_mount( &mnt_dir, @@ -232,6 +239,7 @@ async fn init_process(early_kstack: PRange) { "fat32", MS_RDONLY | MS_NOATIME | MS_NODEV | MS_NOSUID, ) + .await .unwrap(); let init_names = [&b"/init"[..], &b"/sbin/init"[..], &b"/mnt/initsh"[..]]; @@ -239,7 +247,7 @@ async fn init_process(early_kstack: PRange) { let mut init_name = None; let mut init = None; for name in init_names { - if let Ok(dentry) = Dentry::open(fs_context, Path::new(name).unwrap(), true) { + if let Ok(dentry) = Dentry::open(fs_context, Path::new(name).unwrap(), true).await { if dentry.is_valid() { init_name = Some(CString::new(name).unwrap()); init = Some(dentry); @@ -261,6 +269,7 @@ async fn init_process(early_kstack: PRange) { ]; ProgramLoader::parse(fs_context, init_name, init.clone(), argv, envp) + .await .expect("Failed to parse init program") .load() .await diff --git a/src/prelude.rs b/src/prelude.rs index b3dbe2ce..880489da 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -18,34 +18,6 @@ pub(crate) use crate::kernel::console::{ pub(crate) use alloc::{boxed::Box, string::String, vec, vec::Vec}; -pub(crate) use core::{any::Any, fmt::Write, marker::PhantomData, str}; +pub(crate) use core::{fmt::Write, marker::PhantomData, str}; pub use crate::sync::Spin; - -#[allow(dead_code)] -pub trait AsAny: Send + Sync { - fn as_any(&self) -> &dyn Any; - fn as_any_mut(&mut self) -> &mut dyn Any; -} - -macro_rules! impl_any { - ($t:ty) => { - impl AsAny for $t { - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - } - }; -} - -macro_rules! addr_of_mut_field { - ($pointer:expr, $field:ident) => { - core::ptr::addr_of_mut!((*$pointer).$field) - }; -} - -pub(crate) use {addr_of_mut_field, impl_any}; From ea2122331ee265263a23981e230c76199680d866 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 14 Sep 2025 21:46:19 +0800 Subject: [PATCH 002/153] vfs, rcu: rework path walking with new rcu syntax The old path walking algorithm requires recursion, which is not supported in async rust. So we boxed them all as a temporary solution in previous commits. This would introduce mass overhead even for fast path walks just because we might sleep in `readlink()` and `lookup()` calls. The new proposed method is to break the walk into several phases similar to that in Linux: RCU walk and REF walk. The RCU walk will hold the RCU lock and never blocks so the function itself can be non-async. If we hit non-present dentries, we will fallback to REF walk. In REF walks, we clone the Arcs and consult to the VFS layer for an accurate answer. Note that in both the two methods mentioned above, symlinks are not handled and will be returned directly with all path components left untouched. We have a dedicated async function to follow the symlinks by recursive calling the walk function. This can be slow and won't be called frequently. So we wrapped the function with `Box::pin()` to break the recursion chain. After the symlink resolution is done, we return to the original position and continue the walk. We found that the association of an inode to a dentry is one way. So the `data` RCUPointer field is actually unnecessary and we can use the atomic dentry type to sync readers with the writer. This way we can eliminate `DentryData` allocations and improve performance. We also introduced a new RCU read lock syntax. In the RCU walk mentioned above, we need to store dentry references protected by some RCU read lock. With the old syntax, we can't express the lifetime associtated by the common RCU read lock. The new syntax provides a `rcu_read_lock()` method to acquire the RCU read lock. The lock returned has a associated lifetime so we can use it in the RCU session. Signed-off-by: greatbridf --- Cargo.lock | 5 + Cargo.toml | 8 +- crates/arcref/Cargo.lock | 7 + crates/arcref/Cargo.toml | 11 + crates/arcref/src/arcref.rs | 216 +++++++++++++++++ crates/arcref/src/lib.rs | 8 + crates/posix_types/src/result.rs | 6 + src/kernel/vfs/dentry.rs | 394 +++++++++++++------------------ src/kernel/vfs/dentry/dcache.rs | 76 +++++- src/kernel/vfs/dentry/walk.rs | 370 +++++++++++++++++++++++++++++ src/path.rs | 40 ++-- src/rcu.rs | 99 ++++---- 12 files changed, 942 insertions(+), 298 deletions(-) create mode 100644 crates/arcref/Cargo.lock create mode 100644 crates/arcref/Cargo.toml create mode 100644 crates/arcref/src/arcref.rs create mode 100644 crates/arcref/src/lib.rs create mode 100644 src/kernel/vfs/dentry/walk.rs diff --git a/Cargo.lock b/Cargo.lock index 32868677..f85a9d2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,10 @@ dependencies = [ "log", ] +[[package]] +name = "arcref" +version = "0.1.0" + [[package]] name = "async-trait" version = "0.1.89" @@ -155,6 +159,7 @@ dependencies = [ "acpi", "align_ext", "another_ext4", + "arcref", "async-trait", "atomic_unique_refcell", "bitflags", diff --git a/Cargo.toml b/Cargo.toml index bc7e7b0c..5158025e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" crate-type = ["bin"] [dependencies] +arcref = { path = "./crates/arcref", default-features = false, features = [ + "alloc", +] } atomic_unique_refcell = { path = "./crates/atomic_unique_refcell", features = [ "no_std", ] } @@ -35,7 +38,10 @@ stalloc = { version = "0.6.1", default-features = false, features = [ "allocator-api", ] } async-trait = "0.1.89" -futures = { version = "0.3.31", features = ["alloc", "async-await"], default-features = false } +futures = { version = "0.3.31", features = [ + "alloc", + "async-await", +], default-features = false } [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/crates/arcref/Cargo.lock b/crates/arcref/Cargo.lock new file mode 100644 index 00000000..3c4e1567 --- /dev/null +++ b/crates/arcref/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arcref" +version = "0.1.0" diff --git a/crates/arcref/Cargo.toml b/crates/arcref/Cargo.toml new file mode 100644 index 00000000..a0af89f8 --- /dev/null +++ b/crates/arcref/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "arcref" +version = "0.1.0" +edition = "2024" + +[dependencies] + +[features] +alloc = [] +std = ["alloc"] +default = ["std"] diff --git a/crates/arcref/src/arcref.rs b/crates/arcref/src/arcref.rs new file mode 100644 index 00000000..3d01852d --- /dev/null +++ b/crates/arcref/src/arcref.rs @@ -0,0 +1,216 @@ +#[cfg(not(feature = "std"))] +use core::{ + borrow::Borrow, + marker::{PhantomData, Unsize}, + mem::ManuallyDrop, + ops::{Deref, DispatchFromDyn}, +}; + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +extern crate alloc; + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +use alloc::sync::Arc; + +#[cfg(feature = "std")] +use std::{ + borrow::Borrow, + marker::{PhantomData, Unsize}, + mem::ManuallyDrop, + ops::{Deref, DispatchFromDyn}, + sync::Arc, +}; + +pub trait AsArcRef +where + T: ?Sized, +{ + /// Borrow the [`Arc`] and convert the reference into [`ArcRef`]. + fn aref(&self) -> ArcRef<'_, T>; +} + +pub struct ArcRef<'a, T: ?Sized> { + ptr: *const T, + _phantom: PhantomData<&'a ()>, +} + +unsafe impl Send for ArcRef<'_, T> {} +unsafe impl Sync for ArcRef<'_, T> {} + +#[cfg(any(feature = "std", feature = "alloc"))] +impl<'a, T: ?Sized> ArcRef<'a, T> { + pub fn new(arc: &'a Arc) -> Self { + Self { + ptr: Arc::as_ptr(arc), + _phantom: PhantomData, + } + } + + /// Create a new `ArcRef` from a raw pointer. + /// + /// # Safety + /// The given pointer MUST be created by `Arc::as_ptr` or `Arc::into_raw`. + /// The caller is responsible to ensure that the pointer is valid for the + /// lifetime of the `ArcRef`. + pub unsafe fn new_unchecked(arc_ptr: *const T) -> Self { + Self { + ptr: arc_ptr, + _phantom: PhantomData, + } + } + + pub fn with_arc(self, func: Func) -> Out + where + Func: FnOnce(&Arc) -> Out, + { + func(&ManuallyDrop::new(unsafe { Arc::from_raw(self.ptr) })) + } + + pub fn clone_arc(self) -> Arc { + self.with_arc(|arc| arc.clone()) + } + + pub fn ptr_eq_arc(self, other: &Arc) -> bool { + self.with_arc(|arc| Arc::ptr_eq(arc, other)) + } +} + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +impl AsArcRef for Arc +where + T: ?Sized, +{ + fn aref(&self) -> ArcRef<'_, T> { + ArcRef::new(self) + } +} + +impl AsRef for ArcRef<'_, T> +where + T: ?Sized, +{ + fn as_ref(&self) -> &T { + self.deref() + } +} + +impl Borrow for ArcRef<'_, T> +where + T: ?Sized, +{ + fn borrow(&self) -> &T { + self.deref() + } +} + +impl<'a, T> Clone for ArcRef<'a, T> +where + T: ?Sized, +{ + fn clone(&self) -> Self { + Self { + ptr: self.ptr, + _phantom: PhantomData, + } + } +} + +impl Copy for ArcRef<'_, T> where T: ?Sized {} + +impl Deref for ArcRef<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { + // SAFETY: `self.ptr` points to a valid `T` instance because it was + // created from a valid `Arc`. + self.ptr.as_ref().unwrap_unchecked() + } + } +} + +impl<'a, T, U> DispatchFromDyn> for ArcRef<'a, T> +where + T: ?Sized + Unsize, + U: ?Sized, +{ +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn create_from_arc() { + let data = Arc::new(42); + let _arc_ref = ArcRef::new(&data); + } + + #[test] + fn deref() { + let data = Arc::new(42); + let arc_ref = ArcRef::new(&data); + + assert_eq!(*arc_ref, 42); + } + + #[test] + fn clone_into_arc() { + let data = Arc::new(42); + let arc_ref = ArcRef::new(&data); + + let cloned = arc_ref.clone_arc(); + + assert_eq!(Arc::strong_count(&data), 2); + assert_eq!(*cloned, 42); + } + + #[test] + fn dyn_compatible_receiver() { + struct Data(u32); + + trait Trait { + fn foo(self: ArcRef) -> u32; + } + + impl Trait for Data { + fn foo(self: ArcRef) -> u32 { + self.0 + } + } + + let data = Arc::new(Data(42)); + let arc_ref = ArcRef::new(&data); + + assert_eq!(arc_ref.foo(), 42); + } + + #[test] + fn clone_from_train_methods() { + struct Data(u32); + + trait Trait { + fn foo(&self) -> u32; + + fn clone_self(self: ArcRef) -> Arc; + } + + impl Trait for Data { + fn foo(&self) -> u32 { + self.0 + } + + fn clone_self(self: ArcRef) -> Arc { + self.clone_arc() as _ + } + } + + let data = Arc::new(Data(42)); + let arc_ref = ArcRef::new(&data); + + let cloned = arc_ref.clone_self(); + + assert_eq!(arc_ref.foo(), 42); + assert_eq!(cloned.foo(), 42); + } +} diff --git a/crates/arcref/src/lib.rs b/crates/arcref/src/lib.rs new file mode 100644 index 00000000..83a61985 --- /dev/null +++ b/crates/arcref/src/lib.rs @@ -0,0 +1,8 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![feature(arbitrary_self_types)] +#![feature(dispatch_from_dyn)] +#![feature(unsize)] + +mod arcref; + +pub use arcref::{ArcRef, AsArcRef}; diff --git a/crates/posix_types/src/result.rs b/crates/posix_types/src/result.rs index a10ff0ad..1535c444 100644 --- a/crates/posix_types/src/result.rs +++ b/crates/posix_types/src/result.rs @@ -1,14 +1,18 @@ pub enum PosixError { + ENOENT = 2, EFAULT = 14, EXDEV = 18, + ENOTDIR = 20, EINVAL = 22, } impl From for u32 { fn from(error: PosixError) -> Self { match error { + PosixError::ENOENT => 2, PosixError::EFAULT => 14, PosixError::EXDEV => 18, + PosixError::ENOTDIR => 20, PosixError::EINVAL => 22, } } @@ -17,8 +21,10 @@ impl From for u32 { impl core::fmt::Debug for PosixError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { + Self::ENOENT => write!(f, "ENOENT"), Self::EFAULT => write!(f, "EFAULT"), Self::EXDEV => write!(f, "EXDEV"), + Self::ENOTDIR => write!(f, "ENOTDIR"), Self::EINVAL => write!(f, "EINVAL"), } } diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index 5ac4e407..c1eb8cb8 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -1,45 +1,62 @@ pub mod dcache; +mod walk; -use super::{ - inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, - types::{DeviceId, Format, Mode, Permission}, - FsContext, -}; -use crate::{ - hash::KernelHasher, - io::{Buffer, ByteBuffer}, - kernel::{block::BlockDevice, CharDevice}, - path::{Path, PathComponent}, - prelude::*, - rcu::{RCUNode, RCUPointer, RCUReadGuard}, -}; -use crate::{ - io::Stream, - kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, ENOTDIR, EPERM, ERANGE}, -}; -use alloc::sync::Arc; use core::{ + cell::UnsafeCell, fmt, - future::Future, hash::{BuildHasher, BuildHasherDefault, Hasher}, - pin::Pin, - sync::atomic::{AtomicPtr, AtomicU64, Ordering}, + sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}, }; + +use alloc::sync::Arc; +use arcref::AsArcRef; use eonix_sync::LazyLock; use pointers::BorrowedArc; use posix_types::{namei::RenameFlags, open::OpenFlags, result::PosixError, stat::StatX}; -#[derive(PartialEq, Eq)] +use crate::{ + hash::KernelHasher, + io::Buffer, + io::Stream, + kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE}, + kernel::{block::BlockDevice, CharDevice}, + path::Path, + prelude::*, + rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}, +}; + +use super::{ + inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, + types::{DeviceId, Format, Mode, Permission}, + FsContext, +}; + +const D_INVALID: u8 = 0; +const D_REGULAR: u8 = 1; +const D_DIRECTORY: u8 = 2; +const D_SYMLINK: u8 = 3; + +#[derive(Debug, PartialEq, Eq)] enum DentryKind { - Regular, - Directory, - Symlink, - Mountpoint, + Regular = D_REGULAR as isize, + Directory = D_DIRECTORY as isize, + Symlink = D_SYMLINK as isize, } -struct DentryData { - inode: InodeUse, - kind: DentryKind, +/// The [`Inode`] associated with a [`Dentry`]. +/// +/// We could assign an inode to a negative dentry exactly once when the dentry +/// is invalid and we create a file or directory on it, or the dentry is brought +/// to the dcache by [lookup()]. +/// +/// This guarantees that as long as we acquire a non-invalid from [`Self::kind`], +/// we are synced with the writer and can safely read the [`Self::inode`] field +/// without reading torn data. +/// +/// [lookup()]: crate::kernel::vfs::inode::InodeDirOps::lookup +struct AssociatedInode { + kind: UnsafeCell>, + inode: UnsafeCell>>, } /// # Safety @@ -58,8 +75,7 @@ pub struct Dentry { prev: AtomicPtr, next: AtomicPtr, - // RCU Mutable - data: RCUPointer, + inode: AssociatedInode, } pub(super) static DROOT: LazyLock> = LazyLock::new(|| { @@ -69,7 +85,7 @@ pub(super) static DROOT: LazyLock> = LazyLock::new(|| { hash: AtomicU64::new(0), prev: AtomicPtr::default(), next: AtomicPtr::default(), - data: RCUPointer::empty(), + inode: AssociatedInode::new(), }); unsafe { @@ -119,50 +135,19 @@ impl Dentry { self.hash.store(hash, Ordering::Relaxed); } - - async fn find(self: &Arc, name: &[u8]) -> KResult> { - let data = self.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; - - if data.kind != DentryKind::Directory { - return Err(ENOTDIR); - } - - match name { - b"." => Ok(self.clone()), - b".." => Ok(self.parent().clone()), - _ => { - let dentry = Dentry::create(self.clone(), name); - - if let Some(found) = dcache::d_find_fast(&dentry) { - unsafe { - // SAFETY: This is safe because the dentry is never shared with - // others so we can drop them safely. - let _ = dentry.name.swap(None); - let _ = dentry.parent.swap(None); - } - - return Ok(found); - } - - let _ = dcache::d_try_revalidate(&dentry).await; - dcache::d_add(dentry.clone()); - - Ok(dentry) - } - } - } } impl Dentry { pub fn create(parent: Arc, name: &[u8]) -> Arc { + // TODO!!!: don't acquire our parent's refcount here... + let val = Arc::new(Self { parent: RCUPointer::new(parent), name: RCUPointer::new(Arc::new(Arc::from(name))), hash: AtomicU64::new(0), prev: AtomicPtr::default(), next: AtomicPtr::default(), - data: RCUPointer::empty(), + inode: AssociatedInode::new(), }); val.rehash(); @@ -196,27 +181,12 @@ impl Dentry { .map_or(core::ptr::null(), |parent| Arc::as_ptr(&parent)) } - fn save(&self, inode: InodeUse, kind: DentryKind) { - let new = DentryData { inode, kind }; - - // TODO!!!: We don't actually need to use `RCUPointer` here - // Safety: this function may only be called from `create`-like functions which requires the - // superblock's write locks to be held, so only one creation can happen at a time and we - // can't get a reference to the old data. - let old = unsafe { self.data.swap(Some(Arc::new(new))) }; - assert!(old.is_none()); - } - pub fn fill(&self, file: InodeUse) { - match file.format() { - Format::REG | Format::BLK | Format::CHR => self.save(file, DentryKind::Regular), - Format::DIR => self.save(file, DentryKind::Directory), - Format::LNK => self.save(file, DentryKind::Symlink), - } + self.inode.store(file); } pub fn inode(&self) -> Option> { - self.data.load().as_ref().map(|data| data.inode.clone()) + self.inode.load().map(|(_, inode)| inode.clone()) } pub fn get_inode(&self) -> KResult> { @@ -224,181 +194,85 @@ impl Dentry { } pub fn is_directory(&self) -> bool { - let data = self.data.load(); - data.as_ref() - .map_or(false, |data| data.kind == DentryKind::Directory) + self.inode + .load() + .map_or(false, |(kind, _)| kind == DentryKind::Directory) } pub fn is_valid(&self) -> bool { - self.data.load().is_some() + self.inode.load().is_some() } pub async fn open_check(self: &Arc, flags: OpenFlags, perm: Permission) -> KResult<()> { - let data = self.data.load(); - - if data.is_some() { - if flags.contains(OpenFlags::O_CREAT | OpenFlags::O_EXCL) { - Err(EEXIST) - } else { - Ok(()) - } - } else { - if !flags.contains(OpenFlags::O_CREAT) { - return Err(ENOENT); - } - - let parent = self.parent().get_inode()?; - parent.create(self, perm).await - } - } -} - -impl Dentry { - fn resolve_directory( - context: &FsContext, - dentry: Arc, - nrecur: u32, - ) -> Pin>> + use<'_>>> { - Box::pin(async move { - if nrecur >= 16 { - return Err(ELOOP); - } - - let data = dentry.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; - - match data.kind { - DentryKind::Regular => Err(ENOTDIR), - DentryKind::Directory => Ok(dentry), - DentryKind::Symlink => { - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); - - data.inode.readlink(&mut buffer).await?; - let path = Path::new(buffer.data())?; - - let dentry = - Self::open_recursive(context, &dentry.parent(), path, true, nrecur + 1) - .await?; - - Self::resolve_directory(context, dentry, nrecur + 1).await + match self.inode.load() { + Some(_) => { + if flags.contains(OpenFlags::O_CREAT | OpenFlags::O_EXCL) { + Err(EEXIST) + } else { + Ok(()) } - _ => panic!("Invalid dentry flags"), - } - }) - } - - pub fn open_recursive<'r, 'a: 'r, 'b: 'r, 'c: 'r>( - context: &'a FsContext, - cwd: &'b Arc, - path: Path<'c>, - follow: bool, - nrecur: u32, - ) -> Pin>> + 'r>> { - Box::pin(async move { - // too many recursive search layers will cause stack overflow - // so we use 16 for now - if nrecur >= 16 { - return Err(ELOOP); } - - let mut cwd = if path.is_absolute() { - context.fsroot.clone() - } else { - cwd.clone() - }; - - for item in path.iter() { - if let PathComponent::TrailingEmpty = item { - if cwd.data.load().as_ref().is_none() { - return Ok(cwd); - } + None => { + if !flags.contains(OpenFlags::O_CREAT) { + return Err(ENOENT); } - cwd = Self::resolve_directory(context, cwd, nrecur).await?; - - match item { - PathComponent::TrailingEmpty | PathComponent::Current => {} // pass - PathComponent::Parent => { - if !cwd.hash_eq(&context.fsroot) { - let parent = cwd.parent().clone(); - cwd = Self::resolve_directory(context, parent, nrecur).await?; - } - continue; - } - PathComponent::Name(name) => { - cwd = cwd.find(name).await?; - } - } + let parent = self.parent().get_inode()?; + parent.create(self, perm).await } - - if follow { - let data = cwd.data.load(); - - if let Some(data) = data.as_ref() { - if data.kind == DentryKind::Symlink { - let data = cwd.data.load(); - let data = data.as_ref().unwrap(); - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); - - data.inode.readlink(&mut buffer).await?; - let path = Path::new(buffer.data())?; - - let parent = cwd.parent().clone(); - cwd = - Self::open_recursive(context, &parent, path, true, nrecur + 1).await?; - } - } - } - - Ok(cwd) - }) + } } +} +impl Dentry { pub async fn open( context: &FsContext, - path: Path<'_>, + path: &Path, follow_symlinks: bool, ) -> KResult> { let cwd = context.cwd.lock().clone(); - Dentry::open_recursive(context, &cwd, path, follow_symlinks, 0).await + Self::open_at(context, &cwd, path, follow_symlinks).await } pub async fn open_at( context: &FsContext, at: &Arc, - path: Path<'_>, + path: &Path, follow_symlinks: bool, ) -> KResult> { - Dentry::open_recursive(context, at, path, follow_symlinks, 0).await - } + let mut found = context.start_recursive_walk(at, path).await?; - pub fn get_path( - self: &Arc, - context: &FsContext, - buffer: &mut dyn Buffer, - ) -> KResult<()> { - let locked_parent = self.parent(); + if !follow_symlinks { + return Ok(found); + } - let path = { - let mut path = vec![]; + loop { + match found.inode.load() { + Some((DentryKind::Symlink, inode)) => { + found = context.follow_symlink(found.aref(), inode, 0).await?; + } + _ => return Ok(found), + } + } + } - let mut parent = locked_parent.borrow(); - let mut dentry = BorrowedArc::new(self); + pub fn get_path(self: &Arc, context: &FsContext, buffer: &mut dyn Buffer) -> KResult<()> { + let rcu_read = rcu_read_lock(); - while Arc::as_ptr(&dentry) != Arc::as_ptr(&context.fsroot) { - if path.len() > 32 { - return Err(ELOOP); - } + let mut path = vec![]; + + let mut current = self.aref(); + let mut parent = self.parent.dereference(&rcu_read).unwrap(); - path.push(dentry.name().clone()); - dentry = parent; - parent = dentry.parent.load_protected(&locked_parent).unwrap(); + while !current.ptr_eq_arc(&context.fsroot) { + if path.len() > 32 { + return Err(ELOOP); } - path - }; + path.push(current.name.dereference(&rcu_read).unwrap()); + current = parent; + parent = current.parent.dereference(&rcu_read).unwrap(); + } buffer.fill(b"/")?.ok_or(ERANGE)?; for item in path.iter().rev().map(|name| name.as_ref()) { @@ -533,3 +407,71 @@ impl Dentry { old_parent.rename(rename_data).await } } + +impl DentryKind { + fn into_raw(self) -> u8 { + unsafe { core::mem::transmute(self) } + } + + fn from_raw(raw: u8) -> Option { + unsafe { core::mem::transmute(raw) } + } + + fn as_atomic(me: &UnsafeCell>) -> &AtomicU8 { + unsafe { AtomicU8::from_ptr(me.get().cast()) } + } + + fn atomic_acq(me: &UnsafeCell>) -> Option { + Self::from_raw(Self::as_atomic(me).load(Ordering::Acquire)) + } + + fn atomic_swap_acqrel(me: &UnsafeCell>, kind: Option) -> Option { + Self::from_raw(Self::as_atomic(me).swap(kind.map_or(0, Self::into_raw), Ordering::AcqRel)) + } +} + +impl AssociatedInode { + fn new() -> Self { + Self { + inode: UnsafeCell::new(None), + kind: UnsafeCell::new(None), + } + } + + fn store(&self, inode: InodeUse) { + let kind = match inode.format() { + Format::REG | Format::BLK | Format::CHR => DentryKind::Regular, + Format::DIR => DentryKind::Directory, + Format::LNK => DentryKind::Symlink, + }; + + unsafe { + // SAFETY: We should be the first and only one to store the inode as + // is checked below. All other readers reading non-invalid + // kind will see the fully written inode. + self.inode.get().write(Some(inode)); + } + + assert_eq!( + DentryKind::atomic_swap_acqrel(&self.kind, Some(kind)), + None, + "Dentry can only be stored once." + ); + } + + fn kind(&self) -> Option { + DentryKind::atomic_acq(&self.kind) + } + + fn load(&self) -> Option<(DentryKind, &InodeUse)> { + self.kind().map(|kind| unsafe { + let inode = (&*self.inode.get()) + .as_ref() + .expect("Dentry with non-invalid kind has no inode"); + (kind, inode) + }) + } +} + +unsafe impl Send for AssociatedInode {} +unsafe impl Sync for AssociatedInode {} diff --git a/src/kernel/vfs/dentry/dcache.rs b/src/kernel/vfs/dentry/dcache.rs index e2491235..ee7503dc 100644 --- a/src/kernel/vfs/dentry/dcache.rs +++ b/src/kernel/vfs/dentry/dcache.rs @@ -1,11 +1,13 @@ use super::Dentry; use crate::kernel::constants::ENOENT; -use crate::rcu::RCUPointer; +use crate::rcu::{RCUPointer, RCUReadLock}; use crate::{ prelude::*, rcu::{RCUIterator, RCUList}, }; use alloc::sync::Arc; +use arcref::ArcRef; +use core::ops::Deref; use core::sync::atomic::Ordering; use eonix_sync::Mutex; @@ -16,26 +18,47 @@ static DCACHE: [RCUList; 1 << DCACHE_HASH_BITS] = static D_EXCHANGE_LOCK: Mutex<()> = Mutex::new(()); -pub fn d_hinted(dentry: &Dentry) -> &'static RCUList { - let hash = dentry.hash.load(Ordering::Relaxed) as usize & ((1 << DCACHE_HASH_BITS) - 1); +pub trait DCacheItem { + fn d_hash(&self) -> usize; + fn d_parent(&self) -> *const Dentry; + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + rcu_read: &'b RCUReadLock, + ) -> impl Deref + 'r; +} + +fn d_eq(lhs: &impl DCacheItem, rhs: &impl DCacheItem, rcu_read: &RCUReadLock) -> bool { + lhs.d_hash() == rhs.d_hash() + && lhs.d_parent() == rhs.d_parent() + && *lhs.d_name(rcu_read) == *rhs.d_name(rcu_read) +} + +fn d_hinted(item: &impl DCacheItem) -> &'static RCUList { + let hash = item.d_hash() & ((1 << DCACHE_HASH_BITS) - 1); &DCACHE[hash] } -pub fn d_iter_for(dentry: &Dentry) -> RCUIterator<'static, Dentry> { - d_hinted(dentry).iter() +fn d_iter_for<'rcu>( + item: &impl DCacheItem, + rcu_read: &'rcu RCUReadLock, +) -> RCUIterator<'static, 'rcu, Dentry> { + d_hinted(item).iter(rcu_read) +} + +pub fn d_find_rcu<'rcu>( + item: &impl DCacheItem, + rcu_read: &'rcu RCUReadLock, +) -> Option> { + d_iter_for(item, rcu_read).find(|cur_ref| cur_ref.with_arc(|cur| d_eq(cur, item, rcu_read))) } /// Add the dentry to the dcache pub fn d_add(dentry: Arc) { + // TODO: Add `children` field to parent and lock parent dentry to avoid + // concurrent insertion causing duplication. d_hinted(&dentry).insert(dentry); } -pub fn d_find_fast(dentry: &Dentry) -> Option> { - d_iter_for(dentry) - .find(|cur| cur.hash_eq(dentry)) - .map(|dentry| dentry.clone()) -} - /// Call `lookup()` on the parent inode to try find if the dentry points to a valid inode /// /// Silently fail without any side effects @@ -80,3 +103,34 @@ pub async fn d_exchange(old: &Arc, new: &Arc) { d_add(old.clone()); d_add(new.clone()); } + +impl DCacheItem for Arc { + fn d_hash(&self) -> usize { + self.hash.load(Ordering::Relaxed) as usize + } + + fn d_parent(&self) -> *const Dentry { + self.parent_addr() + } + + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + rcu_read: &'b RCUReadLock, + ) -> impl Deref + 'r { + struct Name<'a>(ArcRef<'a, Arc<[u8]>>); + + impl Deref for Name<'_> { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + Name( + self.name + .dereference(rcu_read) + .expect("Dentry should have a non-null name"), + ) + } +} diff --git a/src/kernel/vfs/dentry/walk.rs b/src/kernel/vfs/dentry/walk.rs new file mode 100644 index 00000000..3e401b4b --- /dev/null +++ b/src/kernel/vfs/dentry/walk.rs @@ -0,0 +1,370 @@ +use core::{ + future::Future, + hash::{BuildHasher, BuildHasherDefault, Hasher}, + ops::Deref, + pin::Pin, +}; + +use alloc::{boxed::Box, sync::Arc}; +use arcref::{ArcRef, AsArcRef}; +use posix_types::result::PosixError; + +use crate::{ + hash::KernelHasher, + io::ByteBuffer, + kernel::{ + constants::ELOOP, + vfs::{ + inode::{Inode, InodeUse}, + FsContext, + }, + }, + path::{Path, PathComponent, PathIterator}, + prelude::KResult, + rcu::{rcu_read_lock, RCUReadLock}, +}; + +use super::{ + dcache::{self, DCacheItem}, + Dentry, DentryKind, +}; + +struct DentryFind<'a, 'b> { + parent: &'a Dentry, + name: &'b [u8], + hash: usize, +} + +pub enum WalkResultRcu<'rcu, 'path> { + Err(PosixError), + Ok(ArcRef<'rcu, Dentry>), + Symlink { + symlink: ArcRef<'rcu, Dentry>, + inode: InodeUse, + }, + Miss { + parent: ArcRef<'rcu, Dentry>, + name: &'path [u8], + }, +} + +pub enum WalkResult { + Err(PosixError), + Ok(Arc), + Symlink { + symlink: Arc, + inode: InodeUse, + }, +} + +impl Dentry { + /// Quick path of the dentry find operation. + /// + /// Check invalid and non-directory dentries, return immediately on dot and + /// dotdot component, and do a quick rcu dcache lookup. + /// + /// Note that while `Some(dentry)` guarantees present and valid dentry, + /// returning `None` is acceptable if the actual file exists but is not in + /// the dentry cache. If so, we should check again with `lookup`. + fn find_rcu<'r, 's: 'r>( + self: ArcRef<'s, Self>, + name: &[u8], + rcu_read: &'r RCUReadLock, + ) -> Result>, PosixError> { + match self.inode.load() { + Some((DentryKind::Directory, _)) => {} + Some(_) => return Err(PosixError::ENOTDIR), + None => return Err(PosixError::ENOENT), + } + + match name { + b"." => Ok(Some(self)), + b".." => Ok(Some( + self.parent + .dereference(rcu_read) + .expect("The field `parent` should be non-null"), + )), + _ => { + let dentry_find = DentryFind::new(&self, name); + Ok(dcache::d_find_rcu(&dentry_find, rcu_read)) + } + } + } + + async fn find_slow(self: &Arc, name: &[u8]) -> Result, PosixError> { + let dentry = Dentry::create(self.clone(), name); + + let _ = dcache::d_try_revalidate(&dentry).await; + dcache::d_add(dentry.clone()); + + Ok(dentry) + } + + pub async fn find_full(self: &Arc, name: &[u8]) -> Result, PosixError> { + if let Some(dentry) = self.aref().find_rcu(name, &rcu_read_lock())? { + return Ok(dentry.clone_arc()); + } + + self.find_slow(name).await + } +} + +impl FsContext { + /// Walk the pathname and try to find the corresponding dentry FAST without + /// consulting the VFS for invalid dentries encountered. + fn walk_rcu<'rcu, 'path>( + &self, + mut current: ArcRef<'rcu, Dentry>, + iter: &mut PathIterator<'path>, + rcu_read: &'rcu RCUReadLock, + ) -> WalkResultRcu<'rcu, 'path> { + use PathComponent::*; + + loop { + let inode = current.inode.load(); + + if iter.is_empty() { + break; + } + + // Skip symlink resolution in rcu walk without consuming the iter. + if let Some((DentryKind::Symlink, inode)) = inode { + return WalkResultRcu::Symlink { + symlink: current, + inode: inode.clone(), + }; + } + + let Some(component) = iter.next() else { + break; + }; + + match (inode, component) { + // Skip trailing empty and dot for normal directories. + (Some((DentryKind::Directory, _)), TrailingEmpty | Current) => {} + // Walk to parent directory unless we are at the filesystem root. + (Some((DentryKind::Directory, _)), Parent) => { + if current.ptr_eq_arc(&self.fsroot) { + continue; + } + + current = current + .parent + .dereference(&rcu_read) + .expect("parent should exist"); + } + // Normal directory traversal + (Some((DentryKind::Directory, _)), Name(name)) => { + match current.find_rcu(name, &rcu_read) { + Err(err) => return WalkResultRcu::Err(err), + Ok(Some(found)) => { + current = found; + } + Ok(None) => { + return WalkResultRcu::Miss { + name, + parent: current, + }; + } + } + } + // Not a directory, fail and exit. + (Some(_), _) => return WalkResultRcu::Err(PosixError::ENOTDIR), + // Return invalid trailing entries directly. + (None, TrailingEmpty) => return WalkResultRcu::Ok(current), + // Invalid intermediate entries are not acceptable. + (None, _) => return WalkResultRcu::Err(PosixError::ENOENT), + } + } + + WalkResultRcu::Ok(current) + } + + /// Walk the pathname slowly with refcounts held and VFS lookups. + async fn walk_slow(&self, mut current: Arc, iter: &mut PathIterator<'_>) -> WalkResult { + use PathComponent::*; + + loop { + // `current` should be the parent directory and `component` is the + // next path component we are stepping into. + + if iter.is_empty() { + break; + } + + if let Some((DentryKind::Symlink, inode)) = current.inode.load() { + return WalkResult::Symlink { + inode: inode.clone(), + symlink: current, + }; + } + + let Some(component) = iter.next() else { + break; + }; + + match (current.inode.load(), &component) { + // Normal directory traversal + (Some((DentryKind::Directory, _)), _) => {} + // Not a directory, fail and exit. + (Some(_), _) => return WalkResult::Err(PosixError::ENOTDIR), + // Return invalid trailing entries directly. + (None, TrailingEmpty) => return WalkResult::Ok(current), + // Invalid intermediate entries are not acceptable. + (None, _) => return WalkResult::Err(PosixError::ENOENT), + } + + match component { + PathComponent::TrailingEmpty => {} + PathComponent::Current => {} + PathComponent::Parent => { + if current.hash_eq(&self.fsroot) { + continue; + } + + let parent = current.parent().clone(); + current = parent; + } + PathComponent::Name(name) => { + match current.find_full(name).await { + Ok(found) => current = found, + Err(err) => return WalkResult::Err(err), + }; + } + } + } + + WalkResult::Ok(current) + } + + /// Walk the pathname and get an accurate answer. Stop at symlinks. + async fn walk_full( + &self, + current: ArcRef<'_, Dentry>, + iter: &mut PathIterator<'_>, + ) -> WalkResult { + let (parent_slow, name_slow); + + match self.walk_rcu(current, iter, &rcu_read_lock()) { + WalkResultRcu::Err(error) => return WalkResult::Err(error.into()), + WalkResultRcu::Ok(dentry) => return WalkResult::Ok(dentry.clone_arc()), + WalkResultRcu::Symlink { symlink, inode } => { + return WalkResult::Symlink { + symlink: symlink.clone_arc(), + inode, + }; + } + WalkResultRcu::Miss { parent, name } => { + // Fallback to regular refcounted lookup + parent_slow = parent.clone_arc(); + name_slow = name; + } + } + + match parent_slow.find_slow(name_slow).await { + Ok(found) => self.walk_slow(found, iter).await, + Err(err) => return WalkResult::Err(err), + } + } + + pub async fn follow_symlink( + &self, + symlink: ArcRef<'_, Dentry>, + inode: &InodeUse, + nr_follows: u32, + ) -> KResult> { + let mut target = [0; 256]; + let mut target = ByteBuffer::new(&mut target); + inode.readlink(&mut target).await?; + + self.walk_recursive( + &symlink.parent().clone(), + Path::new(target.data()).unwrap(), + nr_follows + 1, + ) + .await + } + + fn follow_symlink_boxed<'r, 'a: 'r, 'b: 'r, 'c: 'r>( + &'a self, + symlink: ArcRef<'b, Dentry>, + inode: &'c InodeUse, + nr_follows: u32, + ) -> Pin>> + Send + 'r>> { + Box::pin(self.follow_symlink(symlink, inode, nr_follows)) + } + + async fn walk_recursive( + &self, + cwd: &Arc, + path: &Path, + nr_follows: u32, + ) -> KResult> { + const MAX_NR_FOLLOWS: u32 = 16; + + let mut current_owned; + let mut current; + if path.is_absolute() { + current = self.fsroot.aref(); + } else { + current = cwd.aref(); + } + + let mut path_iter = path.iter(); + + loop { + match self.walk_full(current, &mut path_iter).await { + WalkResult::Err(posix_error) => return Err(posix_error.into()), + WalkResult::Ok(dentry) => return Ok(dentry), + WalkResult::Symlink { symlink, inode } => { + if nr_follows >= MAX_NR_FOLLOWS { + return Err(ELOOP); + } + + current_owned = self + .follow_symlink_boxed(symlink.aref(), &inode, nr_follows) + .await?; + current = current_owned.aref(); + } + } + } + } + + pub async fn start_recursive_walk( + &self, + cwd: &Arc, + path: &Path, + ) -> KResult> { + self.walk_recursive(cwd, path, 0).await + } +} + +impl<'a, 'b> DentryFind<'a, 'b> { + fn new(parent: &'a Dentry, name: &'b [u8]) -> Self { + let builder: BuildHasherDefault = Default::default(); + let mut hasher = builder.build_hasher(); + + hasher.write_usize(parent as *const _ as usize); + hasher.write(name); + let hash = hasher.finish() as usize; + + Self { parent, name, hash } + } +} + +impl DCacheItem for DentryFind<'_, '_> { + fn d_hash(&self) -> usize { + self.hash + } + + fn d_parent(&self) -> *const Dentry { + self.parent as *const _ + } + + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + _rcu_read: &'b RCUReadLock, + ) -> impl Deref + 'r { + self.name + } +} diff --git a/src/path.rs b/src/path.rs index 8b740095..b342ef5f 100644 --- a/src/path.rs +++ b/src/path.rs @@ -1,34 +1,30 @@ use crate::{kernel::constants::ENOENT, prelude::*}; use core::fmt::{self, Debug, Formatter}; -pub struct Path<'lt> { - all: &'lt [u8], +#[repr(transparent)] +pub struct Path { + all: [u8], } pub struct PathIterator<'lt> { rem: &'lt [u8], } -#[allow(dead_code)] -impl<'lt> Path<'lt> { - pub fn new(all: &'lt [u8]) -> KResult { +impl Path { + pub fn new(all: &[u8]) -> KResult<&Self> { if all.is_empty() { Err(ENOENT) } else { - Ok(Self { all }) + Ok(unsafe { &*(all as *const [u8] as *const Path) }) } } - pub fn from_str(all: &'lt str) -> KResult { - Self::new(all.as_bytes()) - } - pub fn is_absolute(&self) -> bool { self.all.starts_with(&['/' as u8]) } - pub fn iter(&self) -> PathIterator<'lt> { - PathIterator::new(self.all) + pub fn iter(&self) -> PathIterator { + PathIterator::new(&self.all) } } @@ -46,11 +42,17 @@ pub enum PathComponent<'lt> { Parent, } +impl PathIterator<'_> { + pub fn is_empty(&self) -> bool { + self.rem.is_empty() + } +} + impl<'lt> Iterator for PathIterator<'lt> { type Item = PathComponent<'lt>; fn next(&mut self) -> Option { - if self.rem.is_empty() { + if self.is_empty() { return None; } @@ -71,16 +73,16 @@ impl<'lt> Iterator for PathIterator<'lt> { self.rem = rem; match cur { - cur if cur.is_empty() => Some(PathComponent::TrailingEmpty), - cur if cur == b"." => Some(PathComponent::Current), - cur if cur == b".." => Some(PathComponent::Parent), - cur => Some(PathComponent::Name(cur)), + b"" => Some(PathComponent::TrailingEmpty), + b"." => Some(PathComponent::Current), + b".." => Some(PathComponent::Parent), + name => Some(PathComponent::Name(name)), } } } -impl Debug for Path<'_> { +impl Debug for Path { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "Path({:?})", self.all) + write!(f, "Path({:?})", &self.all) } } diff --git a/src/rcu.rs b/src/rcu.rs index c1645d33..b06db9e2 100644 --- a/src/rcu.rs +++ b/src/rcu.rs @@ -1,21 +1,35 @@ use crate::{kernel::task::block_on, prelude::*}; use alloc::sync::Arc; +use arcref::ArcRef; use core::{ ops::Deref, ptr::NonNull, sync::atomic::{AtomicPtr, Ordering}, }; +use eonix_preempt::PreemptGuard; use eonix_runtime::scheduler::RUNTIME; -use eonix_sync::{Mutex, RwLock, RwLockReadGuard}; +use eonix_sync::{RwLock, RwLockReadGuard}; use pointers::BorrowedArc; +/// The RCU Read Lock. Holding a reference to an instance of the struct assures +/// you that any RCU protected data would not be dropped. +/// +/// The struct cannot be created directly. Instead, use [`rcu_read_lock()`]. +#[derive(Debug)] +pub struct RCUReadLock(); + +pub struct RCUReadGuardNew { + guard: RwLockReadGuard<'static, RCUReadLock>, + _disable_preempt: PreemptGuard<()>, +} + pub struct RCUReadGuard<'data, T: 'data> { value: T, - _guard: RwLockReadGuard<'data, ()>, + _guard: RwLockReadGuard<'static, RCUReadLock>, _phantom: PhantomData<&'data T>, } -static GLOBAL_RCU_SEM: RwLock<()> = RwLock::new(()); +static GLOBAL_RCU_SEM: RwLock = RwLock::new(RCUReadLock()); impl<'data, T> RCUReadGuard<'data, BorrowedArc<'data, T>> { fn lock(value: BorrowedArc<'data, T>) -> Self { @@ -25,14 +39,6 @@ impl<'data, T> RCUReadGuard<'data, BorrowedArc<'data, T>> { _phantom: PhantomData, } } - - pub fn borrow(&self) -> BorrowedArc<'data, T> { - unsafe { - BorrowedArc::from_raw(NonNull::new_unchecked( - &raw const *self.value.borrow() as *mut T - )) - } - } } impl<'data, T: 'data> Deref for RCUReadGuard<'data, T> { @@ -63,17 +69,14 @@ pub trait RCUNode { pub struct RCUList> { head: AtomicPtr, - - reader_lock: RwLock<()>, - update_lock: Mutex<()>, + update_lock: Spin<()>, } impl> RCUList { pub const fn new() -> Self { Self { head: AtomicPtr::new(core::ptr::null_mut()), - reader_lock: RwLock::new(()), - update_lock: Mutex::new(()), + update_lock: Spin::new(()), } } @@ -117,7 +120,6 @@ impl> RCUList { unsafe { Arc::from_raw(me) }; } - let _lck = self.reader_lock.write(); node.rcu_prev() .store(core::ptr::null_mut(), Ordering::Release); node.rcu_next() @@ -152,7 +154,6 @@ impl> RCUList { unsafe { Arc::from_raw(old) }; } - let _lck = self.reader_lock.write(); old_node .rcu_prev() .store(core::ptr::null_mut(), Ordering::Release); @@ -161,36 +162,36 @@ impl> RCUList { .store(core::ptr::null_mut(), Ordering::Release); } - pub fn iter(&self) -> RCUIterator { - let _lck = block_on(self.reader_lock.read()); - + pub fn iter<'a, 'r>(&'a self, _lock: &'r RCUReadLock) -> RCUIterator<'a, 'r, T> { RCUIterator { - // SAFETY: We have a read lock, so the node is still alive. - cur: NonNull::new(self.head.load(Ordering::SeqCst)), - _lock: _lck, + cur: NonNull::new(self.head.load(Ordering::Acquire)), + _phantom: PhantomData, } } } -pub struct RCUIterator<'lt, T: RCUNode> { +pub struct RCUIterator<'list, 'rcu, T: RCUNode> { cur: Option>, - _lock: RwLockReadGuard<'lt, ()>, + _phantom: PhantomData<(&'list (), &'rcu ())>, } -impl<'lt, T: RCUNode> Iterator for RCUIterator<'lt, T> { - type Item = BorrowedArc<'lt, T>; +impl<'rcu, T: RCUNode> Iterator for RCUIterator<'_, 'rcu, T> { + type Item = ArcRef<'rcu, T>; fn next(&mut self) -> Option { - match self.cur { - None => None, - Some(pointer) => { - // SAFETY: We have a read lock, so the node is still alive. - let reference = unsafe { pointer.as_ref() }; + self.cur.map(|pointer| { + let reference = unsafe { + // SAFETY: We have the read lock so the node is still alive. + pointer.as_ref() + }; + + self.cur = NonNull::new(reference.rcu_next().load(Ordering::Acquire)); - self.cur = NonNull::new(reference.rcu_next().load(Ordering::SeqCst)); - Some(unsafe { BorrowedArc::from_raw(pointer) }) + unsafe { + // SAFETY: We have the read lock so the node is still alive. + ArcRef::new_unchecked(pointer.as_ptr()) } - } + }) } } @@ -228,15 +229,16 @@ where } pub fn load<'lt>(&self) -> Option>> { + // BUG: We should acquire the lock before loading the pointer NonNull::new(self.0.load(Ordering::Acquire)) .map(|p| RCUReadGuard::lock(unsafe { BorrowedArc::from_raw(p) })) } - pub fn load_protected<'a, U: 'a>( - &self, - _guard: &RCUReadGuard<'a, U>, - ) -> Option> { - NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { BorrowedArc::from_raw(p) }) + pub fn dereference<'r, 'a: 'r>(&self, _lock: &'a RCUReadLock) -> Option> { + NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { + // SAFETY: We have a read lock, so the node is still alive. + ArcRef::new_unchecked(p.as_ptr()) + }) } /// # Safety @@ -289,3 +291,18 @@ where } } } + +impl Deref for RCUReadGuardNew { + type Target = RCUReadLock; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +pub fn rcu_read_lock() -> RCUReadGuardNew { + RCUReadGuardNew { + guard: block_on(GLOBAL_RCU_SEM.read()), + _disable_preempt: PreemptGuard::new(()), + } +} From 49a49032edc3d071092ec1fdb5f3b9aa070e107b Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 14 Sep 2025 22:22:51 +0800 Subject: [PATCH 003/153] style, vfs: remove unused imports and InodeRef Signed-off-by: greatbridf --- src/kernel/vfs/inode/inode.rs | 5 ----- src/kernel/vfs/inode/mod.rs | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs index 786a31fe..870a045d 100644 --- a/src/kernel/vfs/inode/inode.rs +++ b/src/kernel/vfs/inode/inode.rs @@ -321,11 +321,6 @@ pub struct InodeInfo { pub mtime: Instant, } -#[derive(Clone)] -pub struct InodeRef(Weak) -where - I: Inode + ?Sized; - pub struct InodeUse(Arc) where I: Inode + ?Sized; diff --git a/src/kernel/vfs/inode/mod.rs b/src/kernel/vfs/inode/mod.rs index 786d30fa..08471ef3 100644 --- a/src/kernel/vfs/inode/mod.rs +++ b/src/kernel/vfs/inode/mod.rs @@ -4,7 +4,5 @@ mod ops; mod statx; pub use ino::Ino; -pub use inode::{ - Inode, InodeDir, InodeDirOps, InodeFile, InodeFileOps, InodeInfo, InodeOps, InodeRef, InodeUse, -}; +pub use inode::{Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; pub use ops::{RenameData, WriteOffset}; From 632f1c7882876b7305b90ca5d162172d9e5350b1 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 14 Sep 2025 22:23:09 +0800 Subject: [PATCH 004/153] vfs: fix debug print of `Mode` structs If we have invalid format, we should print a None instead of panicking. Signed-off-by: greatbridf --- src/kernel/vfs/types/mode.rs | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/kernel/vfs/types/mode.rs b/src/kernel/vfs/types/mode.rs index dc1b88ec..a58c8215 100644 --- a/src/kernel/vfs/types/mode.rs +++ b/src/kernel/vfs/types/mode.rs @@ -44,13 +44,20 @@ impl Mode { } pub fn format(&self) -> Format { + match self.try_format() { + None => panic!("unknown format bits: {:#o}", self.format_bits()), + Some(format) => format, + } + } + + pub fn try_format(&self) -> Option { match self.format_bits() { - S_IFREG => Format::REG, - S_IFDIR => Format::DIR, - S_IFLNK => Format::LNK, - S_IFBLK => Format::BLK, - S_IFCHR => Format::CHR, - _ => panic!("unknown format bits: {:#o}", self.format_bits()), + S_IFREG => Some(Format::REG), + S_IFDIR => Some(Format::DIR), + S_IFLNK => Some(Format::LNK), + S_IFBLK => Some(Format::BLK), + S_IFCHR => Some(Format::CHR), + _ => None, } } @@ -58,10 +65,6 @@ impl Mode { Permission::new(self.non_format_bits()) } - pub const fn non_format(&self) -> Self { - Self::new(self.non_format_bits()) - } - pub const fn set_perm(&mut self, perm: Permission) { self.0 = self.format_bits() | perm.bits(); } @@ -100,15 +103,15 @@ impl core::fmt::Debug for Mode { match self.non_format_bits() & !0o777 { 0 => write!( f, - "Mode({format:?}, {perm:#o})", - format = self.format(), - perm = self.non_format_bits() + "Mode({format:?}, {perm:?})", + format = self.try_format(), + perm = Permission::new(self.non_format_bits()), )?, rem => write!( f, - "Mode({format:?}, {perm:#o}, rem={rem:#x})", - format = self.format(), - perm = self.non_format_bits() & 0o777 + "Mode({format:?}, {perm:?}, rem={rem:#x})", + format = self.try_format(), + perm = Permission::new(self.non_format_bits()) )?, } From 210a6693c73e5f38fa9ddacde82018df1712afc8 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 8 Oct 2025 11:10:13 +0800 Subject: [PATCH 005/153] mem, paging: introduce page locks and exclusive pages Introduce the new page locking mechanism to ensure exclusiveness of pages when we access them. The underlying locks are not implemented yet for now because we will change the paging structs in the following few patches. Introduce a new `PageExcl` struct representing a page that conforms with Rust's ownership rules. The page owned exclusively can be accessed without taking page locks. Remove the `MemoryBlock` structs as they are not easy to use and represent barely no semantic meanings. Signed-off-by: greatbridf --- .rustfmt.toml | 4 +- Cargo.lock | 7 + Cargo.toml | 1 + src/driver/ahci/command_table.rs | 61 ++++--- src/driver/ahci/mod.rs | 42 +++-- src/driver/ahci/port.rs | 104 +++++------- src/driver/ahci/slot.rs | 179 +++++++++++++++------ src/driver/e1000e.rs | 135 ++++++---------- src/driver/virtio/virtio_blk.rs | 55 +++---- src/fs/fat32.rs | 64 +++----- src/io.rs | 30 ++-- src/kernel/block.rs | 222 +++++++------------------- src/kernel/mem.rs | 4 +- src/kernel/mem/access.rs | 118 +------------- src/kernel/mem/allocator.rs | 8 +- src/kernel/mem/mm_area.rs | 52 +++--- src/kernel/mem/mm_list.rs | 39 ++--- src/kernel/mem/page_alloc/raw_page.rs | 9 +- src/kernel/mem/page_cache.rs | 37 ++--- src/kernel/mem/paging.rs | 129 ++++++++++++--- src/kernel/vfs/file/mod.rs | 34 ++-- src/lib.rs | 50 +++--- 22 files changed, 633 insertions(+), 751 deletions(-) diff --git a/.rustfmt.toml b/.rustfmt.toml index d69872c6..17b2bbc5 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -29,8 +29,8 @@ fn_single_line = false where_single_line = false imports_indent = "Block" imports_layout = "Mixed" -imports_granularity = "Preserve" -group_imports = "Preserve" +imports_granularity = "Module" +group_imports = "StdExternalCrate" reorder_imports = true reorder_modules = true reorder_impl_items = false diff --git a/Cargo.lock b/Cargo.lock index f85a9d2f..c70190a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,6 +180,7 @@ dependencies = [ "posix_types", "slab_allocator", "stalloc", + "static_assertions", "unwinding", "virtio-drivers", "xmas-elf", @@ -525,6 +526,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a37f0ead4094eeb54c6893316aa139e48b252f1c07511e5124fa1f9414df5b6c" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "syn" version = "2.0.104" diff --git a/Cargo.toml b/Cargo.toml index 5158025e..dca5d34d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ futures = { version = "0.3.31", features = [ "alloc", "async-await", ], default-features = false } +static_assertions = "1.1.0" [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/src/driver/ahci/command_table.rs b/src/driver/ahci/command_table.rs index c77b4abd..7b78d26f 100644 --- a/src/driver/ahci/command_table.rs +++ b/src/driver/ahci/command_table.rs @@ -1,45 +1,54 @@ -use super::{command::Command, PRDTEntry, FISH2D}; -use crate::kernel::mem::{AsMemoryBlock as _, Page}; +use core::ptr::NonNull; + use eonix_mm::address::PAddr; -pub struct CommandTable<'a> { - page: Page, - command_fis: &'a mut FISH2D, +use super::command::Command; +use super::{PRDTEntry, FISH2D}; +use crate::kernel::mem::{Page, PageExt}; - prdt: &'a mut [PRDTEntry; 248], - prdt_entries: Option, +pub struct CommandTable { + page: Page, + cmd_fis: NonNull, + prdt: NonNull<[PRDTEntry; 248]>, + prdt_entries: usize, } -impl CommandTable<'_> { +unsafe impl Send for CommandTable {} +unsafe impl Sync for CommandTable {} + +impl CommandTable { pub fn new() -> Self { let page = Page::alloc(); - let memory = page.as_memblk(); - - let (lhs, prdt) = memory.split_at(0x80); - - let (command_fis, _) = lhs.split_at(size_of::()); - let command_fis = unsafe { command_fis.as_ptr().as_mut() }; - let prdt = unsafe { prdt.as_ptr().as_mut() }; - - Self { - page, - command_fis, - prdt, - prdt_entries: None, + let base = page.get_ptr(); + + unsafe { + Self { + page, + cmd_fis: base.cast(), + prdt: base.byte_add(0x80).cast(), + prdt_entries: 0, + } } } pub fn setup(&mut self, cmd: &impl Command) { - self.command_fis.setup(cmd.cmd(), cmd.lba(), cmd.count()); - self.prdt_entries = Some(cmd.pages().len() as u16); + unsafe { + self.cmd_fis + .as_mut() + .setup(cmd.cmd(), cmd.lba(), cmd.count()); + } + + self.prdt_entries = cmd.pages().len(); for (idx, page) in cmd.pages().iter().enumerate() { - self.prdt[idx].setup(page); + unsafe { + self.prdt.as_mut()[idx].setup(page); + } } } - pub fn prdt_len(&self) -> u16 { - self.prdt_entries.unwrap() + pub fn prdt_len(&self) -> usize { + self.prdt_entries } pub fn base(&self) -> PAddr { diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index ab405829..3ea44ed3 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -1,25 +1,23 @@ -use crate::{ - fs::procfs, - io::Buffer as _, - kernel::{ - block::BlockDevice, - constants::{EINVAL, EIO}, - interrupt::register_irq_handler, - pcie::{self, Header, PCIDevice, PCIDriver, PciError}, - vfs::types::DeviceId, - }, - prelude::*, -}; -use alloc::{format, sync::Arc}; +use alloc::format; +use alloc::sync::Arc; + use async_trait::async_trait; use control::AdapterControl; use defs::*; use eonix_mm::address::{AddrOps as _, PAddr}; use eonix_sync::SpinIrq as _; use port::AdapterPort; - pub(self) use register::Register; +use crate::fs::procfs; +use crate::io::Buffer as _; +use crate::kernel::block::BlockDevice; +use crate::kernel::constants::{EINVAL, EIO}; +use crate::kernel::interrupt::register_irq_handler; +use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; +use crate::kernel::vfs::types::DeviceId; +use crate::prelude::*; + mod command; mod command_table; mod control; @@ -30,7 +28,7 @@ pub(self) mod slot; mod stats; pub struct AHCIDriver { - devices: Spin>>>, + devices: Spin>>, } pub struct BitsIterator { @@ -64,22 +62,22 @@ impl Iterator for BitsIterator { } } -struct Device<'a> { +struct Device { control_base: PAddr, control: AdapterControl, _pcidev: Arc>, /// # Lock /// Might be accessed from irq handler, use with `lock_irq()` - ports: Spin<[Option>>; 32]>, + ports: Spin<[Option>; 32]>, } /// # Safety /// `pcidev` is never accessed from Rust code /// TODO!!!: place *mut pci_device in a safe wrapper -unsafe impl Send for Device<'_> {} -unsafe impl Sync for Device<'_> {} +unsafe impl Send for Device {} +unsafe impl Sync for Device {} -impl Device<'_> { +impl Device { fn handle_interrupt(&self) { // Safety // `self.ports` is accessed inside irq handler @@ -108,8 +106,8 @@ impl Device<'_> { } } -impl Device<'static> { - async fn probe_port(&self, port: Arc>) -> KResult<()> { +impl Device { + async fn probe_port(&self, port: Arc) -> KResult<()> { port.init().await?; { diff --git a/src/driver/ahci/port.rs b/src/driver/ahci/port.rs index 77286ec5..a54bbbba 100644 --- a/src/driver/ahci/port.rs +++ b/src/driver/ahci/port.rs @@ -1,20 +1,18 @@ +use alloc::collections::vec_deque::VecDeque; +use core::task::{Poll, Waker}; + +use async_trait::async_trait; +use eonix_mm::address::{Addr as _, PAddr}; +use eonix_sync::SpinIrq as _; + use super::command::{Command, IdentifyCommand, ReadLBACommand, WriteLBACommand}; -use super::slot::CommandSlot; +use super::slot::CommandList; use super::stats::AdapterPortStats; -use super::{ - CommandHeader, Register, PORT_CMD_CR, PORT_CMD_FR, PORT_CMD_FRE, PORT_CMD_ST, PORT_IE_DEFAULT, -}; +use super::{Register, PORT_CMD_CR, PORT_CMD_FR, PORT_CMD_FRE, PORT_CMD_ST, PORT_IE_DEFAULT}; use crate::driver::ahci::command_table::CommandTable; use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::paging::Page; -use crate::kernel::mem::AsMemoryBlock as _; use crate::prelude::*; -use alloc::collections::vec_deque::VecDeque; -use async_trait::async_trait; -use core::pin::pin; -use eonix_mm::address::{Addr as _, PAddr}; -use eonix_sync::{SpinIrq as _, WaitList}; /// An `AdapterPort` is an HBA device in AHCI mode. /// @@ -55,6 +53,8 @@ pub struct AdapterPortData { struct FreeList { free: VecDeque, working: VecDeque, + + wakers: VecDeque, } impl FreeList { @@ -62,57 +62,32 @@ impl FreeList { Self { free: (0..32).collect(), working: VecDeque::new(), + wakers: VecDeque::new(), } } } -pub struct AdapterPort<'a> { +pub struct AdapterPort { pub nport: u32, regs_base: PAddr, - slots: [CommandSlot<'a>; 32], + cmdlist: CommandList, free_list: Spin, - free_list_wait: WaitList, - - /// Holds the command list. - /// **DO NOT USE IT DIRECTLY** - _page: Page, - - cmdlist_base: PAddr, - fis_base: PAddr, stats: AdapterPortStats, } -impl<'a> AdapterPort<'a> { +impl AdapterPort { pub fn new(base: PAddr, nport: u32) -> Self { - let page = Page::alloc(); - let cmdlist_base = page.start(); - let cmdlist_size = 32 * size_of::(); - let fis_base = cmdlist_base + cmdlist_size; - - let (mut cmdheaders, _) = page.as_memblk().split_at(cmdlist_size); - let slots = core::array::from_fn(move |_| { - let (cmdheader, next) = cmdheaders.split_at(size_of::()); - cmdheaders = next; - CommandSlot::new(unsafe { cmdheader.as_ptr().as_mut() }) - }); - Self { nport, regs_base: base + 0x100 + 0x80 * nport as usize, - slots, + cmdlist: CommandList::new(), free_list: Spin::new(FreeList::new()), - free_list_wait: WaitList::new(), - _page: page, stats: AdapterPortStats::new(), - cmdlist_base, - fis_base, } } -} -impl AdapterPort<'_> { fn command_list_base(&self) -> Register { Register::new(self.regs_base + 0x00) } @@ -146,25 +121,16 @@ impl AdapterPort<'_> { } async fn get_free_slot(&self) -> u32 { - loop { - let mut wait = pin!(self.free_list_wait.prepare_to_wait()); - - { - let mut free_list = self.free_list.lock_irq(); - - if let Some(slot) = free_list.free.pop_front() { - return slot; - } - - wait.as_mut().add_to_wait_list(); - - if let Some(slot) = free_list.free.pop_front() { - return slot; - } + core::future::poll_fn(|ctx| { + let mut free_list = self.free_list.lock_irq(); + if let Some(slot) = free_list.free.pop_front() { + return Poll::Ready(slot); } - wait.await; - } + free_list.wakers.push_back(ctx.waker().clone()); + Poll::Pending + }) + .await } fn save_working(&self, slot: u32) { @@ -172,8 +138,10 @@ impl AdapterPort<'_> { } fn release_free_slot(&self, slot: u32) { - self.free_list.lock_irq().free.push_back(slot); - self.free_list_wait.notify_one(); + let mut free_list = self.free_list.lock_irq(); + + free_list.free.push_back(slot); + free_list.wakers.drain(..).for_each(|waker| waker.wake()); } pub fn handle_interrupt(&self) { @@ -187,7 +155,7 @@ impl AdapterPort<'_> { return true; } - self.slots[n as usize].handle_irq(); + self.cmdlist.get(n as usize).handle_irq(); self.stats.inc_int_fired(); false @@ -216,7 +184,7 @@ impl AdapterPort<'_> { cmdtable.setup(cmd); let slot_index = self.get_free_slot().await; - let slot = &self.slots[slot_index as usize]; + let slot = self.cmdlist.get(slot_index as usize); slot.prepare_command(&cmdtable, cmd.write()); self.save_working(slot_index); @@ -229,10 +197,9 @@ impl AdapterPort<'_> { self.stats.inc_cmd_sent(); - if let Err(_) = slot.wait_finish().await { + slot.wait_finish().await.inspect_err(|_| { self.stats.inc_cmd_error(); - return Err(EIO); - }; + })?; self.release_free_slot(slot_index); Ok(()) @@ -251,8 +218,9 @@ impl AdapterPort<'_> { self.stop_command()?; self.command_list_base() - .write(self.cmdlist_base.addr() as u64); - self.fis_base().write(self.fis_base.addr() as u64); + .write(self.cmdlist.cmdlist_base().addr() as u64); + self.fis_base() + .write(self.cmdlist.recv_fis_base().addr() as u64); self.interrupt_enable().write_once(PORT_IE_DEFAULT); @@ -277,7 +245,7 @@ impl AdapterPort<'_> { } #[async_trait] -impl BlockRequestQueue for AdapterPort<'_> { +impl BlockRequestQueue for AdapterPort { fn max_request_pages(&self) -> u64 { 1024 } diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 60a66de3..06c6f2ec 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -1,20 +1,37 @@ -use super::{command_table::CommandTable, CommandHeader}; +use core::cell::UnsafeCell; +use core::ptr::NonNull; +use core::task::{Poll, Waker}; + +use eonix_mm::address::{Addr as _, PAddr}; +use eonix_sync::{Spin, SpinIrq as _}; + +use super::command_table::CommandTable; +use super::CommandHeader; +use crate::kernel::constants::EIO; +use crate::kernel::mem::paging::AllocZeroed; +use crate::kernel::mem::{Page, PageExt}; use crate::KResult; -use core::pin::pin; -use eonix_mm::address::Addr as _; -use eonix_sync::{Spin, SpinIrq as _, WaitList}; + +pub struct CommandList { + base: NonNull, + _page: Page, +} + +unsafe impl Send for CommandList {} +unsafe impl Sync for CommandList {} pub struct CommandSlot<'a> { - /// # Usage - /// `inner.cmdheader` might be used in irq handler. So in order to wait for - /// commands to finish, we should use `lock_irq` on `inner` - inner: Spin>, - wait_list: WaitList, + cmdheader: &'a UnsafeCell, + /// [`Self::control`] might be used in irq handlers. + control: &'a Spin, } -struct CommandSlotInner<'a> { +unsafe impl Send for CommandSlot<'_> {} +unsafe impl Sync for CommandSlot<'_> {} + +struct SlotControl { state: SlotState, - cmdheader: &'a mut CommandHeader, + waker: Option, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -25,32 +42,103 @@ enum SlotState { Error, } -impl<'a> CommandSlot<'a> { - pub fn new(cmdheader: &'a mut CommandHeader) -> Self { +impl CommandList { + fn cmdheaders(&self) -> &[UnsafeCell; 32] { + unsafe { self.base.cast().as_ref() } + } + + fn controls_ptr(base: NonNull) -> NonNull> { + // 24 bytes for SlotControl and extra 8 bytes for Spin. + const_assert_eq!(size_of::>(), 32); + + unsafe { base.add(size_of::>() * 32).cast() } + } + + fn controls(&self) -> &[Spin; 32] { + unsafe { Self::controls_ptr(self.base).cast().as_ref() } + } + + pub fn cmdlist_base(&self) -> PAddr { + self._page.start() + } + + pub fn recv_fis_base(&self) -> PAddr { + self._page.start() + + (size_of::>() + size_of::>()) * 32 + } + + pub fn get(&self, index: usize) -> CommandSlot { + CommandSlot { + cmdheader: &self.cmdheaders()[index], + control: &self.controls()[index], + } + } + + pub fn new() -> Self { + let page = Page::zeroed(); + let base = page.get_ptr(); + + let controls_ptr = Self::controls_ptr(base); + + for i in 0..32 { + unsafe { + controls_ptr.add(i).write(Spin::new(SlotControl { + state: SlotState::Idle, + waker: None, + })); + } + } + Self { - inner: Spin::new(CommandSlotInner { - state: SlotState::Idle, - cmdheader, - }), - wait_list: WaitList::new(), + base: page.get_ptr(), + _page: page, } } +} +impl Drop for CommandList { + fn drop(&mut self) { + let controls_ptr = Self::controls_ptr(self.base); + + for i in 0..32 { + unsafe { + controls_ptr.add(i).drop_in_place(); + } + } + } +} + +impl CommandSlot<'_> { pub fn handle_irq(&self) { - let mut inner = self.inner.lock(); - debug_assert_eq!(inner.state, SlotState::Working); + // We are already in the IRQ handler. + let mut control = self.control.lock(); + assert_eq!(control.state, SlotState::Working); + + let cmdheader = unsafe { + // SAFETY: The IRQ handler is only called after the command + // is finished. + &mut *self.cmdheader.get() + }; // TODO: Check errors. - inner.state = SlotState::Finished; - inner.cmdheader.bytes_transferred = 0; - inner.cmdheader.prdt_length = 0; + cmdheader.bytes_transferred = 0; + cmdheader.prdt_length = 0; - self.wait_list.notify_all(); + control.state = SlotState::Finished; + + if let Some(waker) = control.waker.take() { + waker.wake(); + } } pub fn prepare_command(&self, cmdtable: &CommandTable, write: bool) { - let mut inner = self.inner.lock_irq(); - let cmdheader = &mut inner.cmdheader; + let mut control = self.control.lock_irq(); + assert_eq!(control.state, SlotState::Idle); + + let cmdheader = unsafe { + // SAFETY: We are in the idle state. + &mut *self.cmdheader.get() + }; cmdheader.first = 0x05; // FIS type @@ -60,36 +148,37 @@ impl<'a> CommandSlot<'a> { cmdheader.second = 0x00; - cmdheader.prdt_length = cmdtable.prdt_len(); + cmdheader.prdt_length = cmdtable.prdt_len() as u16; cmdheader.bytes_transferred = 0; cmdheader.command_table_base = cmdtable.base().addr() as u64; cmdheader._reserved = [0; 4]; - inner.state = SlotState::Working; + control.state = SlotState::Working; } pub async fn wait_finish(&self) -> KResult<()> { - let mut inner = loop { - let mut wait = pin!(self.wait_list.prepare_to_wait()); - - { - let inner = self.inner.lock_irq(); - if inner.state != SlotState::Working { - break inner; + core::future::poll_fn(|ctx| { + let mut control = self.control.lock_irq(); + + match control.state { + SlotState::Idle => unreachable!("Poll called in idle state"), + SlotState::Working => { + control.waker = Some(ctx.waker().clone()); + Poll::Pending } - wait.as_mut().add_to_wait_list(); + SlotState::Finished => { + control.state = SlotState::Idle; + Poll::Ready(Ok(())) + } + SlotState::Error => { + control.state = SlotState::Idle; - if inner.state != SlotState::Working { - break inner; + // TODO: Report errors. + Poll::Ready(Err(EIO)) } } - - wait.await; - }; - - inner.state = SlotState::Idle; - - Ok(()) + }) + .await } } diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index f362f477..73143c2c 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -1,19 +1,18 @@ -use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; -use crate::kernel::interrupt::register_irq_handler; -use crate::kernel::mem::paging::{self, AllocZeroed}; -use crate::kernel::mem::{AsMemoryBlock, PhysAccess}; -use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; -use crate::net::netdev; -use crate::prelude::*; use alloc::boxed::Box; use alloc::sync::Arc; -use alloc::vec::Vec; -use async_trait::async_trait; use core::ptr::NonNull; + +use async_trait::async_trait; use eonix_hal::fence::memory_barrier; use eonix_mm::address::{Addr, PAddr}; use eonix_sync::SpinIrq; -use paging::Page; + +use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; +use crate::kernel::interrupt::register_irq_handler; +use crate::kernel::mem::{PageExcl, PageExt, PhysAccess}; +use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; +use crate::net::netdev; +use crate::prelude::*; mod defs; @@ -55,13 +54,13 @@ struct E1000eDev { id: u32, regs: Registers, - rt_desc_page: Page, + rt_desc_page: PageExcl, rx_head: Option, rx_tail: Option, tx_tail: Option, - rx_buffers: Option>>, - tx_buffers: Option>>, + rx_buffers: Box<[PageExcl; RX_DESC_SIZE]>, + tx_buffers: Box<[Option; TX_DESC_SIZE]>, } fn test(val: u32, bit: u32) -> bool { @@ -196,7 +195,7 @@ impl netdev::Netdev for E1000eDev { break; } - let ref mut desc = self.rx_desc_table()[next_tail as usize]; + let desc = unsafe { &mut self.rx_desc_table()[next_tail as usize] }; if !test(desc.status as u32, defs::RXD_STAT_DD as u32) { Err(EIO)?; } @@ -204,11 +203,8 @@ impl netdev::Netdev for E1000eDev { desc.status = 0; let len = desc.length as usize; - let buffers = self.rx_buffers.as_mut().ok_or(EIO)?; - let data = unsafe { - // SAFETY: No one could be writing to the buffer at this point. - &buffers[next_tail as usize].as_memblk().as_bytes()[..len] - }; + let buffer = &self.rx_buffers[next_tail as usize]; + let data = &buffer.as_bytes()[..len]; println_debug!("e1000e: received {len} bytes, {:?}", PrintableBytes(data)); self.rx_tail = Some(next_tail); @@ -226,20 +222,17 @@ impl netdev::Netdev for E1000eDev { return Err(EAGAIN); } - let ref mut desc = self.tx_desc_table()[tail as usize]; + let desc = unsafe { &mut self.tx_desc_table()[tail as usize] }; if !test(desc.status as u32, defs::TXD_STAT_DD as u32) { return Err(EIO); } - let buffer_page = Page::alloc(); + let mut buffer_page = PageExcl::alloc(); if buf.len() > buffer_page.len() { return Err(EFAULT); } - unsafe { - // SAFETY: We are the only one writing to this memory block. - buffer_page.as_memblk().as_bytes_mut()[..buf.len()].copy_from_slice(buf); - } + buffer_page.as_bytes_mut()[..buf.len()].copy_from_slice(buf); desc.buffer = PAddr::from(buffer_page.pfn()).addr() as u64; desc.length = buf.len() as u16; @@ -249,9 +242,8 @@ impl netdev::Netdev for E1000eDev { self.tx_tail = Some(next_tail); self.regs.write(defs::REG_TDT, next_tail); - // TODO: check if the packets are sent and update self.tx_head state - - Ok(()) + unimplemented!("Check if the packets are sent and update self.tx_head state"); + // Ok(()) } } @@ -324,26 +316,26 @@ impl E1000eDev { Ok(()) } - fn reset(&self) -> Result<(), u32> { + fn reset(regs: &Registers) -> Result<(), u32> { // disable interrupts so we won't mess things up - self.regs.write(defs::REG_IMC, 0xffffffff); + regs.write(defs::REG_IMC, 0xffffffff); - let ctrl = self.regs.read(defs::REG_CTRL); - self.regs.write(defs::REG_CTRL, ctrl | defs::CTRL_GIOD); + let ctrl = regs.read(defs::REG_CTRL); + regs.write(defs::REG_CTRL, ctrl | defs::CTRL_GIOD); - while self.regs.read(defs::REG_STAT) & defs::STAT_GIOE != 0 { + while regs.read(defs::REG_STAT) & defs::STAT_GIOE != 0 { // wait for link up } - let ctrl = self.regs.read(defs::REG_CTRL); - self.regs.write(defs::REG_CTRL, ctrl | defs::CTRL_RST); + let ctrl = regs.read(defs::REG_CTRL); + regs.write(defs::REG_CTRL, ctrl | defs::CTRL_RST); - while self.regs.read(defs::REG_CTRL) & defs::CTRL_RST != 0 { + while regs.read(defs::REG_CTRL) & defs::CTRL_RST != 0 { // wait for reset } // disable interrupts again - self.regs.write(defs::REG_IMC, 0xffffffff); + regs.write(defs::REG_IMC, 0xffffffff); Ok(()) } @@ -360,64 +352,45 @@ impl E1000eDev { Ok(()) } - pub fn new(base: PAddr, irq_no: usize) -> Result { - let page = Page::zeroed(); + pub fn new(base: PAddr, irq_no: usize) -> KResult { + let regs = Registers::new(base); + Self::reset(®s)?; - let mut dev = Self { + let dev = Self { irq_no, - mac: [0; 6], + mac: regs.read_as(0x5400), status: netdev::LinkStatus::Down, speed: netdev::LinkSpeed::SpeedUnknown, id: netdev::alloc_id(), - regs: Registers::new(base), - rt_desc_page: page, + regs, + rt_desc_page: PageExcl::zeroed(), rx_head: None, rx_tail: None, tx_tail: None, - rx_buffers: None, - tx_buffers: None, + rx_buffers: Box::new(core::array::from_fn(|_| PageExcl::alloc_order(2))), + tx_buffers: Box::new([const { None }; 32]), }; - dev.reset()?; - - dev.mac = dev.regs.read_as(0x5400); - dev.tx_buffers = Some(Box::new(Vec::with_capacity(TX_DESC_SIZE))); - - let mut rx_buffers = Box::new(Vec::with_capacity(RX_DESC_SIZE)); - - for index in 0..RX_DESC_SIZE { - let page = Page::alloc_order(2); - - let ref mut desc = dev.rx_desc_table()[index]; - desc.buffer = PAddr::from(page.pfn()).addr() as u64; - desc.status = 0; - - rx_buffers.push(page); - } + unsafe { + for (desc, page) in dev.rx_desc_table().into_iter().zip(dev.rx_buffers.iter()) { + desc.buffer = page.start().addr() as u64; + desc.status = 0; + } - for index in 0..TX_DESC_SIZE { - let ref mut desc = dev.tx_desc_table()[index]; - desc.status = defs::TXD_STAT_DD; + for desc in dev.tx_desc_table() { + desc.status = defs::TXD_STAT_DD; + } } - dev.rx_buffers = Some(rx_buffers); - Ok(dev) } - fn rx_desc_table(&self) -> &mut [RxDescriptor; RX_DESC_SIZE] { - unsafe { - // SAFETY: TODO - self.rt_desc_page.as_memblk().as_ptr().as_mut() - } + unsafe fn rx_desc_table(&self) -> &mut [RxDescriptor; RX_DESC_SIZE] { + self.rt_desc_page.get_ptr().cast().as_mut() } - fn tx_desc_table(&self) -> &mut [TxDescriptor; TX_DESC_SIZE] { - let (_, right) = self.rt_desc_page.as_memblk().split_at(0x200); - unsafe { - // SAFETY: TODO - right.as_ptr().as_mut() - } + unsafe fn tx_desc_table(&self) -> &mut [TxDescriptor; TX_DESC_SIZE] { + self.rt_desc_page.get_ptr().add(0x200).cast().as_mut() } } @@ -425,12 +398,8 @@ impl Drop for E1000eDev { fn drop(&mut self) { assert_eq!(self.status, netdev::LinkStatus::Down); - if let Some(_) = self.rx_buffers.take() {} - - // TODO: we should wait until all packets are sent - if let Some(_) = self.tx_buffers.take() {} - - let _ = self.rt_desc_page; + // TODO: we should wait until all packets are sent before dropping + // tx buffers. } } diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index 86b500b6..c5a3c3d2 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -1,21 +1,19 @@ -use crate::{ - io::Chunks, - kernel::{ - block::{BlockDeviceRequest, BlockRequestQueue}, - constants::EIO, - mem::{AsMemoryBlock, Page}, - }, - prelude::KResult, -}; use alloc::boxed::Box; + use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{Addr, PAddr, PhysAccess}, - paging::PFN, -}; +use eonix_mm::address::{Addr, PAddr, PhysAccess}; +use eonix_mm::paging::PFN; use eonix_sync::Spin; -use virtio_drivers::{device::blk::VirtIOBlk, transport::Transport, Hal}; +use virtio_drivers::device::blk::VirtIOBlk; +use virtio_drivers::transport::Transport; +use virtio_drivers::Hal; + +use crate::io::Chunks; +use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; +use crate::kernel::constants::EIO; +use crate::kernel::mem::{Page, PageExt}; +use crate::prelude::KResult; pub struct HAL; @@ -26,11 +24,10 @@ unsafe impl Hal for HAL { ) -> (virtio_drivers::PhysAddr, core::ptr::NonNull) { let page = Page::alloc_at_least(pages); - let paddr = page.start().addr(); - let ptr = page.as_memblk().as_byte_ptr(); - page.into_raw(); + let ptr = page.get_ptr(); + let pfn = page.into_raw(); - (paddr, ptr) + (PAddr::from(pfn).addr(), ptr) } unsafe fn dma_dealloc( @@ -93,15 +90,14 @@ where buffer, } => { let mut dev = self.lock(); - for ((start, len), buffer_page) in + for ((start, sectors), buffer_page) in Chunks::new(sector as usize, count as usize, 8).zip(buffer.iter()) { - let buffer = unsafe { - // SAFETY: Pages in `req.buffer` are guaranteed to be exclusively owned by us. - &buffer_page.as_memblk().as_bytes()[..len as usize * 512] - }; + let len = sectors * 512; + let pg = buffer_page.lock(); - dev.write_blocks(start, buffer).map_err(|_| EIO)?; + dev.write_blocks(start, &pg.as_bytes()[..len]) + .map_err(|_| EIO)?; } } BlockDeviceRequest::Read { @@ -110,15 +106,14 @@ where buffer, } => { let mut dev = self.lock(); - for ((start, len), buffer_page) in + for ((start, sectors), buffer_page) in Chunks::new(sector as usize, count as usize, 8).zip(buffer.iter()) { - let buffer = unsafe { - // SAFETY: Pages in `req.buffer` are guaranteed to be exclusively owned by us. - &mut buffer_page.as_memblk().as_bytes_mut()[..len as usize * 512] - }; + let len = sectors * 512; + let mut pg = buffer_page.lock(); - dev.read_blocks(start, buffer).map_err(|_| EIO)?; + dev.read_blocks(start, &mut pg.as_bytes_mut()[..len]) + .map_err(|_| EIO)?; } } } diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 1104337d..9a4e03ec 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,45 +1,38 @@ mod dir; mod file; +use alloc::sync::{Arc, Weak}; use core::future::Future; use core::ops::Deref; -use alloc::sync::{Arc, Weak}; use async_trait::async_trait; use dir::{as_raw_dirents, ParseDirent}; use eonix_sync::RwLock; use itertools::Itertools; +use crate::io::{Buffer, ByteBuffer, UninitBuffer}; +use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::{AsMemoryBlock, CachePageStream}; +use crate::kernel::mem::{ + CachePage, CachePageStream, Page, PageCache, PageCacheBackendOps, PageExcl, PageExt, +}; use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{ + Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, +}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; use crate::prelude::*; -use crate::{ - io::{Buffer, ByteBuffer, UninitBuffer}, - kernel::{ - block::{BlockDevice, BlockDeviceRequest}, - mem::{ - paging::Page, - {CachePage, PageCache, PageCacheBackendOps}, - }, - vfs::{ - dentry::Dentry, - inode::{Ino, Inode}, - mount::{register_filesystem, Mount, MountCreator}, - }, - }, - KResult, -}; +use crate::KResult; #[repr(transparent)] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] struct Cluster(u32); #[repr(transparent)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] struct RawCluster(pub u32); impl RawCluster { @@ -70,7 +63,7 @@ impl Cluster { const SECTOR_SIZE: usize = 512; -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] #[repr(C, packed)] struct Bootsector { jmp: [u8; 3], @@ -302,10 +295,8 @@ impl InodeFileOps for FileInode { for cluster in cluster_iter { fs.read_cluster(cluster, &buffer_page).await?; - let data = unsafe { - // SAFETY: We are the only one holding this page. - &buffer_page.as_memblk().as_bytes()[inner_offset..] - }; + let pg = buffer_page.lock(); + let data = &pg.as_bytes()[inner_offset..]; let end = offset + data.len(); let real_end = end.min(self.info.lock().size as usize); @@ -340,7 +331,7 @@ struct DirInode { sb: SbRef, // TODO: Use the new PageCache... - dir_pages: RwLock>, + dir_pages: RwLock>, } impl DirInode { @@ -375,7 +366,7 @@ impl DirInode { let clusters = ClusterIterator::new(fat.as_ref(), self.cluster); for cluster in clusters { - let page = Page::alloc(); + let page = PageExcl::alloc(); fs.read_cluster(cluster, &page).await?; dir_pages.push(page); @@ -384,7 +375,7 @@ impl DirInode { Ok(()) } - async fn get_dir_pages(&self) -> KResult> + use<'_>> { + async fn get_dir_pages(&self) -> KResult> + use<'_>> { { let dir_pages = self.dir_pages.read().await; if !dir_pages.is_empty() { @@ -432,12 +423,7 @@ impl InodeDirOps for DirInode { let sb = self.sb.get()?; let dir_pages = self.get_dir_pages().await?; - let dir_data = dir_pages.iter().map(|page| { - unsafe { - // SAFETY: No one could be writing to it. - page.as_memblk().as_bytes() - } - }); + let dir_data = dir_pages.iter().map(|pg| pg.as_bytes()); let raw_dirents = dir_data .map(as_raw_dirents) @@ -481,12 +467,10 @@ impl InodeDirOps for DirInode { let inner_offset = offset % cluster_size; let inner_raw_dirent_offset = inner_offset / core::mem::size_of::(); - let dir_data = dir_pages.iter().skip(cluster_offset).map(|page| { - unsafe { - // SAFETY: No one could be writing to it. - page.as_memblk().as_bytes() - } - }); + let dir_data = dir_pages + .iter() + .skip(cluster_offset) + .map(|pg| pg.as_bytes()); let raw_dirents = dir_data .map(as_raw_dirents) diff --git a/src/io.rs b/src/io.rs index 85675dea..d7094f6d 100644 --- a/src/io.rs +++ b/src/io.rs @@ -1,6 +1,8 @@ +use core::mem::MaybeUninit; +use core::ops::{Add, AddAssign, Sub}; + use crate::kernel::constants::EFAULT; use crate::prelude::*; -use core::{cmp, mem::MaybeUninit}; #[must_use] #[derive(Debug)] @@ -236,18 +238,26 @@ impl Buffer for ByteBuffer<'_> { } } +pub trait Integer: + Add + Sub + AddAssign + Copy + PartialOrd + Ord +{ +} + +impl Integer for u64 {} +impl Integer for usize {} + /// Iterator that generates chunks of a given length from a start index /// until the end of the total length. /// /// The iterator returns a tuple of (start, len) for each chunk. -pub struct Chunks { - end: usize, - cur: usize, - chunk_len: usize, +pub struct Chunks { + end: T, + cur: T, + chunk_len: T, } -impl Chunks { - pub const fn new(start: usize, total_len: usize, chunk_len: usize) -> Self { +impl Chunks { + pub fn new(start: T, total_len: T, chunk_len: T) -> Self { Self { end: start + total_len, cur: start, @@ -256,8 +266,8 @@ impl Chunks { } } -impl Iterator for Chunks { - type Item = (usize, usize); +impl Iterator for Chunks { + type Item = (T, T); fn next(&mut self) -> Option { if self.cur >= self.end { @@ -265,7 +275,7 @@ impl Iterator for Chunks { } let start = self.cur; - let len = cmp::min(self.chunk_len, self.end - start); + let len = self.chunk_len.min(self.end - start); self.cur += self.chunk_len; Some((start, len)) diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 3e4b65d1..8e017336 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -1,23 +1,20 @@ mod mbr; -use super::{ - constants::ENOENT, - mem::{paging::Page, AsMemoryBlock as _}, - vfs::types::DeviceId, -}; -use crate::kernel::constants::{EEXIST, EINVAL}; -use crate::{ - io::{Buffer, FillResult}, - prelude::*, -}; -use alloc::{ - collections::btree_map::{BTreeMap, Entry}, - sync::Arc, -}; -use async_trait::async_trait; +use alloc::collections::btree_map::{BTreeMap, Entry}; +use alloc::sync::Arc; use core::cmp::Ordering; + +use async_trait::async_trait; use mbr::MBRPartTable; +use super::constants::ENOENT; +use super::mem::paging::Page; +use super::mem::PageExt; +use super::vfs::types::DeviceId; +use crate::io::{Buffer, Chunks, FillResult}; +use crate::kernel::constants::{EEXIST, EINVAL}; +use crate::prelude::*; + pub struct Partition { pub lba_offset: u64, pub sector_count: u64, @@ -193,177 +190,72 @@ impl BlockDevice { /// `offset` - offset in bytes /// pub async fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult { - let mut sector_start = offset as u64 / 512; - let mut first_sector_offset = offset as u64 % 512; - let mut sector_count = (first_sector_offset + buffer.total() as u64 + 511) / 512; - - let mut nfilled = 0; - 'outer: while sector_count != 0 { - let pages: &[Page]; - let page: Option; - let page_vec: Option>; - - let nread; - - match sector_count { - count if count <= 8 => { - nread = count; - - let _page = Page::alloc(); - page = Some(_page); - pages = core::slice::from_ref(page.as_ref().unwrap()); + let sector_start = offset as u64 / 512; + let mut first_sector_offset = offset % 512; + let nr_sectors = (first_sector_offset + buffer.total() + 511) / 512; + + let nr_sectors_per_batch = self.queue().max_request_pages() / 2 * 2 * 8; + + let mut nr_filled = 0; + for (start, nr_batch) in Chunks::new(sector_start, nr_sectors as u64, nr_sectors_per_batch) + { + let (page_slice, page, mut page_vec); + match nr_batch { + ..=8 => { + page = Page::alloc(); + page_slice = core::slice::from_ref(&page); } - count if count <= 16 => { - nread = count; - - let _pages = Page::alloc_order(1); - page = Some(_pages); - pages = core::slice::from_ref(page.as_ref().unwrap()); + ..=16 => { + page = Page::alloc_order(1); + page_slice = core::slice::from_ref(&page); + } + ..=32 => { + page = Page::alloc_order(2); + page_slice = core::slice::from_ref(&page); } count => { - nread = count.min(self.queue().max_request_pages()); + let nr_huge_pages = count as usize / 32; + let nr_small_pages = ((count as usize % 32) + 7) / 8; - let npages = (nread + 15) / 16; - let mut _page_vec = Vec::with_capacity(npages as usize); - for _ in 0..npages { - _page_vec.push(Page::alloc_order(1)); - } - page_vec = Some(_page_vec); - pages = page_vec.as_ref().unwrap().as_slice(); + let nr_pages = nr_huge_pages + nr_small_pages; + page_vec = Vec::with_capacity(nr_pages); + + page_vec.resize_with(nr_huge_pages, || Page::alloc_order(2)); + page_vec.resize_with(nr_pages, || Page::alloc()); + page_slice = &page_vec; } } let req = BlockDeviceRequest::Read { - sector: sector_start, - count: nread, - buffer: &pages, + sector: start, + count: nr_batch, + buffer: page_slice, }; self.commit_request(req).await?; - for page in pages.iter() { - // SAFETY: We are the only owner of the page so no one could be mutating it. - let data = unsafe { &page.as_memblk().as_bytes()[first_sector_offset as usize..] }; + for page in page_slice { + let pg = page.lock(); + let data = &pg.as_bytes()[first_sector_offset..]; first_sector_offset = 0; - match buffer.fill(data)? { - FillResult::Done(n) => nfilled += n, - FillResult::Partial(n) => { - nfilled += n; - break 'outer; - } - FillResult::Full => { - break 'outer; - } - } - } - - sector_start += nread; - sector_count -= nread; - } - - if nfilled == buffer.total() { - Ok(FillResult::Done(nfilled)) - } else { - Ok(FillResult::Partial(nfilled)) - } - } - - /// Write some data to the block device, may involve some copy and fragmentation - /// - /// # Arguments - /// `offset` - offset in bytes - /// `data` - data to write - /// - pub async fn write_some(&self, offset: usize, data: &[u8]) -> KResult { - let mut sector_start = offset as u64 / 512; - let mut first_sector_offset = offset as u64 % 512; - let mut remaining_data = data; - let mut nwritten = 0; - - while !remaining_data.is_empty() { - let pages: &[Page]; - let page: Option; - let page_vec: Option>; - - // Calculate sectors needed for this write - let write_end = first_sector_offset + remaining_data.len() as u64; - let sector_count = ((write_end + 511) / 512).min(self.queue().max_request_pages()); - - match sector_count { - count if count <= 8 => { - let _page = Page::alloc(); - page = Some(_page); - pages = core::slice::from_ref(page.as_ref().unwrap()); - } - count if count <= 16 => { - let _pages = Page::alloc_order(1); - page = Some(_pages); - pages = core::slice::from_ref(page.as_ref().unwrap()); - } - count => { - let npages = (count + 15) / 16; - let mut _page_vec = Vec::with_capacity(npages as usize); - for _ in 0..npages { - _page_vec.push(Page::alloc_order(1)); - } - page_vec = Some(_page_vec); - pages = page_vec.as_ref().unwrap().as_slice(); - } - } - - if first_sector_offset != 0 || remaining_data.len() < (sector_count * 512) as usize { - let read_req = BlockDeviceRequest::Read { - sector: sector_start, - count: sector_count, - buffer: pages, - }; - self.commit_request(read_req).await?; - } - - let mut data_offset = 0; - let mut page_offset = first_sector_offset as usize; + nr_filled += buffer.fill(data)?.allow_partial(); - for page in pages.iter() { - // SAFETY: We own the page and can modify it - let page_data = unsafe { - let memblk = page.as_memblk(); - core::slice::from_raw_parts_mut(memblk.addr().get() as *mut u8, memblk.len()) - }; - - let copy_len = - (remaining_data.len() - data_offset).min(page_data.len() - page_offset); - - if copy_len == 0 { - break; - } - - page_data[page_offset..page_offset + copy_len] - .copy_from_slice(&remaining_data[data_offset..data_offset + copy_len]); - - data_offset += copy_len; - page_offset = 0; // Only first page has offset - - if data_offset >= remaining_data.len() { + if buffer.available() == 0 { break; } } - let write_req = BlockDeviceRequest::Write { - sector: sector_start, - count: sector_count, - buffer: pages, - }; - self.commit_request(write_req).await?; - - let bytes_written = data_offset; - nwritten += bytes_written; - remaining_data = &remaining_data[bytes_written..]; - sector_start += sector_count; - first_sector_offset = 0; + if buffer.available() == 0 { + break; + } } - Ok(nwritten) + if buffer.available() == 0 { + Ok(FillResult::Done(nr_filled)) + } else { + Ok(FillResult::Partial(nr_filled)) + } } } diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index c147306e..bfc826bf 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -8,9 +8,9 @@ mod mm_list; mod page_alloc; mod page_cache; -pub use access::{AsMemoryBlock, MemoryBlock, PhysAccess}; +pub use access::PhysAccess; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}; -pub use paging::{Page, PageBuffer}; +pub use paging::{Page, PageBuffer, PageExcl, PageExt}; diff --git a/src/kernel/mem/access.rs b/src/kernel/mem/access.rs index ce525a0a..328dcfbd 100644 --- a/src/kernel/mem/access.rs +++ b/src/kernel/mem/access.rs @@ -1,22 +1,7 @@ -use core::{num::NonZero, ptr::NonNull}; +use core::ptr::NonNull; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _PhysAccess}; -/// A block of memory starting at a non-zero address and having a specific length. -/// -/// This struct is used to represent a memory block that can be accessed -/// in the kernel space. -pub struct MemoryBlock { - addr: NonZero, - len: usize, -} - -pub trait AsMemoryBlock { - /// Translate the physical page the page object pointing to into kernel - /// accessible pointer. Use it with care. - fn as_memblk(&self) -> MemoryBlock; -} - pub trait PhysAccess { /// Translate the data that this address is pointing to into kernel /// accessible pointer. Use it with care. @@ -30,107 +15,6 @@ pub trait PhysAccess { unsafe fn as_ptr(&self) -> NonNull; } -impl MemoryBlock { - /// Create a new `MemoryBlock` with the given address and length. - /// - /// # Safety - /// The caller must ensure that the address is valid. - /// Otherwise, it may lead to undefined behavior. - pub unsafe fn new(addr: NonZero, len: usize) -> Self { - Self { addr, len } - } - - /// Get the start address of the memory block. - #[allow(dead_code)] - pub fn addr(&self) -> NonZero { - self.addr - } - - /// Get the length of the memory block. - #[allow(dead_code)] - pub fn len(&self) -> usize { - self.len - } - - /// Split the memory block into two parts at the given offset. - pub fn split_at(&self, at: usize) -> (Self, Self) { - if at > self.len { - panic!("Out of bounds"); - } - - let rhs_start = self.addr.checked_add(at).expect("Overflow"); - - let lhs = unsafe { Self::new(self.addr, at) }; - let rhs = unsafe { Self::new(rhs_start, self.len - at) }; - - (lhs, rhs) - } - - /// Provide a pointer to the data. - /// - /// # Safety - /// Using the returned pointer is undefined behavior if the address is not - /// properly aligned or the size is not equal to the size of `T`. - pub unsafe fn as_ptr_unchecked(&self) -> NonNull { - // SAFETY: `self.addr` is a non-zero value. - NonNull::new_unchecked(self.addr.get() as *mut T) - } - - /// Provide a pointer to the data. - /// - /// # Panic - /// Panic if the address is not properly aligned. - pub fn as_ptr(&self) -> NonNull { - let alignment = align_of::(); - - if self.addr.get() % alignment != 0 { - panic!("Alignment error"); - } - - unsafe { - // SAFETY: We've checked that `self.addr` is properly aligned. - self.as_ptr_unchecked() - } - } - - /// Provide a pointer to the bytes. - pub fn as_byte_ptr(&self) -> NonNull { - unsafe { - // SAFETY: No alignment check is needed for bytes. - self.as_ptr_unchecked() - } - } - - /// Provide immutable access to the data it pointed to. - /// - /// # Safety - /// This function is unsafe because it returns an immutable reference with - /// a created lifetime. - /// - /// The caller must ensure that the data has no other mutable aliases while - /// the reference is in use. Otherwise, it may lead to undefined behavior. - pub unsafe fn as_bytes<'a>(&self) -> &'a [u8] { - core::slice::from_raw_parts(self.as_ptr_unchecked().as_ptr(), self.len) - } - - /// Provide mutable access to the data it pointed to. - /// - /// # Panic - /// Panic if the address is not properly aligned or the size is not - /// equal to the size of `T`. - /// - /// # Safety - /// This function is unsafe because it returns a mutable reference with a - /// created lifetime. - /// - /// The caller must ensure that the data has no other immutable or mutable - /// aliases while the reference is in use. - /// Otherwise, it may lead to undefined behavior. - pub unsafe fn as_bytes_mut<'a>(&mut self) -> &'a mut [u8] { - core::slice::from_raw_parts_mut(self.as_ptr_unchecked().as_ptr(), self.len) - } -} - impl PhysAccess for PAddr { unsafe fn as_ptr(&self) -> NonNull { ArchPhysAccess::as_ptr(*self) diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 36b19612..9e5df69b 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -1,13 +1,15 @@ -use super::page_alloc::RawPagePtr; -use super::{AsMemoryBlock, GlobalPageAlloc, Page}; use core::alloc::{GlobalAlloc, Layout}; use core::ptr::NonNull; + use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::PhysAccess; use eonix_mm::paging::{PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; use slab_allocator::SlabAllocator; +use super::page_alloc::RawPagePtr; +use super::{GlobalPageAlloc, Page, PageExt}; + static SLAB_ALLOCATOR: LazyLock> = LazyLock::new(|| SlabAllocator::new_in(GlobalPageAlloc)); @@ -23,7 +25,7 @@ unsafe impl GlobalAlloc for Allocator { let page_count = size >> PAGE_SIZE_BITS; let page = Page::alloc_at_least(page_count); - let ptr = page.as_memblk().as_ptr(); + let ptr = page.get_ptr(); page.into_raw(); ptr.as_ptr() diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index 731c5303..dcbeeb63 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -1,15 +1,17 @@ -use super::mm_list::EMPTY_PAGE; -use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, Mapping, Page, Permission}; -use crate::kernel::constants::EINVAL; -use crate::prelude::KResult; use core::borrow::Borrow; use core::cell::UnsafeCell; use core::cmp; + use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{PAGE_SIZE, PFN}; +use super::mm_list::EMPTY_PAGE; +use super::{Mapping, Page, Permission}; +use crate::kernel::constants::EINVAL; +use crate::kernel::mem::{PageExcl, PageExt}; +use crate::prelude::KResult; + #[derive(Debug)] pub struct MMArea { range: UnsafeCell, @@ -105,25 +107,23 @@ impl MMArea { return; } - let new_page; + let mut new_page; if *pfn == EMPTY_PAGE.pfn() { - new_page = Page::zeroed(); + new_page = PageExcl::zeroed(); } else { - new_page = Page::alloc(); + new_page = PageExcl::alloc(); unsafe { // SAFETY: `page` is CoW, which means that others won't write to it. - let old_page_data = page.as_memblk().as_bytes(); - - // SAFETY: `new_page` is exclusive owned by us. - let new_page_data = new_page.as_memblk().as_bytes_mut(); + let old_page_data = page.get_bytes_ptr().as_ref(); + let new_page_data = new_page.as_bytes_mut(); new_page_data.copy_from_slice(old_page_data); }; } attr.remove(PageAttribute::ACCESSED); - *pfn = new_page.into_raw(); + *pfn = new_page.into_page().into_raw(); } /// # Arguments @@ -156,13 +156,12 @@ impl MMArea { // Bss is embarrassing in pagecache! // We have to assume cnt_to_read < PAGE_SIZE all bss if cnt_to_read < PAGE_SIZE { - let new_page = Page::zeroed(); - unsafe { - let page_data = new_page.as_memblk().as_bytes_mut(); - page_data[..cnt_to_read] - .copy_from_slice(&page.as_memblk().as_bytes()[..cnt_to_read]); - } - *pfn = new_page.into_raw(); + let mut new_page = PageExcl::zeroed(); + + new_page.as_bytes_mut()[..cnt_to_read] + .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); + + *pfn = new_page.into_page().into_raw(); } else { *pfn = page.clone().into_raw(); } @@ -182,13 +181,12 @@ impl MMArea { cache_page.set_dirty(); *pfn = page.clone().into_raw(); } else { - let new_page = Page::zeroed(); - unsafe { - let page_data = new_page.as_memblk().as_bytes_mut(); - page_data[..cnt_to_read] - .copy_from_slice(&page.as_memblk().as_bytes()[..cnt_to_read]); - } - *pfn = new_page.into_raw(); + let mut new_page = PageExcl::zeroed(); + + new_page.as_bytes_mut()[..cnt_to_read] + .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); + + *pfn = new_page.into_page().into_raw(); } attr.insert(PageAttribute::WRITE); diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index ad1e45c2..17dc1b05 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -1,33 +1,30 @@ mod mapping; mod page_fault; -use super::address::{VAddrExt as _, VRangeExt as _}; -use super::page_alloc::GlobalPageAlloc; -use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, MMArea, Page}; -use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; -use crate::kernel::mem::page_alloc::RawPagePtr; -use crate::{prelude::*, sync::ArcSwap}; use alloc::collections::btree_set::BTreeSet; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; + use eonix_hal::mm::{ flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchPagingMode, ArchPhysAccess, GLOBAL_PAGE_TABLE, }; -use eonix_mm::address::{Addr as _, PAddr}; -use eonix_mm::page_table::PageAttribute; -use eonix_mm::paging::PFN; -use eonix_mm::{ - address::{AddrOps as _, VAddr, VRange}, - page_table::{PageTable, RawAttribute, PTE}, - paging::PAGE_SIZE, -}; +use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, PageTable, RawAttribute, PTE}; +use eonix_mm::paging::{PAGE_SIZE, PFN}; use eonix_sync::{LazyLock, Mutex}; - pub use mapping::{FileMapping, Mapping}; pub use page_fault::handle_kernel_page_fault; +use super::address::{VAddrExt as _, VRangeExt as _}; +use super::page_alloc::GlobalPageAlloc; +use super::paging::AllocZeroed as _; +use super::{MMArea, Page, PageExt}; +use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; +use crate::kernel::mem::page_alloc::RawPagePtr; +use crate::prelude::*; +use crate::sync::ArcSwap; + pub static EMPTY_PAGE: LazyLock = LazyLock::new(|| Page::zeroed()); #[derive(Debug, Clone, Copy)] @@ -697,12 +694,10 @@ impl MMList { unsafe { // SAFETY: We are sure that the page is valid and we have the right to access it. Page::with_raw(pte.get_pfn(), |page| { - // SAFETY: The caller guarantees that no one else is using the page. - let page_data = page.as_memblk().as_bytes_mut(); - func( - offset + idx * 0x1000, - &mut page_data[start_offset..end_offset], - ); + let mut pg = page.lock(); + let page_data = &mut pg.as_bytes_mut()[start_offset..end_offset]; + + func(offset + idx * 0x1000, page_data); }); } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 54d4d590..08536693 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -81,7 +81,7 @@ pub struct RawPagePtr(NonNull); impl PageFlags { pub const PRESENT: u32 = 1 << 0; - // pub const LOCKED: u32 = 1 << 1; + pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; pub const SLAB: u32 = 1 << 3; pub const DIRTY: u32 = 1 << 4; @@ -99,6 +99,13 @@ impl PageFlags { pub fn clear(&self, flag: u32) { self.0.fetch_and(!flag, Ordering::Relaxed); } + + /// Set the flag and return whether it was already set. + /// + /// If multiple flags are given, returns true if any of them were already set. + pub fn test_and_set(&self, flag: u32) -> bool { + (self.0.fetch_or(flag, Ordering::Relaxed) & flag) != 0 + } } impl RawPagePtr { diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 9deb50cf..6a1c04ca 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,22 +1,23 @@ -use super::{paging::AllocZeroed, Page}; -use crate::{ - io::{Buffer, FillResult, Stream}, - kernel::mem::page_alloc::RawPagePtr, - prelude::KResult, - GlobalPageAlloc, -}; -use align_ext::AlignExt; use alloc::boxed::Box; -use alloc::{collections::btree_map::BTreeMap, sync::Weak}; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Weak; +use core::future::Future; +use core::mem::ManuallyDrop; + +use align_ext::AlignExt; use async_trait::async_trait; -use core::{future::Future, mem::ManuallyDrop}; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{PAddr, PhysAccess}, - paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}, -}; +use eonix_mm::address::{PAddr, PhysAccess}; +use eonix_mm::paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::Mutex; +use super::paging::AllocZeroed; +use super::Page; +use crate::io::{Buffer, FillResult, Stream}; +use crate::kernel::mem::page_alloc::RawPagePtr; +use crate::prelude::KResult; +use crate::GlobalPageAlloc; + pub struct PageCache { pages: Mutex>, backend: Weak, @@ -315,14 +316,6 @@ impl CachePageStream { pub fn new(page: CachePage) -> Self { Self { page, cur: 0 } } - - pub fn remaining(&self) -> usize { - self.page.valid_size().saturating_sub(self.cur) - } - - pub fn is_drained(&self) -> bool { - self.cur >= self.page.valid_size() - } } impl Stream for CachePageStream { diff --git a/src/kernel/mem/paging.rs b/src/kernel/mem/paging.rs index 8c5f41f2..bca573fb 100644 --- a/src/kernel/mem/paging.rs +++ b/src/kernel/mem/paging.rs @@ -1,41 +1,54 @@ -use super::{access::AsMemoryBlock, page_alloc::GlobalPageAlloc, MemoryBlock, PhysAccess}; +use core::ops::Deref; +use core::ptr::NonNull; + +use eonix_mm::paging::Page as GenericPage; + +use super::page_alloc::GlobalPageAlloc; +use super::PhysAccess; use crate::io::{Buffer, FillResult}; -use eonix_mm::paging::{Page as GenericPage, PageAlloc}; pub type Page = GenericPage; /// A buffer that wraps a page and provides a `Buffer` interface. pub struct PageBuffer { - page: Page, + page: PageExcl, offset: usize, } +pub struct PageLocked<'a> { + page: &'a Page, +} + +/// A page that is exclusively owned. +#[repr(transparent)] +pub struct PageExcl(Page); + pub trait AllocZeroed { fn zeroed() -> Self; } -impl AsMemoryBlock for GenericPage { - fn as_memblk(&self) -> MemoryBlock { - unsafe { - // SAFETY: `self.start()` points to valid memory of length `self.len()`. - MemoryBlock::new(self.start().as_ptr::<()>().addr(), self.len()) - } +pub trait PageExt { + fn lock(&self) -> PageLocked; + + /// Get a vmem pointer to the page data as a byte slice. + fn get_bytes_ptr(&self) -> NonNull<[u8]>; + + /// Get a vmem pointer to the start of the page. + fn get_ptr(&self) -> NonNull { + self.get_bytes_ptr().cast() } } impl PageBuffer { pub fn new() -> Self { Self { - page: Page::alloc(), + page: PageExcl::alloc(), offset: 0, } } pub fn all(&self) -> &[u8] { - unsafe { - // SAFETY: The page is exclusivly owned by us. - self.page.as_memblk().as_bytes() - } + self.page.as_bytes() } pub fn data(&self) -> &[u8] { @@ -43,10 +56,7 @@ impl PageBuffer { } pub fn available_mut(&mut self) -> &mut [u8] { - unsafe { - // SAFETY: The page is exclusivly owned by us. - &mut self.page.as_memblk().as_bytes_mut()[self.offset..] - } + &mut self.page.as_bytes_mut()[self.offset..] } } @@ -80,10 +90,87 @@ impl Buffer for PageBuffer { impl AllocZeroed for Page { fn zeroed() -> Self { let page = Self::alloc(); + + page.lock().as_bytes_mut().fill(0); + + page + } +} + +impl PageExt for Page { + fn lock(&self) -> PageLocked { + // TODO: Actually perform the lock. + PageLocked { page: self } + } + + fn get_bytes_ptr(&self) -> NonNull<[u8]> { unsafe { - // SAFETY: The page is exclusivly owned by us. - page.as_memblk().as_bytes_mut().fill(0); + // SAFETY: `self.start()` can't be null. + NonNull::slice_from_raw_parts(self.start().as_ptr(), self.len()) } - page + } +} + +impl PageLocked<'_> { + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) + } + } +} + +impl Deref for PageLocked<'_> { + type Target = Page; + + fn deref(&self) -> &Self::Target { + self.page + } +} + +impl PageExcl { + pub fn alloc() -> Self { + Self(Page::alloc()) + } + + pub fn alloc_order(order: u32) -> Self { + Self(Page::alloc_order(order)) + } + + pub fn zeroed() -> Self { + Self(Page::zeroed()) + } + + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_ref() + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_mut() + } + } + + pub fn into_page(self) -> Page { + self.0 + } +} + +impl Deref for PageExcl { + type Target = Page; + + fn deref(&self) -> &Self::Target { + &self.0 } } diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index bb1c66ec..eb00cc4c 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -2,29 +2,24 @@ mod inode_file; mod pipe; mod terminal_file; -use crate::{ - io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}, - kernel::{ - constants::{EBADF, EINTR, EINVAL, ENOTTY}, - mem::{AsMemoryBlock, Page}, - task::Thread, - CharDevice, - }, - prelude::KResult, -}; use alloc::sync::Arc; -use bitflags::bitflags; -use core::{ - ops::Deref, - sync::atomic::{AtomicI32, AtomicU32, Ordering}, -}; -use pipe::{PipeReadEnd, PipeWriteEnd}; -use posix_types::open::OpenFlags; +use core::ops::Deref; +use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; +use bitflags::bitflags; pub use inode_file::InodeFile; pub use pipe::Pipe; +use pipe::{PipeReadEnd, PipeWriteEnd}; +use posix_types::open::OpenFlags; pub use terminal_file::TerminalFile; +use crate::io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}; +use crate::kernel::constants::{EBADF, EINTR, EINVAL, ENOTTY}; +use crate::kernel::mem::PageExcl; +use crate::kernel::task::Thread; +use crate::kernel::CharDevice; +use crate::prelude::KResult; + pub enum FileType { Inode(InodeFile), PipeRead(PipeReadEnd), @@ -99,9 +94,8 @@ impl FileType { } pub async fn sendfile(&self, dest_file: &Self, count: usize) -> KResult { - let buffer_page = Page::alloc(); - // SAFETY: We are the only owner of the page. - let buffer = unsafe { buffer_page.as_memblk().as_bytes_mut() }; + let mut buffer_page = PageExcl::alloc(); + let buffer = buffer_page.as_bytes_mut(); self.sendfile_check()?; diff --git a/src/lib.rs b/src/lib.rs index 98e196f8..959cb29f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,9 @@ extern crate alloc; +#[macro_use] +extern crate static_assertions; + #[cfg(any(target_arch = "riscv64", target_arch = "x86_64"))] extern crate unwinding; @@ -28,36 +31,33 @@ mod prelude; mod rcu; mod sync; -use crate::kernel::task::alloc_pid; -use alloc::{ffi::CString, sync::Arc}; -use core::{ - hint::spin_loop, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, -}; -use eonix_hal::{ - arch_exported::bootstrap::shutdown, - context::TaskContext, - processor::{halt, CPU, CPU_COUNT}, - traits::{context::RawTaskContext, trap::IrqState}, - trap::disable_irqs_save, -}; +use alloc::ffi::CString; +use alloc::sync::Arc; +use core::hint::spin_loop; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use eonix_hal::arch_exported::bootstrap::shutdown; +use eonix_hal::context::TaskContext; +use eonix_hal::processor::{halt, CPU, CPU_COUNT}; +use eonix_hal::traits::context::RawTaskContext; +use eonix_hal::traits::trap::IrqState; +use eonix_hal::trap::disable_irqs_save; use eonix_mm::address::PRange; -use eonix_runtime::{executor::Stack, scheduler::RUNTIME}; -use kernel::{ - mem::GlobalPageAlloc, - task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}, - vfs::{ - dentry::Dentry, - mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}, - types::Permission, - FsContext, - }, - CharDevice, -}; +use eonix_runtime::executor::Stack; +use eonix_runtime::scheduler::RUNTIME; +use kernel::mem::GlobalPageAlloc; +use kernel::task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}; +use kernel::vfs::dentry::Dentry; +use kernel::vfs::mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}; +use kernel::vfs::types::Permission; +use kernel::vfs::FsContext; +use kernel::CharDevice; use kernel_init::setup_memory; use path::Path; use prelude::*; +use crate::kernel::task::alloc_pid; + #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] fn do_panic() -> ! { #[cfg(target_arch = "riscv64")] From 922324322b1d0dd75d1b50494785504035d8d598 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 23 Dec 2025 00:12:42 +0800 Subject: [PATCH 006/153] tls: rework of arch's UserTLS design Separate old UserTLS into UserTLS and UserTLSDescriptor. UserTLS is for threads to hold infomation about its storage. Descriptors are used in clone syscalls. Signed-off-by: greatbridf --- Cargo.lock | 5 +- Cargo.toml | 1 + crates/eonix_hal/src/arch/riscv64/cpu.rs | 17 --- crates/eonix_hal/src/lib.rs | 2 +- .../posix_types/src/syscall_no/loongarch64.rs | 3 +- crates/posix_types/src/syscall_no/riscv64.rs | 2 +- src/kernel/interrupt.rs | 12 +- src/kernel/pcie/init.rs | 18 +-- src/kernel/syscall/file_rw.rs | 67 +++++---- src/kernel/syscall/procops.rs | 134 +++++++----------- src/kernel/task.rs | 21 ++- src/kernel/task/clone.rs | 42 +++--- src/kernel/task/thread.rs | 76 ++++------ src/kernel/task/user_tls/mod.rs | 34 +++++ src/kernel/task/user_tls/x86_64.rs | 83 +++++++++++ src/kernel/user/dataflow.rs | 21 ++- 16 files changed, 304 insertions(+), 234 deletions(-) create mode 100644 src/kernel/task/user_tls/mod.rs create mode 100644 src/kernel/task/user_tls/x86_64.rs diff --git a/Cargo.lock b/Cargo.lock index c70190a0..5487f284 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,9 +75,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "critical-section" @@ -164,6 +164,7 @@ dependencies = [ "atomic_unique_refcell", "bitflags", "buddy_allocator", + "cfg-if", "eonix_hal", "eonix_log", "eonix_macros", diff --git a/Cargo.toml b/Cargo.toml index dca5d34d..4fcb6f70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ futures = { version = "0.3.31", features = [ "async-await", ], default-features = false } static_assertions = "1.1.0" +cfg-if = "1.0.4" [target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] virtio-drivers = { version = "0.11.0" } diff --git a/crates/eonix_hal/src/arch/riscv64/cpu.rs b/crates/eonix_hal/src/arch/riscv64/cpu.rs index 9c843eaf..3c58580e 100644 --- a/crates/eonix_hal/src/arch/riscv64/cpu.rs +++ b/crates/eonix_hal/src/arch/riscv64/cpu.rs @@ -27,22 +27,11 @@ static DEFAULT_TRAP_CONTEXT: MaybeUninit = MaybeUninit::uninit(); #[eonix_percpu::define_percpu] static LOCAL_CPU: LazyLock = LazyLock::new(|| CPU::new(CPUID.get())); -#[derive(Debug, Clone)] -pub enum UserTLS { - Base(u64), -} - /// RISC-V Hart pub struct CPU { pub(crate) interrupt: InterruptControl, } -impl UserTLS { - pub fn new(base: u64) -> Self { - Self::Base(base) - } -} - impl CPU { fn new(cpuid: usize) -> Self { Self { @@ -66,12 +55,6 @@ impl CPU { sscratch::write(DEFAULT_TRAP_CONTEXT.as_ptr() as usize); } - pub unsafe fn load_interrupt_stack(self: Pin<&mut Self>, sp: u64) {} - - pub fn set_tls32(self: Pin<&mut Self>, _user_tls: &UserTLS) { - // nothing - } - pub fn local() -> PreemptGuard> { unsafe { // SAFETY: We pass the reference into a `PreemptGuard`, which ensures diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index e789ecbb..b9c7d053 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -19,7 +19,7 @@ pub mod fpu { } pub mod processor { - pub use crate::arch::cpu::{halt, UserTLS, CPU, CPU_COUNT}; + pub use crate::arch::cpu::{halt, CPU, CPU_COUNT}; } /// Re-export the arch module for use in other crates diff --git a/crates/posix_types/src/syscall_no/loongarch64.rs b/crates/posix_types/src/syscall_no/loongarch64.rs index 19776a65..b0d54689 100644 --- a/crates/posix_types/src/syscall_no/loongarch64.rs +++ b/crates/posix_types/src/syscall_no/loongarch64.rs @@ -136,7 +136,7 @@ pub const SYS_RT_SIGSUSPEND: usize = 133; pub const SYS_RT_SIGACTION: usize = 134; pub const SYS_RT_SIGPROCMASK: usize = 135; pub const SYS_RT_SIGPENDING: usize = 136; -pub const SYS_RT_SIGTIMEDWAIT_TIME32: usize = 137; +pub const SYS_RT_SIGTIMEDWAIT: usize = 137; pub const SYS_RT_SIGQUEUEINFO: usize = 138; pub const SYS_RT_SIGRETURN: usize = 139; pub const SYS_SETPRIORITY: usize = 140; @@ -295,7 +295,6 @@ pub const SYS_RECVMMSG: usize = 417; pub const SYS_MQ_TIMEDSEND: usize = 418; pub const SYS_MQ_TIMEDRECEIVE: usize = 419; pub const SYS_SEMTIMEDOP: usize = 420; -pub const SYS_RT_SIGTIMEDWAIT: usize = 421; pub const SYS_FUTEX: usize = 422; pub const SYS_SCHED_RR_GET_INTERVAL: usize = 423; pub const SYS_PIDFD_SEND_SIGNAL: usize = 424; diff --git a/crates/posix_types/src/syscall_no/riscv64.rs b/crates/posix_types/src/syscall_no/riscv64.rs index 4457c20e..076942e9 100644 --- a/crates/posix_types/src/syscall_no/riscv64.rs +++ b/crates/posix_types/src/syscall_no/riscv64.rs @@ -136,7 +136,7 @@ pub const SYS_RT_SIGSUSPEND: usize = 133; pub const SYS_RT_SIGACTION: usize = 134; pub const SYS_RT_SIGPROCMASK: usize = 135; pub const SYS_RT_SIGPENDING: usize = 136; -pub const SYS_RT_SIGTIMEDWAIT_TIME32: usize = 137; +pub const SYS_RT_SIGTIMEDWAIT: usize = 137; pub const SYS_RT_SIGQUEUEINFO: usize = 138; pub const SYS_RT_SIGRETURN: usize = 139; pub const SYS_SETPRIORITY: usize = 140; diff --git a/src/kernel/interrupt.rs b/src/kernel/interrupt.rs index 742727cb..2092bfcb 100644 --- a/src/kernel/interrupt.rs +++ b/src/kernel/interrupt.rs @@ -1,15 +1,17 @@ -use super::mem::handle_kernel_page_fault; -use super::task::block_on; -use super::timer::timer_interrupt; -use crate::kernel::constants::EINVAL; -use crate::prelude::*; use alloc::sync::Arc; + use eonix_hal::traits::fault::Fault; use eonix_hal::traits::trap::{RawTrapContext, TrapType}; use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::SpinIrq as _; +use super::mem::handle_kernel_page_fault; +use super::task::block_on; +use super::timer::timer_interrupt; +use crate::kernel::constants::EINVAL; +use crate::prelude::*; + static IRQ_HANDLERS: Spin<[Vec>; 16]> = Spin::new([const { Vec::new() }; 16]); diff --git a/src/kernel/pcie/init.rs b/src/kernel/pcie/init.rs index c0253f4e..4c183bc5 100644 --- a/src/kernel/pcie/init.rs +++ b/src/kernel/pcie/init.rs @@ -1,13 +1,14 @@ -use super::{ - device::{PCIDevice, SegmentGroup, PCIE_DEVICES}, - error::PciError, -}; -use crate::kernel::{mem::PhysAccess as _, pcie::device::PciMemoryAllocator}; -use acpi::{AcpiHandler, PhysicalMapping}; use alloc::collections::btree_map::Entry; use alloc::vec; + +use acpi::{AcpiHandler, PhysicalMapping}; use eonix_log::println_trace; -use eonix_mm::address::PAddr; +use eonix_mm::address::{PAddr, PRange}; + +use super::device::{PCIDevice, SegmentGroup, PCIE_DEVICES}; +use super::error::PciError; +use crate::kernel::mem::PhysAccess as _; +use crate::kernel::pcie::device::PciMemoryAllocator; #[derive(Clone)] struct AcpiHandlerImpl; @@ -67,10 +68,11 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - use crate::kernel::constants::{EINVAL, EIO, ENOENT}; use eonix_hal::arch_exported::fdt::FDT; use eonix_mm::address::PRange; + use crate::kernel::constants::{EINVAL, EIO, ENOENT}; + let pcie_node = FDT .find_compatible(&["pci-host-ecam-generic"]) .ok_or(ENOENT)?; diff --git a/src/kernel/syscall/file_rw.rs b/src/kernel/syscall/file_rw.rs index db32b0e5..93a543d7 100644 --- a/src/kernel/syscall/file_rw.rs +++ b/src/kernel/syscall/file_rw.rs @@ -1,33 +1,31 @@ +use alloc::sync::Arc; +use core::time::Duration; + +use posix_types::ctypes::{Long, PtrT}; +use posix_types::namei::RenameFlags; +use posix_types::open::{AtFlags, OpenFlags}; +use posix_types::poll::FDSet; +use posix_types::signal::{SigSet, Signal}; +use posix_types::stat::{Stat, StatX, TimeSpec}; +use posix_types::syscall_no::*; + use super::{FromSyscallArg, User}; -use crate::io::IntoStream; +use crate::io::{Buffer, BufferFill, IntoStream}; use crate::kernel::constants::{ EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, SEEK_SET, }; use crate::kernel::syscall::UserMut; use crate::kernel::task::Thread; use crate::kernel::timer::sleep; +use crate::kernel::user::{ + CheckedUserPointer, UserBuffer, UserPointer, UserPointerMut, UserString, +}; +use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::filearray::FD; use crate::kernel::vfs::types::{DeviceId, Mode}; use crate::kernel::vfs::{PollEvent, SeekOption}; -use crate::{ - io::{Buffer, BufferFill}, - kernel::{ - user::{CheckedUserPointer, UserBuffer, UserPointer, UserPointerMut, UserString}, - vfs::dentry::Dentry, - }, - path::Path, - prelude::*, -}; -use alloc::sync::Arc; -use core::time::Duration; -use posix_types::ctypes::{Long, PtrT}; -use posix_types::namei::RenameFlags; -use posix_types::open::{AtFlags, OpenFlags}; -use posix_types::poll::FDSet; -use posix_types::signal::{SigSet, Signal}; -use posix_types::stat::Stat; -use posix_types::stat::{StatX, TimeSpec}; -use posix_types::syscall_no::*; +use crate::path::Path; +use crate::prelude::*; impl FromSyscallArg for OpenFlags { fn from_arg(value: usize) -> Self { @@ -128,7 +126,7 @@ async fn openat(dirfd: FD, pathname: User, flags: OpenFlags, mode: Mode) -> #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_OPEN)] -async fn open(path: User, flags: OpenFlags, mode: u32) -> KResult { +async fn open(path: User, flags: OpenFlags, mode: Mode) -> KResult { sys_openat(thread, FD::AT_FDCWD, path, flags, mode).await } @@ -145,7 +143,10 @@ async fn dup(fd: FD) -> KResult { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_DUP2)] async fn dup2(old_fd: FD, new_fd: FD) -> KResult { - thread.files.dup_to(old_fd, new_fd, OpenFlags::empty()) + thread + .files + .dup_to(old_fd, new_fd, OpenFlags::empty()) + .await } #[eonix_macros::define_syscall(SYS_DUP3)] @@ -172,7 +173,13 @@ async fn pipe(pipe_fd: UserMut<[FD; 2]>) -> KResult<()> { async fn getdents(fd: FD, buffer: UserMut, bufsize: usize) -> KResult { let mut buffer = UserBuffer::new(buffer, bufsize)?; - thread.files.get(fd).ok_or(EBADF)?.getdents(&mut buffer)?; + thread + .files + .get(fd) + .ok_or(EBADF)? + .getdents(&mut buffer) + .await?; + Ok(buffer.wrote()) } @@ -264,7 +271,7 @@ async fn mkdirat(dirfd: FD, pathname: User, mode: Mode) -> KResult<()> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKDIR)] -async fn mkdir(pathname: User, mode: u32) -> KResult<()> { +async fn mkdir(pathname: User, mode: Mode) -> KResult<()> { sys_mkdirat(thread, FD::AT_FDCWD, pathname, mode).await } @@ -280,9 +287,9 @@ async fn truncate(pathname: User, length: usize) -> KResult<()> { let path = UserString::new(pathname)?; let path = Path::new(path.as_cstr().to_bytes())?; - let dentry = Dentry::open(&thread.fs_context, path, true)?; + let dentry = Dentry::open(&thread.fs_context, path, true).await?; - dentry.truncate(length) + dentry.truncate(length).await } #[eonix_macros::define_syscall(SYS_UNLINKAT)] @@ -296,7 +303,7 @@ async fn unlinkat(dirfd: FD, pathname: User) -> KResult<()> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_UNLINK)] async fn unlink(pathname: User) -> KResult<()> { - sys_unlinkat(thread, FD::AT_FDCWD, pathname) + sys_unlinkat(thread, FD::AT_FDCWD, pathname).await } #[eonix_macros::define_syscall(SYS_SYMLINKAT)] @@ -310,7 +317,7 @@ async fn symlinkat(target: User, dirfd: FD, linkpath: User) -> KResult<( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_SYMLINK)] async fn symlink(target: User, linkpath: User) -> KResult<()> { - sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath) + sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath).await } #[derive(Clone, Copy, Debug)] @@ -347,7 +354,7 @@ async fn mknodat(dirfd: FD, pathname: User, mut mode: Mode, dev: UserDeviceI #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKNOD)] -async fn mknod(pathname: User, mode: u32, dev: u32) -> KResult<()> { +async fn mknod(pathname: User, mode: Mode, dev: UserDeviceId) -> KResult<()> { sys_mknodat(thread, FD::AT_FDCWD, pathname, mode, dev).await } @@ -389,7 +396,7 @@ async fn lseek(fd: FD, offset: u64, whence: u32) -> KResult { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_LLSEEK)] -fn llseek( +async fn llseek( fd: FD, offset_high: u32, offset_low: u32, diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index b4d3e449..1359d0ab 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -1,38 +1,37 @@ +use alloc::borrow::ToOwned; +use alloc::ffi::CString; +use core::time::Duration; + +use bitflags::bitflags; +use eonix_hal::traits::trap::RawTrapContext; +use eonix_hal::trap::TrapContext; +use eonix_mm::address::Addr as _; +use eonix_sync::AsProof as _; +use posix_types::ctypes::PtrT; +use posix_types::signal::{SigAction, SigInfo, SigSet, Signal}; +use posix_types::stat::{TimeSpec, TimeVal}; +use posix_types::syscall_no::*; +use posix_types::SIGNAL_NOW; + use super::SyscallNoReturn; use crate::io::Buffer; use crate::kernel::constants::{ - CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, ENOTDIR, ERANGE, ESRCH, -}; -use crate::kernel::constants::{ - ENOSYS, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, + CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, ENOSYS, ENOTDIR, + ERANGE, ESRCH, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, }; use crate::kernel::mem::PageBuffer; use crate::kernel::syscall::{User, UserMut}; use crate::kernel::task::{ - do_clone, futex_wait, futex_wake, yield_now, FutexFlags, FutexOp, ProcessList, ProgramLoader, - RobustListHead, SignalAction, Thread, WaitId, WaitType, + do_clone, futex_wait, futex_wake, parse_futexop, yield_now, CloneArgs, FutexFlags, FutexOp, + ProcessList, ProgramLoader, RobustListHead, SignalAction, Thread, WaitId, WaitType, }; -use crate::kernel::task::{parse_futexop, CloneArgs}; use crate::kernel::timer::sleep; -use crate::kernel::user::UserString; -use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::user::{UserBuffer, UserPointer, UserPointerMut, UserString}; +use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::types::Permission; -use crate::kernel::vfs::{self, dentry::Dentry}; +use crate::kernel::vfs::{self}; use crate::path::Path; -use crate::{kernel::user::UserBuffer, prelude::*}; -use alloc::borrow::ToOwned; -use alloc::ffi::CString; -use bitflags::bitflags; -use core::time::Duration; -use eonix_hal::processor::UserTLS; -use eonix_hal::traits::trap::RawTrapContext; -use eonix_hal::trap::TrapContext; -use eonix_mm::address::Addr as _; -use eonix_sync::AsProof as _; -use posix_types::ctypes::PtrT; -use posix_types::signal::{SigAction, SigInfo, SigSet, Signal}; -use posix_types::stat::TimeVal; -use posix_types::{syscall_no::*, SIGNAL_NOW}; +use crate::prelude::*; #[repr(C)] #[derive(Debug, Clone, Copy)] @@ -366,7 +365,7 @@ async fn wait4( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_WAITPID)] async fn waitpid(waitpid: i32, arg1: UserMut, options: u32) -> KResult { - sys_wait4(thread, waitpid, arg1, options, core::ptr::null_mut()).await + sys_wait4(thread, waitpid, arg1, options, UserMut::null()).await } #[eonix_macros::define_syscall(SYS_SETSID)] @@ -493,51 +492,15 @@ async fn gettid() -> KResult { Ok(thread.tid) } -pub fn parse_user_tls(arch_tls: usize) -> KResult { - #[cfg(target_arch = "x86_64")] - { - let desc = arch_tls as *mut posix_types::x86_64::UserDescriptor; - let desc_pointer = UserPointerMut::new(desc)?; - let mut desc = desc_pointer.read()?; - - // Clear the TLS area if it is not present. - if desc.flags.is_read_exec_only() && !desc.flags.is_present() { - if desc.limit != 0 && desc.base != 0 { - let len = if desc.flags.is_limit_in_pages() { - (desc.limit as usize) << 12 - } else { - desc.limit as usize - }; - - CheckedUserPointer::new(desc.base as _, len)?.zero()?; - } - } - - let (new_tls, entry) = - UserTLS::new32(desc.base, desc.limit, desc.flags.is_limit_in_pages()); - desc.entry = entry; - desc_pointer.write(desc)?; - - Ok(new_tls) - } - - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] - { - Ok(UserTLS::new(arch_tls as u64)) - } -} - #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_SET_THREAD_AREA)] -async fn set_thread_area(arch_tls: usize) -> KResult<()> { - thread.set_user_tls(parse_user_tls(arch_tls)?)?; +async fn set_thread_area(tls: PtrT) -> KResult<()> { + use crate::kernel::task::UserTLSDescriptor; - // SAFETY: Preemption is disabled on calling `load_thread_area32()`. - unsafe { - eonix_preempt::disable(); - thread.load_thread_area32(); - eonix_preempt::enable(); - } + let tls = UserTLSDescriptor::new(tls)?.read()?; + + thread.set_user_tls(tls)?; + thread.activate_tls(); Ok(()) } @@ -651,18 +614,14 @@ async fn rt_sigprocmask( Ok(()) } -#[repr(C)] -#[derive(Clone, Copy)] -struct TimeSpec32 { - tv_sec: i32, - tv_nsec: i32, -} - -#[eonix_macros::define_syscall(SYS_RT_SIGTIMEDWAIT_TIME32)] -async fn rt_sigtimedwait_time32( +#[cfg_attr( + any(target_arch = "riscv64", target_arch = "loongarch64"), + eonix_macros::define_syscall(SYS_RT_SIGTIMEDWAIT) +)] +async fn rt_sigtimedwait( _uthese: User, _uinfo: UserMut, - _uts: User, + _uts: User, ) -> KResult { // TODO Ok(0) @@ -820,7 +779,7 @@ async fn clone( clone_flags: usize, new_sp: usize, parent_tidptr: UserMut, - tls: usize, + tls: PtrT, child_tidptr: UserMut, ) -> KResult { let clone_args = CloneArgs::for_clone(clone_flags, new_sp, child_tidptr, parent_tidptr, tls)?; @@ -925,8 +884,23 @@ async fn sigreturn() -> KResult { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_ARCH_PRCTL)] -async fn arch_prctl(option: u32, addr: u32) -> KResult { - sys_arch_prctl(thread, option, addr).await +async fn arch_prctl(option: u32, addr: PtrT) -> KResult { + match option { + PR_SET_NAME => { + let name = UserPointer::<[u8; 16]>::new(User::with_addr(addr.addr()))?.read()?; + let len = name.iter().position(|&c| c == 0).unwrap_or(15); + thread.set_name(name[..len].into()); + Ok(0) + } + PR_GET_NAME => { + let name = thread.get_name(); + let len = name.len().min(15); + let name: [u8; 16] = core::array::from_fn(|i| if i < len { name[i] } else { 0 }); + UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(addr.addr()))?.write(name)?; + Ok(0) + } + _ => Err(EINVAL), + } } pub fn keep_alive() {} diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 2ef58069..3fe6fe97 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -8,6 +8,7 @@ mod process_list; mod session; mod signal; mod thread; +mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; pub use futex::{futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, RobustListHead}; @@ -19,6 +20,7 @@ pub use process_list::ProcessList; pub use session::Session; pub use signal::SignalAction; pub use thread::{yield_now, Thread, ThreadAlloc, ThreadBuilder}; +pub use user_tls::{UserTLS, UserTLSDescriptor}; fn do_block_on(mut future: core::pin::Pin<&mut F>) -> F::Output where @@ -79,30 +81,25 @@ pub async fn stackful(mut future: F) -> F::Output where F: core::future::Future, { - use crate::kernel::{ - interrupt::{default_fault_handler, default_irq_handler}, - timer::{should_reschedule, timer_interrupt}, - }; use alloc::sync::Arc; use alloc::task::Wake; use core::cell::UnsafeCell; use core::future::Future; use core::pin::Pin; use core::ptr::NonNull; - use core::sync::atomic::AtomicBool; - use core::sync::atomic::Ordering; - use core::task::Context; - use core::task::Poll; - use core::task::Waker; - use eonix_hal::traits::trap::RawTrapContext; - use eonix_hal::traits::trap::TrapReturn; - use eonix_hal::traits::trap::TrapType; + use core::sync::atomic::{AtomicBool, Ordering}; + use core::task::{Context, Poll, Waker}; + + use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; use eonix_hal::trap::TrapContext; use eonix_preempt::assert_preempt_enabled; use eonix_runtime::executor::Stack; use eonix_runtime::task::Task; use thread::wait_for_wakeups; + use crate::kernel::interrupt::{default_fault_handler, default_irq_handler}; + use crate::kernel::timer::{should_reschedule, timer_interrupt}; + let stack = KernelStack::new(); fn execute(mut future: Pin<&mut F>, output_ptr: NonNull>) -> ! diff --git a/src/kernel/task/clone.rs b/src/kernel/task/clone.rs index e0d578c1..dd6f538d 100644 --- a/src/kernel/task/clone.rs +++ b/src/kernel/task/clone.rs @@ -1,18 +1,17 @@ -use crate::{ - kernel::{ - syscall::{procops::parse_user_tls, UserMut}, - task::{alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder}, - user::UserPointerMut, - }, - KResult, -}; -use bitflags::bitflags; use core::num::NonZero; -use eonix_hal::processor::UserTLS; + +use bitflags::bitflags; use eonix_runtime::scheduler::RUNTIME; use eonix_sync::AsProof; +use posix_types::ctypes::PtrT; use posix_types::signal::Signal; +use super::{UserTLS, UserTLSDescriptor}; +use crate::kernel::syscall::UserMut; +use crate::kernel::task::{alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder}; +use crate::kernel::user::UserPointerMut; +use crate::KResult; + bitflags! { #[derive(Debug, Default)] pub struct CloneFlags: usize { @@ -46,12 +45,18 @@ bitflags! { #[derive(Debug)] pub struct CloneArgs { pub flags: CloneFlags, - pub sp: Option>, // Stack pointer for the new thread. - pub exit_signal: Option, // Signal to send to the parent on exit. - pub set_tid_ptr: Option>, // Pointer to set child TID in user space. - pub clear_tid_ptr: Option>, // Pointer to clear child TID in user space. - pub parent_tid_ptr: Option>, // Pointer to parent TID in user space. - pub tls: Option, // Pointer to TLS information. + /// Stack pointer for the new thread. + pub sp: Option>, + /// Signal to send to the parent on exit. + pub exit_signal: Option, + /// Pointer to set child TID in user space. + pub set_tid_ptr: Option>, + /// Pointer to clear child TID in user space. + pub clear_tid_ptr: Option>, + /// Pointer to parent TID in user space. + pub parent_tid_ptr: Option>, + /// Pointer to TLS information. + pub tls: Option, } impl CloneArgs { @@ -62,7 +67,7 @@ impl CloneArgs { sp: usize, child_tid_ptr: UserMut, parent_tid_ptr: UserMut, - tls: usize, + tls: PtrT, ) -> KResult { let clone_flags = CloneFlags::from_bits_truncate(flags & !Self::MASK); let exit_signal = flags & Self::MASK; @@ -87,7 +92,8 @@ impl CloneArgs { .then_some(parent_tid_ptr); let tls = if clone_flags.contains(CloneFlags::CLONE_SETTLS) { - Some(parse_user_tls(tls)?) + let tls_desc = UserTLSDescriptor::new(tls)?; + Some(tls_desc.read()?) } else { None }; diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 11348e51..77e8e618 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -1,43 +1,37 @@ -use super::{ - signal::{RaiseResult, SignalList}, - stackful, Process, ProcessList, WaitType, -}; -use crate::{ - kernel::{ - interrupt::default_irq_handler, - syscall::{syscall_handlers, SyscallHandler, User, UserMut}, - task::{clone::CloneArgs, futex::RobustListHead, CloneFlags}, - timer::{should_reschedule, timer_interrupt}, - user::{UserPointer, UserPointerMut}, - vfs::{filearray::FileArray, FsContext}, - }, - prelude::*, -}; -use alloc::{alloc::Allocator, sync::Arc}; +use alloc::alloc::Allocator; +use alloc::sync::Arc; +use core::future::{poll_fn, Future}; +use core::pin::Pin; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll}; + use atomic_unique_refcell::AtomicUniqueRefCell; -use core::{ - future::{poll_fn, Future}, - pin::Pin, - ptr::NonNull, - sync::atomic::{AtomicBool, Ordering}, - task::{Context, Poll}, -}; -use eonix_hal::{ - fpu::FpuState, - processor::{UserTLS, CPU}, - traits::{ - fault::Fault, - fpu::RawFpuState as _, - trap::{RawTrapContext, TrapReturn, TrapType}, - }, - trap::TrapContext, -}; +use eonix_hal::fpu::FpuState; +use eonix_hal::traits::fault::Fault; +use eonix_hal::traits::fpu::RawFpuState as _; +use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; +use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::AsProofMut as _; use pointers::BorrowedArc; use posix_types::signal::Signal; use stalloc::UnsafeStalloc; +use super::signal::{RaiseResult, SignalList}; +use super::user_tls::UserTLS; +use super::{stackful, Process, ProcessList, WaitType}; +use crate::kernel::interrupt::default_irq_handler; +use crate::kernel::syscall::{syscall_handlers, SyscallHandler, User, UserMut}; +use crate::kernel::task::clone::CloneArgs; +use crate::kernel::task::futex::RobustListHead; +use crate::kernel::task::CloneFlags; +use crate::kernel::timer::{should_reschedule, timer_interrupt}; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::vfs::filearray::FileArray; +use crate::kernel::vfs::FsContext; +use crate::prelude::*; + #[eonix_percpu::define_percpu] static CURRENT_THREAD: Option> = None; @@ -275,12 +269,9 @@ impl Thread { self.signal_list.raise(signal) } - /// # Safety - /// This function is unsafe because it accesses the `current_cpu()`, which needs - /// to be called in a preemption disabled context. - pub unsafe fn load_thread_area32(&self) { + pub fn activate_tls(&self) { if let Some(tls) = self.inner.lock().tls.as_ref() { - CPU::local().as_mut().set_tls32(tls); + tls.activate(); } } @@ -442,14 +433,7 @@ impl Thread { CURRENT_THREAD.set(NonNull::new(&raw const *self as *mut _)); - unsafe { - eonix_preempt::disable(); - - // SAFETY: Preemption is disabled. - self.load_thread_area32(); - - eonix_preempt::enable(); - } + self.activate_tls(); let result = future.as_mut().poll(cx); diff --git a/src/kernel/task/user_tls/mod.rs b/src/kernel/task/user_tls/mod.rs new file mode 100644 index 00000000..2583b580 --- /dev/null +++ b/src/kernel/task/user_tls/mod.rs @@ -0,0 +1,34 @@ +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use x86_64::*; + } else { + use eonix_mm::address::VAddr; + use posix_types::ctypes::PtrT; + + use crate::prelude::KResult; + + + #[derive(Debug, Clone)] + pub struct UserTLS(VAddr); + + #[derive(Debug, Clone)] + pub struct UserTLSDescriptor(VAddr); + + impl UserTLS { + pub fn activate(&self) { + self.0; + } + } + + impl UserTLSDescriptor { + pub fn new(tp: PtrT) -> KResult { + Ok(Self(VAddr::from(tp.addr()))) + } + + pub fn read(&self) -> KResult { + Ok(UserTLS(self.0)) + } + } + } +} diff --git a/src/kernel/task/user_tls/x86_64.rs b/src/kernel/task/user_tls/x86_64.rs new file mode 100644 index 00000000..5bb33b97 --- /dev/null +++ b/src/kernel/task/user_tls/x86_64.rs @@ -0,0 +1,83 @@ +use core::fmt; + +use eonix_hal::arch_exported::gdt::{GDTEntry, GDT}; +use eonix_hal::processor::CPU; +use eonix_mm::address::VAddr; +use posix_types::ctypes::PtrT; +use posix_types::x86_64::UserDescriptor; + +use crate::kernel::syscall::{User, UserMut}; +use crate::kernel::user::{CheckedUserPointer, UserPointerMut}; +use crate::prelude::KResult; + +#[derive(Debug, Clone)] +pub struct UserTLS { + desc: GDTEntry, + base: u64, +} + +pub struct UserTLSDescriptor<'a> { + ptr: UserPointerMut<'a, UserDescriptor>, +} + +impl UserTLS { + fn new(base: u32, limit: u32) -> Self { + Self { + desc: GDTEntry::new_tls(base, limit), + base: base as u64, + } + } + + fn new_page_limit(base: u32, limit_in_pages: u32) -> Self { + Self { + desc: GDTEntry::new_tls_page_limit(base, limit_in_pages), + base: base as u64, + } + } + + pub fn activate(&self) { + CPU::local().as_mut().set_tls32(self.desc, self.base); + } +} + +impl UserTLSDescriptor<'_> { + pub fn new(raw_tls: PtrT) -> KResult { + Ok(Self { + ptr: UserPointerMut::new(UserMut::::with_addr(raw_tls.addr()))?, + }) + } + + pub fn read(&self) -> KResult { + let mut desc = self.ptr.read()?; + + let base = VAddr::from(desc.base as usize); + + // Clear the TLS area if it is not present. + if desc.flags.is_read_exec_only() && !desc.flags.is_present() { + if desc.limit != 0 && base != VAddr::NULL { + let len = if desc.flags.is_limit_in_pages() { + (desc.limit as usize) << 12 + } else { + desc.limit as usize + }; + + CheckedUserPointer::new(User::new(base), len)?.zero()?; + } + } + + desc.entry = GDT::TLS32_INDEX as u32; + self.ptr.write(desc)?; + + Ok(if desc.flags.is_limit_in_pages() { + UserTLS::new_page_limit(desc.base, desc.limit) + } else { + UserTLS::new(desc.base, desc.limit) + }) + } +} + +impl fmt::Debug for UserTLSDescriptor<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("UserTLSDescriptor").finish_non_exhaustive() + } +} diff --git a/src/kernel/user/dataflow.rs b/src/kernel/user/dataflow.rs index 02e7d791..5d8ac167 100644 --- a/src/kernel/user/dataflow.rs +++ b/src/kernel/user/dataflow.rs @@ -1,18 +1,15 @@ -use crate::{ - io::{Buffer, FillResult}, - prelude::*, -}; -use crate::{ - io::{IntoStream, Stream}, - kernel::{ - constants::{EFAULT, EINVAL}, - syscall::{User, UserMut}, - }, -}; -use core::{arch::asm, ffi::CStr, marker::PhantomData}; +use core::arch::asm; +use core::ffi::CStr; +use core::marker::PhantomData; + use eonix_mm::address::Addr; use eonix_preempt::assert_preempt_enabled; +use crate::io::{Buffer, FillResult, IntoStream, Stream}; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::syscall::{User, UserMut}; +use crate::prelude::*; + pub struct CheckedUserPointer<'a> { ptr: User, len: usize, From 841bb379b085494e3975e8f132f8caca0447ff0d Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 6 Jan 2026 22:02:22 +0800 Subject: [PATCH 007/153] mem, slab: rework the slab system Signed-off-by: greatbridf --- .vscode/settings.json | 1 + Cargo.lock | 1 - crates/intrusive_list/src/lib.rs | 2 + crates/slab_allocator/Cargo.toml | 2 - crates/slab_allocator/src/lib.rs | 291 +++++++++++++++++++++--- crates/slab_allocator/src/slab_cache.rs | 164 ------------- src/kernel/mem/allocator.rs | 21 +- src/kernel/mem/page_alloc.rs | 10 +- src/kernel/mem/page_alloc/raw_page.rs | 199 ++++++++++------ 9 files changed, 401 insertions(+), 290 deletions(-) delete mode 100644 crates/slab_allocator/src/slab_cache.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index 634d16af..10b4a8b4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { "makefile.configureOnOpen": false, + "editor.formatOnSave": true, } diff --git a/Cargo.lock b/Cargo.lock index 5487f284..3e8a36bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,7 +518,6 @@ version = "0.1.0" dependencies = [ "eonix_mm", "eonix_sync", - "intrusive_list", ] [[package]] diff --git a/crates/intrusive_list/src/lib.rs b/crates/intrusive_list/src/lib.rs index af8c4f1a..440944d0 100644 --- a/crates/intrusive_list/src/lib.rs +++ b/crates/intrusive_list/src/lib.rs @@ -25,6 +25,8 @@ impl List { } pub fn insert(&mut self, node: &mut Link) { + // TODO: `node` above should be of 'static. + self.head.insert(node); self.count += 1; } diff --git a/crates/slab_allocator/Cargo.toml b/crates/slab_allocator/Cargo.toml index 067b6f53..926ac688 100644 --- a/crates/slab_allocator/Cargo.toml +++ b/crates/slab_allocator/Cargo.toml @@ -6,5 +6,3 @@ edition = "2024" [dependencies] eonix_mm = { path = "../eonix_mm" } eonix_sync = { path = "../eonix_sync" } -intrusive_list = { path = "../intrusive_list" } - diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index ce163183..8a684edd 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -1,69 +1,288 @@ #![no_std] -mod slab_cache; +use core::ptr::NonNull; -use core::{cmp::max, ptr::NonNull}; - -use eonix_mm::paging::{PageAlloc, RawPage}; use eonix_sync::Spin; -use intrusive_list::Link; -use slab_cache::SlabCache; -pub trait SlabRawPage: RawPage { - /// Get the container raw page struct of the list link. +#[repr(C)] +pub union SlabSlot { + slab_slot: Option>, + data: u8, +} + +pub trait SlabPageList: Sized { + type Page: SlabPage; + + fn new() -> Self; + fn is_empty(&self) -> bool; + + fn peek_head(&mut self) -> Option<&mut Self::Page>; + + fn pop_head(&mut self) -> Option<&'static mut Self::Page>; + fn push_tail(&mut self, page: &'static mut Self::Page); + fn remove(&mut self, page: &mut Self::Page); +} + +pub trait SlabPage: Sized + 'static { + fn get_data_ptr(&self) -> NonNull<[u8]>; + + fn get_free_slot(&self) -> Option>; + fn set_free_slot(&mut self, next: Option>); + + fn get_alloc_count(&self) -> usize; + + /// Increase the allocation count by 1 and return the increased value. + fn inc_alloc_count(&mut self) -> usize; + + /// Decrease the allocation count by 1 and return the decreased value. + fn dec_alloc_count(&mut self) -> usize; + + /// Get the [`SlabPage`] that `ptr` is allocated from. /// /// # Safety - /// The caller MUST ensure that the link points to a `RawPage`. - unsafe fn from_link(link: &mut Link) -> Self; + /// The caller MUST ensure that no others could be calling this function and + /// getting the [`SlabPage`] at the same time. + unsafe fn from_allocated(ptr: NonNull) -> &'static mut Self; +} + +pub(crate) trait SlabPageExt { + fn alloc_slot(&mut self) -> Option>; - /// Get the list link of the raw page. - /// /// # Safety - /// The caller MUST ensure that at any time, only one mutable reference - /// to the link exists. - unsafe fn get_link(&self) -> &mut Link; + /// The caller MUST ensure that `slot_data_ptr` points to some position + /// previously allocated by [`SlabPageExt::alloc_slot`]. + unsafe fn free_slot(&mut self, slot_data_ptr: NonNull); + + fn is_empty(&self) -> bool; + fn is_full(&self) -> bool; +} + +impl SlabPageExt for T +where + T: SlabPage, +{ + fn alloc_slot(&mut self) -> Option> { + let mut free_slot = self.get_free_slot()?; + + unsafe { + let free_slot = free_slot.as_mut(); + + let next_slot = free_slot.slab_slot; + // ===== `free_slot` is now safe to be overwritten - fn slab_init(&self, first_free: Option>); + self.set_free_slot(next_slot); + self.inc_alloc_count(); - // which slab page the ptr belong - fn in_which(ptr: *mut u8) -> Self; + Some(NonNull::new_unchecked(&mut free_slot.data)) + } + } + + unsafe fn free_slot(&mut self, slot_data_ptr: NonNull) { + unsafe { + let mut free_slot: NonNull = slot_data_ptr.cast(); + free_slot.as_mut().slab_slot = self.get_free_slot(); - fn real_page_ptr(&self) -> *mut u8; + self.set_free_slot(Some(free_slot)); + self.dec_alloc_count(); + } + } - fn allocated_count(&self) -> &mut u32; + fn is_empty(&self) -> bool { + self.get_alloc_count() == 0 + } - fn next_free(&self) -> &mut Option>; + fn is_full(&self) -> bool { + self.get_free_slot().is_none() + } } -pub struct SlabAllocator { - slabs: [Spin>; SLAB_CACHE_COUNT], - alloc: A, +pub trait SlabPageAlloc { + type Page: SlabPage; + type PageList: SlabPageList; + + /// Allocate a page suitable for slab system use. The page MUST come with + /// its allocation count 0 and next free slot None. + /// + /// # Safety + /// The page returned MUST be properly initialized before its usage. + unsafe fn alloc_uninit(&self) -> &'static mut Self::Page; } -unsafe impl Send for SlabAllocator {} -unsafe impl Sync for SlabAllocator {} +pub(crate) struct SlabList +where + T: SlabPageList, +{ + empty_list: T, + partial_list: T, + full_list: T, + object_size: usize, +} + +pub struct SlabAlloc +where + P: SlabPageAlloc, +{ + slabs: [Spin>; COUNT], + alloc: P, +} + +unsafe impl Send for SlabAlloc where P: SlabPageAlloc {} +unsafe impl Sync for SlabAlloc where P: SlabPageAlloc {} -impl SlabAllocator +impl SlabAlloc where - Raw: SlabRawPage, - Allocator: PageAlloc, + L: SlabPageAlloc, { - pub fn new_in(alloc: Allocator) -> Self { + pub fn new_in(alloc: L) -> Self { Self { - slabs: core::array::from_fn(|i| Spin::new(SlabCache::new_in(1 << (i + 3)))), + slabs: core::array::from_fn(|i| Spin::new(SlabList::new(1 << (i + 3)))), alloc, } } - pub fn alloc(&self, mut size: usize) -> *mut u8 { - size = max(8, size); + pub fn alloc(&self, mut size: usize) -> NonNull { + size = size.max(8); let idx = size.next_power_of_two().trailing_zeros() - 3; self.slabs[idx as usize].lock().alloc(&self.alloc) } - pub fn dealloc(&self, ptr: *mut u8, mut size: usize) { - size = max(8, size); + pub unsafe fn dealloc(&self, ptr: NonNull, mut size: usize) { + size = size.max(8); let idx = size.next_power_of_two().trailing_zeros() - 3; - self.slabs[idx as usize].lock().dealloc(ptr, &self.alloc); + + unsafe { + // SAFETY: + self.slabs[idx as usize].lock().dealloc(ptr, &self.alloc); + } } } + +impl SlabList +where + T: SlabPageList, +{ + fn new(object_size: usize) -> Self { + Self { + empty_list: T::new(), + partial_list: T::new(), + full_list: T::new(), + object_size, + } + } + + fn alloc_from_partial(&mut self) -> NonNull { + let head = self.partial_list.peek_head().unwrap(); + let slot = head.alloc_slot().unwrap(); + + if head.is_full() { + let head = self.partial_list.pop_head().unwrap(); + self.full_list.push_tail(head); + } + + slot + } + + fn alloc_from_empty(&mut self) -> NonNull { + let head = self.empty_list.pop_head().unwrap(); + let slot = head.alloc_slot().unwrap(); + + if head.is_full() { + self.full_list.push_tail(head); + } else { + self.partial_list.push_tail(head); + } + + slot + } + + fn charge(&mut self, alloc: &impl SlabPageAlloc) { + unsafe { + let slab = alloc.alloc_uninit(); + let free_slot = make_slab_page(slab.get_data_ptr(), self.object_size); + + slab.set_free_slot(Some(free_slot)); + + self.empty_list.push_tail(slab); + } + } + + fn alloc(&mut self, alloc: &impl SlabPageAlloc) -> NonNull { + if !self.partial_list.is_empty() { + return self.alloc_from_partial(); + } + + if self.empty_list.is_empty() { + self.charge(alloc); + } + + self.alloc_from_empty() + } + + unsafe fn dealloc(&mut self, ptr: NonNull, _alloc: &impl SlabPageAlloc) { + let slab_page = unsafe { + // SAFETY: + ::from_allocated(ptr) + }; + + let (was_full, is_empty); + + was_full = slab_page.is_full(); + + unsafe { + // SAFETY: + slab_page.free_slot(ptr); + } + + is_empty = slab_page.is_empty(); + + match (was_full, is_empty) { + (false, false) => {} + (false, true) => { + self.partial_list.remove(slab_page); + self.empty_list.push_tail(slab_page); + } + (true, false) => { + self.full_list.remove(slab_page); + self.partial_list.push_tail(slab_page); + } + (true, true) => { + self.full_list.remove(slab_page); + self.empty_list.push_tail(slab_page); + } + } + + // TODO: Check whether we should place some pages back with `alloc` if + // the global free page count is below the watermark. + } +} + +pub fn make_slab_page(page_ptr: NonNull<[u8]>, slot_size: usize) -> NonNull { + assert!( + slot_size >= core::mem::size_of::(), + "The minimum slot size is of a pointer's width" + ); + + let page_size = page_ptr.len(); + let slot_count = page_size / slot_size; + let page_start: NonNull = page_ptr.cast(); + + // Quick checks + assert!( + page_size % slot_size == 0, + "The page's size should be a multiple of the slot size" + ); + + let mut prev_free_slot = None; + for i in (0..slot_count).rev() { + let offset = i * slot_size; + + unsafe { + let mut slot_ptr: NonNull = page_start.add(offset).cast(); + + slot_ptr.as_mut().slab_slot = prev_free_slot; + prev_free_slot = Some(slot_ptr); + } + } + + prev_free_slot.expect("There should be at least one slot.") +} diff --git a/crates/slab_allocator/src/slab_cache.rs b/crates/slab_allocator/src/slab_cache.rs deleted file mode 100644 index 98e27fc8..00000000 --- a/crates/slab_allocator/src/slab_cache.rs +++ /dev/null @@ -1,164 +0,0 @@ -use super::SlabRawPage; -use core::{marker::PhantomData, ptr::NonNull}; -use eonix_mm::paging::{PageAlloc, PAGE_SIZE}; -use intrusive_list::List; - -pub(crate) struct SlabCache { - empty_list: List, - partial_list: List, - full_list: List, - object_size: u32, - _phantom: PhantomData<(T, A)>, -} - -trait SlabRawPageExt { - fn alloc_slot(&self) -> Option>; - fn dealloc_slot(&self, slot_ptr: *mut u8); - fn is_full(&self) -> bool; - fn is_empty(&self) -> bool; - fn slab_page_init(&self, object_size: u32) -> Option>; -} - -impl SlabRawPageExt for T -where - T: SlabRawPage, -{ - fn alloc_slot(&self) -> Option> { - let ptr = self.next_free().clone(); - - let next_free = match ptr { - Some(ptr) => unsafe { ptr.read() as *mut usize }, - None => unreachable!(), - }; - *self.allocated_count() += 1; - *self.next_free() = NonNull::new(next_free); - return ptr; - } - - fn dealloc_slot(&self, slot_ptr: *mut u8) { - let slot_ptr = slot_ptr as *mut usize; - - if let Some(last_free) = self.next_free().clone() { - unsafe { *slot_ptr = last_free.as_ptr() as usize } - } else { - unsafe { *slot_ptr = 0 } - } - - *self.allocated_count() -= 1; - *self.next_free() = NonNull::new(slot_ptr); - } - - fn slab_page_init(&self, object_size: u32) -> Option> { - assert!(object_size >= core::mem::size_of::() as u32); - - let first_free = self.real_page_ptr() as *mut usize; - - let mut slot_ptr = first_free; - let mut slot_count = PAGE_SIZE / object_size as usize; - - // SAFETY: carefully ptr operate - unsafe { - loop { - if slot_count == 1 { - *slot_ptr = 0; - break; - } - - let next_ptr = slot_ptr.byte_add(object_size as usize); - *slot_ptr = next_ptr as usize; - slot_ptr = next_ptr; - slot_count -= 1; - } - } - - NonNull::new(first_free) - } - - fn is_empty(&self) -> bool { - self.allocated_count().clone() == 0 - } - - fn is_full(&self) -> bool { - self.next_free().is_none() - } -} - -impl SlabCache -where - Raw: SlabRawPage, - Allocator: PageAlloc, -{ - pub(crate) const fn new_in(object_size: u32) -> Self { - // avoid unnecessary branch in alloc and dealloc - assert!(object_size <= PAGE_SIZE as u32 / 2); - - Self { - empty_list: List::new(), - partial_list: List::new(), - full_list: List::new(), - object_size: object_size, - _phantom: PhantomData, - } - } - - pub(crate) fn alloc(&mut self, alloc: &Allocator) -> *mut u8 { - if !self.partial_list.is_empty() { - let page_ptr = unsafe { - Raw::from_link( - self.partial_list - .head() - .expect("partial pages should not be empty"), - ) - }; - - let ptr = page_ptr.alloc_slot().expect("should get slot"); - - if page_ptr.is_full() { - self.partial_list.remove(unsafe { page_ptr.get_link() }); - self.full_list.insert(unsafe { page_ptr.get_link() }); - } - return ptr.as_ptr() as *mut u8; - } - - if !self.empty_list.is_empty() { - let page_ptr = unsafe { - Raw::from_link( - self.empty_list - .head() - .expect("empty pages should not be empty"), - ) - }; - - let ptr = page_ptr.alloc_slot().expect("should get slot"); - self.empty_list.remove(unsafe { page_ptr.get_link() }); - self.partial_list.insert(unsafe { page_ptr.get_link() }); - return ptr.as_ptr() as *mut u8; - } - - let new_page_ptr = alloc.alloc().expect("slab_cache get page fail!"); - let first_free = new_page_ptr.slab_page_init(self.object_size); - new_page_ptr.slab_init(first_free); - let ptr = new_page_ptr.alloc_slot().expect("should get slot"); - self.partial_list.insert(unsafe { new_page_ptr.get_link() }); - ptr.as_ptr() as *mut u8 - } - - pub(crate) fn dealloc(&mut self, ptr: *mut u8, _alloc: &Allocator) { - let page_ptr = Raw::in_which(ptr); - - if page_ptr.is_full() { - self.full_list.remove(unsafe { page_ptr.get_link() }); - self.partial_list.insert(unsafe { page_ptr.get_link() }); - } - - page_ptr.dealloc_slot(ptr); - - if page_ptr.is_empty() { - self.partial_list.remove(unsafe { page_ptr.get_link() }); - self.empty_list.insert(unsafe { page_ptr.get_link() }); - } - - // TODO: Check whether we should place some pages back with `alloc` if the global - // free page count is below the watermark. - } -} diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 9e5df69b..a3676ce0 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -5,13 +5,12 @@ use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::PhysAccess; use eonix_mm::paging::{PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; -use slab_allocator::SlabAllocator; +use slab_allocator::SlabAlloc; -use super::page_alloc::RawPagePtr; use super::{GlobalPageAlloc, Page, PageExt}; -static SLAB_ALLOCATOR: LazyLock> = - LazyLock::new(|| SlabAllocator::new_in(GlobalPageAlloc)); +static SLAB_ALLOCATOR: LazyLock> = + LazyLock::new(|| SlabAlloc::new_in(GlobalPageAlloc)); struct Allocator; @@ -28,23 +27,23 @@ unsafe impl GlobalAlloc for Allocator { let ptr = page.get_ptr(); page.into_raw(); - ptr.as_ptr() + ptr }; - if result.is_null() { - core::ptr::null_mut() - } else { - result as *mut u8 - } + result.as_ptr() } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { let size = layout.size().next_power_of_two(); + let ptr = unsafe { + // SAFETY: The memory we've allocated MUST be non-null. + NonNull::new_unchecked(ptr) + }; if size <= 2048 { SLAB_ALLOCATOR.dealloc(ptr, size) } else { - let paddr = ArchPhysAccess::from_ptr(NonNull::new_unchecked(ptr)); + let paddr = ArchPhysAccess::from_ptr(ptr); let pfn = PFN::from(paddr); Page::from_raw(pfn); }; diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index fcbe9bb3..1c018f37 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -1,15 +1,13 @@ mod raw_page; -use buddy_allocator::{BuddyAllocator, BuddyRawPage as _}; use core::sync::atomic::Ordering; -use eonix_mm::{ - address::{AddrOps as _, PRange}, - paging::{GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PFN}, -}; + +use buddy_allocator::{BuddyAllocator, BuddyRawPage as _}; +use eonix_mm::address::{AddrOps as _, PRange}; +use eonix_mm::paging::{GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PFN}; use eonix_sync::{NoContext, Spin}; use intrusive_list::List; use raw_page::PageFlags; - pub use raw_page::{RawPage, RawPagePtr}; const COSTLY_ORDER: u32 = 3; diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 08536693..d793ccd7 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,65 +1,46 @@ -use crate::kernel::mem::page_cache::PageCacheRawPage; -use crate::kernel::mem::PhysAccess; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + use buddy_allocator::BuddyRawPage; -use core::{ - ptr::NonNull, - sync::atomic::{AtomicU32, AtomicUsize, Ordering}, -}; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{PAddr, PhysAccess as _}, - paging::{RawPage as RawPageTrait, PFN}, -}; -use intrusive_list::{container_of, Link}; -use slab_allocator::SlabRawPage; +use eonix_mm::address::{PAddr, PhysAccess as _}; +use eonix_mm::paging::{PageAlloc, RawPage as RawPageTrait, PFN}; +use intrusive_list::{container_of, Link, List}; +use slab_allocator::{SlabPage, SlabPageAlloc, SlabPageList, SlabSlot}; + +use super::GlobalPageAlloc; +use crate::kernel::mem::page_cache::PageCacheRawPage; +use crate::kernel::mem::PhysAccess; const PAGE_ARRAY: NonNull = unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; pub struct PageFlags(AtomicU32); -struct SlabPageInner { - allocated_count: u32, - free_next: Option>, +#[derive(Clone, Copy)] +struct SlabPageData { + allocated_count: usize, + free_next: Option>, } -impl SlabPageInner { - fn new(free_next: Option>) -> Self { +impl SlabPageData { + const fn new() -> Self { Self { allocated_count: 0, - free_next, + free_next: None, } } } -struct PageCacheInner { +#[derive(Clone, Copy)] +struct PageCacheData { valid_size: usize, } -pub struct BuddyPageInner {} - -enum PageType { - Buddy(BuddyPageInner), - Slab(SlabPageInner), - PageCache(PageCacheInner), -} - -impl PageType { - fn slab_data(&mut self) -> &mut SlabPageInner { - if let PageType::Slab(slab_data) = self { - return slab_data; - } else { - unreachable!() - } - } - - fn page_cache_data(&mut self) -> &mut PageCacheInner { - if let PageType::PageCache(cache_data) = self { - return cache_data; - } else { - unreachable!() - } - } +#[repr(C)] +union PageData { + slab: SlabPageData, + page_cache: PageCacheData, } pub struct RawPage { @@ -73,7 +54,7 @@ pub struct RawPage { flags: PageFlags, refcount: AtomicUsize, - shared_data: PageType, + shared_data: PageData, } #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -109,6 +90,13 @@ impl PageFlags { } impl RawPagePtr { + pub const fn from_ref(raw_page_ref: &RawPage) -> Self { + Self::new(unsafe { + // SAFETY: Rust references always points to non-null addresses. + NonNull::new_unchecked(&raw const *raw_page_ref as *mut _) + }) + } + pub const fn new(ptr: NonNull) -> Self { Self(ptr) } @@ -215,50 +203,68 @@ impl BuddyRawPage for RawPagePtr { } } -impl SlabRawPage for RawPagePtr { - unsafe fn from_link(link: &mut Link) -> Self { - let raw_page_ptr = container_of!(link, RawPage, link); - Self(raw_page_ptr) - } +impl SlabPage for RawPage { + fn get_data_ptr(&self) -> NonNull<[u8]> { + let raw_page_ptr = RawPagePtr::from_ref(self); + let paddr_start = PAddr::from(PFN::from(raw_page_ptr)); + let page_data_ptr = unsafe { paddr_start.as_ptr() }; - unsafe fn get_link(&self) -> &mut Link { - &mut self.as_mut().link + NonNull::slice_from_raw_parts(page_data_ptr, 1 << (self.order + 12)) } - fn in_which(ptr: *mut u8) -> RawPagePtr { + fn get_free_slot(&self) -> Option> { unsafe { - // SAFETY: The pointer is allocated from the slab allocator, - // which can't be null. - let ptr = NonNull::new_unchecked(ptr); + // SAFETY: TODO + self.shared_data.slab.free_next + } + } - // SAFETY: The pointer is valid. - let paddr = ArchPhysAccess::from_ptr(ptr); - let pfn = PFN::from(paddr); + fn set_free_slot(&mut self, next: Option>) { + self.shared_data.slab.free_next = next; + } - RawPagePtr::from(pfn) + fn get_alloc_count(&self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count } } - fn allocated_count(&self) -> &mut u32 { - &mut self.as_mut().shared_data.slab_data().allocated_count - } + fn inc_alloc_count(&mut self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count += 1; - fn next_free(&self) -> &mut Option> { - &mut self.as_mut().shared_data.slab_data().free_next + self.shared_data.slab.allocated_count + } } - fn real_page_ptr(&self) -> *mut u8 { - self.real_ptr().as_ptr() + fn dec_alloc_count(&mut self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count -= 1; + + self.shared_data.slab.allocated_count + } } - fn slab_init(&self, first_free: Option>) { - self.as_mut().shared_data = PageType::Slab(SlabPageInner::new(first_free)); + unsafe fn from_allocated(ptr: NonNull) -> &'static mut Self { + unsafe { + // SAFETY: The caller ensures that `ptr` is valid. + let paddr = ArchPhysAccess::from_ptr(ptr); + let pfn = PFN::from(paddr); + + RawPagePtr::from(pfn).as_mut() + } } } impl PageCacheRawPage for RawPagePtr { fn valid_size(&self) -> &mut usize { - &mut self.as_mut().shared_data.page_cache_data().valid_size + unsafe { + // SAFETY: The caller ensures that the page is in some page cache. + &mut self.as_mut().shared_data.page_cache.valid_size + } } fn is_dirty(&self) -> bool { @@ -274,6 +280,59 @@ impl PageCacheRawPage for RawPagePtr { } fn cache_init(&self) { - self.as_mut().shared_data = PageType::PageCache(PageCacheInner { valid_size: 0 }); + self.as_mut().shared_data.page_cache = PageCacheData { valid_size: 0 }; + } +} + +pub struct RawSlabPageList(List); + +impl SlabPageList for RawSlabPageList { + type Page = RawPage; + + fn new() -> Self { + Self(List::new()) + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn peek_head(&mut self) -> Option<&mut Self::Page> { + unsafe { + let link = self.0.head()?; + let mut raw_page_ptr = container_of!(link, RawPage, link); + + Some(raw_page_ptr.as_mut()) + } + } + + fn pop_head(&mut self) -> Option<&'static mut Self::Page> { + unsafe { + let link = self.0.pop()?; + let mut raw_page_ptr = container_of!(link, RawPage, link); + + Some(raw_page_ptr.as_mut()) + } + } + + fn push_tail(&mut self, page: &'static mut Self::Page) { + self.0.insert(&mut page.link); + } + + fn remove(&mut self, page: &mut Self::Page) { + self.0.remove(&mut page.link) + } +} + +impl SlabPageAlloc for GlobalPageAlloc { + type Page = RawPage; + type PageList = RawSlabPageList; + + unsafe fn alloc_uninit(&self) -> &'static mut RawPage { + let raw_page = self.alloc().expect("Out of memory").as_mut(); + raw_page.flags.set(PageFlags::SLAB); + raw_page.shared_data.slab = SlabPageData::new(); + + raw_page } } From f17236f46e26964779a64ec90098e2aa9de9fd38 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 7 Jan 2026 02:16:47 +0800 Subject: [PATCH 008/153] mem, buddy: rework the buddy system Introduce `Zone`s: a Zone is a region of physical memory that all share the same NUMA node. The Zone will hold all RawPage structs. Buddy allocator will now store a reference to the zone that hold all its pages. Thus, we make the buddy allocator independent of underlying physical page frame management framework. Remove unnecessary page flags and structs. Signed-off-by: greatbridf --- Cargo.lock | 1 - crates/buddy_allocator/Cargo.toml | 1 - crates/buddy_allocator/src/free_area.rs | 59 ------ crates/buddy_allocator/src/lib.rs | 265 +++++++++++++++++++----- crates/buddy_allocator/src/zone.rs | 146 ------------- crates/eonix_mm/src/paging.rs | 4 + crates/eonix_mm/src/paging/list.rs | 19 ++ crates/eonix_mm/src/paging/raw_page.rs | 9 +- crates/eonix_mm/src/paging/zone.rs | 20 ++ crates/slab_allocator/src/lib.rs | 35 ++-- src/kernel/mem/page_alloc.rs | 116 +++++------ src/kernel/mem/page_alloc/raw_page.rs | 87 ++++---- src/kernel/mem/page_alloc/zones.rs | 25 +++ src/kernel_init.rs | 24 +-- 14 files changed, 407 insertions(+), 404 deletions(-) delete mode 100644 crates/buddy_allocator/src/free_area.rs delete mode 100644 crates/buddy_allocator/src/zone.rs create mode 100644 crates/eonix_mm/src/paging/list.rs create mode 100644 crates/eonix_mm/src/paging/zone.rs create mode 100644 src/kernel/mem/page_alloc/zones.rs diff --git a/Cargo.lock b/Cargo.lock index 3e8a36bd..896ec493 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,7 +70,6 @@ name = "buddy_allocator" version = "0.1.0" dependencies = [ "eonix_mm", - "intrusive_list", ] [[package]] diff --git a/crates/buddy_allocator/Cargo.toml b/crates/buddy_allocator/Cargo.toml index 51f02295..bdb0a28c 100644 --- a/crates/buddy_allocator/Cargo.toml +++ b/crates/buddy_allocator/Cargo.toml @@ -5,4 +5,3 @@ edition = "2024" [dependencies] eonix_mm = { path = "../eonix_mm" } -intrusive_list = { path = "../intrusive_list" } diff --git a/crates/buddy_allocator/src/free_area.rs b/crates/buddy_allocator/src/free_area.rs deleted file mode 100644 index 837f733f..00000000 --- a/crates/buddy_allocator/src/free_area.rs +++ /dev/null @@ -1,59 +0,0 @@ -use crate::BuddyRawPage; -use core::marker::{PhantomData, Send, Sync}; -use intrusive_list::Link; - -pub struct FreeArea { - free_list: Link, - count: usize, - _phantom: PhantomData, -} - -unsafe impl Send for FreeArea {} -unsafe impl Sync for FreeArea {} - -impl FreeArea -where - Raw: BuddyRawPage, -{ - pub const fn new() -> Self { - Self { - free_list: Link::new(), - count: 0, - _phantom: PhantomData, - } - } - - pub fn get_free_pages(&mut self) -> Option { - self.free_list.next_mut().map(|pages_link| { - assert_ne!(self.count, 0); - - let pages_ptr = unsafe { - // SAFETY: Items in `self.free_list` are guaranteed to be of type `Raw`. - Raw::from_link(pages_link) - }; - - self.count -= 1; - pages_link.remove(); - - pages_ptr - }) - } - - pub fn add_pages(&mut self, pages_ptr: Raw) { - self.count += 1; - pages_ptr.set_free(); - - unsafe { - self.free_list.insert(pages_ptr.get_link()); - } - } - - pub fn del_pages(&mut self, pages_ptr: Raw) { - assert!(self.count >= 1 && pages_ptr.is_free()); - self.count -= 1; - pages_ptr.clear_free(); - unsafe { - pages_ptr.get_link().remove(); - } - } -} diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index f8c8eeda..abe1ef7b 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -1,87 +1,250 @@ #![no_std] -mod free_area; -mod zone; +use core::hint::unreachable_unchecked; -use core::sync::atomic::Ordering; -use eonix_mm::{ - address::PAddr, - paging::{RawPage, PFN}, -}; -use intrusive_list::Link; -use zone::Zone; +use eonix_mm::address::{AddrOps as _, PAddr, PRange}; +use eonix_mm::paging::{PageList, PageListSized, Zone, PFN}; const MAX_ORDER: u32 = 10; -const ZONE_AREAS: usize = const { MAX_ORDER as usize + 1 }; +const AREAS: usize = const { MAX_ORDER as usize + 1 }; -pub trait BuddyRawPage: RawPage { - /// Get the container raw page struct of the list link. - /// - /// # Safety - /// The caller MUST ensure that the link points to a `RawPage`. - unsafe fn from_link(link: &mut Link) -> Self; - - /// Get the list link of the raw page. - /// - /// # Safety - /// The caller MUST ensure that at any time, only one mutable reference - /// to the link exists. - unsafe fn get_link(&self) -> &mut Link; - - fn set_order(&self, order: u32); +pub trait BuddyPage: Sized + 'static { + fn pfn(&self) -> PFN; + fn get_order(&self) -> u32; fn is_buddy(&self) -> bool; - fn is_free(&self) -> bool; - fn set_buddy(&self); - fn set_free(&self); + fn set_order(&mut self, order: u32); + fn set_buddy(&mut self, value: bool); +} - fn clear_buddy(&self); - fn clear_free(&self); +struct FreeArea +where + L: PageList, +{ + free_list: L, + count: usize, } -pub struct BuddyAllocator +unsafe impl Send for FreeArea where L: PageList {} +unsafe impl Sync for FreeArea where L: PageList {} + +pub struct BuddyAllocator where - T: BuddyRawPage, + Z: Zone + 'static, + L: PageList, { - zone: Zone, + zone: &'static Z, + free_areas: [FreeArea; AREAS], } -impl BuddyAllocator +impl BuddyAllocator where - T: BuddyRawPage, + Z: Zone + 'static, + Z::Page: BuddyPage, + L: PageListSized, { - pub const fn new() -> Self { - Self { zone: Zone::new() } + pub const fn new(zone: &'static Z) -> Self { + Self { + zone, + free_areas: [const { FreeArea::new() }; AREAS], + } } +} +impl BuddyAllocator +where + Z: Zone, + L: PageList, + P: BuddyPage + 'static, +{ pub fn create_pages(&mut self, start: PAddr, end: PAddr) { - self.zone.create_pages(start, end); + assert!( + self.zone + .contains_prange(PRange::new(start.ceil(), end.floor())), + "The given address range is not within the zone." + ); + + let mut pfn = PFN::from(start.ceil()); + let end_pfn = PFN::from(end.floor()); + + while pfn < end_pfn { + let mut order = usize::from(pfn).trailing_zeros().min(MAX_ORDER); + let new_end_pfn = loop { + let new_end = pfn + (1 << order); + + if new_end <= end_pfn { + break new_end; + } + + order -= 1; + }; + + unsafe { + // SAFETY: We've checked that the range is within the zone above. + self.add_page_unchecked(pfn, order) + }; + + pfn = new_end_pfn; + } + } + + fn add_page(&mut self, pfn: PFN, order: u32) { + let prange = PRange::from(PAddr::from(pfn)).grow(1 << (order + 12)); + assert!( + self.zone.contains_prange(prange), + "The given page is not within the zone." + ); + + unsafe { + // SAFETY: Checks above. + self.add_page_unchecked(pfn, order); + } + } + + unsafe fn add_page_unchecked(&mut self, pfn: PFN, order: u32) { + let Some(page) = self.zone.get_page(pfn) else { + unsafe { unreachable_unchecked() } + }; + + unsafe { + // SAFETY: The caller ensures that the page is unused. + let page_mut = &mut *page.get(); + self.free_areas[order as usize].add_page(page_mut, order); + } + } + + fn break_page(&mut self, page: &mut P, order: u32, target_order: u32) { + let pfn = page.pfn(); + + for order in (target_order..order).rev() { + let buddy_pfn = pfn + (1 << order); + + unsafe { + // SAFETY: We got the page from `self.free_areas`. Checks are + // done when we've put the page into the buddy system. + self.add_page_unchecked(buddy_pfn, order); + } + } + + page.set_order(target_order); + } + + pub fn alloc_order(&mut self, order: u32) -> Option<&'static mut Z::Page> { + for current_order in order..AREAS as u32 { + let Some(page) = self.free_areas[current_order as usize].get_free_page() else { + continue; + }; + + if current_order > order { + self.break_page(page, current_order, order); + } + + return Some(page); + } + + None + } + + pub unsafe fn dealloc(&mut self, page: &'static mut Z::Page) { + let mut pfn = page.pfn(); + let mut order = page.get_order(); + + assert!( + !page.is_buddy(), + "Trying to free a page that is already in the buddy system: {pfn:?}", + ); + + while order < MAX_ORDER { + let buddy_pfn = pfn.buddy_pfn(order); + let Some(buddy_page) = self.try_get_buddy(buddy_pfn, order) else { + break; + }; + + self.free_areas[order as usize].remove_page(buddy_page); + pfn = pfn.combined_pfn(buddy_pfn); + order += 1; + } + + self.add_page(pfn, order); } - pub fn alloc_order(&mut self, order: u32) -> Option { - let pages_ptr = self.zone.get_free_pages(order); + /// This function checks whether the given page is within our [`Zone`] and + /// is a free buddy page with the specified order. + /// + /// We can assure exclusive access to a buddy page of [`order`] if + /// - the buddy is within the same [`Zone`] as us. + /// - the buddy is a free buddy (in some [`FreeArea`]) + /// - the buddy has order [`order`] + fn try_get_buddy<'a>(&mut self, buddy_pfn: PFN, order: u32) -> Option<&'a mut P> { + let buddy_page = self.zone.get_page(buddy_pfn)?; + + unsafe { + // SAFETY: We just test whether the page is a buddy. + let buddy_page_ref = &*buddy_page.get(); - if let Some(pages_ptr) = pages_ptr { - // SAFETY: Memory order here can be Relaxed is for the same reason as that - // in the copy constructor of `std::shared_ptr`. - pages_ptr.refcount().fetch_add(1, Ordering::Relaxed); - pages_ptr.clear_free(); + if !buddy_page_ref.is_buddy() { + return None; + } + + // Sad... + if buddy_page_ref.get_order() != order { + return None; + } + + // SAFETY: We have the mutable reference to the buddy allocator. + // So all the pages within are exclusively accessible to us. + Some(&mut *buddy_page.get()) + } + } +} + +impl FreeArea +where + L: PageListSized, +{ + const fn new() -> Self { + Self { + free_list: L::NEW, + count: 0, } + } +} + +impl FreeArea +where + L: PageList, + L::Page: BuddyPage + 'static, +{ + pub fn get_free_page(&mut self) -> Option<&'static mut L::Page> { + self.free_list.pop_head().map(|page| { + assert_ne!(self.count, 0, "Oops"); + + page.set_buddy(false); + self.count -= 1; - pages_ptr + page + }) } - pub unsafe fn dealloc(&mut self, page_ptr: T) { - self.zone.free_pages(page_ptr); + pub fn add_page(&mut self, page: &'static mut L::Page, order: u32) { + page.set_order(order); + page.set_buddy(true); + + self.count += 1; + self.free_list.push_tail(page); } - pub fn has_management_over(page_ptr: T) -> bool { - !page_ptr.is_free() && page_ptr.is_buddy() + pub fn remove_page(&mut self, page: &mut L::Page) { + assert_ne!(self.count, 0, "Oops"); + page.set_buddy(false); + + self.count -= 1; + self.free_list.remove(page); } } -pub(self) trait BuddyPFNOps { +trait BuddyPFNOps { fn buddy_pfn(self, order: u32) -> PFN; fn combined_pfn(self, buddy_pfn: PFN) -> PFN; } diff --git a/crates/buddy_allocator/src/zone.rs b/crates/buddy_allocator/src/zone.rs deleted file mode 100644 index 2c850ef9..00000000 --- a/crates/buddy_allocator/src/zone.rs +++ /dev/null @@ -1,146 +0,0 @@ -use super::free_area::FreeArea; -use crate::{BuddyPFNOps as _, BuddyRawPage}; -use core::sync::atomic::Ordering; -use eonix_mm::{ - address::{AddrOps as _, PAddr}, - paging::PFN, -}; - -pub(super) struct Zone { - free_areas: [FreeArea; AREAS], -} - -impl Zone -where - Raw: BuddyRawPage, -{ - pub const fn new() -> Self { - Self { - free_areas: [const { FreeArea::new() }; AREAS], - } - } - - pub fn get_free_pages(&mut self, order: u32) -> Option { - for current_order in order..AREAS as u32 { - let pages_ptr = self.free_areas[current_order as usize].get_free_pages(); - let Some(pages_ptr) = pages_ptr else { continue }; - - pages_ptr.set_order(order); - - if current_order > order { - self.expand(pages_ptr, current_order, order); - } - - assert!( - pages_ptr.is_present(), - "Page {:?} is not present", - pages_ptr.into(), - ); - - assert!( - pages_ptr.is_free(), - "Page {:?} is not free", - pages_ptr.into(), - ); - - return Some(pages_ptr); - } - None - } - - fn expand(&mut self, pages_ptr: Raw, order: u32, target_order: u32) { - let mut offset = 1 << order; - let pages_pfn = Into::::into(pages_ptr); - - for order in (target_order..order).rev() { - offset >>= 1; - - let split_pages_ptr = Raw::from(pages_pfn + offset); - split_pages_ptr.set_order(order); - split_pages_ptr.set_buddy(); - self.free_areas[order as usize].add_pages(split_pages_ptr); - } - } - - pub fn free_pages(&mut self, mut pages_ptr: Raw) { - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - - let mut pfn = Into::::into(pages_ptr); - let mut current_order = pages_ptr.order(); - - assert!( - pages_ptr.is_present(), - "Freeing a page that is not present: {:?}", - pages_ptr.into(), - ); - - assert!( - !pages_ptr.is_free(), - "Freeing a page that is free: {:?}", - pages_ptr.into(), - ); - - while current_order < (AREAS - 1) as u32 { - let buddy_pfn = pfn.buddy_pfn(current_order); - let buddy_pages_ptr = Raw::from(buddy_pfn); - - if !self.buddy_check(buddy_pages_ptr, current_order) { - break; - } - - pages_ptr.clear_buddy(); - buddy_pages_ptr.clear_buddy(); - self.free_areas[current_order as usize].del_pages(buddy_pages_ptr); - - pages_ptr = Raw::from(pfn.combined_pfn(buddy_pfn)); - pfn = pfn.combined_pfn(buddy_pfn); - - pages_ptr.set_buddy(); - current_order += 1; - } - - pages_ptr.set_order(current_order); - self.free_areas[current_order as usize].add_pages(pages_ptr); - } - - /// This function checks whether a page is free && is a buddy - /// we can coalesce a page and its buddy if - /// - the buddy is valid(present) && - /// - the buddy is right now in free_areas && - /// - a page and its buddy have the same order && - /// - a page and its buddy are in the same zone (on smp systems). - fn buddy_check(&self, pages_ptr: Raw, order: u32) -> bool { - if !pages_ptr.is_present() { - return false; - } - if !pages_ptr.is_free() { - return false; - } - if pages_ptr.order() != order { - return false; - } - - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - true - } - - /// Only used on buddy initialization - pub fn create_pages(&mut self, start: PAddr, end: PAddr) { - let mut start_pfn = PFN::from(start.ceil()); - let end_pfn = PFN::from(end.floor()); - - while start_pfn < end_pfn { - let mut order = usize::from(start_pfn) - .trailing_zeros() - .min((AREAS - 1) as u32); - - while start_pfn + (1 << order) as usize > end_pfn { - order -= 1; - } - let page_ptr = Raw::from(start_pfn); - page_ptr.set_buddy(); - self.free_areas[order as usize].add_pages(page_ptr); - start_pfn = start_pfn + (1 << order) as usize; - } - } -} diff --git a/crates/eonix_mm/src/paging.rs b/crates/eonix_mm/src/paging.rs index 88da902e..0c4811f2 100644 --- a/crates/eonix_mm/src/paging.rs +++ b/crates/eonix_mm/src/paging.rs @@ -1,9 +1,13 @@ +mod list; mod page; mod page_alloc; mod pfn; mod raw_page; +mod zone; +pub use list::{PageList, PageListSized}; pub use page::{Page, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; pub use page_alloc::{GlobalPageAlloc, NoAlloc, PageAlloc}; pub use pfn::PFN; pub use raw_page::{RawPage, UnmanagedRawPage}; +pub use zone::Zone; diff --git a/crates/eonix_mm/src/paging/list.rs b/crates/eonix_mm/src/paging/list.rs new file mode 100644 index 00000000..a52cf947 --- /dev/null +++ b/crates/eonix_mm/src/paging/list.rs @@ -0,0 +1,19 @@ +pub trait PageList { + type Page; + + fn is_empty(&self) -> bool; + + fn peek_head(&mut self) -> Option<&mut Self::Page>; + + fn pop_head(&mut self) -> Option<&'static mut Self::Page>; + fn push_tail(&mut self, page: &'static mut Self::Page); + fn remove(&mut self, page: &mut Self::Page); +} + +pub trait PageListSized: PageList + Sized { + const NEW: Self; + + fn new() -> Self { + Self::NEW + } +} diff --git a/crates/eonix_mm/src/paging/raw_page.rs b/crates/eonix_mm/src/paging/raw_page.rs index 7951729d..789e863b 100644 --- a/crates/eonix_mm/src/paging/raw_page.rs +++ b/crates/eonix_mm/src/paging/raw_page.rs @@ -1,14 +1,13 @@ -use super::PFN; use core::sync::atomic::AtomicUsize; +use super::PFN; + /// A `RawPage` represents a page of memory in the kernel. It is a low-level /// representation of a page that is used by the kernel to manage memory. #[doc(notable_trait)] pub trait RawPage: Clone + Copy + From + Into { fn order(&self) -> u32; fn refcount(&self) -> &AtomicUsize; - - fn is_present(&self) -> bool; } #[derive(Clone, Copy)] @@ -45,8 +44,4 @@ impl RawPage for UnmanagedRawPage { fn refcount(&self) -> &AtomicUsize { &UNMANAGED_RAW_PAGE_CLONE_COUNT } - - fn is_present(&self) -> bool { - true - } } diff --git a/crates/eonix_mm/src/paging/zone.rs b/crates/eonix_mm/src/paging/zone.rs new file mode 100644 index 00000000..ec3ed15e --- /dev/null +++ b/crates/eonix_mm/src/paging/zone.rs @@ -0,0 +1,20 @@ +use core::cell::UnsafeCell; + +#[allow(unused_imports)] +use super::{Page, PageAlloc, RawPage, PFN}; +use crate::address::PRange; + +/// A [`Zone`] holds a lot of [`Page`]s that share the same NUMA node or +/// "physical location". +pub trait Zone: Send + Sync { + type Page; + + /// Whether the [`range`] is within this [`Zone`]. + fn contains_prange(&self, range: PRange) -> bool; + + /// Get the [`RawPage`] that [`pfn`] points to. + /// + /// # Return + /// [`None`] if [`pfn`] is not in this [`Zone`]. + fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell>; +} diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index 8a684edd..8597331d 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -2,6 +2,7 @@ use core::ptr::NonNull; +use eonix_mm::paging::{PageList, PageListSized}; use eonix_sync::Spin; #[repr(C)] @@ -10,19 +11,6 @@ pub union SlabSlot { data: u8, } -pub trait SlabPageList: Sized { - type Page: SlabPage; - - fn new() -> Self; - fn is_empty(&self) -> bool; - - fn peek_head(&mut self) -> Option<&mut Self::Page>; - - fn pop_head(&mut self) -> Option<&'static mut Self::Page>; - fn push_tail(&mut self, page: &'static mut Self::Page); - fn remove(&mut self, page: &mut Self::Page); -} - pub trait SlabPage: Sized + 'static { fn get_data_ptr(&self) -> NonNull<[u8]>; @@ -98,7 +86,7 @@ where pub trait SlabPageAlloc { type Page: SlabPage; - type PageList: SlabPageList; + type PageList: PageList; /// Allocate a page suitable for slab system use. The page MUST come with /// its allocation count 0 and next free slot None. @@ -110,7 +98,7 @@ pub trait SlabPageAlloc { pub(crate) struct SlabList where - T: SlabPageList, + T: PageList, { empty_list: T, partial_list: T, @@ -132,6 +120,7 @@ unsafe impl Sync for SlabAlloc where P: SlabPag impl SlabAlloc where L: SlabPageAlloc, + L::PageList: PageListSized, { pub fn new_in(alloc: L) -> Self { Self { @@ -159,17 +148,23 @@ where impl SlabList where - T: SlabPageList, + T: PageListSized, { - fn new(object_size: usize) -> Self { + const fn new(object_size: usize) -> Self { Self { - empty_list: T::new(), - partial_list: T::new(), - full_list: T::new(), + empty_list: T::NEW, + partial_list: T::NEW, + full_list: T::NEW, object_size, } } +} +impl SlabList +where + T: PageList, + T::Page: SlabPage, +{ fn alloc_from_partial(&mut self) -> NonNull { let head = self.partial_list.peek_head().unwrap(); let slot = head.alloc_slot().unwrap(); diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index 1c018f37..9dce4567 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -1,19 +1,25 @@ mod raw_page; +mod zones; use core::sync::atomic::Ordering; -use buddy_allocator::{BuddyAllocator, BuddyRawPage as _}; +use buddy_allocator::BuddyAllocator; use eonix_mm::address::{AddrOps as _, PRange}; -use eonix_mm::paging::{GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PFN}; +use eonix_mm::paging::{ + GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, PFN, +}; +use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; -use intrusive_list::List; -use raw_page::PageFlags; +use raw_page::{PageFlags, RawPageList}; pub use raw_page::{RawPage, RawPagePtr}; +pub use zones::GlobalZone; const COSTLY_ORDER: u32 = 3; +const AREAS: usize = COSTLY_ORDER as usize + 1; const BATCH_SIZE: u32 = 64; -static BUDDY_ALLOC: Spin> = Spin::new(BuddyAllocator::new()); +static BUDDY_ALLOC: Spin> = + Spin::new(BuddyAllocator::new(&GlobalZone())); #[eonix_percpu::define_percpu] static PERCPU_PAGE_ALLOC: PerCpuPageAlloc = PerCpuPageAlloc::new(); @@ -26,58 +32,42 @@ pub struct BuddyPageAlloc(); struct PerCpuPageAlloc { batch: u32, - // TODO: might be used in the future. - // high: u32, - free_areas: [List; COSTLY_ORDER as usize + 1], + free_areas: [RawPageList; AREAS], +} + +pub trait PerCpuPage { + fn set_local(&mut self, val: bool); } impl PerCpuPageAlloc { const fn new() -> Self { Self { batch: BATCH_SIZE, - // high: 0, - free_areas: [const { List::new() }; COSTLY_ORDER as usize + 1], + free_areas: [RawPageList::NEW; AREAS], } } - fn insert_free_pages(&mut self, pages_ptr: RawPagePtr, order: u32) { - let free_area = &mut self.free_areas[order as usize]; - free_area.insert(unsafe { pages_ptr.get_link() }); - } - - fn get_free_pages(&mut self, order: u32) -> Option { - let free_area = &mut self.free_areas[order as usize]; - free_area.pop().map(|node| unsafe { - // SAFETY: `node` is a valid pointer to a `Link` that is not used by anyone. - RawPagePtr::from_link(node) - }) - } - - fn alloc_order(&mut self, order: u32) -> Option { + fn alloc_order(&mut self, order: u32) -> Option<&'static mut RawPage> { assert!(order <= COSTLY_ORDER); - if let Some(pages) = self.get_free_pages(order) { + if let Some(pages) = self.free_areas[order as usize].pop_head() { return Some(pages); } let batch = self.batch >> order; for _ in 0..batch { - if let Some(pages_ptr) = BUDDY_ALLOC.lock().alloc_order(order) { - pages_ptr.flags().set(PageFlags::LOCAL); - self.insert_free_pages(pages_ptr, order); - } else { + let Some(page) = BUDDY_ALLOC.lock().alloc_order(order) else { break; }; + + page.set_local(true); + self.free_areas[order as usize].push_tail(page); } - self.get_free_pages(order) + self.free_areas[order as usize].pop_head() } - fn free_pages(&mut self, pages_ptr: RawPagePtr, order: u32) { - assert_eq!(pages_ptr.order(), order); - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - - pages_ptr.refcount().store(1, Ordering::Relaxed); - self.insert_free_pages(pages_ptr, order); + fn free_pages(&mut self, page: &'static mut RawPage, order: u32) { + self.free_areas[order as usize].push_tail(page); } } @@ -87,16 +77,6 @@ impl GlobalPageAlloc { BuddyPageAlloc() } - pub fn mark_present(range: PRange) { - let mut pfn = PFN::from(range.start().ceil()); - let end_pfn = PFN::from(range.end().floor()); - - while pfn < end_pfn { - RawPagePtr::from(pfn).flags().set(PageFlags::PRESENT); - pfn = pfn + 1; - } - } - /// Add the pages in the PAddr range `range` to the global allocator. /// /// This function is only to be called on system initialization when `eonix_preempt` @@ -116,34 +96,47 @@ impl PageAlloc for GlobalPageAlloc { type RawPage = RawPagePtr; fn alloc_order(&self, order: u32) -> Option { - if order > COSTLY_ORDER { + let raw_page = if order > COSTLY_ORDER { BUDDY_ALLOC.lock().alloc_order(order) } else { unsafe { eonix_preempt::disable(); - let page_ptr = PERCPU_PAGE_ALLOC.as_mut().alloc_order(order); + let page = PERCPU_PAGE_ALLOC.as_mut().alloc_order(order); eonix_preempt::enable(); - page_ptr + + page } - } + }; + + raw_page.map(|raw_page| { + // SAFETY: Memory order here can be Relaxed is for the same reason + // as that in the copy constructor of `std::shared_ptr`. + raw_page.refcount.fetch_add(1, Ordering::Relaxed); + + RawPagePtr::from_ref(raw_page) + }) } unsafe fn dealloc(&self, page_ptr: RawPagePtr) { + assert_eq!( + page_ptr.refcount().load(Ordering::Relaxed), + 0, + "Trying to free a page with refcount > 0" + ); + if page_ptr.order() > COSTLY_ORDER { - BUDDY_ALLOC.lock().dealloc(page_ptr); + BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); } else { let order = page_ptr.order(); + unsafe { - eonix_preempt::disable(); - PERCPU_PAGE_ALLOC.as_mut().free_pages(page_ptr, order); - eonix_preempt::enable(); + PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()).free_pages(page_ptr.as_mut(), order); } } } fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - BuddyAllocator::has_management_over(page_ptr) - && (page_ptr.order() > COSTLY_ORDER || page_ptr.flags().has(PageFlags::LOCAL)) + page_ptr.order() > COSTLY_ORDER || page_ptr.flags().has(PageFlags::LOCAL) } } @@ -157,14 +150,17 @@ impl PageAlloc for BuddyPageAlloc { type RawPage = RawPagePtr; fn alloc_order(&self, order: u32) -> Option { - BUDDY_ALLOC.lock().alloc_order(order) + BUDDY_ALLOC + .lock() + .alloc_order(order) + .map(|raw_page| RawPagePtr::from_ref(raw_page)) } unsafe fn dealloc(&self, page_ptr: RawPagePtr) { - BUDDY_ALLOC.lock().dealloc(page_ptr); + BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); } - fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - BuddyAllocator::has_management_over(page_ptr) + fn has_management_over(&self, _: RawPagePtr) -> bool { + true } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index d793ccd7..0baa7b9a 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,18 +1,18 @@ use core::ptr::NonNull; use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; -use buddy_allocator::BuddyRawPage; +use buddy_allocator::BuddyPage; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _}; -use eonix_mm::paging::{PageAlloc, RawPage as RawPageTrait, PFN}; +use eonix_mm::paging::{PageAlloc, PageList, PageListSized, RawPage as RawPageTrait, PFN}; use intrusive_list::{container_of, Link, List}; -use slab_allocator::{SlabPage, SlabPageAlloc, SlabPageList, SlabSlot}; +use slab_allocator::{SlabPage, SlabPageAlloc, SlabSlot}; -use super::GlobalPageAlloc; +use super::{GlobalPageAlloc, PerCpuPage}; use crate::kernel::mem::page_cache::PageCacheRawPage; use crate::kernel::mem::PhysAccess; -const PAGE_ARRAY: NonNull = +pub const PAGE_ARRAY: NonNull = unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; pub struct PageFlags(AtomicU32); @@ -52,21 +52,23 @@ pub struct RawPage { /// This field is only used in buddy system and is protected by the global lock. order: u32, flags: PageFlags, - refcount: AtomicUsize, + pub refcount: AtomicUsize, shared_data: PageData, } +// XXX: introduce Folio and remove this. +unsafe impl Send for RawPage {} +unsafe impl Sync for RawPage {} + #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct RawPagePtr(NonNull); impl PageFlags { - pub const PRESENT: u32 = 1 << 0; pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; pub const SLAB: u32 = 1 << 3; pub const DIRTY: u32 = 1 << 4; - pub const FREE: u32 = 1 << 5; pub const LOCAL: u32 = 1 << 6; pub fn has(&self, flag: u32) -> bool { @@ -158,48 +160,31 @@ impl RawPageTrait for RawPagePtr { fn refcount(&self) -> &AtomicUsize { self.refcount() } - - fn is_present(&self) -> bool { - self.flags().has(PageFlags::PRESENT) - } } -impl BuddyRawPage for RawPagePtr { - unsafe fn from_link(link: &mut Link) -> Self { - let raw_page_ptr = container_of!(link, RawPage, link); - Self(raw_page_ptr) - } - - fn set_order(&self, order: u32) { - self.as_mut().order = order; +impl BuddyPage for RawPage { + fn pfn(&self) -> PFN { + PFN::from(RawPagePtr::from_ref(self)) } - unsafe fn get_link(&self) -> &mut Link { - &mut self.as_mut().link + fn get_order(&self) -> u32 { + self.order } fn is_buddy(&self) -> bool { - self.flags().has(PageFlags::BUDDY) - } - - fn is_free(&self) -> bool { - self.flags().has(PageFlags::FREE) - } - - fn set_buddy(&self) { - self.flags().set(PageFlags::BUDDY); - } - - fn set_free(&self) { - self.flags().set(PageFlags::FREE); + self.flags.has(PageFlags::BUDDY) } - fn clear_buddy(&self) { - self.flags().clear(PageFlags::BUDDY); + fn set_order(&mut self, order: u32) { + self.order = order; } - fn clear_free(&self) { - self.flags().clear(PageFlags::FREE); + fn set_buddy(&mut self, val: bool) { + if val { + self.flags.set(PageFlags::BUDDY); + } else { + self.flags.clear(PageFlags::BUDDY) + } } } @@ -284,15 +269,21 @@ impl PageCacheRawPage for RawPagePtr { } } -pub struct RawSlabPageList(List); +impl PerCpuPage for RawPage { + fn set_local(&mut self, val: bool) { + if val { + self.flags.set(PageFlags::LOCAL) + } else { + self.flags.clear(PageFlags::LOCAL) + } + } +} + +pub struct RawPageList(List); -impl SlabPageList for RawSlabPageList { +impl PageList for RawPageList { type Page = RawPage; - fn new() -> Self { - Self(List::new()) - } - fn is_empty(&self) -> bool { self.0.is_empty() } @@ -324,9 +315,13 @@ impl SlabPageList for RawSlabPageList { } } +impl PageListSized for RawPageList { + const NEW: Self = RawPageList(List::new()); +} + impl SlabPageAlloc for GlobalPageAlloc { type Page = RawPage; - type PageList = RawSlabPageList; + type PageList = RawPageList; unsafe fn alloc_uninit(&self) -> &'static mut RawPage { let raw_page = self.alloc().expect("Out of memory").as_mut(); diff --git a/src/kernel/mem/page_alloc/zones.rs b/src/kernel/mem/page_alloc/zones.rs new file mode 100644 index 00000000..7a2e4e33 --- /dev/null +++ b/src/kernel/mem/page_alloc/zones.rs @@ -0,0 +1,25 @@ +use core::cell::UnsafeCell; + +use eonix_mm::address::PRange; +use eonix_mm::paging::{Zone, PFN}; + +use super::RawPage; +use crate::kernel::mem::page_alloc::RawPagePtr; + +pub struct GlobalZone(); + +impl Zone for GlobalZone { + type Page = RawPage; + + fn contains_prange(&self, _: PRange) -> bool { + true + } + + fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell> { + unsafe { + // SAFETY: The pointer returned by [`RawPagePtr::as_ptr()`] is valid. + // And so is it wrapped with [`UnsafeCell`] + Some(&*(RawPagePtr::from(pfn).as_ptr() as *const UnsafeCell)) + } + } +} diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 3d8be90f..93b6da20 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -1,14 +1,11 @@ +use eonix_hal::bootstrap::BootStrapData; +use eonix_hal::mm::{ArchMemory, ArchPagingMode, GLOBAL_PAGE_TABLE}; +use eonix_hal::traits::mm::Memory; +use eonix_mm::address::{Addr as _, AddrOps as _, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, PagingMode as _, PTE}; +use eonix_mm::paging::{Page as GenericPage, PAGE_SIZE, PFN}; + use crate::kernel::mem::{GlobalPageAlloc, RawPage}; -use eonix_hal::{ - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, GLOBAL_PAGE_TABLE}, - traits::mm::Memory, -}; -use eonix_mm::{ - address::{Addr as _, AddrOps as _, VAddr, VRange}, - page_table::{PageAttribute, PagingMode as _, PTE}, - paging::{Page as GenericPage, PAGE_SIZE, PFN}, -}; pub fn setup_memory(data: &mut BootStrapData) { let addr_max = ArchMemory::present_ram() @@ -50,9 +47,10 @@ pub fn setup_memory(data: &mut BootStrapData) { ); } - for range in ArchMemory::present_ram() { - GlobalPageAlloc::mark_present(range); - } + // TODO!!!: Construct the global zone with all present ram. + // for range in ArchMemory::present_ram() { + // GlobalPageAlloc::mark_present(range); + // } if let Some(early_alloc) = data.take_alloc() { for range in early_alloc.into_iter() { From 45268e7d335e5403b7ea868379b5491bb5d0f26d Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 14 Jan 2026 01:09:49 +0800 Subject: [PATCH 009/153] vfs: rework of inode and page cache system - Bump rust compiler version to nightly-2026-01-09. - Inode rework: add a generic Inode struct. - Add a macro to help function tweaks. - PageCache rework: reduce complexity and try to decouple. - Adapt fat32, tmpfs to the new page cache system. - Change the way we process mapped pages and load ELF executables. - Refine handling flags in `MMArea::handle_mmap`. Signed-off-by: greatbridf --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 62 +- rust-toolchain | 2 +- src/fs/fat32.rs | 276 ++++----- src/fs/procfs.rs | 155 +++-- src/fs/tmpfs/dir.rs | 287 +++++----- src/fs/tmpfs/file.rs | 331 +++++------ src/fs/tmpfs/mod.rs | 23 +- src/kernel/mem.rs | 2 +- src/kernel/mem/mm_area.rs | 97 ++-- src/kernel/mem/mm_list/mapping.rs | 15 +- src/kernel/mem/mm_list/page_fault.rs | 5 +- src/kernel/mem/page_alloc.rs | 4 +- src/kernel/mem/page_alloc/raw_page.rs | 29 +- src/kernel/mem/page_cache.rs | 416 +++++--------- src/kernel/syscall/mm.rs | 21 +- src/kernel/task/loader/elf.rs | 70 +-- src/kernel/vfs/dentry.rs | 66 +-- src/kernel/vfs/dentry/walk.rs | 52 +- src/kernel/vfs/file/inode_file.rs | 36 +- src/kernel/vfs/filearray.rs | 46 +- src/kernel/vfs/inode/inode.rs | 531 +++++++++--------- src/kernel/vfs/inode/mod.rs | 2 +- src/kernel/vfs/inode/ops.rs | 5 +- src/kernel/vfs/inode/statx.rs | 32 +- src/kernel/vfs/mount.rs | 28 +- src/kernel/vfs/superblock.rs | 45 +- src/lib.rs | 1 - 27 files changed, 1152 insertions(+), 1487 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index 0f1dff63..7b3dc043 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -1,39 +1,31 @@ -use super::{ - config::{self, mm::*}, - console::write_str, - cpu::{CPUID, CPU_COUNT}, - time::set_next_timer, -}; -use crate::{ - arch::{ - cpu::CPU, - fdt::{init_dtb_and_fdt, FdtExt, FDT}, - mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE}, - }, - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}, -}; -use core::{ - alloc::Allocator, - arch::asm, - cell::RefCell, - sync::atomic::{AtomicBool, AtomicUsize}, -}; -use core::{ - arch::{global_asm, naked_asm}, - hint::spin_loop, - sync::atomic::{AtomicPtr, Ordering}, -}; +use core::alloc::Allocator; +use core::arch::{asm, global_asm, naked_asm}; +use core::cell::RefCell; +use core::hint::spin_loop; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + use eonix_hal_traits::mm::Memory; -use eonix_mm::{ - address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}, - page_table::{PageAttribute, PagingMode, PTE as _}, - paging::{Page, PageAccess, PageAlloc, PAGE_SIZE, PFN}, -}; +use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, PagingMode, PTE as _}; +use eonix_mm::paging::{Page, PageAccess, PageAlloc, PAGE_SIZE, PFN}; use eonix_percpu::PercpuArea; use fdt::Fdt; -use riscv::{asm::sfence_vma_all, register::satp}; -use sbi::{hsm::hart_start, legacy::console_putchar, PhysicalAddress}; +use riscv::asm::sfence_vma_all; +use riscv::register::satp; +use sbi::hsm::hart_start; +use sbi::legacy::console_putchar; +use sbi::PhysicalAddress; + +use super::config::mm::*; +use super::config::{self}; +use super::console::write_str; +use super::cpu::{CPUID, CPU_COUNT}; +use super::time::set_next_timer; +use crate::arch::cpu::CPU; +use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; +use crate::arch::mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE}; +use crate::bootstrap::BootStrapData; +use crate::mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -78,7 +70,7 @@ static AP_SEM: AtomicBool = AtomicBool::new(false); #[unsafe(naked)] #[unsafe(no_mangle)] #[unsafe(link_section = ".bootstrap.entry")] -unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) -> ! { +unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) { naked_asm!( " ld sp, 2f @@ -289,7 +281,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { #[unsafe(naked)] #[unsafe(no_mangle)] #[unsafe(link_section = ".bootstrap.apentry")] -unsafe extern "C" fn _ap_start(hart_id: usize) -> ! { +unsafe extern "C" fn _ap_start(hart_id: usize) { naked_asm!( " la sp, 1f // set temp stack diff --git a/rust-toolchain b/rust-toolchain index 8adb8e58..11ad5efd 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2025-05-16 +nightly-2026-01-09 diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 9a4e03ec..b19c9908 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,26 +1,22 @@ mod dir; mod file; -use alloc::sync::{Arc, Weak}; -use core::future::Future; +use alloc::sync::Arc; use core::ops::Deref; use async_trait::async_trait; use dir::{as_raw_dirents, ParseDirent}; +use eonix_mm::paging::PAGE_SIZE; use eonix_sync::RwLock; use itertools::Itertools; use crate::io::{Buffer, ByteBuffer, UninitBuffer}; use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::{ - CachePage, CachePageStream, Page, PageCache, PageCacheBackendOps, PageExcl, PageExt, -}; +use crate::kernel::mem::{CachePage, Page, PageExcl, PageExt, PageOffset}; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; -use crate::kernel::vfs::inode::{ - Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, -}; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; @@ -56,6 +52,10 @@ impl Cluster { Ino::new(self.0 as _) } + pub fn from_ino(ino: Ino) -> Self { + Self(ino.as_raw() as u32) + } + fn normalized(self) -> Self { Self(self.0 - 2) } @@ -130,7 +130,7 @@ impl FatFs { } impl FatFs { - pub async fn create(device: DeviceId) -> KResult<(SbUse, InodeUse)> { + pub async fn create(device: DeviceId) -> KResult<(SbUse, InodeUse)> { let device = BlockDevice::get(device)?; let mut info = UninitBuffer::::new(); @@ -217,18 +217,15 @@ impl<'fat> Iterator for ClusterIterator<'fat> { } } -struct FileInode { - cluster: Cluster, - info: Spin, - sb: SbRef, - page_cache: PageCache, -} +struct FileInode; impl FileInode { - fn new(cluster: Cluster, sb: SbRef, size: u32) -> InodeUse { - InodeUse::new_cyclic(|weak: &Weak| Self { - cluster, - info: Spin::new(InodeInfo { + fn new(cluster: Cluster, sb: SbRef, size: u32) -> InodeUse { + InodeUse::new( + sb, + cluster.as_ino(), + Format::REG, + InodeInfo { size: size as u64, nlink: 1, uid: 0, @@ -237,108 +234,75 @@ impl FileInode { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - sb, - page_cache: PageCache::new(weak.clone()), - }) + }, + Self, + ) } } impl InodeOps for FileInode { type SuperBlock = FatFs; - fn ino(&self) -> Ino { - self.cluster.as_ino() - } - - fn format(&self) -> Format { - Format::REG - } - - fn info(&self) -> &Spin { - &self.info - } - - fn super_block(&self) -> &SbRef { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.page_cache) - } -} - -impl InodeDirOps for FileInode {} -impl InodeFileOps for FileInode { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - self.page_cache.read(buffer, offset).await + async fn read( + &self, + _: SbUse, + inode: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult { + inode.get_page_cache().read(buffer, offset).await } - async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - let sb = self.sb.get()?; + async fn read_page( + &self, + sb: SbUse, + inode: &InodeUse, + page: &mut CachePage, + offset: PageOffset, + ) -> KResult<()> { let fs = &sb.backend; let fat = sb.backend.fat.read().await; - if offset >= self.info.lock().size as usize { - return Ok(0); + if offset >= PageOffset::from_byte_ceil(inode.info.lock().size as usize) { + unreachable!("read_page called with offset beyond file size"); } let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; - assert!(cluster_size <= 0x1000, "Cluster size is too large"); - - let skip_clusters = offset / cluster_size; - let inner_offset = offset % cluster_size; - - let cluster_iter = ClusterIterator::new(fat.as_ref(), self.cluster).skip(skip_clusters); - - let buffer_page = Page::alloc(); - for cluster in cluster_iter { - fs.read_cluster(cluster, &buffer_page).await?; - - let pg = buffer_page.lock(); - let data = &pg.as_bytes()[inner_offset..]; - - let end = offset + data.len(); - let real_end = end.min(self.info.lock().size as usize); - let real_size = real_end - offset; - - if buffer.fill(&data[..real_size])?.should_stop() { - break; - } + if cluster_size != PAGE_SIZE { + unimplemented!("cluster size != PAGE_SIZE"); } - Ok(buffer.wrote()) - } -} + // XXX: Ugly and inefficient O(n^2) algorithm for sequential file read. + let cluster = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)) + .skip(offset.page_count()) + .next() + .ok_or(EIO)?; -impl PageCacheBackendOps for FileInode { - async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult { - self.read_direct(page, offset).await - } + let page = page.get_page(); + fs.read_cluster(cluster, &page).await?; - async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult { - todo!() - } + let real_len = (inode.info.lock().size as usize) - offset.byte_count(); + if real_len < PAGE_SIZE { + let mut page = page.lock(); + page.as_bytes_mut()[real_len..].fill(0); + } - fn size(&self) -> usize { - self.info.lock().size as usize + Ok(()) } } struct DirInode { - cluster: Cluster, - info: Spin, - sb: SbRef, - // TODO: Use the new PageCache... dir_pages: RwLock>, } impl DirInode { - fn new(cluster: Cluster, sb: SbRef, size: u32) -> InodeUse { - InodeUse::new(Self { - cluster, - info: Spin::new(InodeInfo { + fn new(cluster: Cluster, sb: SbRef, size: u32) -> InodeUse { + InodeUse::new( + sb, + cluster.as_ino(), + Format::DIR, + InodeInfo { size: size as u64, nlink: 2, // '.' and '..' uid: 0, @@ -347,23 +311,23 @@ impl DirInode { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - sb, - dir_pages: RwLock::new(Vec::new()), - }) + }, + Self { + dir_pages: RwLock::new(Vec::new()), + }, + ) } - async fn read_dir_pages(&self) -> KResult<()> { + async fn read_dir_pages(&self, sb: &SbUse, inode: &InodeUse) -> KResult<()> { let mut dir_pages = self.dir_pages.write().await; if !dir_pages.is_empty() { return Ok(()); } - let sb = self.sb.get()?; let fs = &sb.backend; let fat = fs.fat.read().await; - let clusters = ClusterIterator::new(fat.as_ref(), self.cluster); + let clusters = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)); for cluster in clusters { let page = PageExcl::alloc(); @@ -375,7 +339,11 @@ impl DirInode { Ok(()) } - async fn get_dir_pages(&self) -> KResult> + use<'_>> { + async fn get_dir_pages( + &self, + sb: &SbUse, + inode: &InodeUse, + ) -> KResult> + use<'_>> { { let dir_pages = self.dir_pages.read().await; if !dir_pages.is_empty() { @@ -383,7 +351,7 @@ impl DirInode { } } - self.read_dir_pages().await?; + self.read_dir_pages(sb, inode).await?; if let Some(dir_pages) = self.dir_pages.try_read() { return Ok(dir_pages); @@ -396,32 +364,13 @@ impl DirInode { impl InodeOps for DirInode { type SuperBlock = FatFs; - fn ino(&self) -> Ino { - self.cluster.as_ino() - } - - fn format(&self) -> Format { - Format::DIR - } - - fn info(&self) -> &Spin { - &self.info - } - - fn super_block(&self) -> &SbRef { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } -} - -impl InodeFileOps for DirInode {} -impl InodeDirOps for DirInode { - async fn lookup(&self, dentry: &Arc) -> KResult>> { - let sb = self.sb.get()?; - let dir_pages = self.get_dir_pages().await?; + async fn lookup( + &self, + sb: SbUse, + inode: &InodeUse, + dentry: &Arc, + ) -> KResult> { + let dir_pages = self.get_dir_pages(&sb, inode).await?; let dir_data = dir_pages.iter().map(|pg| pg.as_bytes()); @@ -451,48 +400,47 @@ impl InodeDirOps for DirInode { Ok(None) } - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn readdir( + &self, + sb: SbUse, + inode: &InodeUse, offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), - ) -> impl Future>> + Send + 'r { - async move { - let sb = self.sb.get()?; - let fs = &sb.backend; - let dir_pages = self.get_dir_pages().await?; + callback: &mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> KResult> { + let fs = &sb.backend; + let dir_pages = self.get_dir_pages(&sb, inode).await?; - let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; - let cluster_offset = offset / cluster_size; - let inner_offset = offset % cluster_size; - let inner_raw_dirent_offset = inner_offset / core::mem::size_of::(); + let cluster_offset = offset / cluster_size; + let inner_offset = offset % cluster_size; + let inner_raw_dirent_offset = inner_offset / core::mem::size_of::(); - let dir_data = dir_pages - .iter() - .skip(cluster_offset) - .map(|pg| pg.as_bytes()); - - let raw_dirents = dir_data - .map(as_raw_dirents) - .take_while_inclusive(Result::is_ok) - .flatten_ok() - .skip(inner_raw_dirent_offset); - - let mut dirents = futures::stream::iter(raw_dirents); - - let mut nread = 0; - while let Some(result) = dirents.next_dirent().await { - let entry = result?; - - match callback(&entry.filename, entry.cluster.as_ino()) { - Err(err) => return Ok(Err(err)), - Ok(true) => nread += entry.entry_offset as usize, - Ok(false) => break, - } - } + let dir_data = dir_pages + .iter() + .skip(cluster_offset) + .map(|pg| pg.as_bytes()); - Ok(Ok(nread)) + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok() + .skip(inner_raw_dirent_offset); + + let mut dirents = futures::stream::iter(raw_dirents); + + let mut nread = 0; + while let Some(result) = dirents.next_dirent().await { + let entry = result?; + + match callback(&entry.filename, entry.cluster.as_ino()) { + Err(err) => return Ok(Err(err)), + Ok(true) => nread += entry.entry_offset as usize, + Ok(false) => break, + } } + + Ok(Ok(nread)) } } diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 57b881df..32ede420 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -1,30 +1,21 @@ +use alloc::sync::Arc; +use core::sync::atomic::{AtomicU64, Ordering}; + +use async_trait::async_trait; +use eonix_sync::{LazyLock, RwLock}; + +use crate::io::Buffer; use crate::kernel::constants::{EACCES, EISDIR, ENOTDIR}; +use crate::kernel::mem::paging::PageBuffer; use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::mount::{dump_mounts, register_filesystem, Mount, MountCreator}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; -use crate::{ - io::Buffer, - kernel::{ - mem::paging::PageBuffer, - vfs::{ - dentry::Dentry, - inode::{Ino, Inode}, - mount::{dump_mounts, register_filesystem, Mount, MountCreator}, - }, - }, - prelude::*, -}; -use alloc::sync::Arc; -use async_trait::async_trait; -use core::future::Future; -use core::sync::atomic::{AtomicU64, Ordering}; -use eonix_sync::{LazyLock, RwLock}; +use crate::prelude::*; struct Node { - ino: Ino, - sb: SbRef, - info: Spin, kind: NodeKind, } @@ -39,38 +30,19 @@ struct FileInode { } struct DirInode { - entries: RwLock, InodeUse)>>, + entries: RwLock, InodeUse)>>, } impl InodeOps for Node { type SuperBlock = ProcFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - match &self.kind { - NodeKind::File(_) => Format::REG, - NodeKind::Dir(_) => Format::DIR, - } - } - - fn info(&self) -> &Spin { - &self.info - } - - fn super_block(&self) -> &SbRef { - &self.sb - } - - fn page_cache(&self) -> Option<&crate::kernel::mem::PageCache> { - None - } -} - -impl InodeFileOps for Node { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + async fn read( + &self, + _: SbUse, + _: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult { let NodeKind::File(file_inode) = &self.kind else { return Err(EISDIR); }; @@ -88,10 +60,13 @@ impl InodeFileOps for Node { Ok(buffer.fill(data)?.allow_partial()) } -} -impl InodeDirOps for Node { - async fn lookup(&self, dentry: &Arc) -> KResult>> { + async fn lookup( + &self, + _: SbUse, + _: &InodeUse, + dentry: &Arc, + ) -> KResult> { let NodeKind::Dir(dir) = &self.kind else { return Err(ENOTDIR); }; @@ -108,29 +83,29 @@ impl InodeDirOps for Node { Ok(None) } - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn readdir( + &self, + _: SbUse, + _: &InodeUse, offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), - ) -> impl Future>> + Send + 'r { - Box::pin(async move { - let NodeKind::Dir(dir) = &self.kind else { - return Err(ENOTDIR); - }; - - let entries = dir.entries.read().await; - - let mut count = 0; - for (name, node) in entries.iter().skip(offset) { - match callback(name.as_ref(), node.ino) { - Err(err) => return Ok(Err(err)), - Ok(true) => count += 1, - Ok(false) => break, - } + callback: &mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> KResult> { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; + + let entries = dir.entries.read().await; + + let mut count = 0; + for (name, node) in entries.iter().skip(offset) { + match callback(name.as_ref(), node.ino) { + Err(err) => return Ok(Err(err)), + Ok(true) => count += 1, + Ok(false) => break, } + } - Ok(Ok(count)) - }) + Ok(Ok(count)) } } @@ -139,11 +114,12 @@ impl Node { ino: Ino, sb: SbRef, read: impl Fn(&mut PageBuffer) -> KResult<()> + Send + Sync + 'static, - ) -> InodeUse { - InodeUse::new(Self { - ino, + ) -> InodeUse { + InodeUse::new( sb, - info: Spin::new(InodeInfo { + ino, + Format::REG, + InodeInfo { size: 0, nlink: 1, uid: 0, @@ -152,16 +128,19 @@ impl Node { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - kind: NodeKind::File(FileInode::new(Box::new(read))), - }) + }, + Self { + kind: NodeKind::File(FileInode::new(Box::new(read))), + }, + ) } - fn new_dir(ino: Ino, sb: SbRef) -> InodeUse { - InodeUse::new(Self { - ino, + fn new_dir(ino: Ino, sb: SbRef) -> InodeUse { + InodeUse::new( sb, - info: Spin::new(InodeInfo { + ino, + Format::DIR, + InodeInfo { size: 0, nlink: 1, uid: 0, @@ -170,9 +149,11 @@ impl Node { atime: Instant::UNIX_EPOCH, ctime: Instant::UNIX_EPOCH, mtime: Instant::UNIX_EPOCH, - }), - kind: NodeKind::Dir(DirInode::new()), - }) + }, + Self { + kind: NodeKind::Dir(DirInode::new()), + }, + ) } } @@ -194,7 +175,7 @@ impl DirInode { } pub struct ProcFs { - root: InodeUse, + root: InodeUse, next_ino: AtomicU64, } @@ -240,7 +221,7 @@ where F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, { let procfs = &GLOBAL_PROCFS.backend; - let root = &procfs.root; + let root = &procfs.root.get_priv::(); let NodeKind::Dir(root) = &root.kind else { unreachable!(); diff --git a/src/fs/tmpfs/dir.rs b/src/fs/tmpfs/dir.rs index e2be1d12..4dd64d52 100644 --- a/src/fs/tmpfs/dir.rs +++ b/src/fs/tmpfs/dir.rs @@ -1,72 +1,51 @@ -use core::{any::Any, future::Future}; +use alloc::sync::Arc; +use alloc::vec; +use alloc::vec::Vec; -use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; use eonix_log::println_warn; -use eonix_sync::{LazyLock, RwLock, Spin}; - -use crate::{ - kernel::{ - constants::{EEXIST, EINVAL, EISDIR, ENOENT, ENOSYS, ENOTDIR}, - mem::PageCache, - timer::Instant, - vfs::{ - dentry::{dcache, Dentry}, - inode::{ - Ino, Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, RenameData, - }, - types::{DeviceId, Format, Mode, Permission}, - SbRef, - }, - }, - prelude::KResult, -}; - -use super::{ - file::{DeviceInode, FileInode, SymlinkInode}, - TmpFs, -}; +use eonix_sync::{LazyLock, RwLock}; + +use super::file::{DeviceInode, FileInode, SymlinkInode}; +use super::TmpFs; +use crate::kernel::constants::{EEXIST, EINVAL, EISDIR, ENOENT, ENOSYS, ENOTDIR}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::{dcache, Dentry}; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse, RenameData}; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse}; +use crate::prelude::KResult; pub struct DirectoryInode { - sb: SbRef, - ino: Ino, - info: Spin, entries: RwLock, Ino)>>, } -impl InodeOps for DirectoryInode { - type SuperBlock = TmpFs; - - fn ino(&self) -> Ino { - self.ino - } +fn link(dir: &InodeUse, entries: &mut Vec<(Arc<[u8]>, Ino)>, name: Arc<[u8]>, file: &InodeUse) { + let mut dir_info = dir.info.lock(); + let mut file_info = file.info.lock(); - fn format(&self) -> Format { - Format::DIR - } + let now = Instant::now(); - fn info(&self) -> &Spin { - &self.info - } + file_info.nlink += 1; + file_info.ctime = now; - fn super_block(&self) -> &SbRef { - &self.sb - } + dir_info.size += 1; + dir_info.mtime = now; + dir_info.ctime = now; - fn page_cache(&self) -> Option<&PageCache> { - None - } + entries.push((name, file.ino)); } impl DirectoryInode { - pub fn new(ino: Ino, sb: SbRef, perm: Permission) -> InodeUse { + pub fn new(ino: Ino, sb: SbRef, perm: Permission) -> InodeUse { static DOT: LazyLock> = LazyLock::new(|| Arc::from(b".".as_slice())); let now = Instant::now(); - InodeUse::new(Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + Format::DIR, + InodeInfo { size: 1, nlink: 1, // link from `.` to itself perm, @@ -75,35 +54,16 @@ impl DirectoryInode { atime: now, uid: 0, gid: 0, - }), - entries: RwLock::new(vec![(DOT.clone(), ino)]), - }) - } - - fn link( - &self, - entries: &mut Vec<(Arc<[u8]>, Ino)>, - name: Arc<[u8]>, - file: &InodeUse, - ) { - let mut self_info = self.info.lock(); - let mut file_info = file.info().lock(); - - let now = Instant::now(); - - file_info.nlink += 1; - file_info.ctime = now; - - self_info.size += 1; - self_info.mtime = now; - self_info.ctime = now; - - entries.push((name, file.ino())); + }, + Self { + entries: RwLock::new(vec![(DOT.clone(), ino)]), + }, + ) } fn do_unlink( &self, - file: &InodeUse, + file: &InodeUse, filename: &[u8], entries: &mut Vec<(Arc<[u8]>, Ino)>, now: Instant, @@ -112,11 +72,11 @@ impl DirectoryInode { file_info: &mut InodeInfo, ) -> KResult<()> { // SAFETY: `file_lock` has done the synchronization - if file.format() == Format::DIR { + if file.format == Format::DIR { return Err(EISDIR); } - let file_ino = file.ino(); + let file_ino = file.ino; entries.retain(|(name, ino)| *ino != file_ino || name.as_ref() != filename); if decrease_size { @@ -138,87 +98,114 @@ impl DirectoryInode { } } -impl InodeDirOps for DirectoryInode { - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, +impl InodeOps for DirectoryInode { + type SuperBlock = TmpFs; + + async fn readdir( + &self, + sb: SbUse, + _: &InodeUse, offset: usize, - for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), - ) -> impl Future>> + Send + 'r { - Box::pin(async move { - let _sb = self.sb.get()?; - let entries = self.entries.read().await; - - let mut count = 0; - for entry in entries.iter().skip(offset) { - match for_each_entry(&entry.0, entry.1) { - Err(err) => return Ok(Err(err)), - Ok(false) => break, - Ok(true) => count += 1, - } + for_each_entry: &mut (dyn FnMut(&[u8], Ino) -> KResult + Send), + ) -> KResult> { + let _sb = sb; + let entries = self.entries.read().await; + + let mut count = 0; + for entry in entries.iter().skip(offset) { + match for_each_entry(&entry.0, entry.1) { + Err(err) => return Ok(Err(err)), + Ok(false) => break, + Ok(true) => count += 1, } + } - Ok(Ok(count)) - }) + Ok(Ok(count)) } - async fn create(&self, at: &Arc, perm: Permission) -> KResult<()> { - let sb = self.sb.get()?; + async fn create( + &self, + sb: SbUse, + inode: &InodeUse, + at: &Arc, + perm: Permission, + ) -> KResult<()> { let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let file: InodeUse = FileInode::new(ino, self.sb.clone(), 0, perm); + let file = FileInode::new(ino, sb.get_ref(), 0, perm); - self.link(&mut entries, at.get_name(), &file); + link(inode, &mut entries, at.get_name(), &file); at.fill(file); Ok(()) } - async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + async fn mknod( + &self, + sb: SbUse, + inode: &InodeUse, + at: &Dentry, + mode: Mode, + dev: DeviceId, + ) -> KResult<()> { if !mode.is_chr() && !mode.is_blk() { return Err(EINVAL); } - let sb = self.sb.get()?; let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let file: InodeUse = DeviceInode::new(ino, self.sb.clone(), mode, dev); + let file = DeviceInode::new(ino, sb.get_ref(), mode, dev); - self.link(&mut entries, at.get_name(), &file); + link(inode, &mut entries, at.get_name(), &file); at.fill(file); Ok(()) } - async fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()> { - let sb = self.sb.get()?; + async fn symlink( + &self, + sb: SbUse, + inode: &InodeUse, + at: &Arc, + target: &[u8], + ) -> KResult<()> { let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let file: InodeUse = SymlinkInode::new(ino, self.sb.clone(), target.into()); + let file = SymlinkInode::new(ino, sb.get_ref(), target.into()); - self.link(&mut entries, at.get_name(), &file); + link(inode, &mut entries, at.get_name(), &file); at.fill(file); Ok(()) } - async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { - let sb = self.sb.get()?; + async fn mkdir( + &self, + sb: SbUse, + inode: &InodeUse, + at: &Dentry, + perm: Permission, + ) -> KResult<()> { let mut entries = self.entries.write().await; let ino = sb.backend.assign_ino(); - let new_dir: InodeUse = DirectoryInode::new(ino, self.sb.clone(), perm); + let new_dir = DirectoryInode::new(ino, sb.get_ref(), perm); - self.link(&mut entries, at.get_name(), &new_dir); + link(inode, &mut entries, at.get_name(), &new_dir); at.fill(new_dir); Ok(()) } - async fn unlink(&self, at: &Arc) -> KResult<()> { - let _sb = self.sb.get()?; + async fn unlink( + &self, + _sb: SbUse, + inode: &InodeUse, + at: &Arc, + ) -> KResult<()> { let mut entries = self.entries.write().await; let file = at.get_inode()?; @@ -230,8 +217,8 @@ impl InodeDirOps for DirectoryInode { &mut entries, Instant::now(), true, - &mut self.info.lock(), - &mut file.info().lock(), + &mut inode.info.lock(), + &mut file.info.lock(), )?; // Remove the dentry from the dentry cache immediately @@ -241,8 +228,12 @@ impl InodeDirOps for DirectoryInode { Ok(()) } - async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { - let sb = self.sb.get()?; + async fn rename( + &self, + sb: SbUse, + inode: &InodeUse, + rename_data: RenameData<'_, '_>, + ) -> KResult<()> { let _rename_lock = sb.backend.rename_lock.lock().await; let mut self_entries = self.entries.write().await; @@ -266,11 +257,11 @@ impl InodeDirOps for DirectoryInode { return Err(EEXIST); } - if new_parent.as_raw() == &raw const *self { + if inode == &new_parent { // Same directory rename // Remove from old location and add to new location - let old_ino = old_file.ino(); - let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_ino = old_file.ino; + let new_ino = new_file.as_ref().map(|f| f.ino); let old_name = old_dentry.get_name(); let new_name = new_dentry.get_name(); @@ -299,7 +290,7 @@ impl InodeDirOps for DirectoryInode { // Replace existing file (i.e. rename the old and unlink the new) let new_file = new_file.unwrap(); - match (new_file.format(), old_file.format()) { + match (new_file.format, old_file.format) { (Format::DIR, _) => return Err(EISDIR), (_, Format::DIR) => return Err(ENOTDIR), _ => {} @@ -307,12 +298,12 @@ impl InodeDirOps for DirectoryInode { self_entries.remove(new_idx); - self.info.lock().size -= 1; + inode.info.lock().size -= 1; // The last reference to the inode is held by some dentry // and will be released when the dentry is released - let mut new_info = new_file.info().lock(); + let mut new_info = new_file.info.lock(); new_info.nlink -= 1; new_info.mtime = now; @@ -322,24 +313,21 @@ impl InodeDirOps for DirectoryInode { let (name, _) = &mut self_entries[old_ent_idx]; *name = new_dentry.get_name(); - let mut self_info = self.info.lock(); + let mut self_info = inode.info.lock(); self_info.mtime = now; self_info.ctime = now; } else { // Cross-directory rename - handle similar to same directory case // Get new parent directory - let new_parent_inode = new_dentry.parent().get_inode()?; - assert_eq!(new_parent_inode.format(), Format::DIR); - - let new_parent = (&new_parent_inode as &dyn Any) - .downcast_ref::() - .expect("new parent must be a DirectoryInode"); + let new_parent = new_dentry.parent().get_inode()?; + assert_eq!(new_parent.format, Format::DIR); - let mut new_entries = new_parent.entries.write().await; + let new_parent_priv = new_parent.get_priv::(); + let mut new_entries = new_parent_priv.entries.write().await; - let old_ino = old_file.ino(); - let new_ino = new_file.as_ref().map(|f| f.ino()); + let old_ino = old_file.ino; + let new_ino = new_file.as_ref().map(|f| f.ino); let old_name = old_dentry.get_name(); let new_name = new_dentry.get_name(); @@ -361,26 +349,28 @@ impl InodeDirOps for DirectoryInode { // Replace existing file (i.e. move the old and unlink the new) let new_file = new_file.unwrap(); - match (old_file.format(), new_file.format()) { + match (old_file.format, new_file.format) { (Format::DIR, Format::DIR) => {} (Format::DIR, _) => return Err(ENOTDIR), (_, _) => {} } // Unlink the old file that was replaced - new_parent.do_unlink( + new_parent_priv.do_unlink( &new_file, &new_name, &mut new_entries, now, false, &mut new_parent.info.lock(), - &mut new_file.info().lock(), + &mut new_file.info.lock(), )?; } else { - new_parent.info.lock().size += 1; - new_parent.info.lock().mtime = now; - new_parent.info.lock().ctime = now; + let mut info = new_parent.info.lock(); + + info.size += 1; + info.mtime = now; + info.ctime = now; } // Remove from old directory @@ -389,7 +379,7 @@ impl InodeDirOps for DirectoryInode { // Add new entry new_entries.push((new_name, old_ino)); - let mut self_info = self.info.lock(); + let mut self_info = inode.info.lock(); self_info.size -= 1; self_info.mtime = now; self_info.ctime = now; @@ -398,17 +388,16 @@ impl InodeDirOps for DirectoryInode { dcache::d_exchange(old_dentry, new_dentry).await; Ok(()) } -} - -impl InodeFileOps for DirectoryInode { - async fn chmod(&self, perm: Permission) -> KResult<()> { - let _sb = self.sb.get()?; - { - let mut info = self.info.lock(); - info.perm = perm; - info.ctime = Instant::now(); - } + async fn chmod( + &self, + _sb: SbUse, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); Ok(()) } diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index 624112e0..a1755908 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -1,39 +1,26 @@ +use alloc::collections::btree_map::BTreeMap; use alloc::sync::Arc; -use eonix_mm::paging::PAGE_SIZE; -use eonix_sync::{RwLock, Spin}; - -use crate::{ - io::{Buffer, Stream}, - kernel::{ - mem::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}, - timer::Instant, - vfs::{ - inode::{Ino, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse, WriteOffset}, - types::{DeviceId, Format, Mode, Permission}, - SbRef, - }, - }, - prelude::KResult, -}; use super::TmpFs; +use crate::io::{Buffer, Stream}; +use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse, WriteOffset}; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse}; +use crate::prelude::KResult; -pub struct FileInode { - sb: SbRef, - ino: Ino, - info: Spin, - rwsem: RwLock<()>, - pages: PageCache, -} +pub struct FileInode; impl FileInode { - pub fn new(ino: Ino, sb: SbRef, size: usize, perm: Permission) -> InodeUse { + pub fn new(ino: Ino, sb: SbRef, size: usize, perm: Permission) -> InodeUse { let now = Instant::now(); - InodeUse::new_cyclic(|weak| Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + Format::REG, + InodeInfo { size: size as _, nlink: 1, uid: 0, @@ -42,60 +29,34 @@ impl FileInode { atime: now, ctime: now, mtime: now, - }), - rwsem: RwLock::new(()), - pages: PageCache::new(weak.clone() as _), - }) - } -} - -impl PageCacheBackendOps for FileInode { - async fn read_page(&self, _cache_page: &mut CachePage, _offset: usize) -> KResult { - Ok(PAGE_SIZE) - } - - async fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult { - Ok(PAGE_SIZE) - } - - fn size(&self) -> usize { - self.info.lock().size as usize + }, + Self, + ) } } impl InodeOps for FileInode { type SuperBlock = TmpFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - Format::REG - } - - fn info(&self) -> &Spin { - &self.info - } - - fn super_block(&self) -> &SbRef { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.pages) - } -} - -impl InodeDirOps for FileInode {} -impl InodeFileOps for FileInode { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - let _lock = self.rwsem.read().await; - self.pages.read(buffer, offset).await + async fn read( + &self, + _: SbUse, + inode: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult { + let _lock = inode.rwsem.read().await; + inode.get_page_cache().read(buffer, offset).await } - async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult { - let _lock = self.rwsem.write().await; + async fn write( + &self, + _: SbUse, + inode: &InodeUse, + stream: &mut dyn Stream, + offset: WriteOffset<'_>, + ) -> KResult { + let _lock = inode.rwsem.write().await; let mut store_new_end = None; let offset = match offset { @@ -104,74 +65,131 @@ impl InodeFileOps for FileInode { store_new_end = Some(end); // `info.size` won't change since we are holding the write lock. - self.info.lock().size as usize + inode.info.lock().size as usize } }; - let wrote = self.pages.write(stream, offset).await?; + let page_cache = inode.get_page_cache(); + + if Arc::strong_count(&page_cache) == 1 { + // XXX: A temporary workaround here. Change this ASAP... + // Prevent the page cache from being dropped during the write. + let _ = Arc::into_raw(page_cache.clone()); + } + + let wrote = page_cache.write(stream, offset).await?; let cursor_end = offset + wrote; if let Some(store_end) = store_new_end { *store_end = cursor_end; } - { - let now = Instant::now(); - let mut info = self.info.lock(); - info.mtime = now; - info.ctime = now; - info.size = info.size.max(cursor_end as u64); - } - Ok(wrote) } - async fn truncate(&self, length: usize) -> KResult<()> { - let _lock = self.rwsem.write().await; + async fn truncate( + &self, + _: SbUse, + inode: &InodeUse, + length: usize, + ) -> KResult<()> { + let _lock = inode.rwsem.write().await; - self.pages.resize(length).await?; + let now = Instant::now(); + let mut info = inode.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = length as u64; - { - let now = Instant::now(); - let mut info = self.info.lock(); - info.mtime = now; - info.ctime = now; - info.size = length as u64; - } + Ok(()) + } + + async fn chmod( + &self, + _sb: SbUse, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + + info.perm = perm; + info.ctime = Instant::now(); + + Ok(()) + } + async fn read_page( + &self, + _: SbUse, + _: &InodeUse, + page: &mut CachePage, + _: PageOffset, + ) -> KResult<()> { + page.as_bytes_mut().fill(0); Ok(()) } - async fn chmod(&self, perm: Permission) -> KResult<()> { - let _sb = self.sb.get()?; + async fn write_page( + &self, + _: SbUse, + _: &InodeUse, + _: &mut CachePage, + _: PageOffset, + ) -> KResult<()> { + // XXX: actually we should refuse to do the writeback. + // think of a way to inform that of the page cache. + Ok(()) + } - { - let mut info = self.info.lock(); + async fn write_begin<'a>( + &self, + _: SbUse, + _: &InodeUse, + page_cache: &PageCache, + pages: &'a mut BTreeMap, + offset: usize, + _: usize, + ) -> KResult<&'a mut CachePage> { + // TODO: Remove dependency on `page_cache`. + page_cache + .get_page_locked(pages, PageOffset::from_byte_floor(offset)) + .await + } - info.perm = perm; - info.ctime = Instant::now(); - } + async fn write_end( + &self, + _: SbUse, + inode: &InodeUse, + _: &PageCache, + _: &mut BTreeMap, + offset: usize, + _: usize, + copied: usize, + ) -> KResult<()> { + let now = Instant::now(); + let mut info = inode.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = info.size.max((offset + copied) as u64); Ok(()) } } pub struct DeviceInode { - sb: SbRef, - ino: Ino, - info: Spin, is_block: bool, devid: DeviceId, } impl DeviceInode { - pub fn new(ino: Ino, sb: SbRef, mode: Mode, devid: DeviceId) -> InodeUse { + pub fn new(ino: Ino, sb: SbRef, mode: Mode, devid: DeviceId) -> InodeUse { let now = Instant::now(); - InodeUse::new(Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + mode.format(), + InodeInfo { size: 0, nlink: 1, uid: 0, @@ -180,76 +198,49 @@ impl DeviceInode { atime: now, ctime: now, mtime: now, - }), - is_block: mode.format() == Format::BLK, - devid, - }) + }, + Self { + is_block: mode.format() == Format::BLK, + devid, + }, + ) } } impl InodeOps for DeviceInode { type SuperBlock = TmpFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - if self.is_block { - Format::BLK - } else { - Format::CHR - } - } - - fn info(&self) -> &Spin { - &self.info - } - - fn super_block(&self) -> &SbRef { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } -} - -impl InodeDirOps for DeviceInode {} -impl InodeFileOps for DeviceInode { - async fn chmod(&self, perm: Permission) -> KResult<()> { - let _sb = self.sb.get()?; - - { - let mut info = self.info.lock(); - - info.perm = perm; - info.ctime = Instant::now(); - } + async fn chmod( + &self, + _sb: SbUse, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); Ok(()) } - fn devid(&self) -> KResult { + fn devid(&self, _: SbUse, _: &InodeUse) -> KResult { Ok(self.devid) } } pub struct SymlinkInode { - sb: SbRef, - ino: Ino, - info: Spin, target: Arc<[u8]>, } impl SymlinkInode { - pub fn new(ino: Ino, sb: SbRef, target: Arc<[u8]>) -> InodeUse { + pub fn new(ino: Ino, sb: SbRef, target: Arc<[u8]>) -> InodeUse { let now = Instant::now(); - InodeUse::new(Self { + InodeUse::new( sb, ino, - info: Spin::new(InodeInfo { + Format::LNK, + InodeInfo { size: target.len() as _, nlink: 1, uid: 0, @@ -258,39 +249,21 @@ impl SymlinkInode { atime: now, ctime: now, mtime: now, - }), - target, - }) + }, + Self { target }, + ) } } -impl InodeDirOps for SymlinkInode {} impl InodeOps for SymlinkInode { type SuperBlock = TmpFs; - fn ino(&self) -> Ino { - self.ino - } - - fn format(&self) -> Format { - Format::LNK - } - - fn info(&self) -> &Spin { - &self.info - } - - fn super_block(&self) -> &SbRef { - &self.sb - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } -} - -impl InodeFileOps for SymlinkInode { - async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { + async fn readlink( + &self, + _sb: SbUse, + _inode: &InodeUse, + buffer: &mut dyn Buffer, + ) -> KResult { buffer .fill(self.target.as_ref()) .map(|result| result.allow_partial()) diff --git a/src/fs/tmpfs/mod.rs b/src/fs/tmpfs/mod.rs index 2bef67b6..62a0dfc2 100644 --- a/src/fs/tmpfs/mod.rs +++ b/src/fs/tmpfs/mod.rs @@ -1,23 +1,20 @@ mod dir; mod file; -use crate::kernel::vfs::inode::{Ino, InodeUse}; -use crate::kernel::vfs::types::{DeviceId, Permission}; -use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; -use crate::{ - kernel::vfs::{ - dentry::Dentry, - mount::{register_filesystem, Mount, MountCreator}, - }, - prelude::*, -}; use alloc::sync::Arc; +use core::sync::atomic::{AtomicU64, Ordering}; + use async_trait::async_trait; -use core::sync::atomic::AtomicU64; -use core::sync::atomic::Ordering; use dir::DirectoryInode; use eonix_sync::Mutex; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{Ino, InodeUse}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; +use crate::kernel::vfs::types::{DeviceId, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::prelude::*; + pub struct TmpFs { next_ino: AtomicU64, rename_lock: Mutex<()>, @@ -30,7 +27,7 @@ impl TmpFs { Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) } - fn create() -> KResult<(SbUse, InodeUse)> { + fn create() -> KResult<(SbUse, InodeUse)> { let tmpfs = SbUse::new( SuperBlockInfo { io_blksize: 4096, diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index bfc826bf..f8b5dc0b 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -12,5 +12,5 @@ pub use access::PhysAccess; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; -pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackendOps}; +pub use page_cache::{CachePage, PageCache, PageOffset}; pub use paging::{Page, PageBuffer, PageExcl, PageExt}; diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index dcbeeb63..2891dad8 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -4,12 +4,12 @@ use core::cmp; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; -use eonix_mm::paging::{PAGE_SIZE, PFN}; +use eonix_mm::paging::PFN; use super::mm_list::EMPTY_PAGE; use super::{Mapping, Page, Permission}; -use crate::kernel::constants::EINVAL; -use crate::kernel::mem::{PageExcl, PageExt}; +use crate::kernel::mem::page_cache::PageOffset; +use crate::kernel::mem::{CachePage, PageExcl, PageExt}; use crate::prelude::KResult; #[derive(Debug)] @@ -141,59 +141,48 @@ impl MMArea { assert!(offset < file_mapping.length, "Offset out of range"); - let Some(page_cache) = file_mapping.file.page_cache() else { - panic!("Mapping file should have pagecache"); + let file_offset = file_mapping.offset + offset; + + let map_page = |page: &Page, cache_page: &CachePage| { + if !self.permission.write { + assert!(!write, "Write fault on read-only mapping"); + + *pfn = page.clone().into_raw(); + return; + } + + if self.is_shared { + // We don't process dirty flags in write faults. + // Simply assume that page will eventually be dirtied. + // So here we can set the dirty flag now. + cache_page.set_dirty(true); + attr.insert(PageAttribute::WRITE); + *pfn = page.clone().into_raw(); + return; + } + + if !write { + // Delay the copy-on-write until write fault happens. + attr.insert(PageAttribute::COPY_ON_WRITE); + *pfn = page.clone().into_raw(); + return; + } + + // XXX: Change this. Let's handle mapped pages before CoW pages. + // Nah, we are writing to a mapped private mapping... + let mut new_page = PageExcl::zeroed(); + new_page + .as_bytes_mut() + .copy_from_slice(page.lock().as_bytes()); + + attr.insert(PageAttribute::WRITE); + *pfn = new_page.into_page().into_raw(); }; - let file_offset = file_mapping.offset + offset; - let cnt_to_read = (file_mapping.length - offset).min(0x1000); - - page_cache - .with_page(file_offset, |page, cache_page| { - // Non-write faults: we find page in pagecache and do mapping - // Write fault: we need to care about shared or private mapping. - if !write { - // Bss is embarrassing in pagecache! - // We have to assume cnt_to_read < PAGE_SIZE all bss - if cnt_to_read < PAGE_SIZE { - let mut new_page = PageExcl::zeroed(); - - new_page.as_bytes_mut()[..cnt_to_read] - .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); - - *pfn = new_page.into_page().into_raw(); - } else { - *pfn = page.clone().into_raw(); - } - - if self.permission.write { - if self.is_shared { - // The page may will not be written, - // But we simply assume page will be dirty - cache_page.set_dirty(); - attr.insert(PageAttribute::WRITE); - } else { - attr.insert(PageAttribute::COPY_ON_WRITE); - } - } - } else { - if self.is_shared { - cache_page.set_dirty(); - *pfn = page.clone().into_raw(); - } else { - let mut new_page = PageExcl::zeroed(); - - new_page.as_bytes_mut()[..cnt_to_read] - .copy_from_slice(&page.lock().as_bytes()[..cnt_to_read]); - - *pfn = new_page.into_page().into_raw(); - } - - attr.insert(PageAttribute::WRITE); - } - }) - .await? - .ok_or(EINVAL)?; + file_mapping + .page_cache + .with_page(PageOffset::from_byte_floor(file_offset), map_page) + .await?; attr.insert(PageAttribute::PRESENT); attr.remove(PageAttribute::MAPPED); diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 5446ae42..2b837ae7 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,9 +1,12 @@ -use crate::kernel::vfs::inode::{Inode, InodeUse}; +use alloc::sync::Arc; + use eonix_mm::paging::PAGE_SIZE; +use crate::kernel::mem::PageCache; + #[derive(Debug, Clone)] pub struct FileMapping { - pub file: InodeUse, + pub page_cache: Arc, /// Offset in the file, aligned to 4KB boundary. pub offset: usize, /// Length of the mapping. Exceeding part will be zeroed. @@ -19,10 +22,10 @@ pub enum Mapping { } impl FileMapping { - pub fn new(file: InodeUse, offset: usize, length: usize) -> Self { + pub fn new(page_cache: Arc, offset: usize, length: usize) -> Self { assert_eq!(offset & (PAGE_SIZE - 1), 0); Self { - file, + page_cache, offset, length, } @@ -30,10 +33,10 @@ impl FileMapping { pub fn offset(&self, offset: usize) -> Self { if self.length <= offset { - Self::new(self.file.clone(), self.offset + self.length, 0) + Self::new(self.page_cache.clone(), self.offset + self.length, 0) } else { Self::new( - self.file.clone(), + self.page_cache.clone(), self.offset + offset, self.length - offset, ) diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index 6f14583d..7aac141d 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -1,11 +1,12 @@ -use super::{MMList, VAddr}; -use crate::kernel::task::Thread; use eonix_hal::mm::flush_tlb; use eonix_hal::traits::fault::PageFaultErrorCode; use eonix_mm::address::{Addr as _, AddrOps as _, VRange}; use eonix_mm::paging::PAGE_SIZE; use posix_types::signal::Signal; +use super::{MMList, VAddr}; +use crate::kernel::task::Thread; + #[repr(C)] struct FixEntry { start: u64, diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index 9dce4567..37344fc2 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -4,9 +4,9 @@ mod zones; use core::sync::atomic::Ordering; use buddy_allocator::BuddyAllocator; -use eonix_mm::address::{AddrOps as _, PRange}; +use eonix_mm::address::PRange; use eonix_mm::paging::{ - GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, PFN, + GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, }; use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 0baa7b9a..074f82c7 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -32,15 +32,9 @@ impl SlabPageData { } } -#[derive(Clone, Copy)] -struct PageCacheData { - valid_size: usize, -} - #[repr(C)] union PageData { slab: SlabPageData, - page_cache: PageCacheData, } pub struct RawPage { @@ -245,27 +239,16 @@ impl SlabPage for RawPage { } impl PageCacheRawPage for RawPagePtr { - fn valid_size(&self) -> &mut usize { - unsafe { - // SAFETY: The caller ensures that the page is in some page cache. - &mut self.as_mut().shared_data.page_cache.valid_size - } - } - fn is_dirty(&self) -> bool { self.flags().has(PageFlags::DIRTY) } - fn clear_dirty(&self) { - self.flags().clear(PageFlags::DIRTY); - } - - fn set_dirty(&self) { - self.flags().set(PageFlags::DIRTY); - } - - fn cache_init(&self) { - self.as_mut().shared_data.page_cache = PageCacheData { valid_size: 0 }; + fn set_dirty(&self, dirty: bool) { + if dirty { + self.flags().set(PageFlags::DIRTY); + } else { + self.flags().clear(PageFlags::DIRTY); + } } } diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 6a1c04ca..214c65a5 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,26 +1,27 @@ -use alloc::boxed::Box; -use alloc::collections::btree_map::BTreeMap; -use alloc::sync::Weak; +use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; use core::mem::ManuallyDrop; -use align_ext::AlignExt; -use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess}; use eonix_mm::paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::Mutex; -use super::paging::AllocZeroed; use super::Page; -use crate::io::{Buffer, FillResult, Stream}; +use crate::io::{Buffer, Stream}; +use crate::kernel::constants::EINVAL; use crate::kernel::mem::page_alloc::RawPagePtr; +use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; use crate::GlobalPageAlloc; +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct PageOffset(usize); + pub struct PageCache { - pages: Mutex>, - backend: Weak, + pages: Mutex>, + inode: InodeUse, } unsafe impl Send for PageCache {} @@ -30,70 +31,46 @@ unsafe impl Sync for PageCache {} pub struct CachePage(RawPagePtr); unsafe impl Send for CachePage {} +unsafe impl Sync for CachePage {} -impl Buffer for CachePage { - fn total(&self) -> usize { - PAGE_SIZE +impl PageOffset { + pub const fn from_byte_floor(offset: usize) -> Self { + Self(offset >> PAGE_SIZE_BITS) } - fn wrote(&self) -> usize { - self.valid_size() + pub const fn from_byte_ceil(offset: usize) -> Self { + Self((offset + PAGE_SIZE - 1) >> PAGE_SIZE_BITS) } - fn fill(&mut self, data: &[u8]) -> KResult { - let valid_size = self.valid_size(); - let available = &mut self.all_mut()[valid_size..]; - if available.len() == 0 { - return Ok(FillResult::Full); - } - - let len = core::cmp::min(data.len(), available.len()); - available[..len].copy_from_slice(&data[..len]); + pub fn iter_till(self, end: PageOffset) -> impl Iterator { + (self.0..end.0).map(PageOffset) + } - *self.0.valid_size() += len; + pub fn page_count(self) -> usize { + self.0 + } - if len < data.len() { - Ok(FillResult::Partial(len)) - } else { - Ok(FillResult::Done(len)) - } + pub fn byte_count(self) -> usize { + self.page_count() * PAGE_SIZE } } impl CachePage { pub fn new() -> Self { - let page = GlobalPageAlloc.alloc().unwrap(); - page.cache_init(); - Self(page) - } - - pub fn new_zeroed() -> Self { - let page = Page::zeroed(); - let raw_page_ptr = RawPagePtr::from(page.into_raw()); - - raw_page_ptr.cache_init(); - Self(raw_page_ptr) - } - - pub fn valid_size(&self) -> usize { - *self.0.valid_size() + Self(GlobalPageAlloc.alloc().unwrap()) } - pub fn set_valid_size(&mut self, valid_size: usize) { - *self.0.valid_size() = valid_size; - } - - pub fn all(&self) -> &[u8] { + pub fn as_bytes(&self) -> &[u8] { unsafe { core::slice::from_raw_parts( - // SAFETY: The page is exclusively owned by us, so we can safely access its data. + // SAFETY: The page is owned by us, so we can safely access its data. ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), PAGE_SIZE, ) } } - pub fn all_mut(&mut self) -> &mut [u8] { + pub fn as_bytes_mut(&mut self) -> &mut [u8] { unsafe { core::slice::from_raw_parts_mut( // SAFETY: The page is exclusively owned by us, so we can safely access its data. @@ -103,306 +80,171 @@ impl CachePage { } } - pub fn valid_data(&self) -> &[u8] { - &self.all()[..self.valid_size()] - } - pub fn is_dirty(&self) -> bool { self.0.is_dirty() } - pub fn set_dirty(&self) { - self.0.set_dirty(); + pub fn set_dirty(&self, dirty: bool) { + self.0.set_dirty(dirty); } - pub fn clear_dirty(&self) { - self.0.clear_dirty(); + pub fn get_page(&self) -> Page { + unsafe { Page::with_raw(PFN::from(self.0), |page| page.clone()) } } } impl PageCache { - pub fn new(backend: Weak) -> Self { + pub fn new(inode: InodeUse) -> Self { Self { pages: Mutex::new(BTreeMap::new()), - backend: backend, + inode, } } - pub async fn read(&self, buffer: &mut dyn Buffer, mut offset: usize) -> KResult { - let mut pages = self.pages.lock().await; - let size = self.backend.upgrade().unwrap().size(); - - loop { - if offset >= size { - break; - } - let page_id = offset >> PAGE_SIZE_BITS; - let page = pages.get(&page_id); - - match page { - Some(page) => { - let inner_offset = offset % PAGE_SIZE; - let available_in_file = size.saturating_sub(offset); - - // TODO: still cause unnecessary IO if valid_size < PAGESIZE - // and fill result is Done - let page_data = &page.valid_data()[inner_offset..]; - let read_size = page_data.len().min(available_in_file); - - if read_size == 0 - || buffer.fill(&page_data[..read_size])?.should_stop() - || buffer.available() == 0 - { - break; - } - offset += read_size; - } - None => { + pub fn get_page_locked<'a>( + &self, + pages: &'a mut BTreeMap, + pgoff: PageOffset, + ) -> impl Future> + Send + use<'_, 'a> { + async move { + match pages.entry(pgoff) { + Entry::Occupied(ent) => Ok(ent.into_mut()), + Entry::Vacant(vacant_entry) => { let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) - .await?; - pages.insert(page_id, new_page); + self.inode.read_page(&mut new_page, pgoff).await?; + + Ok(vacant_entry.insert(new_page)) } } } + } - Ok(buffer.wrote()) + fn len(&self) -> usize { + self.inode.info.lock().size as usize } - pub async fn write(&self, stream: &mut dyn Stream, mut offset: usize) -> KResult { + // TODO: Remove this. + pub async fn with_page( + &self, + pgoff: PageOffset, + func: impl FnOnce(&Page, &CachePage), + ) -> KResult<()> { let mut pages = self.pages.lock().await; - let old_size = self.backend.upgrade().unwrap().size(); - let mut wrote = 0; - - loop { - let page_id = offset >> PAGE_SIZE_BITS; - let page = pages.get_mut(&page_id); - - match page { - Some(page) => { - let inner_offset = offset % PAGE_SIZE; - let cursor_end = match stream.poll_data(&mut page.all_mut()[inner_offset..])? { - Some(buf) => { - wrote += buf.len(); - inner_offset + buf.len() - } - None => { - break; - } - }; - - if page.valid_size() < cursor_end { - page.set_valid_size(cursor_end); - } - page.set_dirty(); - offset += PAGE_SIZE - inner_offset; - } - None => { - let new_page = if (offset >> PAGE_SIZE_BITS) > (old_size >> PAGE_SIZE_BITS) { - let new_page = CachePage::new_zeroed(); - new_page - } else { - let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE)) - .await?; - new_page - }; - - pages.insert(page_id, new_page); - } - } + if pgoff > PageOffset::from_byte_ceil(self.len()) { + return Err(EINVAL); } - Ok(wrote) - } + let cache_page = self.get_page_locked(&mut pages, pgoff).await?; - pub async fn fsync(&self) -> KResult<()> { - let pages = self.pages.lock().await; - for (page_id, page) in pages.iter() { - if page.is_dirty() { - self.backend - .upgrade() - .unwrap() - .write_page(&mut CachePageStream::new(*page), page_id << PAGE_SIZE_BITS) - .await?; - page.clear_dirty(); - } + unsafe { + let page = ManuallyDrop::new(Page::from_raw_unchecked(PFN::from(cache_page.0))); + + func(&page, cache_page); } + Ok(()) } - // This function is used for extend write or truncate - pub async fn resize(&self, new_size: usize) -> KResult<()> { + pub async fn read(&self, buffer: &mut dyn Buffer, mut offset: usize) -> KResult { let mut pages = self.pages.lock().await; - let old_size = self.backend.upgrade().unwrap().size(); + let total_len = self.len(); - if new_size < old_size { - let begin = new_size.align_down(PAGE_SIZE) >> PAGE_SIZE_BITS; - let end = old_size.align_up(PAGE_SIZE) >> PAGE_SIZE_BITS; + if offset >= total_len { + return Ok(0); + } - for page_id in begin..end { - pages.remove(&page_id); - } - } else if new_size > old_size { - let begin = old_size.align_down(PAGE_SIZE) >> PAGE_SIZE_BITS; - let end = new_size.align_up(PAGE_SIZE) >> PAGE_SIZE_BITS; + let pgoff_start = PageOffset::from_byte_floor(offset); + let pgoff_end = PageOffset::from_byte_ceil(total_len); - pages.remove(&begin); + for pgoff in pgoff_start.iter_till(pgoff_end) { + let page = self.get_page_locked(&mut pages, pgoff).await?; - for page_id in begin..end { - let mut new_page = CachePage::new_zeroed(); + let end_offset = (offset + PAGE_SIZE) / PAGE_SIZE * PAGE_SIZE; + let real_end = end_offset.min(total_len); - if page_id != end - 1 { - new_page.set_valid_size(PAGE_SIZE); - } else { - new_page.set_valid_size(new_size % PAGE_SIZE); - } - new_page.set_dirty(); - pages.insert(page_id, new_page); + let inner_offset = offset % PAGE_SIZE; + let data_len = real_end - offset; + + if buffer + .fill(&page.as_bytes()[inner_offset..inner_offset + data_len])? + .should_stop() + || buffer.available() == 0 + { + break; } + + offset = real_end; } - Ok(()) + Ok(buffer.wrote()) } - pub async fn with_page(&self, offset: usize, func: F) -> KResult> - where - F: FnOnce(&Page, &CachePage) -> O, - { - let offset_aligin = offset.align_down(PAGE_SIZE); - let page_id = offset_aligin >> PAGE_SIZE_BITS; - let size = self.backend.upgrade().unwrap().size(); - - if offset_aligin > size { - return Ok(None); - } - + pub async fn write(&self, stream: &mut dyn Stream, mut offset: usize) -> KResult { let mut pages = self.pages.lock().await; + let mut total_written = 0; - let raw_page_ptr = match pages.get(&page_id) { - Some(CachePage(raw_page_ptr)) => *raw_page_ptr, - None => { - let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset_aligin) - .await?; - pages.insert(page_id, new_page); - new_page.0 + loop { + let end_offset = (offset + PAGE_SIZE) / PAGE_SIZE * PAGE_SIZE; + let len = end_offset - offset; + + // TODO: Rewrite to return a write state object. + let page = self + .inode + .write_begin(self, &mut pages, offset, len) + .await?; + + let inner_offset = offset % PAGE_SIZE; + let written = stream + .poll_data(&mut page.as_bytes_mut()[inner_offset..])? + .map(|b| b.len()) + .unwrap_or(0); + + page.set_dirty(true); + self.inode + .write_end(self, &mut pages, offset, len, written) + .await?; + + if written == 0 { + break; } - }; - - unsafe { - let page = ManuallyDrop::new(Page::from_raw_unchecked(PFN::from(raw_page_ptr))); - Ok(Some(func(&page, &CachePage(raw_page_ptr)))) + total_written += written; + offset += written; } - } -} -pub struct CachePageStream { - page: CachePage, - cur: usize, -} - -impl CachePageStream { - pub fn new(page: CachePage) -> Self { - Self { page, cur: 0 } + Ok(total_written) } -} - -impl Stream for CachePageStream { - fn poll_data<'a>(&mut self, buf: &'a mut [u8]) -> KResult> { - if self.cur >= self.page.valid_size() { - return Ok(None); - } - - let page_data = &self.page.all()[self.cur..self.page.valid_size()]; - let to_read = buf.len().min(page_data.len()); - buf[..to_read].copy_from_slice(&page_data[..to_read]); - self.cur += to_read; + pub async fn fsync(&self) -> KResult<()> { + let mut pages = self.pages.lock().await; - Ok(Some(&mut buf[..to_read])) - } + for (&pgoff, page) in pages.iter_mut() { + if !page.is_dirty() { + continue; + } - fn ignore(&mut self, len: usize) -> KResult> { - if self.cur >= self.page.valid_size() { - return Ok(None); + self.inode.write_page(page, pgoff).await?; + page.set_dirty(false); } - let to_ignore = len.min(self.page.valid_size() - self.cur); - self.cur += to_ignore; - Ok(Some(to_ignore)) + Ok(()) } } -// with this trait, "page cache" and "block cache" are unified, -// for fs, offset is file offset (floor algin to PAGE_SIZE) -// for blkdev, offset is block idx (floor align to PAGE_SIZE / BLK_SIZE) -// Oh no, this would make unnecessary cache -pub trait PageCacheBackendOps: Sized { - fn read_page( - &self, - page: &mut CachePage, - offset: usize, - ) -> impl Future> + Send; - - fn write_page( - &self, - page: &mut CachePageStream, - offset: usize, - ) -> impl Future> + Send; - - fn size(&self) -> usize; -} - -#[async_trait] -pub trait PageCacheBackend: Send + Sync { - async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult; - async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult; - fn size(&self) -> usize; -} - -#[async_trait] -impl PageCacheBackend for T -where - T: PageCacheBackendOps + Send + Sync + 'static, -{ - async fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult { - self.read_page(page, offset).await - } - - async fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult { - self.write_page(page, offset).await - } - - fn size(&self) -> usize { - self.size() +impl core::fmt::Debug for PageCache { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PageCache").finish() } } pub trait PageCacheRawPage: RawPage { - fn valid_size(&self) -> &mut usize; - fn is_dirty(&self) -> bool; - - fn set_dirty(&self); - - fn clear_dirty(&self); - - fn cache_init(&self); + fn set_dirty(&self, dirty: bool); } impl Drop for PageCache { fn drop(&mut self) { - let _ = self.fsync(); + // TODO: Write back dirty pages... + // let _ = self.fsync(); } } diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index 4cb7908c..825440ef 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -1,20 +1,15 @@ -use super::FromSyscallArg; -use crate::kernel::constants::{EBADF, EINVAL}; -use crate::kernel::mem::FileMapping; -use crate::kernel::task::Thread; -use crate::kernel::vfs::filearray::FD; -use crate::{ - kernel::{ - constants::{UserMmapFlags, UserMmapProtocol}, - mem::{Mapping, Permission}, - }, - prelude::*, -}; use align_ext::AlignExt; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr}; use eonix_mm::paging::PAGE_SIZE; use posix_types::syscall_no::*; +use super::FromSyscallArg; +use crate::kernel::constants::{UserMmapFlags, UserMmapProtocol, EBADF, EINVAL}; +use crate::kernel::mem::{FileMapping, Mapping, Permission}; +use crate::kernel::task::Thread; +use crate::kernel::vfs::filearray::FD; +use crate::prelude::*; + impl FromSyscallArg for UserMmapProtocol { fn from_arg(value: usize) -> UserMmapProtocol { UserMmapProtocol::from_bits_truncate(value as u32) @@ -74,7 +69,7 @@ async fn do_mmap2( .get_inode()? .ok_or(EBADF)?; - Mapping::File(FileMapping::new(file, pgoffset, len)) + Mapping::File(FileMapping::new(file.get_page_cache(), pgoffset, len)) }; let permission = Permission { diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index 36d139a8..9f8aa166 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -1,27 +1,22 @@ +use alloc::ffi::CString; +use alloc::sync::Arc; +use alloc::vec::Vec; + +use align_ext::AlignExt; +use eonix_mm::address::{Addr, AddrOps as _, VAddr}; +use eonix_mm::paging::PAGE_SIZE; +use xmas_elf::header::{self, Class, HeaderPt1, Machine_}; +use xmas_elf::program::{self, ProgramHeader32, ProgramHeader64}; + use super::{LoadInfo, ELF_MAGIC}; -use crate::io::UninitBuffer; +use crate::io::{ByteBuffer, UninitBuffer}; +use crate::kernel::constants::ENOEXEC; +use crate::kernel::mem::{FileMapping, MMList, Mapping, Permission}; use crate::kernel::task::loader::aux_vec::{AuxKey, AuxVec}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::FsContext; use crate::path::Path; -use crate::{ - io::ByteBuffer, - kernel::{ - constants::ENOEXEC, - mem::{FileMapping, MMList, Mapping, Permission}, - vfs::{dentry::Dentry, FsContext}, - }, - prelude::*, -}; -use align_ext::AlignExt; -use alloc::vec::Vec; -use alloc::{ffi::CString, sync::Arc}; -use eonix_mm::{ - address::{Addr, AddrOps as _, VAddr}, - paging::PAGE_SIZE, -}; -use xmas_elf::{ - header::{self, Class, HeaderPt1, Machine_}, - program::{self, ProgramHeader32, ProgramHeader64}, -}; +use crate::prelude::*; const INIT_STACK_SIZE: usize = 0x80_0000; @@ -366,7 +361,7 @@ impl Elf { vmap_start, file_len, Mapping::File(FileMapping::new( - self.file.get_inode()?, + self.file.get_inode()?.get_page_cache(), file_offset, real_file_length, )), @@ -376,16 +371,27 @@ impl Elf { .await?; } - if vmem_len > file_len { - mm_list - .mmap_fixed( - vmap_start + file_len, - vmem_len - file_len, - Mapping::Anonymous, - permission, - false, - ) - .await?; + if vmem_vaddr_end > load_vaddr_end { + if load_vaddr_end.page_offset() != 0 { + let mut zero_len = PAGE_SIZE - load_vaddr_end.page_offset(); + zero_len = zero_len.min(vmem_vaddr_end - load_vaddr_end); + + mm_list + .access_mut(load_vaddr_end, zero_len, |_, data| data.fill(0)) + .await?; + } + + if vmem_len - file_len > 0 { + mm_list + .mmap_fixed( + vmap_start + file_len, + vmem_len - file_len, + Mapping::Anonymous, + permission, + false, + ) + .await?; + } } Ok(vmap_start + vmem_len) diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index c1eb8cb8..22760de9 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -1,35 +1,31 @@ pub mod dcache; mod walk; -use core::{ - cell::UnsafeCell, - fmt, - hash::{BuildHasher, BuildHasherDefault, Hasher}, - sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}, -}; - use alloc::sync::Arc; +use core::cell::UnsafeCell; +use core::fmt; +use core::hash::{BuildHasher, BuildHasherDefault, Hasher}; +use core::sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}; + use arcref::AsArcRef; use eonix_sync::LazyLock; use pointers::BorrowedArc; -use posix_types::{namei::RenameFlags, open::OpenFlags, result::PosixError, stat::StatX}; - -use crate::{ - hash::KernelHasher, - io::Buffer, - io::Stream, - kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE}, - kernel::{block::BlockDevice, CharDevice}, - path::Path, - prelude::*, - rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}, -}; - -use super::{ - inode::{Ino, Inode, InodeUse, RenameData, WriteOffset}, - types::{DeviceId, Format, Mode, Permission}, - FsContext, -}; +use posix_types::namei::RenameFlags; +use posix_types::open::OpenFlags; +use posix_types::result::PosixError; +use posix_types::stat::StatX; + +use super::inode::{Ino, InodeUse, RenameData, WriteOffset}; +use super::types::{DeviceId, Format, Mode, Permission}; +use super::FsContext; +use crate::hash::KernelHasher; +use crate::io::{Buffer, Stream}; +use crate::kernel::block::BlockDevice; +use crate::kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE}; +use crate::kernel::CharDevice; +use crate::path::Path; +use crate::prelude::*; +use crate::rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}; const D_INVALID: u8 = 0; const D_REGULAR: u8 = 1; @@ -56,7 +52,7 @@ enum DentryKind { /// [lookup()]: crate::kernel::vfs::inode::InodeDirOps::lookup struct AssociatedInode { kind: UnsafeCell>, - inode: UnsafeCell>>, + inode: UnsafeCell>, } /// # Safety @@ -181,15 +177,15 @@ impl Dentry { .map_or(core::ptr::null(), |parent| Arc::as_ptr(&parent)) } - pub fn fill(&self, file: InodeUse) { + pub fn fill(&self, file: InodeUse) { self.inode.store(file); } - pub fn inode(&self) -> Option> { + pub fn inode(&self) -> Option { self.inode.load().map(|(_, inode)| inode.clone()) } - pub fn get_inode(&self) -> KResult> { + pub fn get_inode(&self) -> KResult { self.inode().ok_or(ENOENT) } @@ -291,7 +287,7 @@ impl Dentry { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.format() { + match inode.format { Format::DIR => Err(EISDIR), Format::REG => inode.read(buffer, offset).await, Format::BLK => { @@ -309,7 +305,7 @@ impl Dentry { pub async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.format() { + match inode.format { Format::DIR => Err(EISDIR), Format::REG => inode.write(stream, offset).await, Format::BLK => Err(EINVAL), // TODO @@ -375,7 +371,7 @@ impl Dentry { } pub async fn chmod(&self, mode: Mode) -> KResult<()> { - self.get_inode()?.chmod(mode).await + self.get_inode()?.chmod(mode.perm()).await } pub async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { @@ -438,8 +434,8 @@ impl AssociatedInode { } } - fn store(&self, inode: InodeUse) { - let kind = match inode.format() { + fn store(&self, inode: InodeUse) { + let kind = match inode.format { Format::REG | Format::BLK | Format::CHR => DentryKind::Regular, Format::DIR => DentryKind::Directory, Format::LNK => DentryKind::Symlink, @@ -463,7 +459,7 @@ impl AssociatedInode { DentryKind::atomic_acq(&self.kind) } - fn load(&self) -> Option<(DentryKind, &InodeUse)> { + fn load(&self) -> Option<(DentryKind, &InodeUse)> { self.kind().map(|kind| unsafe { let inode = (&*self.inode.get()) .as_ref() diff --git a/src/kernel/vfs/dentry/walk.rs b/src/kernel/vfs/dentry/walk.rs index 3e401b4b..7b1060ac 100644 --- a/src/kernel/vfs/dentry/walk.rs +++ b/src/kernel/vfs/dentry/walk.rs @@ -1,33 +1,23 @@ -use core::{ - future::Future, - hash::{BuildHasher, BuildHasherDefault, Hasher}, - ops::Deref, - pin::Pin, -}; - -use alloc::{boxed::Box, sync::Arc}; +use alloc::boxed::Box; +use alloc::sync::Arc; +use core::future::Future; +use core::hash::{BuildHasher, BuildHasherDefault, Hasher}; +use core::ops::Deref; +use core::pin::Pin; + use arcref::{ArcRef, AsArcRef}; use posix_types::result::PosixError; -use crate::{ - hash::KernelHasher, - io::ByteBuffer, - kernel::{ - constants::ELOOP, - vfs::{ - inode::{Inode, InodeUse}, - FsContext, - }, - }, - path::{Path, PathComponent, PathIterator}, - prelude::KResult, - rcu::{rcu_read_lock, RCUReadLock}, -}; - -use super::{ - dcache::{self, DCacheItem}, - Dentry, DentryKind, -}; +use super::dcache::{self, DCacheItem}; +use super::{Dentry, DentryKind}; +use crate::hash::KernelHasher; +use crate::io::ByteBuffer; +use crate::kernel::constants::ELOOP; +use crate::kernel::vfs::inode::InodeUse; +use crate::kernel::vfs::FsContext; +use crate::path::{Path, PathComponent, PathIterator}; +use crate::prelude::KResult; +use crate::rcu::{rcu_read_lock, RCUReadLock}; struct DentryFind<'a, 'b> { parent: &'a Dentry, @@ -40,7 +30,7 @@ pub enum WalkResultRcu<'rcu, 'path> { Ok(ArcRef<'rcu, Dentry>), Symlink { symlink: ArcRef<'rcu, Dentry>, - inode: InodeUse, + inode: InodeUse, }, Miss { parent: ArcRef<'rcu, Dentry>, @@ -53,7 +43,7 @@ pub enum WalkResult { Ok(Arc), Symlink { symlink: Arc, - inode: InodeUse, + inode: InodeUse, }, } @@ -270,7 +260,7 @@ impl FsContext { pub async fn follow_symlink( &self, symlink: ArcRef<'_, Dentry>, - inode: &InodeUse, + inode: &InodeUse, nr_follows: u32, ) -> KResult> { let mut target = [0; 256]; @@ -288,7 +278,7 @@ impl FsContext { fn follow_symlink_boxed<'r, 'a: 'r, 'b: 'r, 'c: 'r>( &'a self, symlink: ArcRef<'b, Dentry>, - inode: &'c InodeUse, + inode: &'c InodeUse, nr_follows: u32, ) -> Pin>> + Send + 'r>> { Box::pin(self.follow_symlink(symlink, inode, nr_follows)) diff --git a/src/kernel/vfs/file/inode_file.rs b/src/kernel/vfs/file/inode_file.rs index 96526ee9..d302079c 100644 --- a/src/kernel/vfs/file/inode_file.rs +++ b/src/kernel/vfs/file/inode_file.rs @@ -1,23 +1,17 @@ -use super::{File, FileType, SeekOption}; -use crate::{ - io::{Buffer, BufferFill, Stream}, - kernel::{ - constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}, - vfs::{ - dentry::Dentry, - inode::{Inode, InodeUse, WriteOffset}, - types::Format, - }, - }, - prelude::KResult, -}; use alloc::sync::Arc; + use eonix_sync::Mutex; -use posix_types::{ - getdent::{UserDirent, UserDirent64}, - open::OpenFlags, - stat::StatX, -}; +use posix_types::getdent::{UserDirent, UserDirent64}; +use posix_types::open::OpenFlags; +use posix_types::stat::StatX; + +use super::{File, FileType, SeekOption}; +use crate::io::{Buffer, BufferFill, Stream}; +use crate::kernel::constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{InodeUse, WriteOffset}; +use crate::kernel::vfs::types::Format; +use crate::prelude::KResult; pub struct InodeFile { pub r: bool, @@ -34,7 +28,7 @@ impl InodeFile { pub fn new(dentry: Arc, flags: OpenFlags) -> File { // SAFETY: `dentry` used to create `InodeFile` is valid. // SAFETY: `mode` should never change with respect to the `S_IFMT` fields. - let format = dentry.inode().expect("dentry should be invalid").format(); + let format = dentry.inode().expect("dentry should be invalid").format; let (r, w, a) = flags.as_rwa(); @@ -98,7 +92,7 @@ impl InodeFile { } impl File { - pub fn get_inode(&self) -> KResult>> { + pub fn get_inode(&self) -> KResult> { if let FileType::Inode(inode_file) = &**self { Ok(Some(inode_file.dentry.get_inode()?)) } else { @@ -191,7 +185,7 @@ impl File { SeekOption::Set(n) => n, SeekOption::End(off) => { let inode = inode_file.dentry.get_inode()?; - let size = inode.info().lock().size as usize; + let size = inode.info.lock().size as usize; size.checked_add_signed(off).ok_or(EOVERFLOW)? } }; diff --git a/src/kernel/vfs/filearray.rs b/src/kernel/vfs/filearray.rs index 1862a3e1..609d969c 100644 --- a/src/kernel/vfs/filearray.rs +++ b/src/kernel/vfs/filearray.rs @@ -1,28 +1,23 @@ -use super::{ - file::{File, InodeFile, Pipe}, - types::{Format, Permission}, - Spin, TerminalFile, -}; -use crate::kernel::{ - constants::{ - EBADF, EISDIR, ENOTDIR, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, - }, - syscall::{FromSyscallArg, SyscallRetVal}, -}; -use crate::{ - kernel::{console::get_console, constants::ENXIO, vfs::dentry::Dentry, CharDevice}, - prelude::*, -}; use alloc::sync::Arc; -use intrusive_collections::{ - intrusive_adapter, rbtree::Entry, Bound, KeyAdapter, RBTree, RBTreeAtomicLink, -}; -use itertools::{ - FoldWhile::{Continue, Done}, - Itertools, -}; + +use intrusive_collections::rbtree::Entry; +use intrusive_collections::{intrusive_adapter, Bound, KeyAdapter, RBTree, RBTreeAtomicLink}; +use itertools::FoldWhile::{Continue, Done}; +use itertools::Itertools; use posix_types::open::{FDFlags, OpenFlags}; +use super::file::{File, InodeFile, Pipe}; +use super::types::{Format, Permission}; +use super::{Spin, TerminalFile}; +use crate::kernel::console::get_console; +use crate::kernel::constants::{ + EBADF, EISDIR, ENOTDIR, ENXIO, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, +}; +use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::CharDevice; +use crate::prelude::*; + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct FD(u32); @@ -291,20 +286,19 @@ impl FileArray { let fdflag = flags.as_fd_flags(); let inode = dentry.get_inode()?; - let file_format = inode.format(); - match (flags.directory(), file_format, flags.write()) { + match (flags.directory(), inode.format, flags.write()) { (true, Format::DIR, _) => {} (true, _, _) => return Err(ENOTDIR), (false, Format::DIR, true) => return Err(EISDIR), _ => {} } - if flags.truncate() && flags.write() && file_format == Format::REG { + if flags.truncate() && flags.write() && inode.format == Format::REG { inode.truncate(0).await?; } - let file = if file_format == Format::CHR { + let file = if inode.format == Format::CHR { let device = CharDevice::get(inode.devid()?).ok_or(ENXIO)?; device.open(flags)? } else { diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs index 870a045d..5f0b98c2 100644 --- a/src/kernel/vfs/inode/inode.rs +++ b/src/kernel/vfs/inode/inode.rs @@ -1,52 +1,149 @@ use alloc::boxed::Box; -use core::{ - any::Any, - future::Future, - marker::Unsize, - ops::{CoerceUnsized, Deref}, - pin::Pin, -}; -use eonix_sync::Spin; - +use alloc::collections::btree_map::BTreeMap; use alloc::sync::{Arc, Weak}; -use async_trait::async_trait; +use core::any::Any; +use core::future::Future; +use core::ops::Deref; -use crate::{ - io::{Buffer, Stream}, - kernel::{ - constants::{EINVAL, EPERM}, - mem::PageCache, - timer::Instant, - vfs::{ - dentry::Dentry, - types::{DeviceId, Format, Mode, Permission}, - SbRef, SbUse, SuperBlock, - }, - }, - prelude::KResult, -}; +use async_trait::async_trait; +use eonix_sync::{RwLock, Spin}; use super::{Ino, RenameData, WriteOffset}; +use crate::io::{Buffer, Stream}; +use crate::kernel::constants::{EINVAL, EPERM}; +use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock}; +use crate::prelude::KResult; + +pub struct Inode { + pub ino: Ino, + pub format: Format, + pub info: Spin, + pub rwsem: RwLock<()>, + page_cache: Spin>, + sb: SbRef, + ops: Box, +} -pub trait InodeOps: Sized + Send + Sync + 'static { - type SuperBlock: SuperBlock + Sized; - - fn ino(&self) -> Ino; - fn format(&self) -> Format; - fn info(&self) -> &Spin; - - fn super_block(&self) -> &SbRef; +macro_rules! return_type { + ($type:ty) => { + $type + }; + () => { + () + }; +} - fn page_cache(&self) -> Option<&PageCache>; +macro_rules! define_inode_ops { + { + $( + $(#[$attr:meta])* + async fn $method:ident $(<$($lt:lifetime),+>)? (&self $(,)? $($name:ident : $type:ty $(,)?)*) $(-> $ret:ty)? + $body:block + )* + + --- + + $( + $(#[$attr1:meta])* + fn $method1:ident $(<$($lt1:lifetime),+>)? (&self $(,)? $($name1:ident : $type1:ty $(,)?)*) $(-> $ret1:ty)? + $body1:block + )* + } => { + #[allow(unused_variables)] + pub trait InodeOps: Sized + Send + Sync + 'static { + type SuperBlock: SuperBlock + Sized; + + $( + $(#[$attr])* + fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse, + inode: &InodeUse, + $($name : $type),* + ) -> impl Future + Send { + async { $body } + })* + + $( + $(#[$attr1])* + fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + $body1 + })* + } + + #[async_trait] + trait InodeOpsErased: Any + Send + Sync + 'static { + $(async fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse, + inode: &InodeUse, + $($name : $type),* + ) -> return_type!($($ret)?);)* + + $(fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?);)* + } + + #[async_trait] + impl InodeOpsErased for T + where + T: InodeOps, + { + $(async fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse, + inode: &InodeUse, + $($name : $type),* + ) -> return_type!($($ret)?) { + self.$method(sb.downcast(), inode, $($name),*).await + })* + + $(fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + self.$method1(sb.downcast(), inode, $($name1),*) + })* + } + + impl InodeUse { + $(pub async fn $method $(<$($lt),+>)? ( + &self, + $($name : $type),* + ) -> return_type!($($ret)?) { + self.ops.$method(self.sbget()?, self, $($name),*).await + })* + + $(pub fn $method1 $(<$($lt1),+>)? ( + &self, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + self.ops.$method1(self.sbget()?, self, $($name1),*) + })* + } + }; } -#[allow(unused_variables)] -pub trait InodeDirOps: InodeOps { - fn lookup( - &self, - dentry: &Arc, - ) -> impl Future>>> + Send { - async { Err(EPERM) } +define_inode_ops! { + // DIRECTORY OPERATIONS + + async fn lookup(&self, dentry: &Arc) -> KResult> { + Err(EPERM) } /// Read directory entries and call the given closure for each entry. @@ -55,255 +152,114 @@ pub trait InodeDirOps: InodeOps { /// - Ok(count): The number of entries read. /// - Ok(Err(err)): Some error occurred while calling the given closure. /// - Err(err): An error occurred while reading the directory. - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn readdir( + &self, offset: usize, - for_each_entry: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), - ) -> impl Future>> + Send + 'r { - async { Err(EPERM) } + for_each_entry: &mut (dyn (for<'a> FnMut(&'a [u8], Ino) -> KResult) + Send), + ) -> KResult> { + Err(EPERM) } - fn create( - &self, - at: &Arc, - mode: Permission, - ) -> impl Future> + Send { - async { Err(EPERM) } + async fn create(&self, at: &Arc, mode: Permission) -> KResult<()> { + Err(EPERM) } - fn mkdir(&self, at: &Dentry, mode: Permission) -> impl Future> + Send { - async { Err(EPERM) } + async fn mkdir(&self, at: &Dentry, mode: Permission) -> KResult<()> { + Err(EPERM) } - fn mknod( - &self, - at: &Dentry, - mode: Mode, - dev: DeviceId, - ) -> impl Future> + Send { - async { Err(EPERM) } + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + Err(EPERM) } - fn unlink(&self, at: &Arc) -> impl Future> + Send { - async { Err(EPERM) } + async fn unlink(&self, at: &Arc) -> KResult<()> { + Err(EPERM) } - fn symlink(&self, at: &Arc, target: &[u8]) -> impl Future> + Send { - async { Err(EPERM) } + async fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()> { + Err(EPERM) } - fn rename(&self, rename_data: RenameData<'_, '_>) -> impl Future> + Send { - async { Err(EPERM) } + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { + Err(EPERM) } -} -#[allow(unused_variables)] -pub trait InodeFileOps: InodeOps { - fn read( - &self, - buffer: &mut dyn Buffer, - offset: usize, - ) -> impl Future> + Send { - async { Err(EINVAL) } + // FILE OPERATIONS + + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + Err(EINVAL) } - fn read_direct( - &self, - buffer: &mut dyn Buffer, - offset: usize, - ) -> impl Future> + Send { - async { Err(EINVAL) } + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + Err(EINVAL) } - fn write( + async fn write( &self, stream: &mut dyn Stream, - offset: WriteOffset<'_>, - ) -> impl Future> + Send { - async { Err(EINVAL) } + offset: WriteOffset<'_> + ) -> KResult { + Err(EINVAL) } - fn write_direct( + async fn write_direct( &self, stream: &mut dyn Stream, offset: usize, - ) -> impl Future> + Send { - async { Err(EINVAL) } - } - - fn devid(&self) -> KResult { + ) -> KResult { Err(EINVAL) } - fn readlink(&self, buffer: &mut dyn Buffer) -> impl Future> + Send { - async { Err(EINVAL) } - } - - fn truncate(&self, length: usize) -> impl Future> + Send { - async { Err(EPERM) } - } - - fn chmod(&self, perm: Permission) -> impl Future> + Send { - async { Err(EPERM) } - } - - fn chown(&self, uid: u32, gid: u32) -> impl Future> + Send { - async { Err(EPERM) } - } -} - -#[async_trait] -pub trait InodeDir { - async fn lookup(&self, dentry: &Arc) -> KResult>>; - async fn create(&self, at: &Arc, perm: Permission) -> KResult<()>; - async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()>; - async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()>; - async fn unlink(&self, at: &Arc) -> KResult<()>; - async fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()>; - async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()>; - - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, - offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), - ) -> Pin>> + Send + 'r>>; -} - -#[async_trait] -pub trait InodeFile { - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult; - async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult; - async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult; - async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult; - fn devid(&self) -> KResult; - async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult; - async fn truncate(&self, length: usize) -> KResult<()>; - async fn chmod(&self, mode: Mode) -> KResult<()>; - async fn chown(&self, uid: u32, gid: u32) -> KResult<()>; -} - -pub trait Inode: InodeFile + InodeDir + Any + Send + Sync + 'static { - fn ino(&self) -> Ino; - fn format(&self) -> Format; - fn info(&self) -> &Spin; - - // TODO: This might should be removed... Temporary workaround for now. - fn page_cache(&self) -> Option<&PageCache>; - - fn sbref(&self) -> SbRef; - fn sbget(&self) -> KResult>; -} - -#[async_trait] -impl InodeFile for T -where - T: InodeFileOps, -{ - async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - self.read(buffer, offset).await - } - - async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - self.read_direct(buffer, offset).await - } - - async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult { - self.write(stream, offset).await - } - - async fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult { - self.write_direct(stream, offset).await - } - - fn devid(&self) -> KResult { - self.devid() - } - async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { - self.readlink(buffer).await + Err(EINVAL) } async fn truncate(&self, length: usize) -> KResult<()> { - self.truncate(length).await + Err(EPERM) } - async fn chmod(&self, mode: Mode) -> KResult<()> { - self.chmod(Permission::new(mode.non_format_bits())).await + async fn chmod(&self, perm: Permission) -> KResult<()> { + Err(EPERM) } async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - self.chown(uid, gid).await - } -} - -#[async_trait] -impl InodeDir for T -where - T: InodeDirOps, -{ - async fn lookup(&self, dentry: &Arc) -> KResult>> { - self.lookup(dentry).await - } - - async fn create(&self, at: &Arc, perm: Permission) -> KResult<()> { - self.create(at, perm).await - } - - async fn mkdir(&self, at: &Dentry, perm: Permission) -> KResult<()> { - self.mkdir(at, perm).await + Err(EPERM) } - async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { - self.mknod(at, mode, dev).await - } - - async fn unlink(&self, at: &Arc) -> KResult<()> { - self.unlink(at).await - } - - async fn symlink(&self, at: &Arc, target: &[u8]) -> KResult<()> { - self.symlink(at, target).await + // PAGE CACHE OPERATIONS + async fn read_page(&self, page: &mut CachePage, offset: PageOffset) -> KResult<()> { + Err(EINVAL) } - async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { - self.rename(rename_data).await + async fn write_page(&self, page: &mut CachePage, offset: PageOffset) -> KResult<()> { + Err(EINVAL) } - fn readdir<'r, 'a: 'r, 'b: 'r>( - &'a self, + async fn write_begin<'a>( + &self, + page_cache: &PageCache, + pages: &'a mut BTreeMap, offset: usize, - callback: &'b mut (dyn FnMut(&[u8], Ino) -> KResult + Send), - ) -> Pin>> + Send + 'r>> { - Box::pin(self.readdir(offset, callback)) - } -} - -impl Inode for T -where - T: InodeOps + InodeFile + InodeDir, -{ - fn ino(&self) -> Ino { - self.ino() - } - - fn format(&self) -> Format { - self.format() - } - - fn info(&self) -> &Spin { - self.info() + len: usize, + ) -> KResult<&'a mut CachePage> { + Err(EINVAL) } - fn page_cache(&self) -> Option<&PageCache> { - self.page_cache() + async fn write_end( + &self, + page_cache: &PageCache, + pages: &mut BTreeMap, + offset: usize, + len: usize, + copied: usize + ) -> KResult<()> { + Err(EINVAL) } - fn sbref(&self) -> SbRef { - self.super_block().clone() - } + --- - fn sbget(&self) -> KResult> { - self.super_block().get().map(|sb| sb as _) + fn devid(&self) -> KResult { + Err(EINVAL) } } @@ -321,64 +277,87 @@ pub struct InodeInfo { pub mtime: Instant, } -pub struct InodeUse(Arc) -where - I: Inode + ?Sized; +#[repr(transparent)] +pub struct InodeUse(Arc); + +impl InodeUse { + pub fn new( + sb: SbRef, + ino: Ino, + format: Format, + info: InodeInfo, + ops: impl InodeOps, + ) -> Self { + let inode = Inode { + sb, + ino, + format, + info: Spin::new(info), + rwsem: RwLock::new(()), + page_cache: Spin::new(Weak::new()), + ops: Box::new(ops), + }; -impl InodeUse -where - I: Inode, -{ - pub fn new(inode: I) -> Self { Self(Arc::new(inode)) } - pub fn new_cyclic(inode_func: impl FnOnce(&Weak) -> I) -> Self { - Self(Arc::new_cyclic(inode_func)) + pub fn sbref(&self) -> SbRef { + self.sb.clone() } -} -impl InodeUse -where - I: Inode + ?Sized, -{ - pub fn as_raw(&self) -> *const I { - Arc::as_ptr(&self.0) + pub fn sbget(&self) -> KResult> { + self.sb.get().map(|sb| sb as _) + } + + pub fn get_priv(&self) -> &I + where + I: InodeOps, + { + let ops = (&*self.ops) as &dyn Any; + + ops.downcast_ref() + .expect("InodeUse::private: InodeOps type mismatch") } -} -impl CoerceUnsized> for InodeUse -where - T: Inode + Unsize + ?Sized, - U: Inode + ?Sized, -{ + pub fn get_page_cache(&self) -> Arc { + if let Some(cache) = self.page_cache.lock().upgrade() { + return cache; + } + + // Slow path... + let cache = Arc::new(PageCache::new(self.clone())); + let mut page_cache = self.page_cache.lock(); + if let Some(cache) = page_cache.upgrade() { + return cache; + } + + *page_cache = Arc::downgrade(&cache); + cache + } } -impl Clone for InodeUse -where - I: Inode + ?Sized, -{ +impl Clone for InodeUse { fn clone(&self) -> Self { Self(self.0.clone()) } } -impl core::fmt::Debug for InodeUse -where - I: Inode + ?Sized, -{ +impl core::fmt::Debug for InodeUse { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "InodeUse(ino={})", self.ino()) + write!(f, "InodeUse(ino={})", self.ino) } } -impl Deref for InodeUse -where - I: Inode + ?Sized, -{ - type Target = I; +impl Deref for InodeUse { + type Target = Inode; fn deref(&self) -> &Self::Target { self.0.deref() } } + +impl PartialEq for InodeUse { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} diff --git a/src/kernel/vfs/inode/mod.rs b/src/kernel/vfs/inode/mod.rs index 08471ef3..6f4f041a 100644 --- a/src/kernel/vfs/inode/mod.rs +++ b/src/kernel/vfs/inode/mod.rs @@ -4,5 +4,5 @@ mod ops; mod statx; pub use ino::Ino; -pub use inode::{Inode, InodeDirOps, InodeFileOps, InodeInfo, InodeOps, InodeUse}; +pub use inode::{Inode, InodeInfo, InodeOps, InodeUse}; pub use ops::{RenameData, WriteOffset}; diff --git a/src/kernel/vfs/inode/ops.rs b/src/kernel/vfs/inode/ops.rs index baab1a80..7bf00ce5 100644 --- a/src/kernel/vfs/inode/ops.rs +++ b/src/kernel/vfs/inode/ops.rs @@ -1,9 +1,8 @@ use alloc::sync::Arc; +use super::inode::InodeUse; use crate::kernel::vfs::dentry::Dentry; -use super::{inode::InodeUse, Inode}; - pub enum WriteOffset<'end> { Position(usize), End(&'end mut usize), @@ -12,7 +11,7 @@ pub enum WriteOffset<'end> { pub struct RenameData<'a, 'b> { pub old_dentry: &'a Arc, pub new_dentry: &'b Arc, - pub new_parent: InodeUse, + pub new_parent: InodeUse, pub is_exchange: bool, pub no_replace: bool, } diff --git a/src/kernel/vfs/inode/statx.rs b/src/kernel/vfs/inode/statx.rs index a85ef3af..feb2a1b5 100644 --- a/src/kernel/vfs/inode/statx.rs +++ b/src/kernel/vfs/inode/statx.rs @@ -1,25 +1,17 @@ use posix_types::stat::StatX; -use crate::{ - kernel::{ - constants::{ - STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, STATX_MODE, STATX_MTIME, - STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, - }, - vfs::types::Format, - }, - prelude::KResult, +use super::inode::InodeUse; +use crate::kernel::constants::{ + STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, STATX_MODE, STATX_MTIME, + STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, }; +use crate::kernel::vfs::types::Format; +use crate::prelude::KResult; -use super::{inode::InodeUse, Inode}; - -impl InodeUse -where - I: Inode + ?Sized, -{ +impl InodeUse { pub fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { let sb = self.sbget()?; - let info = self.info().lock(); + let info = self.info.lock(); if mask & STATX_NLINK != 0 { stat.stx_nlink = info.nlink as _; @@ -53,10 +45,8 @@ where } if mask & STATX_TYPE != 0 { - let format = self.format(); - - stat.stx_mode |= format.as_raw() as u16; - if let Format::BLK | Format::CHR = format { + stat.stx_mode |= self.format.as_raw() as u16; + if let Format::BLK | Format::CHR = self.format { let devid = self.devid()?; stat.stx_rdev_major = devid.major as _; stat.stx_rdev_minor = devid.minor as _; @@ -65,7 +55,7 @@ where } if mask & STATX_INO != 0 { - stat.stx_ino = self.ino().as_raw(); + stat.stx_ino = self.ino.as_raw(); stat.stx_mask |= STATX_INO; } diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 213acae9..6b171f81 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -1,17 +1,17 @@ -use super::{ - dentry::{dcache, Dentry, DROOT}, - inode::{Inode, InodeUse}, - SbUse, SuperBlock, -}; -use crate::kernel::{ - constants::{EEXIST, ENODEV, ENOTDIR}, - task::block_on, -}; -use crate::prelude::*; -use alloc::{collections::btree_map::BTreeMap, string::ToString as _, sync::Arc}; +use alloc::collections::btree_map::BTreeMap; +use alloc::string::ToString as _; +use alloc::sync::Arc; + use async_trait::async_trait; use eonix_sync::LazyLock; +use super::dentry::{dcache, Dentry, DROOT}; +use super::inode::InodeUse; +use super::{SbUse, SuperBlock}; +use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; +use crate::kernel::task::block_on; +use crate::prelude::*; + pub const MS_RDONLY: u64 = 1 << 0; pub const MS_NOSUID: u64 = 1 << 1; pub const MS_NODEV: u64 = 1 << 2; @@ -39,11 +39,7 @@ pub struct Mount { } impl Mount { - pub fn new( - mp: &Dentry, - sb: SbUse, - root_inode: InodeUse, - ) -> KResult { + pub fn new(mp: &Dentry, sb: SbUse, root_inode: InodeUse) -> KResult { let root_dentry = Dentry::create(mp.parent().clone(), &mp.get_name()); root_dentry.fill(root_inode); diff --git a/src/kernel/vfs/superblock.rs b/src/kernel/vfs/superblock.rs index 85b28c01..e3be5cef 100644 --- a/src/kernel/vfs/superblock.rs +++ b/src/kernel/vfs/superblock.rs @@ -1,16 +1,15 @@ -use core::{ - marker::Unsize, - ops::{CoerceUnsized, Deref}, -}; - use alloc::sync::{Arc, Weak}; -use eonix_sync::RwLock; +use core::any::{Any, TypeId}; +use core::marker::Unsize; +use core::ops::{CoerceUnsized, Deref}; -use crate::{kernel::constants::EIO, prelude::KResult}; +use eonix_sync::RwLock; use super::types::DeviceId; +use crate::kernel::constants::EIO; +use crate::prelude::KResult; -pub trait SuperBlock: Send + Sync + 'static {} +pub trait SuperBlock: Any + Send + Sync + 'static {} #[derive(Debug, Clone)] pub struct SuperBlockInfo { @@ -83,6 +82,36 @@ where } } +impl SbUse +where + S: SuperBlock + ?Sized, +{ + pub fn get_ref(&self) -> SbRef { + SbRef(Arc::downgrade(&self.0)) + } +} + +impl SbUse { + /// Downcast the superblock to a specific type. + /// + /// # Panics + /// Panics if the downcast fails. + pub fn downcast(self) -> SbUse { + let Self(sb_complex) = self; + if (&sb_complex.backend as &dyn Any).type_id() != TypeId::of::() { + panic!("Downcast failed: type mismatch"); + } + + unsafe { + // SAFETY: We have checked the type above and unsized coercion says + // that Arc has the same layout as Arc if T: Unsize. + SbUse(Arc::from_raw( + Arc::into_raw(sb_complex) as *const SuperBlockComplex + )) + } + } +} + impl Clone for SbRef where S: SuperBlock + ?Sized, diff --git a/src/lib.rs b/src/lib.rs index 959cb29f..8457169c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,6 @@ #![feature(allocator_api)] #![feature(c_size_t)] #![feature(coerce_unsized)] -#![feature(concat_idents)] #![feature(arbitrary_self_types)] #![feature(get_mut_unchecked)] #![feature(macro_metavar_expr)] From 913f71f81970bf4dd7f30e60f2f0166eaf13785a Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 18 Jan 2026 00:56:10 +0800 Subject: [PATCH 010/153] mem: introduce new `Folio` abstraction - Remove struct `Page` and add `Folio`s to represent adjacent pages. - Introduce `Zone`s similar to that in Linux. Looking forward to removing all occurrence of `ArchPhysAccess` and so on. - Adapt existing code to new `Folio` interface in a dirty and rough way. Signed-off-by: greatbridf --- .vscode/tasks.json | 2 +- crates/buddy_allocator/src/lib.rs | 112 +++---- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 62 ++-- crates/eonix_hal/src/arch/riscv64/mm.rs | 51 +-- crates/eonix_hal/src/mm.rs | 38 ++- crates/eonix_mm/src/address/paddr.rs | 10 +- crates/eonix_mm/src/page_table.rs | 2 +- crates/eonix_mm/src/page_table/page_table.rs | 149 ++++----- crates/eonix_mm/src/page_table/pte.rs | 3 +- .../eonix_mm/src/page_table/pte_iterator.rs | 138 ++++---- crates/eonix_mm/src/page_table/walk.rs | 210 ++++++++++++ crates/eonix_mm/src/paging.rs | 8 +- crates/eonix_mm/src/paging/list.rs | 14 +- crates/eonix_mm/src/paging/page.rs | 313 +++--------------- crates/eonix_mm/src/paging/page_alloc.rs | 79 +---- crates/eonix_mm/src/paging/raw_page.rs | 47 --- crates/eonix_mm/src/paging/zone.rs | 7 +- crates/slab_allocator/src/lib.rs | 44 ++- src/driver/ahci/command.rs | 28 +- src/driver/ahci/command_table.rs | 7 +- src/driver/ahci/defs.rs | 6 +- src/driver/ahci/slot.rs | 10 +- src/driver/e1000e.rs | 19 +- src/driver/virtio/virtio_blk.rs | 8 +- src/fs/fat32.rs | 12 +- src/fs/fat32/file.rs | 24 -- src/fs/tmpfs/file.rs | 2 +- src/kernel/block.rs | 17 +- src/kernel/mem.rs | 4 +- src/kernel/mem/allocator.rs | 26 +- src/kernel/mem/folio.rs | 210 ++++++++++++ src/kernel/mem/mm_area.rs | 39 ++- src/kernel/mem/mm_list.rs | 64 ++-- src/kernel/mem/mm_list/page_table.rs | 40 +++ src/kernel/mem/page_alloc.rs | 91 ++--- src/kernel/mem/page_alloc/raw_page.rs | 136 ++------ src/kernel/mem/page_alloc/zones.rs | 22 +- src/kernel/mem/page_cache.rs | 97 +++--- src/kernel/mem/paging.rs | 122 +------ src/kernel/task/kernel_stack.rs | 25 +- src/kernel/vfs/file/mod.rs | 4 +- src/kernel_init.rs | 58 ++-- 42 files changed, 1116 insertions(+), 1244 deletions(-) create mode 100644 crates/eonix_mm/src/page_table/walk.rs delete mode 100644 crates/eonix_mm/src/paging/raw_page.rs delete mode 100644 src/fs/fat32/file.rs create mode 100644 src/kernel/mem/folio.rs create mode 100644 src/kernel/mem/mm_list/page_table.rs diff --git a/.vscode/tasks.json b/.vscode/tasks.json index a85ea0cf..e7a54791 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,7 +6,7 @@ { "label": "debug run riscv64", "type": "shell", - "command": "make srun ARCH=riscv64 IMG=/Volumes/oscomp/sdcard-rv.img", + "command": "make srun ARCH=riscv64", "isBackground": true, "problemMatcher": [ { diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index abe1ef7b..82a7d6c5 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -3,12 +3,12 @@ use core::hint::unreachable_unchecked; use eonix_mm::address::{AddrOps as _, PAddr, PRange}; -use eonix_mm::paging::{PageList, PageListSized, Zone, PFN}; +use eonix_mm::paging::{FolioList, FolioListSized, Zone, PFN}; const MAX_ORDER: u32 = 10; const AREAS: usize = const { MAX_ORDER as usize + 1 }; -pub trait BuddyPage: Sized + 'static { +pub trait BuddyFolio: Sized + 'static { fn pfn(&self) -> PFN; fn get_order(&self) -> u32; @@ -20,19 +20,19 @@ pub trait BuddyPage: Sized + 'static { struct FreeArea where - L: PageList, + L: FolioList, { free_list: L, count: usize, } -unsafe impl Send for FreeArea where L: PageList {} -unsafe impl Sync for FreeArea where L: PageList {} +unsafe impl Send for FreeArea where L: FolioList {} +unsafe impl Sync for FreeArea where L: FolioList {} pub struct BuddyAllocator where Z: Zone + 'static, - L: PageList, + L: FolioList, { zone: &'static Z, free_areas: [FreeArea; AREAS], @@ -41,8 +41,8 @@ where impl BuddyAllocator where Z: Zone + 'static, - Z::Page: BuddyPage, - L: PageListSized, + Z::Page: BuddyFolio, + L: FolioListSized, { pub const fn new(zone: &'static Z) -> Self { Self { @@ -52,13 +52,13 @@ where } } -impl BuddyAllocator +impl BuddyAllocator where - Z: Zone, - L: PageList, - P: BuddyPage + 'static, + Z: Zone, + L: FolioList, + F: BuddyFolio + 'static, { - pub fn create_pages(&mut self, start: PAddr, end: PAddr) { + pub fn create_folios(&mut self, start: PAddr, end: PAddr) { assert!( self.zone .contains_prange(PRange::new(start.ceil(), end.floor())), @@ -82,40 +82,40 @@ where unsafe { // SAFETY: We've checked that the range is within the zone above. - self.add_page_unchecked(pfn, order) + self.add_folio_unchecked(pfn, order) }; pfn = new_end_pfn; } } - fn add_page(&mut self, pfn: PFN, order: u32) { + fn add_folio(&mut self, pfn: PFN, order: u32) { let prange = PRange::from(PAddr::from(pfn)).grow(1 << (order + 12)); assert!( self.zone.contains_prange(prange), - "The given page is not within the zone." + "The given folio is not within the zone." ); unsafe { // SAFETY: Checks above. - self.add_page_unchecked(pfn, order); + self.add_folio_unchecked(pfn, order); } } - unsafe fn add_page_unchecked(&mut self, pfn: PFN, order: u32) { - let Some(page) = self.zone.get_page(pfn) else { + unsafe fn add_folio_unchecked(&mut self, pfn: PFN, order: u32) { + let Some(mut folio) = self.zone.get_page(pfn) else { unsafe { unreachable_unchecked() } }; unsafe { // SAFETY: The caller ensures that the page is unused. - let page_mut = &mut *page.get(); - self.free_areas[order as usize].add_page(page_mut, order); + let folio_mut = folio.as_mut(); + self.free_areas[order as usize].add_folio(folio_mut, order); } } - fn break_page(&mut self, page: &mut P, order: u32, target_order: u32) { - let pfn = page.pfn(); + fn break_folio(&mut self, folio: &mut F, order: u32, target_order: u32) { + let pfn = folio.pfn(); for order in (target_order..order).rev() { let buddy_pfn = pfn + (1 << order); @@ -123,50 +123,50 @@ where unsafe { // SAFETY: We got the page from `self.free_areas`. Checks are // done when we've put the page into the buddy system. - self.add_page_unchecked(buddy_pfn, order); + self.add_folio_unchecked(buddy_pfn, order); } } - page.set_order(target_order); + folio.set_order(target_order); } pub fn alloc_order(&mut self, order: u32) -> Option<&'static mut Z::Page> { for current_order in order..AREAS as u32 { - let Some(page) = self.free_areas[current_order as usize].get_free_page() else { + let Some(folio) = self.free_areas[current_order as usize].get_free_folio() else { continue; }; if current_order > order { - self.break_page(page, current_order, order); + self.break_folio(folio, current_order, order); } - return Some(page); + return Some(folio); } None } - pub unsafe fn dealloc(&mut self, page: &'static mut Z::Page) { - let mut pfn = page.pfn(); - let mut order = page.get_order(); + pub unsafe fn dealloc(&mut self, folio: &'static mut Z::Page) { + let mut pfn = folio.pfn(); + let mut order = folio.get_order(); assert!( - !page.is_buddy(), - "Trying to free a page that is already in the buddy system: {pfn:?}", + !folio.is_buddy(), + "Trying to free a folio that is already in the buddy system: {pfn:?}", ); while order < MAX_ORDER { let buddy_pfn = pfn.buddy_pfn(order); - let Some(buddy_page) = self.try_get_buddy(buddy_pfn, order) else { + let Some(buddy) = self.try_get_buddy(buddy_pfn, order) else { break; }; - self.free_areas[order as usize].remove_page(buddy_page); + self.free_areas[order as usize].remove_folio(buddy); pfn = pfn.combined_pfn(buddy_pfn); order += 1; } - self.add_page(pfn, order); + self.add_folio(pfn, order); } /// This function checks whether the given page is within our [`Zone`] and @@ -176,32 +176,32 @@ where /// - the buddy is within the same [`Zone`] as us. /// - the buddy is a free buddy (in some [`FreeArea`]) /// - the buddy has order [`order`] - fn try_get_buddy<'a>(&mut self, buddy_pfn: PFN, order: u32) -> Option<&'a mut P> { - let buddy_page = self.zone.get_page(buddy_pfn)?; + fn try_get_buddy<'a>(&mut self, buddy_pfn: PFN, order: u32) -> Option<&'a mut F> { + let mut buddy = self.zone.get_page(buddy_pfn)?; unsafe { // SAFETY: We just test whether the page is a buddy. - let buddy_page_ref = &*buddy_page.get(); + let buddy_ref = buddy.as_ref(); - if !buddy_page_ref.is_buddy() { + if !buddy_ref.is_buddy() { return None; } // Sad... - if buddy_page_ref.get_order() != order { + if buddy_ref.get_order() != order { return None; } // SAFETY: We have the mutable reference to the buddy allocator. // So all the pages within are exclusively accessible to us. - Some(&mut *buddy_page.get()) + Some(buddy.as_mut()) } } } impl FreeArea where - L: PageListSized, + L: FolioListSized, { const fn new() -> Self { Self { @@ -213,34 +213,34 @@ where impl FreeArea where - L: PageList, - L::Page: BuddyPage + 'static, + L: FolioList, + L::Folio: BuddyFolio + 'static, { - pub fn get_free_page(&mut self) -> Option<&'static mut L::Page> { - self.free_list.pop_head().map(|page| { + pub fn get_free_folio(&mut self) -> Option<&'static mut L::Folio> { + self.free_list.pop_head().map(|folio| { assert_ne!(self.count, 0, "Oops"); - page.set_buddy(false); + folio.set_buddy(false); self.count -= 1; - page + folio }) } - pub fn add_page(&mut self, page: &'static mut L::Page, order: u32) { - page.set_order(order); - page.set_buddy(true); + pub fn add_folio(&mut self, folio: &'static mut L::Folio, order: u32) { + folio.set_order(order); + folio.set_buddy(true); self.count += 1; - self.free_list.push_tail(page); + self.free_list.push_tail(folio); } - pub fn remove_page(&mut self, page: &mut L::Page) { + pub fn remove_folio(&mut self, folio: &mut L::Folio) { assert_ne!(self.count, 0, "Oops"); - page.set_buddy(false); + folio.set_buddy(false); self.count -= 1; - self.free_list.remove(page); + self.free_list.remove(folio); } } diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index 7b3dc043..b2305f99 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -2,12 +2,13 @@ use core::alloc::Allocator; use core::arch::{asm, global_asm, naked_asm}; use core::cell::RefCell; use core::hint::spin_loop; +use core::ptr::NonNull; use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; use eonix_hal_traits::mm::Memory; use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PagingMode, PTE as _}; -use eonix_mm::paging::{Page, PageAccess, PageAlloc, PAGE_SIZE, PFN}; +use eonix_mm::page_table::{PageAttribute, PageTable, PagingMode, TableAttribute, PTE as _}; +use eonix_mm::paging::{Folio, FrameAlloc, PageAccess, PageBlock, PAGE_SIZE, PFN}; use eonix_percpu::PercpuArea; use fdt::Fdt; use riscv::asm::sfence_vma_all; @@ -23,9 +24,12 @@ use super::cpu::{CPUID, CPU_COUNT}; use super::time::set_next_timer; use crate::arch::cpu::CPU; use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; -use crate::arch::mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE}; +use crate::arch::mm::{ + ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, RawPageTableSv48, + GLOBAL_PAGE_TABLE, +}; use crate::bootstrap::BootStrapData; -use crate::mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; +use crate::mm::{ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -38,26 +42,26 @@ static TEMP_AP_STACK: [u8; 256] = [0; 256]; static TEMP_AP_STACK_START: &'static [u8; 256] = &TEMP_AP_STACK; #[repr(C, align(4096))] -struct PageTable([u64; PTES_PER_PAGE]); +struct BootPageTable([u64; PTES_PER_PAGE]); /// map 0x8000 0000 to itself and 0xffff ffff 8000 0000 #[unsafe(link_section = ".bootstrap.page_table.1")] -static BOOT_PAGE_TABLE: PageTable = { +static BOOT_PAGE_TABLE: BootPageTable = { let mut arr: [u64; PTES_PER_PAGE] = [0; PTES_PER_PAGE]; arr[0] = 0 | 0x2f; arr[510] = 0 | 0x2f; arr[511] = (0x80202 << 10) | 0x21; - PageTable(arr) + BootPageTable(arr) }; #[unsafe(link_section = ".bootstrap.page_table.2")] #[used] -static PT1: PageTable = { +static PT1: BootPageTable = { let mut arr: [u64; PTES_PER_PAGE] = [0; PTES_PER_PAGE]; arr[510] = (0x80000 << 10) | 0x2f; - PageTable(arr) + BootPageTable(arr) }; static BSP_PAGE_ALLOC: AtomicPtr> = AtomicPtr::new(core::ptr::null_mut()); @@ -111,7 +115,7 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { real_allocator.borrow_mut().add_range(range); } - setup_kernel_page_table(&alloc); + setup_kernel_page_table(alloc.clone()); unsafe { init_dtb_and_fdt(dtb_addr); } @@ -148,8 +152,12 @@ unsafe extern "C" { /// TODO: /// 对kernel image添加更细的控制，或者不加也行 -fn setup_kernel_page_table(alloc: impl PageAlloc) { - let global_page_table = &GLOBAL_PAGE_TABLE; +fn setup_kernel_page_table(alloc: BasicPageAllocRef) { + let global_page_table = PageTable::::new( + GLOBAL_PAGE_TABLE.clone(), + alloc.clone(), + PageAccessImpl, + ); let attr = PageAttribute::WRITE | PageAttribute::READ @@ -160,18 +168,11 @@ fn setup_kernel_page_table(alloc: impl PageAlloc) { const KERNEL_BSS_START: VAddr = VAddr::from(0xffffffff40000000); // Map kernel BSS - for pte in global_page_table.iter_kernel_in( - VRange::from(KERNEL_BSS_START).grow(BSS_LENGTH as usize), - ArchPagingMode::LEVELS, - &alloc, - ) { - let page = Page::alloc_in(&alloc); - - let attr = { - let mut attr = attr.clone(); - attr.remove(PageAttribute::EXECUTE); - attr - }; + let bss_range = VRange::from(KERNEL_BSS_START).grow(BSS_LENGTH as usize); + for pte in global_page_table.iter_kernel(bss_range) { + let page = alloc.alloc().unwrap(); + let attr = attr.difference(PageAttribute::EXECUTE); + pte.set(page.into_raw(), attr.into()); } @@ -189,17 +190,22 @@ fn setup_kernel_page_table(alloc: impl PageAlloc) { ); } sfence_vma_all(); + + core::mem::forget(global_page_table); } /// set up tp register to percpu -fn setup_cpu(alloc: impl PageAlloc, hart_id: usize) { +fn setup_cpu(alloc: impl FrameAlloc, hart_id: usize) { CPU_COUNT.fetch_add(1, Ordering::Relaxed); let mut percpu_area = PercpuArea::new(|layout| { let page_count = layout.size().div_ceil(PAGE_SIZE); - let page = Page::alloc_at_least_in(page_count, alloc); + let page = alloc.alloc_at_least(page_count).unwrap(); - let ptr = ArchPhysAccess::get_ptr_for_page(&page).cast(); + let ptr = unsafe { + // TODO: safety + ArchPhysAccess::as_ptr(page.start()) + }; page.into_raw(); ptr @@ -243,7 +249,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { for hart_id in FDT.harts().filter(|&id| id != local_hart_id) { let stack_range = { let page_alloc = BasicPageAllocRef::new(&page_alloc); - let ap_stack = Page::alloc_order_in(4, page_alloc); + let ap_stack = page_alloc.alloc_order(4).unwrap(); let stack_range = ap_stack.range(); ap_stack.into_raw(); stack_range diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 46dd9437..f67646cf 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -1,31 +1,25 @@ -use super::{ - config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}, - fdt::{FdtExt, FDT}, -}; -use crate::{arch::riscv64::config::mm::KIMAGE_OFFSET, traits::mm::Memory}; -use core::{marker::PhantomData, ptr::NonNull}; -use eonix_mm::{ - address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}, - page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, - }, - paging::{NoAlloc, Page, PageBlock, PFN}, +use core::marker::PhantomData; +use core::ptr::NonNull; + +use eonix_hal_traits::mm::Memory; +use eonix_mm::address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, + TableAttribute, PTE, }; +use eonix_mm::paging::{BasicFolio, Folio, PageAccess, PageBlock, PFN}; use eonix_sync_base::LazyLock; use fdt::Fdt; -use riscv::{ - asm::{sfence_vma, sfence_vma_all}, - register::satp, -}; +use riscv::asm::{sfence_vma, sfence_vma_all}; +use riscv::register::satp; -pub const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); -pub static GLOBAL_PAGE_TABLE: LazyLock> = - LazyLock::new(|| unsafe { - Page::with_raw(PAGE_TABLE_BASE, |root_table_page| { - PageTable::with_root_table(root_table_page.clone()) - }) - }); +use super::config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}; +use super::fdt::{FdtExt, FDT}; +use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::mm::BasicPageAlloc; + +const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); +pub const GLOBAL_PAGE_TABLE: BasicFolio = BasicFolio::new(PAGE_TABLE_BASE, 0); pub const PA_V: u64 = 0b1 << 0; pub const PA_R: u64 = 0b1 << 1; @@ -61,6 +55,9 @@ pub struct ArchPhysAccess; pub struct ArchMemory; +#[derive(Clone)] +pub struct PageAccessImpl; + impl PTE for PTE64 { type Attr = PageAttribute64; @@ -261,6 +258,12 @@ impl PhysAccess for ArchPhysAccess { } } +impl PageAccess for PageAccessImpl { + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull { + unsafe { ArchPhysAccess::as_ptr(PAddr::from(pfn)) } + } +} + impl Memory for ArchMemory { fn present_ram() -> impl Iterator { FDT.present_ram() diff --git a/crates/eonix_hal/src/mm.rs b/crates/eonix_hal/src/mm.rs index 0a5597ac..c4b9bb74 100644 --- a/crates/eonix_hal/src/mm.rs +++ b/crates/eonix_hal/src/mm.rs @@ -1,16 +1,14 @@ -use core::{ - alloc::{AllocError, Allocator, Layout}, - cell::RefCell, - ptr::NonNull, -}; -use eonix_mm::{ - address::{AddrOps as _, PRange}, - paging::{PageAlloc, UnmanagedRawPage, PAGE_SIZE, PFN}, -}; +use core::alloc::{AllocError, Allocator, Layout}; +use core::cell::RefCell; +use core::ptr::NonNull; + +use eonix_mm::address::{AddrOps as _, PRange}; +use eonix_mm::page_table::PageTableAlloc; +use eonix_mm::paging::{BasicFolio, FrameAlloc, PAGE_SIZE, PFN}; pub use crate::arch::mm::{ flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchMemory, - ArchPagingMode, ArchPhysAccess, GLOBAL_PAGE_TABLE, + ArchPhysAccess, GLOBAL_PAGE_TABLE, }; pub struct BasicPageAlloc { @@ -118,19 +116,23 @@ impl<'a> BasicPageAllocRef<'a> { } } -impl PageAlloc for BasicPageAllocRef<'_> { - type RawPage = UnmanagedRawPage; +impl FrameAlloc for BasicPageAllocRef<'_> { + type Folio = BasicFolio; - fn alloc_order(&self, order: u32) -> Option { - Some(Self::RawPage::new(self.0.borrow_mut().alloc(order), order)) + fn alloc_order(&self, order: u32) -> Option { + Some(BasicFolio::new(self.0.borrow_mut().alloc(order), order)) } +} + +impl PageTableAlloc for BasicPageAllocRef<'_> { + type Folio = BasicFolio; - unsafe fn dealloc(&self, _: Self::RawPage) { - panic!("Dealloc is not supported in BasicPageAlloc"); + fn alloc(&self) -> Self::Folio { + FrameAlloc::alloc(self).unwrap() } - fn has_management_over(&self, _: Self::RawPage) -> bool { - true + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio { + BasicFolio::new(pfn, 0) } } diff --git a/crates/eonix_mm/src/address/paddr.rs b/crates/eonix_mm/src/address/paddr.rs index 6fadbd2a..bbfa299e 100644 --- a/crates/eonix_mm/src/address/paddr.rs +++ b/crates/eonix_mm/src/address/paddr.rs @@ -1,11 +1,11 @@ +use core::fmt; +use core::ops::{Add, Sub}; +use core::ptr::NonNull; + use super::addr::Addr; use crate::paging::{PAGE_SIZE_BITS, PFN}; -use core::{ - fmt, - ops::{Add, Sub}, - ptr::NonNull, -}; +/// Convert PAddr to VAddr. pub trait PhysAccess { /// Translate the data that this address is pointing to into kernel /// accessible pointer. Use it with care. diff --git a/crates/eonix_mm/src/page_table.rs b/crates/eonix_mm/src/page_table.rs index 55732f72..f3528060 100644 --- a/crates/eonix_mm/src/page_table.rs +++ b/crates/eonix_mm/src/page_table.rs @@ -3,7 +3,7 @@ mod paging_mode; mod pte; mod pte_iterator; -pub use page_table::{PageTable, RawPageTable}; +pub use page_table::{PageTable, PageTableAlloc, RawPageTable}; pub use paging_mode::{PageTableLevel, PagingMode}; pub use pte::{PageAttribute, RawAttribute, TableAttribute, PTE}; pub use pte_iterator::PageTableIterator; diff --git a/crates/eonix_mm/src/page_table/page_table.rs b/crates/eonix_mm/src/page_table/page_table.rs index 8318049f..80be63b9 100644 --- a/crates/eonix_mm/src/page_table/page_table.rs +++ b/crates/eonix_mm/src/page_table/page_table.rs @@ -1,15 +1,12 @@ -use super::{ - paging_mode::PageTableLevel, - pte::{RawAttribute, TableAttribute}, - pte_iterator::{KernelIterator, UserIterator}, - PagingMode, PTE, -}; -use crate::{ - address::{PAddr, VRange}, - page_table::PageTableIterator, - paging::{GlobalPageAlloc, Page, PageAccess, PageAlloc, PageBlock}, -}; -use core::{marker::PhantomData, ptr::NonNull}; +use core::marker::PhantomData; +use core::ptr::NonNull; + +use super::paging_mode::PageTableLevel; +use super::pte::{RawAttribute, TableAttribute}; +use super::{PagingMode, PTE}; +use crate::address::{PAddr, VRange}; +use crate::page_table::PageTableIterator; +use crate::paging::{Folio, PageAccess, PageBlock, PFN}; pub trait RawPageTable<'a>: Send + 'a { type Entry: PTE + 'a; @@ -24,45 +21,60 @@ pub trait RawPageTable<'a>: Send + 'a { unsafe fn from_ptr(ptr: NonNull) -> Self; } +pub trait PageTableAlloc: Clone { + type Folio: Folio; + + fn alloc(&self) -> Self::Folio; + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio; +} + +pub trait GlobalPageTableAlloc: PageTableAlloc { + const GLOBAL: Self; +} + pub struct PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { - root_table_page: Page, - phantom: PhantomData<&'a (M, X)>, + root_table_page: A::Folio, + alloc: A, + access: X, + phantom: PhantomData<&'a M>, } impl<'a, M, A, X> PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { - pub fn with_root_table(root_table_page: Page) -> Self { + pub fn new(root_table_page: A::Folio, alloc: A, access: X) -> Self { Self { root_table_page, + alloc, + access, phantom: PhantomData, } } pub fn clone_global<'b, B>(&self) -> PageTable<'b, M, B, X> where - B: GlobalPageAlloc, + B: GlobalPageTableAlloc, { - self.clone_in(B::global()) + self.clone_in(B::GLOBAL) } pub fn clone_in<'b, B>(&self, alloc: B) -> PageTable<'b, M, B, X> where - B: PageAlloc, + B: PageTableAlloc, { - let new_root_table_page = Page::alloc_in(alloc); - let new_table_data = X::get_ptr_for_page(&new_root_table_page); - let kernel_table_data = X::get_ptr_for_page(&self.root_table_page); + let new_root_table_page = alloc.alloc(); + let new_table_data = self.access.get_ptr_for_page(&new_root_table_page); + let kernel_table_data = self.access.get_ptr_for_page(&self.root_table_page); unsafe { // SAFETY: `new_table_data` and `kernel_table_data` are both valid pointers @@ -82,7 +94,7 @@ where root_page_table.index_mut(idx).take(); } - PageTable::with_root_table(new_root_table_page) + PageTable::new(new_root_table_page, alloc, self.access.clone()) } pub fn addr(&self) -> PAddr { @@ -90,100 +102,59 @@ where } pub fn iter_user(&self, range: VRange) -> impl Iterator { - let alloc = self.root_table_page.allocator(); - let page_table_ptr = X::get_ptr_for_page(&self.root_table_page); + let page_table_ptr = self.access.get_ptr_for_page(&self.root_table_page); let root_page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) }; - PageTableIterator::::new(root_page_table, range, alloc.clone()) - } - - pub fn iter_kernel(&self, range: VRange) -> impl Iterator { - self.iter_kernel_levels(range, M::LEVELS) + PageTableIterator::::new( + root_page_table, + range, + TableAttribute::USER, + self.alloc.clone(), + self.access.clone(), + ) } - /// Iterates over the kernel space entries in the page table for the specified levels. - /// - /// # Parameters - /// - `range`: The virtual address range to iterate over. - /// - `levels`: A slice of `PageTableLevel` that specifies which levels of the page table - /// should be included in the iteration. Each level corresponds to a level in the page - /// table hierarchy, and the iterator will traverse entries at these levels. + /// Iterates over the kernel space entries in the page table. /// /// # Returns /// An iterator over mutable references to the page table entries (`M::Entry`) within the - /// specified range and levels. + /// specified range. /// /// # Example /// ``` /// let range = VRange::new(0x1234000, 0x1300000); - /// let levels = &M::LEVELS[..2]; - /// for pte in page_table.iter_kernel_levels(range, levels) { + /// for pte in page_table.iter_kernel(range) { /// // Process each entry /// } /// ``` - pub fn iter_kernel_levels( - &self, - range: VRange, - levels: &'static [PageTableLevel], - ) -> impl Iterator { - self.iter_kernel_in(range, levels, self.root_table_page.allocator()) - } - - /// Iterates over the kernel space entries in the page table for the specified levels - /// with a given page allocator. - /// - /// # Parameters - /// - `range`: The virtual address range to iterate over. - /// - `levels`: A slice of `PageTableLevel` that specifies which levels of the page table - /// should be included in the iteration. Each level corresponds to a level in the page - /// table hierarchy, and the iterator will traverse entries at these levels. - /// - `alloc`: A page allocator that provides memory for the page table entries. - /// - /// # Returns - /// An iterator over mutable references to the page table entries (`M::Entry`) within the - /// specified range and levels. - /// - /// # Example - /// ```no_run - /// let range = VRange::new(0x1234000, 0x1300000); - /// let levels = &M::LEVELS[..2]; - /// for pte in page_table.iter_kernel_in(range, levels, NoAlloc) { - /// // Process each entry - /// } - /// ``` - pub fn iter_kernel_in( - &self, - range: VRange, - levels: &'static [PageTableLevel], - alloc: A1, - ) -> impl Iterator { - let page_table_ptr = X::get_ptr_for_page(&self.root_table_page); + pub fn iter_kernel(&self, range: VRange) -> impl Iterator { + let page_table_ptr = self.access.get_ptr_for_page(&self.root_table_page); let root_page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) }; - PageTableIterator::::with_levels( + PageTableIterator::::with_levels( root_page_table, range, - alloc, - levels, + TableAttribute::GLOBAL, + self.alloc.clone(), + self.access.clone(), + M::LEVELS, ) } - fn drop_page_table_recursive(page_table: &Page, levels: &[PageTableLevel]) { + fn drop_page_table_recursive(&self, page_table: &A::Folio, levels: &[PageTableLevel]) { let [level, remaining_levels @ ..] = levels else { return }; if remaining_levels.is_empty() { // We reached the last level, no need to go deeper. return; } - let alloc = page_table.allocator(); - - let page_table_ptr = X::get_ptr_for_page(page_table); + let page_table_ptr = self.access.get_ptr_for_page(page_table); let mut page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) @@ -201,10 +172,10 @@ where let page_table = unsafe { // SAFETY: We got the pfn from a valid page table entry, so it should be valid. - Page::from_raw_in(pfn, alloc.clone()) + self.alloc.from_raw(pfn) }; - Self::drop_page_table_recursive(&page_table, remaining_levels); + self.drop_page_table_recursive(&page_table, remaining_levels); } } } @@ -213,10 +184,10 @@ impl<'a, M, A, X> Drop for PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { fn drop(&mut self) { - Self::drop_page_table_recursive(&self.root_table_page, M::LEVELS); + self.drop_page_table_recursive(&self.root_table_page, M::LEVELS); } } diff --git a/crates/eonix_mm/src/page_table/pte.rs b/crates/eonix_mm/src/page_table/pte.rs index e067d207..c14d5738 100644 --- a/crates/eonix_mm/src/page_table/pte.rs +++ b/crates/eonix_mm/src/page_table/pte.rs @@ -1,6 +1,7 @@ -use crate::paging::PFN; use bitflags::bitflags; +use crate::paging::PFN; + bitflags! { #[derive(Clone, Copy, PartialEq)] pub struct TableAttribute: usize { diff --git a/crates/eonix_mm/src/page_table/pte_iterator.rs b/crates/eonix_mm/src/page_table/pte_iterator.rs index 89b9fb9f..a9e4ff46 100644 --- a/crates/eonix_mm/src/page_table/pte_iterator.rs +++ b/crates/eonix_mm/src/page_table/pte_iterator.rs @@ -1,62 +1,14 @@ -use super::{ - pte::{RawAttribute, TableAttribute}, - PageTableLevel, PagingMode, RawPageTable as _, PTE, -}; -use crate::{ - address::{AddrOps as _, VRange}, - paging::{Page, PageAccess, PageAlloc}, -}; -use core::{marker::PhantomData}; - -pub struct KernelIterator; -pub struct UserIterator; - -pub trait IteratorType { - fn page_table_attributes() -> TableAttribute; - - fn get_page_table<'a, A, X>(pte: &mut M::Entry, alloc: &A) -> M::RawTable<'a> - where - A: PageAlloc, - X: PageAccess, - { - let attr = pte.get_attr().as_table_attr().expect("Not a page table"); - - if attr.contains(TableAttribute::PRESENT) { - let pfn = pte.get_pfn(); - unsafe { - // SAFETY: We are creating a pointer to a page referenced to in - // some page table, which should be valid. - let page_table_ptr = X::get_ptr_for_pfn(pfn); - // SAFETY: `page_table_ptr` is a valid pointer to a page table. - M::RawTable::from_ptr(page_table_ptr) - } - } else { - let page = Page::alloc_in(alloc.clone()); - let page_table_ptr = X::get_ptr_for_page(&page); - - unsafe { - // SAFETY: `page_table_ptr` is good for writing and properly aligned. - page_table_ptr.write_bytes(0, 1); - } - - pte.set( - page.into_raw(), - ::Attr::from(Self::page_table_attributes()), - ); - - unsafe { - // SAFETY: `page_table_ptr` is a valid pointer to a page table. - M::RawTable::from_ptr(page_table_ptr) - } - } - } -} +use super::page_table::PageTableAlloc; +use super::pte::{RawAttribute, TableAttribute}; +use super::{PageTableLevel, PagingMode, RawPageTable as _, PTE}; +use crate::address::{AddrOps as _, VRange}; +use crate::paging::{Folio, PageAccess}; -pub struct PageTableIterator<'a, M, A, X, K> +pub struct PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { /// Specifies the hierarchy of page table levels to iterate over. @@ -69,19 +21,19 @@ where indicies: [u16; 8], tables: [Option>; 8], + fill_entry_attr: TableAttribute, + alloc: A, - _phantom: PhantomData<&'a (X, K)>, + access: X, } -impl<'a, M, A, X, K> PageTableIterator<'a, M, A, X, K> +impl<'a, M, A, X> PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, - K: IteratorType, { - fn parse_tables_starting_from(&mut self, idx_level: usize) { for (idx, &pt_idx) in self .indicies @@ -98,18 +50,58 @@ where }; let parent_table = parent_table.as_mut().expect("Parent table is None"); let next_pte = parent_table.index_mut(pt_idx); - child_table.replace(K::get_page_table::(next_pte, &self.alloc)); + + child_table.replace({ + let attr = next_pte + .get_attr() + .as_table_attr() + .expect("Not a page table"); + + if attr.contains(TableAttribute::PRESENT) { + let pfn = next_pte.get_pfn(); + unsafe { + // SAFETY: We are creating a pointer to a page referenced to in + // some page table, which should be valid. + let page_table_ptr = self.access.get_ptr_for_pfn(pfn); + // SAFETY: `page_table_ptr` is a valid pointer to a page table. + M::RawTable::from_ptr(page_table_ptr) + } + } else { + let page = self.alloc.alloc(); + let page_table_ptr = self.access.get_ptr_for_page(&page); + + unsafe { + // SAFETY: `page_table_ptr` is good for writing and properly aligned. + page_table_ptr.write_bytes(0, 1); + } + + next_pte.set(page.into_raw(), self.fill_entry_attr.into()); + + unsafe { + // SAFETY: `page_table_ptr` is a valid pointer to a page table. + M::RawTable::from_ptr(page_table_ptr) + } + } + }); } } - pub fn new(page_table: M::RawTable<'a>, range: VRange, alloc: A) -> Self { - Self::with_levels(page_table, range, alloc, M::LEVELS) + pub fn new( + page_table: M::RawTable<'a>, + range: VRange, + fill_entry_attr: TableAttribute, + alloc: A, + access: X, + ) -> Self { + Self::with_levels(page_table, range, fill_entry_attr, alloc, access, M::LEVELS) } pub fn with_levels( page_table: M::RawTable<'a>, range: VRange, + fill_entry_attr: TableAttribute, alloc: A, + access: X, levels: &'static [PageTableLevel], ) -> Self { let start = range.start().floor(); @@ -122,8 +114,9 @@ where remaining: (end - start) / last_level.page_size(), indicies: [0; 8], tables: [const { None }; 8], + fill_entry_attr: fill_entry_attr.union(TableAttribute::PRESENT), alloc, - _phantom: PhantomData, + access, }; for (i, level) in levels.iter().enumerate() { @@ -137,13 +130,12 @@ where } } -impl<'a, M, A, X, K> Iterator for PageTableIterator<'a, M, A, X, K> +impl<'a, M, A, X> Iterator for PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, - K: IteratorType, { type Item = &'a mut M::Entry; @@ -178,15 +170,3 @@ where Some(retval) } } - -impl IteratorType for KernelIterator { - fn page_table_attributes() -> TableAttribute { - TableAttribute::PRESENT | TableAttribute::GLOBAL - } -} - -impl IteratorType for UserIterator { - fn page_table_attributes() -> TableAttribute { - TableAttribute::PRESENT | TableAttribute::USER - } -} diff --git a/crates/eonix_mm/src/page_table/walk.rs b/crates/eonix_mm/src/page_table/walk.rs new file mode 100644 index 00000000..aba80b09 --- /dev/null +++ b/crates/eonix_mm/src/page_table/walk.rs @@ -0,0 +1,210 @@ +use super::pte::{RawAttribute, TableAttribute}; +use super::{PageTableLevel, PTE}; +use crate::address::{AddrOps, VAddr, VRange}; +use crate::paging::PFN; + +pub enum WalkState { + Next, + Skip, + Break, +} + +pub trait PageTable: Sized { + type Entry: PTE; + const LEVELS: &'static [PageTableLevel]; + + fn index(&self, index: usize) -> &Self::Entry; + fn index_mut(&mut self, index: usize) -> &mut Self::Entry; + + fn from_pfn(pfn: PFN) -> Self; + unsafe fn take_pfn(pfn: PFN) -> Self; +} + +pub struct PageTableWalk<'a, T, D> +where + T: PageTable, +{ + levels: &'a [PageTableLevel], + fill_entry: &'a [fn(&mut D, &mut T::Entry) -> Option], + walk_entry: &'a [fn(&mut D, &mut T::Entry) -> WalkState], + data: D, +} + +fn try_get_table( + entry: &mut T::Entry, + data: &mut D, + fill_entry: fn(&mut D, &mut T::Entry) -> Option, +) -> Option +where + T: PageTable, +{ + let (mut pfn, attr) = entry.get(); + + // Always skip huge page entries + let attr = attr.as_table_attr()?; + + // For normal entries, check present flags + if !attr.contains(TableAttribute::PRESENT) { + // Skip entries filled with nothing + pfn = fill_entry(data, entry)?; + } + + Some(T::from_pfn(pfn)) +} + +fn _walk_page_table( + walk: &mut PageTableWalk, + cur_level: usize, + table: &mut T, + range: VRange, +) where + T: PageTable, +{ + let level = walk.levels[cur_level]; + + let page_size = level.page_size(); + let mut addr = range.start(); + + while addr < range.end() { + let idx = level.index_of(addr); + let entry = table.index_mut(idx); + + let mut next_table = None; + if cur_level < walk.levels.len() - 1 { + next_table = try_get_table(entry, &mut walk.data, walk.fill_entry[cur_level]); + } + + match ( + walk.walk_entry[cur_level](&mut walk.data, entry), + &mut next_table, + ) { + (WalkState::Break, _) => break, + (WalkState::Next, Some(next_table)) => _walk_page_table( + walk, + cur_level + 1, + next_table, + VRange::new(addr, range.end()), + ), + // `fill_entry` says that we shouldn't continue. + (WalkState::Next, None) => {} + _ => {} + } + + addr = addr.floor_to(page_size) + page_size; + } +} + +pub fn walk_page_table(walk: &mut PageTableWalk, table: &mut T, range: VRange) +where + T: PageTable, +{ + _walk_page_table(walk, 0, table, range); +} + +pub fn drop_user_page_table(mut root_page_table: T) +where + T: PageTable, +{ + fn walk(_: &mut (), entry: &mut T::Entry) -> WalkState { + let (pfn, attr) = entry.get(); + let Some(attr) = attr.as_table_attr() else { + return WalkState::Skip; + }; + + if !attr.contains(TableAttribute::USER) { + return WalkState::Skip; + } + + unsafe { + // Check `_walk_page_table`: We will and only will touch the next level of table with + // `next_table` holding a refcount. We take the table away from the parent table now. + T::take_pfn(pfn); + } + + entry.set(PFN::from_val(0), TableAttribute::empty().into()); + + if LEVEL == 2 { + WalkState::Skip + } else { + WalkState::Next + } + } + + let mut walk = PageTableWalk { + levels: T::LEVELS, + fill_entry: &[no_fill::, no_fill::, no_fill::], + walk_entry: &[walk::, walk::, walk::, skip_walk::], + data: (), + }; + + walk_page_table( + &mut walk, + &mut root_page_table, + VRange::new(VAddr::from(0), VAddr::from(0x0000_8000_0000_0000)), + ); +} + +pub fn iter_pte( + page_table: &mut T, + range: VRange, + fill_func: impl FnMut(&mut T::Entry) -> Option, + for_each: impl FnMut(&mut T::Entry), +) { + let walker = (fill_func, for_each); + + fn fill_entry( + (fill, _): &mut ( + impl FnMut(&mut T::Entry) -> Option, + impl FnMut(&mut T::Entry), + ), + entry: &mut T::Entry, + ) -> Option { + fill(entry) + } + + fn walk_entry( + (_, for_each): &mut ( + impl FnMut(&mut T::Entry) -> Option, + impl FnMut(&mut T::Entry), + ), + entry: &mut T::Entry, + ) -> WalkState { + for_each(entry); + WalkState::Next + } + + let mut walk = PageTableWalk { + levels: T::LEVELS, + fill_entry: &[fill_entry::, fill_entry::, fill_entry::], + walk_entry: &[ + cont_walk::, + cont_walk::, + cont_walk::, + walk_entry::, + ], + data: walker, + }; + + walk_page_table(&mut walk, page_table, range); +} + +pub fn no_fill(_: &mut D, _: &mut T::Entry) -> Option +where + T: PageTable, +{ + None +} + +pub fn skip_walk(_: &mut D, _: &mut T::Entry) -> WalkState +where + T: PageTable, +{ + WalkState::Skip +} + +pub fn cont_walk(_: &mut D, _: &mut T::Entry) -> WalkState +where + T: PageTable, +{ + WalkState::Next +} diff --git a/crates/eonix_mm/src/paging.rs b/crates/eonix_mm/src/paging.rs index 0c4811f2..f0166cf3 100644 --- a/crates/eonix_mm/src/paging.rs +++ b/crates/eonix_mm/src/paging.rs @@ -2,12 +2,10 @@ mod list; mod page; mod page_alloc; mod pfn; -mod raw_page; mod zone; -pub use list::{PageList, PageListSized}; -pub use page::{Page, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; -pub use page_alloc::{GlobalPageAlloc, NoAlloc, PageAlloc}; +pub use list::{FolioList, FolioListSized}; +pub use page::{BasicFolio, Folio, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; +pub use page_alloc::{FrameAlloc, GlobalFrameAlloc}; pub use pfn::PFN; -pub use raw_page::{RawPage, UnmanagedRawPage}; pub use zone::Zone; diff --git a/crates/eonix_mm/src/paging/list.rs b/crates/eonix_mm/src/paging/list.rs index a52cf947..2dd557c9 100644 --- a/crates/eonix_mm/src/paging/list.rs +++ b/crates/eonix_mm/src/paging/list.rs @@ -1,16 +1,16 @@ -pub trait PageList { - type Page; +pub trait FolioList { + type Folio; fn is_empty(&self) -> bool; - fn peek_head(&mut self) -> Option<&mut Self::Page>; + fn peek_head(&mut self) -> Option<&mut Self::Folio>; - fn pop_head(&mut self) -> Option<&'static mut Self::Page>; - fn push_tail(&mut self, page: &'static mut Self::Page); - fn remove(&mut self, page: &mut Self::Page); + fn pop_head(&mut self) -> Option<&'static mut Self::Folio>; + fn push_tail(&mut self, page: &'static mut Self::Folio); + fn remove(&mut self, page: &mut Self::Folio); } -pub trait PageListSized: PageList + Sized { +pub trait FolioListSized: FolioList + Sized { const NEW: Self; fn new() -> Self { diff --git a/crates/eonix_mm/src/paging/page.rs b/crates/eonix_mm/src/paging/page.rs index c5a14b5e..8b067e43 100644 --- a/crates/eonix_mm/src/paging/page.rs +++ b/crates/eonix_mm/src/paging/page.rs @@ -1,6 +1,8 @@ -use super::{GlobalPageAlloc, PageAlloc, RawPage as _, PFN}; -use crate::address::{AddrRange, PAddr, PhysAccess}; -use core::{fmt, mem::ManuallyDrop, ptr::NonNull, sync::atomic::Ordering}; +use core::mem::ManuallyDrop; +use core::ptr::NonNull; + +use super::PFN; +use crate::address::{PAddr, PRange}; pub const PAGE_SIZE: usize = 4096; pub const PAGE_SIZE_BITS: u32 = PAGE_SIZE.trailing_zeros(); @@ -15,306 +17,81 @@ pub struct PageBlock([u8; PAGE_SIZE]); /// A trait that provides the kernel access to the page. #[doc(notable_trait)] -pub trait PageAccess { +pub trait PageAccess: Clone { /// Returns a kernel-accessible pointer to the page referenced by the given /// physical frame number. /// /// # Safety /// This function is unsafe because calling this function on some non-existing /// pfn will cause undefined behavior. - unsafe fn get_ptr_for_pfn(pfn: PFN) -> NonNull; + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull; /// Returns a kernel-accessible pointer to the given page. - fn get_ptr_for_page(page: &Page) -> NonNull { + fn get_ptr_for_page(&self, page: &F) -> NonNull { unsafe { // SAFETY: `page.pfn()` is guaranteed to be valid. - Self::get_ptr_for_pfn(page.pfn()) + self.get_ptr_for_pfn(page.pfn()) } } } -/// A Page allocated in allocator `A`. -#[derive(PartialEq, Eq, PartialOrd, Ord)] -pub struct Page { - raw_page: A::RawPage, - alloc: A, -} - -unsafe impl Send for Page {} -unsafe impl Sync for Page {} - -impl Page -where - A: GlobalPageAlloc, -{ - /// Allocate a page of the given *order*. - pub fn alloc_order(order: u32) -> Self { - Self::alloc_order_in(order, A::global()) - } - - /// Allocate exactly one page. - pub fn alloc() -> Self { - Self::alloc_in(A::global()) - } +/// A [`Folio`] represents one page or a bunch of adjacent pages. +pub trait Folio { + /// Returns the physical frame number of the folio, which is aligned with + /// the folio's size and valid. + fn pfn(&self) -> PFN; - /// Allocate a contiguous block of pages that can contain at least `count` pages. - pub fn alloc_at_least(count: usize) -> Self { - Self::alloc_at_least_in(count, A::global()) - } + /// Returns the folio's *order* (log2 of the number of pages contained in + /// the folio). + fn order(&self) -> u32; - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to a valid page allocated through `alloc_order()` and that the - /// page have not been freed or deallocated yet. - /// - /// No checks are done. Any violation of this assumption may lead to undefined behavior. - pub unsafe fn from_raw_unchecked(pfn: PFN) -> Self { - unsafe { Self::from_raw_unchecked_in(pfn, A::global()) } - } - - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// This function is a safe wrapper around `from_paddr_unchecked()` that does **some sort - /// of** checks to ensure that the page is valid and managed by the allocator. - /// - /// # Panic - /// This function will panic if the page is not valid or if the page is not managed by - /// the allocator. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to an existing page (A.K.A. inside the global page array) and the - /// page will not be freed or deallocated during the call. - pub unsafe fn from_raw(pfn: PFN) -> Self { - unsafe { Self::from_raw_in(pfn, A::global()) } - } - - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_in()`. - /// - /// # Safety - /// Check `from_raw()` for the safety requirements. - pub unsafe fn with_raw(pfn: PFN, func: F) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { Self::with_raw_in(pfn, A::global(), func) } - } - - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_unchecked_in()`. - /// - /// # Safety - /// Check `from_raw_unchecked()` for the safety requirements. - pub unsafe fn with_raw_unchecked(pfn: PFN, func: F, alloc: A) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { Self::with_raw_unchecked_in(pfn, func, alloc) } - } -} - -impl Page -where - A: PageAlloc, -{ - /// Allocate a page of the given *order*. - pub fn alloc_order_in(order: u32, alloc: A) -> Self { - Self { - raw_page: alloc.alloc_order(order).expect("Out of memory"), - alloc, - } - } - - /// Allocate exactly one page. - pub fn alloc_in(alloc: A) -> Self { - Self { - raw_page: alloc.alloc().expect("Out of memory"), - alloc, - } - } - - /// Allocate a contiguous block of pages that can contain at least `count` pages. - pub fn alloc_at_least_in(count: usize, alloc: A) -> Self { - Self { - raw_page: alloc.alloc_at_least(count).expect("Out of memory"), - alloc, - } - } - - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to a valid page managed by `alloc` and that the page have not - /// been freed or deallocated yet. - /// - /// No checks are done. Any violation of this assumption may lead to undefined behavior. - pub unsafe fn from_raw_unchecked_in(pfn: PFN, alloc: A) -> Self { - Self { - raw_page: A::RawPage::from(pfn), - alloc, - } + /// Returns the total size of the folio in bytes. + fn len(&self) -> usize { + 1 << (self.order() + PAGE_SIZE_BITS) } - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// This function is a safe wrapper around `from_paddr_unchecked()` that does **some sort - /// of** checks to ensure that the page is valid and managed by the allocator. - /// - /// # Panic - /// This function will panic if the page is not valid or if the page is not managed by - /// the allocator. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to an existing page (A.K.A. inside the global page array) and the - /// page will not be freed or deallocated during the call. - pub unsafe fn from_raw_in(pfn: PFN, alloc: A) -> Self { - unsafe { - // SAFETY: The caller guarantees that the page is inside the global page array. - assert!(alloc.has_management_over(A::RawPage::from(pfn))); - - // SAFETY: We've checked that the validity of the page. And the caller guarantees - // that the page will not be freed or deallocated during the call. - Self::from_raw_unchecked_in(pfn, alloc) - } + /// Returns the start physical address of the folio, which is guaranteed to + /// be aligned to the folio's size and valid. + fn start(&self) -> PAddr { + PAddr::from(self.pfn()) } - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_in()`. - /// - /// # Safety - /// Check `from_raw_in()` for the safety requirements. - pub unsafe fn with_raw_in(pfn: PFN, alloc: A, func: F) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { - let me = ManuallyDrop::new(Self::from_raw_in(pfn, alloc)); - func(&me) - } + /// Returns the physical address range of the ifolio, which is guaranteed to + /// be aligned to the folio's size and valid. + fn range(&self) -> PRange { + PRange::from(self.start()).grow(self.len()) } - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_unchecked_in()`. - /// - /// # Safety - /// Check `from_raw_unchecked_in()` for the safety requirements. - pub unsafe fn with_raw_unchecked_in(pfn: PFN, func: F, alloc: A) -> O + /// Consumes the folio and returns the PFN without dropping the reference + /// count the folio holds. + fn into_raw(self) -> PFN where - F: FnOnce(&Self) -> O, + Self: Sized, { - unsafe { - let me = ManuallyDrop::new(Self::from_raw_unchecked_in(pfn, alloc)); - func(&me) - } - } - - /// Whether we are the only owner of the page. - pub fn is_exclusive(&self) -> bool { - self.raw_page.refcount().load(Ordering::Acquire) == 1 - } - - /// Returns the *order* of the page, which is the log2 of the number of pages - /// contained in the page object. - pub fn order(&self) -> u32 { - self.raw_page.order() - } - - /// Returns the total size of the page in bytes. - pub fn len(&self) -> usize { - 1 << (self.order() + PAGE_SIZE_BITS) - } - - /// Consumes the `Page` and returns the physical frame number without dropping - /// the reference count the page holds. - pub fn into_raw(self) -> PFN { let me = ManuallyDrop::new(self); me.pfn() } - - /// Returns the physical frame number of the page, which is aligned with the - /// page size and valid. - pub fn pfn(&self) -> PFN { - Into::::into(self.raw_page) - } - - /// Returns the start physical address of the page, which is guaranteed to be - /// aligned to the page size and valid. - pub fn start(&self) -> PAddr { - PAddr::from(self.pfn()) - } - - /// Returns the physical address range of the page, which is guaranteed to be - /// aligned to the page size and valid. - pub fn range(&self) -> AddrRange { - AddrRange::from(self.start()).grow(self.len()) - } - - /// Get the allocator that manages this page. - pub fn allocator(&self) -> &A { - &self.alloc - } } -impl Clone for Page -where - A: PageAlloc, -{ - fn clone(&self) -> Self { - // SAFETY: Memory order here can be Relaxed is for the same reason as that - // in the copy constructor of `std::shared_ptr`. - self.raw_page.refcount().fetch_add(1, Ordering::Relaxed); - - Self { - raw_page: self.raw_page, - alloc: self.alloc.clone(), - } - } +/// A simple [`Folio`] with no reference counting or other ownership mechanism. +#[derive(Clone)] +pub struct BasicFolio { + pfn: PFN, + order: u32, } -impl Drop for Page -where - A: PageAlloc, -{ - fn drop(&mut self) { - match self.raw_page.refcount().fetch_sub(1, Ordering::AcqRel) { - 0 => panic!("Refcount for an in-use page is 0"), - 1 => unsafe { - // SAFETY: `self.raw_page` points to a valid page inside the global page array. - assert!(self.alloc.has_management_over(self.raw_page)); - - // SAFETY: `self.raw_page` is managed by the allocator and we're dropping the page. - self.alloc.dealloc(self.raw_page) - }, - _ => {} - } +impl BasicFolio { + pub const fn new(pfn: PFN, order: u32) -> Self { + Self { pfn, order } } } -impl fmt::Debug for Page { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Page({:?}, order={})", - Into::::into(self.raw_page), - self.order() - ) +impl Folio for BasicFolio { + fn pfn(&self) -> PFN { + self.pfn } -} -impl PageAccess for T -where - T: PhysAccess, -{ - unsafe fn get_ptr_for_pfn(pfn: PFN) -> NonNull { - unsafe { - // SAFETY: The physical address of a existing page must be - // aligned to the page size. - T::as_ptr(PAddr::from(pfn)) - } + fn order(&self) -> u32 { + self.order } } diff --git a/crates/eonix_mm/src/paging/page_alloc.rs b/crates/eonix_mm/src/paging/page_alloc.rs index fe222605..267d3ccb 100644 --- a/crates/eonix_mm/src/paging/page_alloc.rs +++ b/crates/eonix_mm/src/paging/page_alloc.rs @@ -1,89 +1,44 @@ -use super::{raw_page::UnmanagedRawPage, RawPage}; +use super::Folio; -/// A trait for allocating and deallocating pages of memory. +/// A trait for allocating and deallocating folios. /// /// Note that the instances of this trait should provide pointer-like or reference-like /// behavior, meaning that the allocators are to be passed around by value and stored in /// managed data structures. This is because the allocator may be used to deallocate the /// pages it allocates. -#[doc(notable_trait)] -pub trait PageAlloc: Clone { - type RawPage: RawPage; +pub trait FrameAlloc: Clone { + type Folio: Folio; - /// Allocate a page of the given *order*. - fn alloc_order(&self, order: u32) -> Option; + /// Allocate a folio of the given *order*. + fn alloc_order(&self, order: u32) -> Option; - /// Allocate exactly one page. - fn alloc(&self) -> Option { + /// Allocate exactly one folio. + fn alloc(&self) -> Option { self.alloc_order(0) } - /// Allocate a contiguous block of pages that can contain at least `count` pages. - fn alloc_at_least(&self, count: usize) -> Option { + /// Allocate a folio that can contain at least [`count`] contiguous pages. + fn alloc_at_least(&self, count: usize) -> Option { let order = count.next_power_of_two().trailing_zeros(); self.alloc_order(order) } - - /// Deallocate a page. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller MUST ensure that - /// `raw_page` is allocated in this allocator and never used after this call. - unsafe fn dealloc(&self, raw_page: Self::RawPage); - - /// Check whether the page is allocated and managed by the allocator. - fn has_management_over(&self, page_ptr: Self::RawPage) -> bool; } /// A trait for global page allocators. /// /// Global means that we can get an instance of the allocator from anywhere in the kernel. -#[doc(notable_trait)] -pub trait GlobalPageAlloc: PageAlloc + 'static { - /// Get the global page allocator. - fn global() -> Self; +pub trait GlobalFrameAlloc: FrameAlloc + 'static { + /// The global page allocator. + const GLOBAL: Self; } -#[derive(Clone)] -pub struct NoAlloc; - -impl<'a, A> PageAlloc for &'a A +impl<'a, A> FrameAlloc for &'a A where - A: PageAlloc, + A: FrameAlloc, { - type RawPage = A::RawPage; + type Folio = A::Folio; - fn alloc_order(&self, order: u32) -> Option { + fn alloc_order(&self, order: u32) -> Option { (*self).alloc_order(order) } - - unsafe fn dealloc(&self, raw_page: Self::RawPage) { - unsafe { (*self).dealloc(raw_page) } - } - - fn has_management_over(&self, raw_page: Self::RawPage) -> bool { - (*self).has_management_over(raw_page) - } -} - -impl PageAlloc for NoAlloc { - type RawPage = UnmanagedRawPage; - - fn alloc_order(&self, _: u32) -> Option { - panic!("`NoAlloc` cannot allocate pages"); - } - - unsafe fn dealloc(&self, _: Self::RawPage) { - panic!("`NoAlloc` cannot free pages"); - } - - fn has_management_over(&self, _: Self::RawPage) -> bool { - true - } -} - -impl GlobalPageAlloc for NoAlloc { - fn global() -> Self { - Self - } } diff --git a/crates/eonix_mm/src/paging/raw_page.rs b/crates/eonix_mm/src/paging/raw_page.rs deleted file mode 100644 index 789e863b..00000000 --- a/crates/eonix_mm/src/paging/raw_page.rs +++ /dev/null @@ -1,47 +0,0 @@ -use core::sync::atomic::AtomicUsize; - -use super::PFN; - -/// A `RawPage` represents a page of memory in the kernel. It is a low-level -/// representation of a page that is used by the kernel to manage memory. -#[doc(notable_trait)] -pub trait RawPage: Clone + Copy + From + Into { - fn order(&self) -> u32; - fn refcount(&self) -> &AtomicUsize; -} - -#[derive(Clone, Copy)] -pub struct UnmanagedRawPage(PFN, u32); - -/// Unmanaged raw pages should always have a non-zero refcount to -/// avoid `free()` from being called. -static UNMANAGED_RAW_PAGE_CLONE_COUNT: AtomicUsize = AtomicUsize::new(1); - -impl UnmanagedRawPage { - pub const fn new(pfn: PFN, order: u32) -> Self { - Self(pfn, order) - } -} - -impl From for UnmanagedRawPage { - fn from(value: PFN) -> Self { - Self::new(value, 0) - } -} - -impl Into for UnmanagedRawPage { - fn into(self) -> PFN { - let Self(pfn, _) = self; - pfn - } -} - -impl RawPage for UnmanagedRawPage { - fn order(&self) -> u32 { - self.1 - } - - fn refcount(&self) -> &AtomicUsize { - &UNMANAGED_RAW_PAGE_CLONE_COUNT - } -} diff --git a/crates/eonix_mm/src/paging/zone.rs b/crates/eonix_mm/src/paging/zone.rs index ec3ed15e..a2e85343 100644 --- a/crates/eonix_mm/src/paging/zone.rs +++ b/crates/eonix_mm/src/paging/zone.rs @@ -1,7 +1,6 @@ -use core::cell::UnsafeCell; +use core::ptr::NonNull; -#[allow(unused_imports)] -use super::{Page, PageAlloc, RawPage, PFN}; +use super::PFN; use crate::address::PRange; /// A [`Zone`] holds a lot of [`Page`]s that share the same NUMA node or @@ -16,5 +15,5 @@ pub trait Zone: Send + Sync { /// /// # Return /// [`None`] if [`pfn`] is not in this [`Zone`]. - fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell>; + fn get_page(&self, pfn: PFN) -> Option>; } diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index 8597331d..c3e7f392 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -2,7 +2,7 @@ use core::ptr::NonNull; -use eonix_mm::paging::{PageList, PageListSized}; +use eonix_mm::paging::{FolioList, FolioListSized}; use eonix_sync::Spin; #[repr(C)] @@ -84,21 +84,21 @@ where } } -pub trait SlabPageAlloc { +/// Allocate a page suitable for slab system use. The page MUST come with +/// its allocation count 0 and next free slot None. +/// +/// # Safety +/// The page returned MUST have been properly initialized after allocation. +pub unsafe trait SlabPageAlloc { type Page: SlabPage; - type PageList: PageList; + type PageList: FolioList; - /// Allocate a page suitable for slab system use. The page MUST come with - /// its allocation count 0 and next free slot None. - /// - /// # Safety - /// The page returned MUST be properly initialized before its usage. - unsafe fn alloc_uninit(&self) -> &'static mut Self::Page; + fn alloc_slab_page(&self) -> &'static mut Self::Page; } pub(crate) struct SlabList where - T: PageList, + T: FolioList, { empty_list: T, partial_list: T, @@ -120,7 +120,7 @@ unsafe impl Sync for SlabAlloc where P: SlabPag impl SlabAlloc where L: SlabPageAlloc, - L::PageList: PageListSized, + L::PageList: FolioListSized, { pub fn new_in(alloc: L) -> Self { Self { @@ -148,7 +148,7 @@ where impl SlabList where - T: PageListSized, + T: FolioListSized, { const fn new(object_size: usize) -> Self { Self { @@ -162,8 +162,8 @@ where impl SlabList where - T: PageList, - T::Page: SlabPage, + T: FolioList, + T::Folio: SlabPage, { fn alloc_from_partial(&mut self) -> NonNull { let head = self.partial_list.peek_head().unwrap(); @@ -190,18 +190,16 @@ where slot } - fn charge(&mut self, alloc: &impl SlabPageAlloc) { - unsafe { - let slab = alloc.alloc_uninit(); - let free_slot = make_slab_page(slab.get_data_ptr(), self.object_size); + fn charge(&mut self, alloc: &impl SlabPageAlloc) { + let slab = alloc.alloc_slab_page(); + let free_slot = make_slab_page(slab.get_data_ptr(), self.object_size); - slab.set_free_slot(Some(free_slot)); + slab.set_free_slot(Some(free_slot)); - self.empty_list.push_tail(slab); - } + self.empty_list.push_tail(slab); } - fn alloc(&mut self, alloc: &impl SlabPageAlloc) -> NonNull { + fn alloc(&mut self, alloc: &impl SlabPageAlloc) -> NonNull { if !self.partial_list.is_empty() { return self.alloc_from_partial(); } @@ -216,7 +214,7 @@ where unsafe fn dealloc(&mut self, ptr: NonNull, _alloc: &impl SlabPageAlloc) { let slab_page = unsafe { // SAFETY: - ::from_allocated(ptr) + ::from_allocated(ptr) }; let (was_full, is_empty); diff --git a/src/driver/ahci/command.rs b/src/driver/ahci/command.rs index c83339b7..4609d38d 100644 --- a/src/driver/ahci/command.rs +++ b/src/driver/ahci/command.rs @@ -1,9 +1,11 @@ +use eonix_mm::paging::Folio as _; + use crate::kernel::constants::EINVAL; -use crate::kernel::mem::paging::Page; +use crate::kernel::mem::Folio; use crate::prelude::*; pub trait Command { - fn pages(&self) -> &[Page]; + fn pages(&self) -> &[Folio]; fn lba(&self) -> u64; // in sectors @@ -14,19 +16,19 @@ pub trait Command { } pub struct IdentifyCommand { - page: Page, + page: Folio, } impl IdentifyCommand { pub fn new() -> Self { Self { - page: Page::alloc(), + page: Folio::alloc(), } } } impl Command for IdentifyCommand { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { core::slice::from_ref(&self.page) } @@ -47,14 +49,14 @@ impl Command for IdentifyCommand { } } -pub struct ReadLBACommand<'lt> { - pages: &'lt [Page], +pub struct ReadLBACommand<'a> { + pages: &'a [Folio], lba: u64, count: u16, } -impl<'lt> ReadLBACommand<'lt> { - pub fn new(pages: &'lt [Page], lba: u64, count: u16) -> KResult { +impl<'a> ReadLBACommand<'a> { + pub fn new(pages: &'a [Folio], lba: u64, count: u16) -> KResult { if pages.len() > 248 { return Err(EINVAL); } @@ -69,7 +71,7 @@ impl<'lt> ReadLBACommand<'lt> { } impl Command for ReadLBACommand<'_> { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { self.pages } @@ -91,13 +93,13 @@ impl Command for ReadLBACommand<'_> { } pub struct WriteLBACommand<'a> { - pages: &'a [Page], + pages: &'a [Folio], lba: u64, count: u16, } impl<'a> WriteLBACommand<'a> { - pub fn new(pages: &'a [Page], lba: u64, count: u16) -> KResult { + pub fn new(pages: &'a [Folio], lba: u64, count: u16) -> KResult { if pages.len() > 248 { return Err(EINVAL); } @@ -112,7 +114,7 @@ impl<'a> WriteLBACommand<'a> { } impl Command for WriteLBACommand<'_> { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { self.pages } diff --git a/src/driver/ahci/command_table.rs b/src/driver/ahci/command_table.rs index 7b78d26f..00fc8a0b 100644 --- a/src/driver/ahci/command_table.rs +++ b/src/driver/ahci/command_table.rs @@ -1,13 +1,14 @@ use core::ptr::NonNull; use eonix_mm::address::PAddr; +use eonix_mm::paging::Folio as _; use super::command::Command; use super::{PRDTEntry, FISH2D}; -use crate::kernel::mem::{Page, PageExt}; +use crate::kernel::mem::FolioOwned; pub struct CommandTable { - page: Page, + page: FolioOwned, cmd_fis: NonNull, prdt: NonNull<[PRDTEntry; 248]>, prdt_entries: usize, @@ -18,7 +19,7 @@ unsafe impl Sync for CommandTable {} impl CommandTable { pub fn new() -> Self { - let page = Page::alloc(); + let page = FolioOwned::alloc(); let base = page.get_ptr(); unsafe { diff --git a/src/driver/ahci/defs.rs b/src/driver/ahci/defs.rs index c5440246..66841da8 100644 --- a/src/driver/ahci/defs.rs +++ b/src/driver/ahci/defs.rs @@ -1,7 +1,9 @@ #![allow(dead_code)] -use crate::kernel::mem::paging::Page; use eonix_mm::address::Addr as _; +use eonix_mm::paging::Folio as _; + +use crate::kernel::mem::Folio; pub const VENDOR_INTEL: u16 = 0x8086; pub const DEVICE_AHCI: u16 = 0x2922; @@ -239,7 +241,7 @@ pub struct PRDTEntry { } impl PRDTEntry { - pub fn setup(&mut self, page: &Page) { + pub fn setup(&mut self, page: &Folio) { self.base = page.start().addr() as u64; self._reserved1 = 0; diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 06c6f2ec..dd096f57 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -3,18 +3,18 @@ use core::ptr::NonNull; use core::task::{Poll, Waker}; use eonix_mm::address::{Addr as _, PAddr}; +use eonix_mm::paging::Folio as _; use eonix_sync::{Spin, SpinIrq as _}; use super::command_table::CommandTable; use super::CommandHeader; use crate::kernel::constants::EIO; -use crate::kernel::mem::paging::AllocZeroed; -use crate::kernel::mem::{Page, PageExt}; +use crate::kernel::mem::FolioOwned; use crate::KResult; pub struct CommandList { base: NonNull, - _page: Page, + _page: FolioOwned, } unsafe impl Send for CommandList {} @@ -75,7 +75,9 @@ impl CommandList { } pub fn new() -> Self { - let page = Page::zeroed(); + let mut page = FolioOwned::alloc(); + page.as_bytes_mut().fill(0); + let base = page.get_ptr(); let controls_ptr = Self::controls_ptr(base); diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index 73143c2c..6d6ca353 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -5,11 +5,12 @@ use core::ptr::NonNull; use async_trait::async_trait; use eonix_hal::fence::memory_barrier; use eonix_mm::address::{Addr, PAddr}; +use eonix_mm::paging::Folio as _; use eonix_sync::SpinIrq; use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; use crate::kernel::interrupt::register_irq_handler; -use crate::kernel::mem::{PageExcl, PageExt, PhysAccess}; +use crate::kernel::mem::{FolioOwned, PhysAccess}; use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; use crate::net::netdev; use crate::prelude::*; @@ -54,13 +55,13 @@ struct E1000eDev { id: u32, regs: Registers, - rt_desc_page: PageExcl, + rt_desc_page: FolioOwned, rx_head: Option, rx_tail: Option, tx_tail: Option, - rx_buffers: Box<[PageExcl; RX_DESC_SIZE]>, - tx_buffers: Box<[Option; TX_DESC_SIZE]>, + rx_buffers: Box<[FolioOwned; RX_DESC_SIZE]>, + tx_buffers: Box<[Option; TX_DESC_SIZE]>, } fn test(val: u32, bit: u32) -> bool { @@ -227,7 +228,7 @@ impl netdev::Netdev for E1000eDev { return Err(EIO); } - let mut buffer_page = PageExcl::alloc(); + let mut buffer_page = FolioOwned::alloc(); if buf.len() > buffer_page.len() { return Err(EFAULT); } @@ -363,11 +364,15 @@ impl E1000eDev { speed: netdev::LinkSpeed::SpeedUnknown, id: netdev::alloc_id(), regs, - rt_desc_page: PageExcl::zeroed(), + rt_desc_page: { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio + }, rx_head: None, rx_tail: None, tx_tail: None, - rx_buffers: Box::new(core::array::from_fn(|_| PageExcl::alloc_order(2))), + rx_buffers: Box::new(core::array::from_fn(|_| FolioOwned::alloc_order(2))), tx_buffers: Box::new([const { None }; 32]), }; diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index c5a3c3d2..5dfed88a 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -3,7 +3,7 @@ use alloc::boxed::Box; use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{Addr, PAddr, PhysAccess}; -use eonix_mm::paging::PFN; +use eonix_mm::paging::{Folio as _, PFN}; use eonix_sync::Spin; use virtio_drivers::device::blk::VirtIOBlk; use virtio_drivers::transport::Transport; @@ -12,7 +12,7 @@ use virtio_drivers::Hal; use crate::io::Chunks; use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::EIO; -use crate::kernel::mem::{Page, PageExt}; +use crate::kernel::mem::Folio; use crate::prelude::KResult; pub struct HAL; @@ -22,7 +22,7 @@ unsafe impl Hal for HAL { pages: usize, _direction: virtio_drivers::BufferDirection, ) -> (virtio_drivers::PhysAddr, core::ptr::NonNull) { - let page = Page::alloc_at_least(pages); + let page = Folio::alloc_at_least(pages); let ptr = page.get_ptr(); let pfn = page.into_raw(); @@ -40,7 +40,7 @@ unsafe impl Hal for HAL { unsafe { // SAFETY: The caller ensures that the pfn corresponds to a valid // page allocated by `dma_alloc`. - Page::from_raw(pfn); + Folio::from_raw(pfn); } 0 diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index b19c9908..c1feebdf 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,5 +1,4 @@ mod dir; -mod file; use alloc::sync::Arc; use core::ops::Deref; @@ -13,7 +12,7 @@ use itertools::Itertools; use crate::io::{Buffer, ByteBuffer, UninitBuffer}; use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::{CachePage, Page, PageExcl, PageExt, PageOffset}; +use crate::kernel::mem::{CachePage, Folio, FolioOwned, PageOffset}; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; @@ -114,7 +113,7 @@ struct FatFs { impl SuperBlock for FatFs {} impl FatFs { - async fn read_cluster(&self, mut cluster: Cluster, buf: &Page) -> KResult<()> { + async fn read_cluster(&self, mut cluster: Cluster, buf: &Folio) -> KResult<()> { cluster = cluster.normalized(); let rq = BlockDeviceRequest::Read { @@ -278,7 +277,6 @@ impl InodeOps for FileInode { .next() .ok_or(EIO)?; - let page = page.get_page(); fs.read_cluster(cluster, &page).await?; let real_len = (inode.info.lock().size as usize) - offset.byte_count(); @@ -293,7 +291,7 @@ impl InodeOps for FileInode { struct DirInode { // TODO: Use the new PageCache... - dir_pages: RwLock>, + dir_pages: RwLock>, } impl DirInode { @@ -330,7 +328,7 @@ impl DirInode { let clusters = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)); for cluster in clusters { - let page = PageExcl::alloc(); + let page = FolioOwned::alloc(); fs.read_cluster(cluster, &page).await?; dir_pages.push(page); @@ -343,7 +341,7 @@ impl DirInode { &self, sb: &SbUse, inode: &InodeUse, - ) -> KResult> + use<'_>> { + ) -> KResult> + use<'_>> { { let dir_pages = self.dir_pages.read().await; if !dir_pages.is_empty() { diff --git a/src/fs/fat32/file.rs b/src/fs/fat32/file.rs deleted file mode 100644 index 2df69728..00000000 --- a/src/fs/fat32/file.rs +++ /dev/null @@ -1,24 +0,0 @@ -use futures::Stream; - -use crate::{kernel::mem::Page, prelude::KResult}; - -use super::{ClusterIterator, FatFs}; - -pub trait ReadClusters { - fn read_clusters(self, fs: &FatFs) -> impl Stream> + Send; -} - -impl ReadClusters for ClusterIterator<'_> { - fn read_clusters(self, fs: &FatFs) -> impl Stream> + Send { - futures::stream::unfold(self, move |mut me| async { - let cluster = me.next()?; - let page = Page::alloc(); - - if let Err(err) = fs.read_cluster(cluster, &page).await { - return Some((Err(err), me)); - } - - Some((Ok(page), me)) - }) - } -} diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index a1755908..d560a672 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -125,7 +125,7 @@ impl InodeOps for FileInode { page: &mut CachePage, _: PageOffset, ) -> KResult<()> { - page.as_bytes_mut().fill(0); + page.lock().as_bytes_mut().fill(0); Ok(()) } diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 8e017336..be2146f8 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -8,8 +8,7 @@ use async_trait::async_trait; use mbr::MBRPartTable; use super::constants::ENOENT; -use super::mem::paging::Page; -use super::mem::PageExt; +use super::mem::Folio; use super::vfs::types::DeviceId; use crate::io::{Buffer, Chunks, FillResult}; use crate::kernel::constants::{EEXIST, EINVAL}; @@ -202,15 +201,15 @@ impl BlockDevice { let (page_slice, page, mut page_vec); match nr_batch { ..=8 => { - page = Page::alloc(); + page = Folio::alloc(); page_slice = core::slice::from_ref(&page); } ..=16 => { - page = Page::alloc_order(1); + page = Folio::alloc_order(1); page_slice = core::slice::from_ref(&page); } ..=32 => { - page = Page::alloc_order(2); + page = Folio::alloc_order(2); page_slice = core::slice::from_ref(&page); } count => { @@ -220,8 +219,8 @@ impl BlockDevice { let nr_pages = nr_huge_pages + nr_small_pages; page_vec = Vec::with_capacity(nr_pages); - page_vec.resize_with(nr_huge_pages, || Page::alloc_order(2)); - page_vec.resize_with(nr_pages, || Page::alloc()); + page_vec.resize_with(nr_huge_pages, || Folio::alloc_order(2)); + page_vec.resize_with(nr_pages, || Folio::alloc()); page_slice = &page_vec; } } @@ -266,7 +265,7 @@ pub enum BlockDeviceRequest<'lt> { /// Number of sectors to read count: u64, /// Buffer pages to read into - buffer: &'lt [Page], + buffer: &'lt [Folio], }, Write { /// Sector to write to, in 512-byte blocks @@ -274,6 +273,6 @@ pub enum BlockDeviceRequest<'lt> { /// Number of sectors to write count: u64, /// Buffer pages to write from - buffer: &'lt [Page], + buffer: &'lt [Folio], }, } diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index f8b5dc0b..47b864bb 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -3,14 +3,16 @@ pub mod paging; mod access; mod address; mod allocator; +mod folio; mod mm_area; mod mm_list; mod page_alloc; mod page_cache; pub use access::PhysAccess; +pub use folio::{Folio, FolioOwned, LockedFolio}; pub(self) use mm_area::MMArea; pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache, PageOffset}; -pub use paging::{Page, PageBuffer, PageExcl, PageExt}; +pub use paging::PageBuffer; diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index a3676ce0..3a70a8c2 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -3,11 +3,12 @@ use core::ptr::NonNull; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::PhysAccess; -use eonix_mm::paging::{PAGE_SIZE_BITS, PFN}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; use slab_allocator::SlabAlloc; -use super::{GlobalPageAlloc, Page, PageExt}; +use super::folio::Folio; +use super::GlobalPageAlloc; static SLAB_ALLOCATOR: LazyLock> = LazyLock::new(|| SlabAlloc::new_in(GlobalPageAlloc)); @@ -18,19 +19,15 @@ unsafe impl GlobalAlloc for Allocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { let size = layout.size().next_power_of_two(); - let result = if size <= 2048 { - SLAB_ALLOCATOR.alloc(size) + if size <= 2048 { + SLAB_ALLOCATOR.alloc(size).as_ptr() } else { - let page_count = size >> PAGE_SIZE_BITS; - let page = Page::alloc_at_least(page_count); - - let ptr = page.get_ptr(); - page.into_raw(); + let folio = Folio::alloc_at_least(size >> PAGE_SIZE_BITS); + let ptr = folio.get_ptr(); + folio.into_raw(); - ptr - }; - - result.as_ptr() + ptr.as_ptr() + } } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { @@ -45,7 +42,8 @@ unsafe impl GlobalAlloc for Allocator { } else { let paddr = ArchPhysAccess::from_ptr(ptr); let pfn = PFN::from(paddr); - Page::from_raw(pfn); + + Folio::from_raw(pfn); }; } } diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs new file mode 100644 index 00000000..6647e1af --- /dev/null +++ b/src/kernel/mem/folio.rs @@ -0,0 +1,210 @@ +use core::fmt; +use core::mem::ManuallyDrop; +use core::ops::Deref; +use core::ptr::NonNull; +use core::sync::atomic::Ordering; + +use eonix_mm::paging::{Folio as FolioTrait, FrameAlloc, GlobalFrameAlloc, Zone, PFN}; + +use super::page_alloc::ZONE; +use super::{GlobalPageAlloc, PhysAccess as _, RawPage}; + +#[repr(transparent)] +pub struct Folio(NonNull); + +#[derive(Debug)] +#[repr(transparent)] +pub struct FolioOwned(Folio); + +#[repr(transparent)] +pub struct LockedFolio<'a>(&'a Folio); + +unsafe impl Send for Folio {} +unsafe impl Sync for Folio {} + +impl Folio { + pub(super) const fn from_mut_page(raw_page: &'static mut RawPage) -> Self { + Self(NonNull::new(raw_page).unwrap()) + } + + /// Allocate a folio of the given *order*. + pub fn alloc_order(order: u32) -> Self { + GlobalPageAlloc::GLOBAL + .alloc_order(order) + .expect("Out of memory") + } + + /// Allocate a folio of order 0 + pub fn alloc() -> Self { + Self::alloc_order(0) + } + + /// Allocate a folio consisting of at least [`count`] pages. + pub fn alloc_at_least(count: usize) -> Self { + GlobalPageAlloc::GLOBAL + .alloc_at_least(count) + .expect("Out of memory") + } + + /// Acquire the ownership of the folio pointed to by [`pfn`], leaving + /// [`refcount`] untouched. + /// + /// # Panic + /// This function will panic if the folio is not within the global zone. + /// + /// # Safety + /// This function is unsafe because it assumes that the caller has to ensure + /// that [`pfn`] points to a valid folio allocated through [`Self::alloc()`] + /// and that the folio have not been freed or deallocated yet. + pub unsafe fn from_raw(pfn: PFN) -> Self { + unsafe { + // SAFETY: The caller ensures that [`pfn`] points to a folio within + // the global zone. + Self(ZONE.get_page(pfn).unwrap_unchecked()) + } + } + + /// Do some work with the folio without touching the reference count with + /// the same restrictions as [`Self::from_raw()`]. + /// + /// # Safety + /// Check [`Self::from_raw()`] for safety requirements. + pub unsafe fn with_raw(pfn: PFN, func: F) -> O + where + F: FnOnce(&Self) -> O, + { + unsafe { + let me = ManuallyDrop::new(Self::from_raw(pfn)); + func(&me) + } + } + + pub fn lock(&self) -> LockedFolio { + // TODO: actually perform the lock... + LockedFolio(self) + } + + /// Get a vmem pointer to the folio data as a byte slice. + pub fn get_bytes_ptr(&self) -> NonNull<[u8]> { + unsafe { + // SAFETY: `self.start()` can't be null. + NonNull::slice_from_raw_parts(self.start().as_ptr(), self.len()) + } + } + + /// Get a vmem pointer to the start of the folio. + pub fn get_ptr(&self) -> NonNull { + self.get_bytes_ptr().cast() + } +} + +impl Deref for Folio { + type Target = RawPage; + + fn deref(&self) -> &Self::Target { + unsafe { + // SAFETY: We don't expose mutable references to the folio. + self.0.as_ref() + } + } +} + +impl Clone for Folio { + fn clone(&self) -> Self { + // SAFETY: Memory order here can be Relaxed is for the same reason as + // that in the copy constructor of `std::shared_ptr`. + self.refcount.fetch_add(1, Ordering::Relaxed); + + Self(self.0) + } +} + +impl Drop for Folio { + fn drop(&mut self) { + match self.refcount.fetch_sub(1, Ordering::AcqRel) { + 0 => unreachable!("Refcount for an in-use page is 0"), + 1 => unsafe { GlobalPageAlloc::GLOBAL.dealloc_raw(self.0.as_mut()) }, + _ => {} + } + } +} + +impl fmt::Debug for Folio { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Page({:?}, order={})", self.pfn(), self.order) + } +} + +impl FolioTrait for Folio { + fn pfn(&self) -> PFN { + ZONE.get_pfn(self.0.as_ptr()) + } + + fn order(&self) -> u32 { + self.order + } +} + +impl LockedFolio<'_> { + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) + } + } +} + +impl Deref for LockedFolio<'_> { + type Target = Folio; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl FolioOwned { + pub fn alloc() -> Self { + Self(Folio::alloc()) + } + + pub fn alloc_order(order: u32) -> Self { + Self(Folio::alloc_order(order)) + } + + pub fn alloc_at_least(count: usize) -> Self { + Self(Folio::alloc_at_least(count)) + } + + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_ref() + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_mut() + } + } + + pub fn share(self) -> Folio { + self.0 + } +} + +impl Deref for FolioOwned { + type Target = Folio; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index 2891dad8..782c5ef7 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -1,15 +1,16 @@ use core::borrow::Borrow; use core::cell::UnsafeCell; use core::cmp; +use core::sync::atomic::Ordering; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; -use eonix_mm::paging::PFN; +use eonix_mm::paging::{Folio as _, PFN}; use super::mm_list::EMPTY_PAGE; -use super::{Mapping, Page, Permission}; -use crate::kernel::mem::page_cache::PageOffset; -use crate::kernel::mem::{CachePage, PageExcl, PageExt}; +use super::{Mapping, Permission}; +use crate::kernel::mem::folio::Folio; +use crate::kernel::mem::{CachePage, FolioOwned, PageOffset}; use crate::prelude::KResult; #[derive(Debug)] @@ -98,8 +99,10 @@ impl MMArea { attr.remove(PageAttribute::COPY_ON_WRITE); attr.set(PageAttribute::WRITE, self.permission.write); - let page = unsafe { Page::from_raw(*pfn) }; - if page.is_exclusive() { + let page = unsafe { Folio::from_raw(*pfn) }; + + // XXX: Change me!!! + if page.refcount.load(Ordering::Relaxed) == 1 { // SAFETY: This is actually safe. If we read `1` here and we have `MMList` lock // held, there couldn't be neither other processes sharing the page, nor other // threads making the page COW at the same time. @@ -109,9 +112,13 @@ impl MMArea { let mut new_page; if *pfn == EMPTY_PAGE.pfn() { - new_page = PageExcl::zeroed(); + new_page = { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio + }; } else { - new_page = PageExcl::alloc(); + new_page = FolioOwned::alloc(); unsafe { // SAFETY: `page` is CoW, which means that others won't write to it. @@ -123,7 +130,7 @@ impl MMArea { } attr.remove(PageAttribute::ACCESSED); - *pfn = new_page.into_page().into_raw(); + *pfn = new_page.share().into_raw(); } /// # Arguments @@ -143,11 +150,11 @@ impl MMArea { let file_offset = file_mapping.offset + offset; - let map_page = |page: &Page, cache_page: &CachePage| { + let map_page = |cache_page: &CachePage| { if !self.permission.write { assert!(!write, "Write fault on read-only mapping"); - *pfn = page.clone().into_raw(); + *pfn = cache_page.add_mapping(); return; } @@ -157,26 +164,26 @@ impl MMArea { // So here we can set the dirty flag now. cache_page.set_dirty(true); attr.insert(PageAttribute::WRITE); - *pfn = page.clone().into_raw(); + *pfn = cache_page.add_mapping(); return; } if !write { // Delay the copy-on-write until write fault happens. attr.insert(PageAttribute::COPY_ON_WRITE); - *pfn = page.clone().into_raw(); + *pfn = cache_page.add_mapping(); return; } // XXX: Change this. Let's handle mapped pages before CoW pages. // Nah, we are writing to a mapped private mapping... - let mut new_page = PageExcl::zeroed(); + let mut new_page = FolioOwned::alloc(); new_page .as_bytes_mut() - .copy_from_slice(page.lock().as_bytes()); + .copy_from_slice(cache_page.lock().as_bytes()); attr.insert(PageAttribute::WRITE); - *pfn = new_page.into_page().into_raw(); + *pfn = new_page.share().into_raw(); }; file_mapping diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 17dc1b05..5221c73b 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -1,31 +1,33 @@ mod mapping; mod page_fault; +mod page_table; use alloc::collections::btree_set::BTreeSet; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; use eonix_hal::mm::{ - flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchPagingMode, - ArchPhysAccess, GLOBAL_PAGE_TABLE, + flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, GLOBAL_PAGE_TABLE, }; use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PageTable, RawAttribute, PTE}; -use eonix_mm::paging::{PAGE_SIZE, PFN}; +use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; use eonix_sync::{LazyLock, Mutex}; pub use mapping::{FileMapping, Mapping}; pub use page_fault::handle_kernel_page_fault; +use page_table::KernelPageTable; use super::address::{VAddrExt as _, VRangeExt as _}; -use super::page_alloc::GlobalPageAlloc; -use super::paging::AllocZeroed as _; -use super::{MMArea, Page, PageExt}; +use super::{Folio, FolioOwned, MMArea}; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; -use crate::kernel::mem::page_alloc::RawPagePtr; use crate::prelude::*; use crate::sync::ArcSwap; -pub static EMPTY_PAGE: LazyLock = LazyLock::new(|| Page::zeroed()); +pub static EMPTY_PAGE: LazyLock = LazyLock::new(|| { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio.share() +}); #[derive(Debug, Clone, Copy)] pub struct Permission { @@ -34,23 +36,21 @@ pub struct Permission { pub execute: bool, } -pub type KernelPageTable<'a> = PageTable<'a, ArchPagingMode, GlobalPageAlloc, ArchPhysAccess>; - -struct MMListInner<'a> { +struct MMListInner { areas: BTreeSet, - page_table: KernelPageTable<'a>, + page_table: KernelPageTable, break_start: Option, break_pos: Option, } pub struct MMList { - inner: ArcSwap>>, + inner: ArcSwap>, user_count: AtomicUsize, /// Only used in kernel space to switch page tables on context switch. root_page_table: AtomicUsize, } -impl MMListInner<'_> { +impl MMListInner { fn overlapping_addr(&self, addr: VAddr) -> Option<&MMArea> { self.areas.get(&VRange::from(addr)) } @@ -96,7 +96,7 @@ impl MMListInner<'_> { } } - fn unmap(&mut self, start: VAddr, len: usize) -> KResult> { + fn unmap(&mut self, start: VAddr, len: usize) -> KResult> { assert_eq!(start.floor(), start); let end = (start + len).ceil(); let range_to_unmap = VRange::new(start, end); @@ -120,7 +120,7 @@ impl MMListInner<'_> { let (pfn, _) = pte.take(); pages_to_free.push(unsafe { // SAFETY: We got the pfn from a valid page table entry, so it should be valid. - Page::from_raw(pfn) + Folio::from_raw(pfn) }); } @@ -275,23 +275,23 @@ impl MMListInner<'_> { } } -impl Drop for MMListInner<'_> { +impl Drop for MMListInner { fn drop(&mut self) { // May buggy for area in &self.areas { if area.is_shared { for pte in self.page_table.iter_user(area.range()) { - let (pfn, _) = pte.take(); - let raw_page = RawPagePtr::from(pfn); - if raw_page.refcount().fetch_sub(1, Ordering::Relaxed) == 1 { - // Wrong here - // unsafe { Page::from_raw(pfn) }; - } + // XXX: Fix me + let _ = pte.take(); + // let raw_page = RawPagePtr::from(pfn); + // if raw_page.refcount().fetch_sub(1, Ordering::Relaxed) == 1 { + // unsafe { Page::from_raw(pfn) }; + // } } } else { for pte in self.page_table.iter_user(area.range()) { let (pfn, _) = pte.take(); - unsafe { Page::from_raw(pfn) }; + unsafe { Folio::from_raw(pfn) }; } } } @@ -327,7 +327,7 @@ impl MMList { } pub fn new() -> Self { - let page_table = GLOBAL_PAGE_TABLE.clone_global(); + let page_table = KernelPageTable::new(); Self { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), @@ -344,7 +344,7 @@ impl MMList { let inner = self.inner.borrow(); let mut inner = inner.lock().await; - let page_table = GLOBAL_PAGE_TABLE.clone_global(); + let page_table = KernelPageTable::new(); let list = Self { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), @@ -392,7 +392,7 @@ impl MMList { } pub fn deactivate(&self) { - set_root_page_table_pfn(PFN::from(GLOBAL_PAGE_TABLE.addr())); + set_root_page_table_pfn(PFN::from(GLOBAL_PAGE_TABLE.start())); let old_user_count = self.user_count.fetch_sub(1, Ordering::Release); assert_ne!(old_user_count, 0); @@ -444,7 +444,7 @@ impl MMList { let new_root_page_table = match &new { Some(new_mm) => new_mm.root_page_table.load(Ordering::Relaxed), - None => GLOBAL_PAGE_TABLE.addr().addr(), + None => GLOBAL_PAGE_TABLE.start().addr(), }; set_root_page_table_pfn(PFN::from(PAddr::from(new_root_page_table))); @@ -693,7 +693,7 @@ impl MMList { unsafe { // SAFETY: We are sure that the page is valid and we have the right to access it. - Page::with_raw(pte.get_pfn(), |page| { + Folio::with_raw(pte.get_pfn(), |page| { let mut pg = page.lock(); let page_data = &mut pg.as_bytes_mut()[start_offset..end_offset]; @@ -724,7 +724,7 @@ trait PageTableExt { fn set_copied(&self, from: &Self, range: VRange); } -impl PageTableExt for KernelPageTable<'_> { +impl PageTableExt for KernelPageTable { fn set_anonymous(&self, range: VRange, permission: Permission) { for pte in self.iter_user(range) { pte.set_anonymous(permission.execute); @@ -805,7 +805,7 @@ where let pfn = unsafe { // SAFETY: We get the pfn from a valid page table entry, so it should be valid as well. - Page::with_raw(from.get_pfn(), |page| page.clone().into_raw()) + Folio::with_raw(from.get_pfn(), |page| page.clone().into_raw()) }; self.set(pfn, T::Attr::from(from_attr & !PageAttribute::ACCESSED)); diff --git a/src/kernel/mem/mm_list/page_table.rs b/src/kernel/mem/mm_list/page_table.rs new file mode 100644 index 00000000..8a2acc13 --- /dev/null +++ b/src/kernel/mem/mm_list/page_table.rs @@ -0,0 +1,40 @@ +use core::ops::Deref; + +use eonix_hal::arch_exported::mm::{ArchPagingMode, PageAccessImpl}; +use eonix_hal::mm::GLOBAL_PAGE_TABLE; +use eonix_mm::page_table::PageTable; +use eonix_mm::paging::{Folio, GlobalFrameAlloc}; + +use crate::kernel::mem::{FolioOwned, GlobalPageAlloc, PhysAccess}; + +#[repr(transparent)] +pub struct KernelPageTable(PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>); + +impl KernelPageTable { + pub fn new() -> Self { + let global_page_table = unsafe { + // SAFETY: The region is valid and read only after initialization. + GLOBAL_PAGE_TABLE.start().as_ptr::<[u8; 4096]>().as_ref() + }; + + let mut table_page = FolioOwned::alloc(); + let entries = table_page.as_bytes_mut().len(); + table_page.as_bytes_mut()[..(entries / 2)].fill(0); + table_page.as_bytes_mut()[(entries / 2)..] + .copy_from_slice(&global_page_table[(entries / 2)..]); + + Self(PageTable::new( + table_page.share(), + GlobalPageAlloc::GLOBAL, + PageAccessImpl, + )) + } +} + +impl Deref for KernelPageTable { + type Target = PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index 37344fc2..ac2485da 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -5,14 +5,14 @@ use core::sync::atomic::Ordering; use buddy_allocator::BuddyAllocator; use eonix_mm::address::PRange; -use eonix_mm::paging::{ - GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PageList, PageListSized as _, -}; +use eonix_mm::page_table::PageTableAlloc; +use eonix_mm::paging::{FolioList, FolioListSized as _, FrameAlloc, GlobalFrameAlloc, PFN}; use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; -use raw_page::{PageFlags, RawPageList}; -pub use raw_page::{RawPage, RawPagePtr}; -pub use zones::GlobalZone; +pub use raw_page::{PageFlags, RawPage, RawPageList}; +pub use zones::{GlobalZone, ZONE}; + +use super::folio::Folio; const COSTLY_ORDER: u32 = 3; const AREAS: usize = COSTLY_ORDER as usize + 1; @@ -27,9 +27,6 @@ static PERCPU_PAGE_ALLOC: PerCpuPageAlloc = PerCpuPageAlloc::new(); #[derive(Clone)] pub struct GlobalPageAlloc; -#[derive(Clone)] -pub struct BuddyPageAlloc(); - struct PerCpuPageAlloc { batch: u32, free_areas: [RawPageList; AREAS], @@ -72,11 +69,6 @@ impl PerCpuPageAlloc { } impl GlobalPageAlloc { - #[allow(dead_code)] - pub const fn buddy_alloc() -> BuddyPageAlloc { - BuddyPageAlloc() - } - /// Add the pages in the PAddr range `range` to the global allocator. /// /// This function is only to be called on system initialization when `eonix_preempt` @@ -88,15 +80,11 @@ impl GlobalPageAlloc { pub unsafe fn add_pages(range: PRange) { BUDDY_ALLOC .lock_with_context(NoContext) - .create_pages(range.start(), range.end()) + .create_folios(range.start(), range.end()) } -} - -impl PageAlloc for GlobalPageAlloc { - type RawPage = RawPagePtr; - fn alloc_order(&self, order: u32) -> Option { - let raw_page = if order > COSTLY_ORDER { + pub fn alloc_raw_order(&self, order: u32) -> Option<&'static mut RawPage> { + if order > COSTLY_ORDER { BUDDY_ALLOC.lock().alloc_order(order) } else { unsafe { @@ -106,61 +94,54 @@ impl PageAlloc for GlobalPageAlloc { page } - }; - - raw_page.map(|raw_page| { - // SAFETY: Memory order here can be Relaxed is for the same reason - // as that in the copy constructor of `std::shared_ptr`. - raw_page.refcount.fetch_add(1, Ordering::Relaxed); - - RawPagePtr::from_ref(raw_page) - }) + } } - unsafe fn dealloc(&self, page_ptr: RawPagePtr) { + pub unsafe fn dealloc_raw(&self, raw_page: &'static mut RawPage) { assert_eq!( - page_ptr.refcount().load(Ordering::Relaxed), + raw_page.refcount.load(Ordering::Relaxed), 0, "Trying to free a page with refcount > 0" ); - if page_ptr.order() > COSTLY_ORDER { - BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); + if raw_page.order > COSTLY_ORDER { + BUDDY_ALLOC.lock().dealloc(raw_page); } else { - let order = page_ptr.order(); + let order = raw_page.order; unsafe { - PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()).free_pages(page_ptr.as_mut(), order); + PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()).free_pages(raw_page, order); } } } - - fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - page_ptr.order() > COSTLY_ORDER || page_ptr.flags().has(PageFlags::LOCAL) - } } -impl GlobalPageAllocTrait for GlobalPageAlloc { - fn global() -> Self { - GlobalPageAlloc +impl FrameAlloc for GlobalPageAlloc { + type Folio = Folio; + + fn alloc_order(&self, order: u32) -> Option { + self.alloc_raw_order(order).map(|raw_page| { + // SAFETY: Memory order here can be Relaxed is for the same reason + // as that in the copy constructor of `std::shared_ptr`. + + raw_page.refcount.fetch_add(1, Ordering::Relaxed); + Folio::from_mut_page(raw_page) + }) } } -impl PageAlloc for BuddyPageAlloc { - type RawPage = RawPagePtr; +impl GlobalFrameAlloc for GlobalPageAlloc { + const GLOBAL: Self = GlobalPageAlloc; +} - fn alloc_order(&self, order: u32) -> Option { - BUDDY_ALLOC - .lock() - .alloc_order(order) - .map(|raw_page| RawPagePtr::from_ref(raw_page)) - } +impl PageTableAlloc for GlobalPageAlloc { + type Folio = Folio; - unsafe fn dealloc(&self, page_ptr: RawPagePtr) { - BUDDY_ALLOC.lock().dealloc(page_ptr.as_mut()); + fn alloc(&self) -> Self::Folio { + FrameAlloc::alloc(self).unwrap() } - fn has_management_over(&self, _: RawPagePtr) -> bool { - true + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio { + unsafe { Folio::from_raw(pfn) } } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 074f82c7..0d775245 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,20 +1,17 @@ use core::ptr::NonNull; use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; -use buddy_allocator::BuddyPage; +use buddy_allocator::BuddyFolio; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _}; -use eonix_mm::paging::{PageAlloc, PageList, PageListSized, RawPage as RawPageTrait, PFN}; +use eonix_mm::paging::{FolioList, FolioListSized, Zone, PFN}; use intrusive_list::{container_of, Link, List}; use slab_allocator::{SlabPage, SlabPageAlloc, SlabSlot}; +use super::zones::ZONE; use super::{GlobalPageAlloc, PerCpuPage}; -use crate::kernel::mem::page_cache::PageCacheRawPage; use crate::kernel::mem::PhysAccess; -pub const PAGE_ARRAY: NonNull = - unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; - pub struct PageFlags(AtomicU32); #[derive(Clone, Copy)] @@ -41,11 +38,11 @@ pub struct RawPage { /// This can be used for LRU page swap in the future. /// /// Now only used for free page links in the buddy system. - link: Link, + pub link: Link, /// # Safety /// This field is only used in buddy system and is protected by the global lock. - order: u32, - flags: PageFlags, + pub order: u32, + pub flags: PageFlags, pub refcount: AtomicUsize, shared_data: PageData, @@ -55,9 +52,6 @@ pub struct RawPage { unsafe impl Send for RawPage {} unsafe impl Sync for RawPage {} -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct RawPagePtr(NonNull); - impl PageFlags { pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; @@ -85,80 +79,9 @@ impl PageFlags { } } -impl RawPagePtr { - pub const fn from_ref(raw_page_ref: &RawPage) -> Self { - Self::new(unsafe { - // SAFETY: Rust references always points to non-null addresses. - NonNull::new_unchecked(&raw const *raw_page_ref as *mut _) - }) - } - - pub const fn new(ptr: NonNull) -> Self { - Self(ptr) - } - - /// Get a raw pointer to the underlying `RawPage` struct. - /// - /// # Safety - /// Doing arithmetic on the pointer returned will cause immediate undefined behavior. - pub const unsafe fn as_ptr(self) -> *mut RawPage { - self.0.as_ptr() - } - - pub const fn as_ref<'a>(self) -> &'a RawPage { - unsafe { &*self.as_ptr() } - } - - pub const fn as_mut<'a>(self) -> &'a mut RawPage { - unsafe { &mut *self.as_ptr() } - } - - pub const fn order(&self) -> u32 { - self.as_ref().order - } - - pub const fn flags(&self) -> &PageFlags { - &self.as_ref().flags - } - - pub const fn refcount(&self) -> &AtomicUsize { - &self.as_ref().refcount - } - - // return the ptr point to the actually raw page - pub fn real_ptr(&self) -> NonNull { - let pfn = unsafe { PFN::from(RawPagePtr(NonNull::new_unchecked(self.as_ptr()))) }; - unsafe { PAddr::from(pfn).as_ptr::() } - } -} - -impl From for PFN { - fn from(value: RawPagePtr) -> Self { - let idx = unsafe { value.as_ptr().offset_from(PAGE_ARRAY.as_ptr()) as usize }; - Self::from(idx) - } -} - -impl From for RawPagePtr { - fn from(pfn: PFN) -> Self { - let raw_page_ptr = unsafe { PAGE_ARRAY.add(usize::from(pfn)) }; - Self::new(raw_page_ptr) - } -} - -impl RawPageTrait for RawPagePtr { - fn order(&self) -> u32 { - self.order() - } - - fn refcount(&self) -> &AtomicUsize { - self.refcount() - } -} - -impl BuddyPage for RawPage { +impl BuddyFolio for RawPage { fn pfn(&self) -> PFN { - PFN::from(RawPagePtr::from_ref(self)) + ZONE.get_pfn(self) } fn get_order(&self) -> u32 { @@ -184,8 +107,7 @@ impl BuddyPage for RawPage { impl SlabPage for RawPage { fn get_data_ptr(&self) -> NonNull<[u8]> { - let raw_page_ptr = RawPagePtr::from_ref(self); - let paddr_start = PAddr::from(PFN::from(raw_page_ptr)); + let paddr_start = PAddr::from(ZONE.get_pfn(self)); let page_data_ptr = unsafe { paddr_start.as_ptr() }; NonNull::slice_from_raw_parts(page_data_ptr, 1 << (self.order + 12)) @@ -233,21 +155,9 @@ impl SlabPage for RawPage { let paddr = ArchPhysAccess::from_ptr(ptr); let pfn = PFN::from(paddr); - RawPagePtr::from(pfn).as_mut() - } - } -} - -impl PageCacheRawPage for RawPagePtr { - fn is_dirty(&self) -> bool { - self.flags().has(PageFlags::DIRTY) - } - - fn set_dirty(&self, dirty: bool) { - if dirty { - self.flags().set(PageFlags::DIRTY); - } else { - self.flags().clear(PageFlags::DIRTY); + ZONE.get_page(pfn) + .expect("Page outside of the global zone") + .as_mut() } } } @@ -264,14 +174,16 @@ impl PerCpuPage for RawPage { pub struct RawPageList(List); -impl PageList for RawPageList { - type Page = RawPage; +unsafe impl Send for RawPageList {} + +impl FolioList for RawPageList { + type Folio = RawPage; fn is_empty(&self) -> bool { self.0.is_empty() } - fn peek_head(&mut self) -> Option<&mut Self::Page> { + fn peek_head(&mut self) -> Option<&mut Self::Folio> { unsafe { let link = self.0.head()?; let mut raw_page_ptr = container_of!(link, RawPage, link); @@ -280,7 +192,7 @@ impl PageList for RawPageList { } } - fn pop_head(&mut self) -> Option<&'static mut Self::Page> { + fn pop_head(&mut self) -> Option<&'static mut Self::Folio> { unsafe { let link = self.0.pop()?; let mut raw_page_ptr = container_of!(link, RawPage, link); @@ -289,25 +201,25 @@ impl PageList for RawPageList { } } - fn push_tail(&mut self, page: &'static mut Self::Page) { + fn push_tail(&mut self, page: &'static mut Self::Folio) { self.0.insert(&mut page.link); } - fn remove(&mut self, page: &mut Self::Page) { + fn remove(&mut self, page: &mut Self::Folio) { self.0.remove(&mut page.link) } } -impl PageListSized for RawPageList { +impl FolioListSized for RawPageList { const NEW: Self = RawPageList(List::new()); } -impl SlabPageAlloc for GlobalPageAlloc { +unsafe impl SlabPageAlloc for GlobalPageAlloc { type Page = RawPage; type PageList = RawPageList; - unsafe fn alloc_uninit(&self) -> &'static mut RawPage { - let raw_page = self.alloc().expect("Out of memory").as_mut(); + fn alloc_slab_page(&self) -> &'static mut RawPage { + let raw_page = self.alloc_raw_order(0).expect("Out of memory"); raw_page.flags.set(PageFlags::SLAB); raw_page.shared_data.slab = SlabPageData::new(); diff --git a/src/kernel/mem/page_alloc/zones.rs b/src/kernel/mem/page_alloc/zones.rs index 7a2e4e33..032b9cd0 100644 --- a/src/kernel/mem/page_alloc/zones.rs +++ b/src/kernel/mem/page_alloc/zones.rs @@ -1,13 +1,23 @@ -use core::cell::UnsafeCell; +use core::ptr::NonNull; use eonix_mm::address::PRange; use eonix_mm::paging::{Zone, PFN}; use super::RawPage; -use crate::kernel::mem::page_alloc::RawPagePtr; + +pub static ZONE: GlobalZone = GlobalZone(); + +const PAGE_ARRAY: NonNull = + unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; pub struct GlobalZone(); +impl GlobalZone { + pub fn get_pfn(&self, page_ptr: *const RawPage) -> PFN { + PFN::from(unsafe { page_ptr.offset_from(PAGE_ARRAY.as_ptr()) as usize }) + } +} + impl Zone for GlobalZone { type Page = RawPage; @@ -15,11 +25,7 @@ impl Zone for GlobalZone { true } - fn get_page(&self, pfn: PFN) -> Option<&UnsafeCell> { - unsafe { - // SAFETY: The pointer returned by [`RawPagePtr::as_ptr()`] is valid. - // And so is it wrapped with [`UnsafeCell`] - Some(&*(RawPagePtr::from(pfn).as_ptr() as *const UnsafeCell)) - } + fn get_page(&self, pfn: PFN) -> Option> { + Some(unsafe { PAGE_ARRAY.add(usize::from(pfn)) }) } } diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 214c65a5..3fe33d5b 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,19 +1,16 @@ use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; -use core::mem::ManuallyDrop; +use core::ops::{Deref, DerefMut}; -use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::address::{PAddr, PhysAccess}; -use eonix_mm::paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::Mutex; -use super::Page; +use super::page_alloc::PageFlags; +use super::{Folio, FolioOwned}; use crate::io::{Buffer, Stream}; use crate::kernel::constants::EINVAL; -use crate::kernel::mem::page_alloc::RawPagePtr; use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; -use crate::GlobalPageAlloc; #[repr(transparent)] #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -24,14 +21,7 @@ pub struct PageCache { inode: InodeUse, } -unsafe impl Send for PageCache {} -unsafe impl Sync for PageCache {} - -#[derive(Clone, Copy)] -pub struct CachePage(RawPagePtr); - -unsafe impl Send for CachePage {} -unsafe impl Sync for CachePage {} +pub struct CachePage(Folio); impl PageOffset { pub const fn from_byte_floor(offset: usize) -> Self { @@ -57,39 +47,47 @@ impl PageOffset { impl CachePage { pub fn new() -> Self { - Self(GlobalPageAlloc.alloc().unwrap()) + CachePage(Folio::alloc()) } - pub fn as_bytes(&self) -> &[u8] { - unsafe { - core::slice::from_raw_parts( - // SAFETY: The page is owned by us, so we can safely access its data. - ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), - PAGE_SIZE, - ) - } - } + pub fn new_zeroed() -> Self { + CachePage({ + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { - core::slice::from_raw_parts_mut( - // SAFETY: The page is exclusively owned by us, so we can safely access its data. - ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), - PAGE_SIZE, - ) - } + folio.share() + }) } pub fn is_dirty(&self) -> bool { - self.0.is_dirty() + self.flags.has(PageFlags::DIRTY) } pub fn set_dirty(&self, dirty: bool) { - self.0.set_dirty(dirty); + if dirty { + self.flags.set(PageFlags::DIRTY); + } else { + self.flags.clear(PageFlags::DIRTY); + } + } + + pub fn add_mapping(&self) -> PFN { + // TODO: Increase map_count + self.0.clone().into_raw() } +} + +impl Deref for CachePage { + type Target = Folio; - pub fn get_page(&self) -> Page { - unsafe { Page::with_raw(PFN::from(self.0), |page| page.clone()) } + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for CachePage { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 } } @@ -124,11 +122,7 @@ impl PageCache { } // TODO: Remove this. - pub async fn with_page( - &self, - pgoff: PageOffset, - func: impl FnOnce(&Page, &CachePage), - ) -> KResult<()> { + pub async fn with_page(&self, pgoff: PageOffset, func: impl FnOnce(&CachePage)) -> KResult<()> { let mut pages = self.pages.lock().await; if pgoff > PageOffset::from_byte_ceil(self.len()) { return Err(EINVAL); @@ -136,11 +130,7 @@ impl PageCache { let cache_page = self.get_page_locked(&mut pages, pgoff).await?; - unsafe { - let page = ManuallyDrop::new(Page::from_raw_unchecked(PFN::from(cache_page.0))); - - func(&page, cache_page); - } + func(cache_page); Ok(()) } @@ -166,7 +156,7 @@ impl PageCache { let data_len = real_end - offset; if buffer - .fill(&page.as_bytes()[inner_offset..inner_offset + data_len])? + .fill(&page.lock().as_bytes()[inner_offset..inner_offset + data_len])? .should_stop() || buffer.available() == 0 { @@ -195,7 +185,7 @@ impl PageCache { let inner_offset = offset % PAGE_SIZE; let written = stream - .poll_data(&mut page.as_bytes_mut()[inner_offset..])? + .poll_data(&mut page.lock().as_bytes_mut()[inner_offset..])? .map(|b| b.len()) .unwrap_or(0); @@ -237,14 +227,9 @@ impl core::fmt::Debug for PageCache { } } -pub trait PageCacheRawPage: RawPage { - fn is_dirty(&self) -> bool; - fn set_dirty(&self, dirty: bool); -} - impl Drop for PageCache { fn drop(&mut self) { - // TODO: Write back dirty pages... - // let _ = self.fsync(); + // XXX: Send the PageCache to some flusher worker. + let _ = self.fsync(); } } diff --git a/src/kernel/mem/paging.rs b/src/kernel/mem/paging.rs index bca573fb..1b95ce79 100644 --- a/src/kernel/mem/paging.rs +++ b/src/kernel/mem/paging.rs @@ -1,48 +1,22 @@ -use core::ops::Deref; -use core::ptr::NonNull; +use eonix_mm::paging::Folio as _; -use eonix_mm::paging::Page as GenericPage; - -use super::page_alloc::GlobalPageAlloc; -use super::PhysAccess; +use super::folio::FolioOwned; use crate::io::{Buffer, FillResult}; -pub type Page = GenericPage; - /// A buffer that wraps a page and provides a `Buffer` interface. pub struct PageBuffer { - page: PageExcl, + page: FolioOwned, offset: usize, } -pub struct PageLocked<'a> { - page: &'a Page, -} - -/// A page that is exclusively owned. -#[repr(transparent)] -pub struct PageExcl(Page); - pub trait AllocZeroed { fn zeroed() -> Self; } -pub trait PageExt { - fn lock(&self) -> PageLocked; - - /// Get a vmem pointer to the page data as a byte slice. - fn get_bytes_ptr(&self) -> NonNull<[u8]>; - - /// Get a vmem pointer to the start of the page. - fn get_ptr(&self) -> NonNull { - self.get_bytes_ptr().cast() - } -} - impl PageBuffer { pub fn new() -> Self { Self { - page: PageExcl::alloc(), + page: FolioOwned::alloc(), offset: 0, } } @@ -86,91 +60,3 @@ impl Buffer for PageBuffer { } } } - -impl AllocZeroed for Page { - fn zeroed() -> Self { - let page = Self::alloc(); - - page.lock().as_bytes_mut().fill(0); - - page - } -} - -impl PageExt for Page { - fn lock(&self) -> PageLocked { - // TODO: Actually perform the lock. - PageLocked { page: self } - } - - fn get_bytes_ptr(&self) -> NonNull<[u8]> { - unsafe { - // SAFETY: `self.start()` can't be null. - NonNull::slice_from_raw_parts(self.start().as_ptr(), self.len()) - } - } -} - -impl PageLocked<'_> { - pub fn as_bytes(&self) -> &[u8] { - unsafe { - // SAFETY: `self.start()` points to valid memory of length `self.len()`. - core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) - } - } - - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { - // SAFETY: `self.start()` points to valid memory of length `self.len()`. - core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) - } - } -} - -impl Deref for PageLocked<'_> { - type Target = Page; - - fn deref(&self) -> &Self::Target { - self.page - } -} - -impl PageExcl { - pub fn alloc() -> Self { - Self(Page::alloc()) - } - - pub fn alloc_order(order: u32) -> Self { - Self(Page::alloc_order(order)) - } - - pub fn zeroed() -> Self { - Self(Page::zeroed()) - } - - pub fn as_bytes(&self) -> &[u8] { - unsafe { - // SAFETY: The page is exclusively owned by us. - self.get_bytes_ptr().as_ref() - } - } - - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { - // SAFETY: The page is exclusively owned by us. - self.get_bytes_ptr().as_mut() - } - } - - pub fn into_page(self) -> Page { - self.0 - } -} - -impl Deref for PageExcl { - type Target = Page; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/src/kernel/task/kernel_stack.rs b/src/kernel/task/kernel_stack.rs index d3e9de2f..f00b91bd 100644 --- a/src/kernel/task/kernel_stack.rs +++ b/src/kernel/task/kernel_stack.rs @@ -1,11 +1,12 @@ -use crate::kernel::mem::{paging::Page, PhysAccess as _}; -use core::{num::NonZero, ptr::NonNull}; +use core::ptr::NonNull; + use eonix_runtime::executor::Stack; +use crate::kernel::mem::FolioOwned; + #[derive(Debug)] pub struct KernelStack { - _pages: Page, - bottom: NonZero, + folio: FolioOwned, } impl KernelStack { @@ -14,15 +15,8 @@ impl KernelStack { const KERNEL_STACK_ORDER: u32 = 7; pub fn new() -> Self { - let pages = Page::alloc_order(Self::KERNEL_STACK_ORDER); - let bottom = unsafe { - // SAFETY: The paddr is from a page, which should be valid. - pages.range().end().as_ptr::().addr() - }; - Self { - _pages: pages, - bottom, + folio: FolioOwned::alloc_order(Self::KERNEL_STACK_ORDER), } } } @@ -33,7 +27,10 @@ impl Stack for KernelStack { } fn get_bottom(&self) -> NonNull<()> { - // SAFETY: The stack is allocated and `bottom` is non-zero. - unsafe { NonNull::new_unchecked(self.bottom.get() as *mut _) } + let ptr = self.folio.get_bytes_ptr(); + let len = ptr.len(); + + // SAFETY: The vaddr of the folio is guaranteed to be non-zero. + unsafe { ptr.cast().byte_add(len) } } } diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index eb00cc4c..799b9848 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -15,7 +15,7 @@ pub use terminal_file::TerminalFile; use crate::io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}; use crate::kernel::constants::{EBADF, EINTR, EINVAL, ENOTTY}; -use crate::kernel::mem::PageExcl; +use crate::kernel::mem::FolioOwned; use crate::kernel::task::Thread; use crate::kernel::CharDevice; use crate::prelude::KResult; @@ -94,7 +94,7 @@ impl FileType { } pub async fn sendfile(&self, dest_file: &Self, count: usize) -> KResult { - let mut buffer_page = PageExcl::alloc(); + let mut buffer_page = FolioOwned::alloc(); let buffer = buffer_page.as_bytes_mut(); self.sendfile_check()?; diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 93b6da20..65af41e4 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -1,32 +1,26 @@ +use eonix_hal::arch_exported::mm::{ArchPagingMode, PageAccessImpl}; use eonix_hal::bootstrap::BootStrapData; -use eonix_hal::mm::{ArchMemory, ArchPagingMode, GLOBAL_PAGE_TABLE}; +use eonix_hal::mm::{ArchMemory, BasicPageAllocRef, GLOBAL_PAGE_TABLE}; use eonix_hal::traits::mm::Memory; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PagingMode as _, PTE}; -use eonix_mm::paging::{Page as GenericPage, PAGE_SIZE, PFN}; +use eonix_mm::page_table::{PageAttribute, PageTable, PTE}; +use eonix_mm::paging::{Folio as _, FrameAlloc, PAGE_SIZE, PFN}; use crate::kernel::mem::{GlobalPageAlloc, RawPage}; -pub fn setup_memory(data: &mut BootStrapData) { - let addr_max = ArchMemory::present_ram() - .map(|range| range.end()) - .max() - .expect("No free memory"); - - let pfn_max = PFN::from(addr_max.ceil()); - let len_bytes_page_array = usize::from(pfn_max) * size_of::(); - let count_pages = len_bytes_page_array.div_ceil(PAGE_SIZE); - - let alloc = data.get_alloc().unwrap(); +fn setup_kernel_page_array(alloc: BasicPageAllocRef, count_pages: usize) { + // TODO: This should be done by the global Zone + let global_page_table = PageTable::::new( + GLOBAL_PAGE_TABLE.clone(), + alloc.clone(), + PageAccessImpl, + ); // Map kernel page array. const V_KERNEL_PAGE_ARRAY_START: VAddr = VAddr::from(0xffffff8040000000); - for pte in GLOBAL_PAGE_TABLE.iter_kernel_in( - VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages), - ArchPagingMode::LEVELS, - &alloc, - ) { + let range = VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages); + for pte in global_page_table.iter_kernel(range) { let attr = PageAttribute::PRESENT | PageAttribute::WRITE | PageAttribute::READ @@ -34,10 +28,15 @@ pub fn setup_memory(data: &mut BootStrapData) { | PageAttribute::ACCESSED | PageAttribute::DIRTY; - let page = GenericPage::alloc_in(&alloc); + let page = alloc.alloc().unwrap(); pte.set(page.into_raw(), attr.into()); } + // TODO!!!: Construct the global zone with all present ram. + // for range in ArchMemory::present_ram() { + // GlobalPageAlloc::mark_present(range); + // } + unsafe { // SAFETY: We've just mapped the area with sufficient length. core::ptr::write_bytes( @@ -47,10 +46,21 @@ pub fn setup_memory(data: &mut BootStrapData) { ); } - // TODO!!!: Construct the global zone with all present ram. - // for range in ArchMemory::present_ram() { - // GlobalPageAlloc::mark_present(range); - // } + core::mem::forget(global_page_table); +} + +pub fn setup_memory(data: &mut BootStrapData) { + let addr_max = ArchMemory::present_ram() + .map(|range| range.end()) + .max() + .expect("No free memory"); + + let pfn_max = PFN::from(addr_max.ceil()); + let len_bytes_page_array = usize::from(pfn_max) * size_of::(); + let count_pages = len_bytes_page_array.div_ceil(PAGE_SIZE); + + let alloc = data.get_alloc().unwrap(); + setup_kernel_page_array(alloc, count_pages); if let Some(early_alloc) = data.take_alloc() { for range in early_alloc.into_iter() { From cfc959e53dabfcde855cea830610ff0a783c82d5 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 18 Jan 2026 01:09:33 +0800 Subject: [PATCH 011/153] script: add a script to help translate stacktraces - Add script/backtrace to translate backtraces. - Add a cut sign in the kernel panic routine to indicate the start of stack backtrace. Signed-off-by: greatbridf --- script/backtrace | 100 +++++++++++++++++++++++++++++++++++++++++++++++ src/panic.rs | 6 +++ 2 files changed, 106 insertions(+) create mode 100755 script/backtrace diff --git a/script/backtrace b/script/backtrace new file mode 100755 index 00000000..8a60c829 --- /dev/null +++ b/script/backtrace @@ -0,0 +1,100 @@ +#!/bin/bash + +ADDR2LINE=${ADDR2LINE:-riscv64-unknown-elf-addr2line} + +ksym=build/riscv64gc-unknown-none-elf/debug/eonix_kernel + +usage() { + cat < Use the given kernel symbol file + -o, --only-gbos Show kernel function calls only + -h, --help Show this message +EOF + exit "$1" +} + +# $1: instruction address +parse_pos() { + addr="$1" + shift + + "$ADDR2LINE" -e "$ksym" -i "$addr" "$@" 2>/dev/null +} + +filter_col() { + [ "$1" -eq 0 ] || awk "{ print \$$1; }" +} + +str_contains() { + grep -E "$1" >/dev/null 2>&1 +} + +filter_stacktrace() { + NL=$'\n' + _state=nonstart + _out= + while [ $_state != "end" ]; do + read -r _line + case $_state in + nonstart) + str_contains "8< CUT HERE" <<< "$_line" && _state=save + ;; + save) + if str_contains "8< CUT HERE" <<< "$_line"; then + _state=end + else + _out="$_out$_line$NL" + fi + ;; + esac + done + + echo "$_out" +} + +while [ "$#" -gt 0 ]; do + case "$1" in + -s|--ksym) + shift + ksym="$1" + ;; + -o|--only-gbos) + only_gb=y + ;; + --) + shift + break + ;; + -h|--help) + usage 0 + ;; + *) + usage 1 + ;; + esac + shift +done + +stacktrace="$(filter_stacktrace)" + +i=1 +for addr in $(filter_col 3 <<< "$stacktrace"); do + pos="$(parse_pos "$addr" "$@")" + + if [ -n "$only_gb" ]; then + if ! str_contains "greatbridf_os" <<< "$pos"; then + continue + fi + fi + + printf "========== %4d ==========\n" "$i" + + parse_pos "$addr" "$@" + + i=$((i + 1)) +done diff --git a/src/panic.rs b/src/panic.rs index 3c9c5f34..4a9ef92c 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -24,6 +24,12 @@ pub fn stack_trace() { UnwindReasonCode::NO_REASON } + println_fatal!("--------------8< CUT HERE 8<--------------"); + println_fatal!("Stacktrace:"); + println_fatal!(); + let mut data = CallbackData { counter: 0 }; _Unwind_Backtrace(callback, &raw mut data as *mut c_void); + + println_fatal!("--------------8< CUT HERE 8<--------------"); } From b0c8ef4ccc367cbbcb83ca9901e19ffad20e2758 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 18 Jan 2026 02:35:05 +0800 Subject: [PATCH 012/153] style: fix or suppress warnings No functional changes - Add `{extern_,}symbol_addr` macro to retrieve symbol address. - Remove manual impl Send and Sync for RawPage - Make elided '_ lifetimes in return types explicit - Suppress unused warnings by allowing them - Remove really unused functions - Refactor `println_trace` macro to suppress unused variable warnings Signed-off-by: greatbridf --- crates/eonix_hal/src/lib.rs | 26 +++++++++ crates/eonix_log/src/lib.rs | 22 ++++++-- crates/eonix_percpu/src/lib.rs | 31 +++++------ crates/eonix_runtime/src/scheduler.rs | 35 +++++------- crates/eonix_sync/eonix_spin/src/lib.rs | 14 +++-- .../eonix_sync/eonix_sync_rt/src/spin_irq.rs | 4 +- crates/posix_types/src/poll.rs | 2 + macros/src/lib.rs | 4 +- src/driver/ahci/slot.rs | 4 +- src/driver/e1000e.rs | 2 + src/driver/serial/io.rs | 15 ++++-- src/fs/procfs.rs | 2 + src/fs/tmpfs/file.rs | 6 +-- src/kernel/constants.rs | 2 +- src/kernel/mem/folio.rs | 2 +- src/kernel/mem/mm_list/page_fault.rs | 19 +++---- src/kernel/mem/page_alloc/raw_page.rs | 4 -- src/kernel/pcie/device.rs | 24 +++++---- src/kernel/pcie/header.rs | 17 +++--- src/kernel/pcie/init.rs | 5 +- src/kernel/syscall.rs | 23 ++++---- src/kernel/task.rs | 3 +- src/kernel/task/process.rs | 54 +++++++++---------- src/kernel/vfs/dentry.rs | 6 ++- src/kernel/vfs/types/device_id.rs | 11 ---- src/lib.rs | 5 +- src/path.rs | 6 ++- src/sync/arcswap.rs | 11 ++-- 28 files changed, 188 insertions(+), 171 deletions(-) diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index b9c7d053..d3bf7825 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -43,3 +43,29 @@ pub mod arch_exported { pub use eonix_hal_macros::{ap_main, default_trap_handler, main}; pub use eonix_hal_traits as traits; + +#[macro_export] +macro_rules! symbol_addr { + ($sym:expr) => {{ + ($sym) as *const () as usize + }}; + ($sym:expr, $type:ty) => {{ + ($sym) as *const () as *const $type + }}; +} + +#[macro_export] +macro_rules! extern_symbol_addr { + ($sym:ident) => {{ + unsafe extern "C" { + fn $sym(); + } + $crate::symbol_addr!($sym) + }}; + ($sym:ident, $type:ty) => {{ + unsafe extern "C" { + fn $sym(); + } + $crate::symbol_addr!($sym, $type) + }}; +} diff --git a/crates/eonix_log/src/lib.rs b/crates/eonix_log/src/lib.rs index 92b1639f..01b6a587 100644 --- a/crates/eonix_log/src/lib.rs +++ b/crates/eonix_log/src/lib.rs @@ -2,6 +2,7 @@ use alloc::sync::Arc; use core::fmt::{self, Write}; + use eonix_sync::{Spin, SpinIrq as _}; extern crate alloc; @@ -91,18 +92,31 @@ macro_rules! println_fatal { #[macro_export] macro_rules! println_trace { - ($feat:literal) => { + (feat:$feat:literal) => { #[deny(unexpected_cfgs)] { #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] ") + $crate::println!("[kernel:trace]") } }; - ($feat:literal, $($arg:tt)*) => {{ + (feat:$feat:literal, $fmt:literal) => {{ #[deny(unexpected_cfgs)] { #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] {}", format_args!($($arg)*)) + $crate::println!(concat!("[kernel:trace] ", $feat)) } }}; + (feat:$feat:literal, $fmt:literal, $($arg:expr $(,)?)*) => { + #[deny(unexpected_cfgs)] + { + // Suppress unused variables warning + #[cfg(not(feature = $feat))] + { + $(let _ = $arg;)* + } + + #[cfg(feature = $feat)] + $crate::println!("[kernel:trace] {}", format_args!($fmt, $($arg,)*)) + } + }; } diff --git a/crates/eonix_percpu/src/lib.rs b/crates/eonix_percpu/src/lib.rs index 1fc7ffb8..a00b5c05 100644 --- a/crates/eonix_percpu/src/lib.rs +++ b/crates/eonix_percpu/src/lib.rs @@ -1,28 +1,21 @@ #![no_std] use core::alloc::Layout; -use core::ptr::null_mut; -use core::ptr::NonNull; -use core::sync::atomic::AtomicPtr; -use core::sync::atomic::Ordering; - -#[cfg(target_arch = "x86_64")] -pub use eonix_percpu_macros::define_percpu_x86_64 as define_percpu; - -#[cfg(target_arch = "x86_64")] -pub use eonix_percpu_macros::define_percpu_shared_x86_64 as define_percpu_shared; - -#[cfg(target_arch = "riscv64")] -pub use eonix_percpu_macros::define_percpu_riscv64 as define_percpu; - -#[cfg(target_arch = "riscv64")] -pub use eonix_percpu_macros::define_percpu_shared_riscv64 as define_percpu_shared; +use core::ptr::{null_mut, NonNull}; +use core::sync::atomic::{AtomicPtr, Ordering}; #[cfg(target_arch = "loongarch64")] pub use eonix_percpu_macros::define_percpu_loongarch64 as define_percpu; - +#[cfg(target_arch = "riscv64")] +pub use eonix_percpu_macros::define_percpu_riscv64 as define_percpu; #[cfg(target_arch = "loongarch64")] pub use eonix_percpu_macros::define_percpu_shared_loongarch64 as define_percpu_shared; +#[cfg(target_arch = "riscv64")] +pub use eonix_percpu_macros::define_percpu_shared_riscv64 as define_percpu_shared; +#[cfg(target_arch = "x86_64")] +pub use eonix_percpu_macros::define_percpu_shared_x86_64 as define_percpu_shared; +#[cfg(target_arch = "x86_64")] +pub use eonix_percpu_macros::define_percpu_x86_64 as define_percpu; const MAX_CPUS: usize = 256; @@ -41,7 +34,7 @@ impl PercpuArea { unsafe extern "C" { fn PERCPU_LENGTH(); } - let len = PERCPU_LENGTH as usize; + let len = PERCPU_LENGTH as *const () as usize; assert_ne!(len, 0, "Percpu length should not be zero."); len @@ -52,7 +45,7 @@ impl PercpuArea { fn PERCPU_DATA_START(); } - let addr = PERCPU_DATA_START as usize; + let addr = PERCPU_DATA_START as *const () as usize; NonNull::new(addr as *mut _).expect("Percpu data should not be null.") } diff --git a/crates/eonix_runtime/src/scheduler.rs b/crates/eonix_runtime/src/scheduler.rs index 3f72fbf4..b4b7960d 100644 --- a/crates/eonix_runtime/src/scheduler.rs +++ b/crates/eonix_runtime/src/scheduler.rs @@ -1,20 +1,19 @@ -use crate::{ - executor::OutputHandle, - ready_queue::{local_rq, ReadyQueue}, - task::{Task, TaskAdapter, TaskHandle, TaskState}, -}; -use alloc::{sync::Arc, task::Wake}; -use core::{ - ops::{Deref, DerefMut}, - ptr::NonNull, - task::{Context, Poll, Waker}, -}; +use alloc::sync::Arc; +use alloc::task::Wake; +use core::ops::{Deref, DerefMut}; +use core::ptr::NonNull; +use core::task::{Context, Poll, Waker}; + use eonix_hal::processor::halt; use eonix_log::println_trace; use eonix_sync::{LazyLock, Spin, SpinIrq as _}; use intrusive_collections::RBTree; use pointers::BorrowedArc; +use crate::executor::OutputHandle; +use crate::ready_queue::{local_rq, ReadyQueue}; +use crate::task::{Task, TaskAdapter, TaskHandle, TaskState}; + #[eonix_percpu::define_percpu] static CURRENT_TASK: Option> = None; @@ -93,12 +92,6 @@ impl Runtime { } } - fn current(&self) -> Option> { - CURRENT_TASK - .get() - .map(|ptr| unsafe { BorrowedArc::from_raw(ptr) }) - } - fn remove_and_enqueue_current(&self, rq: &mut impl DerefMut) { let Some(current) = CURRENT_TASK .swap(None) @@ -116,7 +109,7 @@ impl Runtime { }) { Ok(TaskState::READY_RUNNING) => { println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Re-enqueueing task {:?} (CPU{})", current.id, eonix_hal::processor::CPU::local().cpuid(), @@ -126,7 +119,7 @@ impl Runtime { } Ok(_) => { println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Current task {:?} (CPU{}) is blocked, not re-enqueueing", current.id, eonix_hal::processor::CPU::local().cpuid(), @@ -184,7 +177,7 @@ impl Runtime { }; println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Switching to task {:?} (CPU{})", next.id, eonix_hal::processor::CPU::local().cpuid(), @@ -212,7 +205,7 @@ impl Runtime { ); println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Task {:?} finished execution, removing...", Task::current().id, ); diff --git a/crates/eonix_sync/eonix_spin/src/lib.rs b/crates/eonix_sync/eonix_spin/src/lib.rs index 4718b867..7225aceb 100644 --- a/crates/eonix_sync/eonix_spin/src/lib.rs +++ b/crates/eonix_sync/eonix_spin/src/lib.rs @@ -2,13 +2,11 @@ mod guard; -use core::{ - cell::UnsafeCell, - marker::PhantomData, - sync::atomic::{AtomicBool, Ordering}, -}; -use eonix_sync_base::{Relax, SpinRelax}; +use core::cell::UnsafeCell; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicBool, Ordering}; +use eonix_sync_base::{Relax, SpinRelax}; pub use guard::{SpinGuard, UnlockedSpinGuard}; pub trait SpinContext { @@ -84,7 +82,7 @@ where T: ?Sized, R: Relax, { - pub fn lock_with_context(&self, context: C) -> SpinGuard + pub fn lock_with_context(&self, context: C) -> SpinGuard<'_, T, C, R> where C: SpinContext, { @@ -100,7 +98,7 @@ where ) } - pub fn lock(&self) -> SpinGuard { + pub fn lock(&self) -> SpinGuard<'_, T, DisablePreemption, R> { self.lock_with_context(DisablePreemption::save()) } diff --git a/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs b/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs index 76a28682..b70cdc3d 100644 --- a/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs +++ b/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs @@ -12,7 +12,7 @@ pub trait SpinIrq { type Context: SpinContext; type Relax; - fn lock_irq(&self) -> SpinGuard; + fn lock_irq(&self) -> SpinGuard<'_, Self::Value, Self::Context, Self::Relax>; } impl SpinContext for IrqContext { @@ -50,7 +50,7 @@ where type Context = IrqContext; type Relax = R; - fn lock_irq(&self) -> SpinGuard { + fn lock_irq(&self) -> SpinGuard<'_, Self::Value, Self::Context, Self::Relax> { self.lock_with_context(IrqContext::save()) } } diff --git a/crates/posix_types/src/poll.rs b/crates/posix_types/src/poll.rs index 781f589f..dcf5f9b2 100644 --- a/crates/posix_types/src/poll.rs +++ b/crates/posix_types/src/poll.rs @@ -1,5 +1,7 @@ pub const FDSET_LENGTH: usize = 1024 / (8 * size_of::()); +// TODO: Implement syscall pselect +#[allow(unused)] pub struct FDSet { fds_bits: [usize; FDSET_LENGTH], } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 722fa5da..09e12f99 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -123,7 +123,7 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { Box::new_in( async move { eonix_log::println_trace!( - "trace_syscall", + feat: "trace_syscall", "tid{}: {}({}) => {{", thd.tid, #syscall_name_str, @@ -133,7 +133,7 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { let retval = #real_fn(thd, #(#args_call),*).await.into_retval(); eonix_log::println_trace!( - "trace_syscall", + feat: "trace_syscall", "}} => {:x?}", retval, ); diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index dd096f57..fdb61f96 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -39,6 +39,8 @@ enum SlotState { Idle, Working, Finished, + // TODO: Implement AHCI error handling + #[allow(unused)] Error, } @@ -67,7 +69,7 @@ impl CommandList { + (size_of::>() + size_of::>()) * 32 } - pub fn get(&self, index: usize) -> CommandSlot { + pub fn get(&self, index: usize) -> CommandSlot<'_> { CommandSlot { cmdheader: &self.cmdheaders()[index], control: &self.controls()[index], diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index 6d6ca353..923a4594 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -61,6 +61,8 @@ struct E1000eDev { tx_tail: Option, rx_buffers: Box<[FolioOwned; RX_DESC_SIZE]>, + // TODO: Implement E1000e send + #[allow(unused)] tx_buffers: Box<[Option; TX_DESC_SIZE]>, } diff --git a/src/driver/serial/io.rs b/src/driver/serial/io.rs index aec18f20..57e61c56 100644 --- a/src/driver/serial/io.rs +++ b/src/driver/serial/io.rs @@ -1,10 +1,11 @@ -use super::SerialRegister; use core::ptr::NonNull; -use eonix_hal::{fence::memory_barrier, mm::ArchPhysAccess}; -use eonix_mm::address::{PAddr, PhysAccess}; #[cfg(target_arch = "x86_64")] use eonix_hal::arch_exported::io::Port8; +use eonix_hal::mm::ArchPhysAccess; +use eonix_mm::address::{PAddr, PhysAccess}; + +use super::SerialRegister; #[cfg(target_arch = "x86_64")] pub struct SerialIO { @@ -73,10 +74,12 @@ impl SerialIO { self.line_status } + #[allow(unused)] pub fn modem_status(&self) -> impl SerialRegister { self.modem_status } + #[allow(unused)] pub fn scratch(&self) -> impl SerialRegister { self.scratch } @@ -100,7 +103,7 @@ impl SerialRegister for NonNull { let retval = unsafe { self.as_ptr().read_volatile() }; #[cfg(target_arch = "loongarch64")] - memory_barrier(); + eonix_hal::fence::memory_barrier(); retval } @@ -110,7 +113,7 @@ impl SerialRegister for NonNull { unsafe { self.as_ptr().write_volatile(data) }; #[cfg(target_arch = "loongarch64")] - memory_barrier(); + eonix_hal::fence::memory_barrier(); } } @@ -155,10 +158,12 @@ impl SerialIO { unsafe { self.base_addr.add(5) } } + #[allow(unused)] pub fn modem_status(&self) -> impl SerialRegister { unsafe { self.base_addr.add(6) } } + #[allow(unused)] pub fn scratch(&self) -> impl SerialRegister { unsafe { self.base_addr.add(7) } } diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 32ede420..9a3933bb 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -26,6 +26,8 @@ enum NodeKind { struct FileInode { read: Option KResult<()> + Send + Sync>>, + // TODO: Implement writes to procfs files + #[allow(unused)] write: Option<()>, } diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index d560a672..aafae539 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -177,7 +177,6 @@ impl InodeOps for FileInode { } pub struct DeviceInode { - is_block: bool, devid: DeviceId, } @@ -199,10 +198,7 @@ impl DeviceInode { ctime: now, mtime: now, }, - Self { - is_block: mode.format() == Format::BLK, - devid, - }, + Self { devid }, ) } } diff --git a/src/kernel/constants.rs b/src/kernel/constants.rs index 4e11d66e..b96387b0 100644 --- a/src/kernel/constants.rs +++ b/src/kernel/constants.rs @@ -36,7 +36,7 @@ pub const ENOTDIR: u32 = 20; pub const EISDIR: u32 = 21; pub const EINVAL: u32 = 22; pub const ENOTTY: u32 = 25; -pub const ENOSPC: u32 = 28; +// pub const ENOSPC: u32 = 28; pub const ESPIPE: u32 = 29; // pub const EROFS: u32 = 30; pub const EPIPE: u32 = 32; diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs index 6647e1af..8ab4d6be 100644 --- a/src/kernel/mem/folio.rs +++ b/src/kernel/mem/folio.rs @@ -79,7 +79,7 @@ impl Folio { } } - pub fn lock(&self) -> LockedFolio { + pub fn lock(&self) -> LockedFolio<'_> { // TODO: actually perform the lock... LockedFolio(self) } diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index 7aac141d..5a56efbc 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -1,3 +1,4 @@ +use eonix_hal::extern_symbol_addr; use eonix_hal::mm::flush_tlb; use eonix_hal::traits::fault::PageFaultErrorCode; use eonix_mm::address::{Addr as _, AddrOps as _, VRange}; @@ -24,27 +25,19 @@ impl FixEntry { VAddr::from((self.start + self.length) as usize) } - #[allow(dead_code)] - fn range(&self) -> VRange { - VRange::new(self.start(), self.end()) - } - fn jump_address(&self) -> VAddr { VAddr::from(self.jump_address as usize) } fn entries() -> &'static [FixEntry] { - extern "C" { - fn FIX_START(); - fn FIX_END(); - } + let fix_seg_len_bytes = extern_symbol_addr!(FIX_END) - extern_symbol_addr!(FIX_START); unsafe { - // SAFETY: `FIX_START` and `FIX_END` are defined in the - // linker script in `.rodata` section. + // SAFETY: `FIX_START` and `FIX_END` are defined in the linker script + // in `.rodata` section. core::slice::from_raw_parts( - FIX_START as usize as *const FixEntry, - (FIX_END as usize - FIX_START as usize) / size_of::(), + extern_symbol_addr!(FIX_START, FixEntry), + fix_seg_len_bytes / size_of::(), ) } } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 0d775245..16d57714 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -48,10 +48,6 @@ pub struct RawPage { shared_data: PageData, } -// XXX: introduce Folio and remove this. -unsafe impl Send for RawPage {} -unsafe impl Sync for RawPage {} - impl PageFlags { pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; diff --git a/src/kernel/pcie/device.rs b/src/kernel/pcie/device.rs index 085e7b9a..2a8f150d 100644 --- a/src/kernel/pcie/device.rs +++ b/src/kernel/pcie/device.rs @@ -1,14 +1,17 @@ -use super::{ - header::{Bar, Command}, - CommonHeader, Header, -}; -use crate::kernel::mem::PhysAccess as _; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::num::NonZero; +use core::ops::RangeInclusive; + use align_ext::AlignExt; -use alloc::{collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; -use core::{num::NonZero, ops::RangeInclusive}; use eonix_mm::address::{Addr, PAddr, PRange}; use eonix_sync::Spin; +use super::header::{Bar, Command}; +use super::{CommonHeader, Header}; +use crate::kernel::mem::PhysAccess as _; + pub(super) static PCIE_DEVICES: Spin>>> = Spin::new(BTreeMap::new()); @@ -20,7 +23,7 @@ pub struct PCIDevice<'a> { pub device_id: u16, } -#[allow(dead_code)] +#[allow(unused)] #[derive(Clone)] pub struct SegmentGroup { id: usize, @@ -28,6 +31,7 @@ pub struct SegmentGroup { base_address: PAddr, } +#[allow(unused)] #[derive(Clone)] pub struct ConfigSpace { pub bus: u8, @@ -180,10 +184,12 @@ impl PCIDevice<'_> { ); } + #[allow(unused)] pub fn config_space(&self) -> &ConfigSpace { &self.config_space } + #[allow(unused)] pub fn segment_group(&self) -> &SegmentGroup { &self.segment_group } @@ -209,7 +215,7 @@ impl PciMemoryAllocator { self.start += size; eonix_log::println_trace!( - "trace_pci", + feat: "trace_pci", "PciMemoryAllocator: Allocated {} bytes at {:#x}", size, base diff --git a/src/kernel/pcie/header.rs b/src/kernel/pcie/header.rs index 889795d3..0a44ea28 100644 --- a/src/kernel/pcie/header.rs +++ b/src/kernel/pcie/header.rs @@ -1,10 +1,9 @@ +use core::marker::PhantomData; +use core::num::NonZero; +use core::ops::{BitAnd, BitOr, Deref, Not}; +use core::sync::atomic::{AtomicU16, AtomicU32, Ordering}; + use bitflags::bitflags; -use core::{ - marker::PhantomData, - num::NonZero, - ops::{BitAnd, BitOr, Deref, Not}, - sync::atomic::{AtomicU16, AtomicU32, Ordering}, -}; use eonix_hal::fence::memory_barrier; pub trait BitFlag: Sized + Copy { @@ -215,14 +214,14 @@ where } impl CommonHeader { - pub fn command(&self) -> Register { + pub fn command(&self) -> Register<'_, Command> { Register { register: unsafe { AtomicU16::from_ptr((&raw const self._command) as *mut u16) }, _phantom: PhantomData, } } - pub fn status(&self) -> Register { + pub fn status(&self) -> Register<'_, Status> { Register { register: unsafe { AtomicU16::from_ptr((&raw const self._status) as *mut u16) }, _phantom: PhantomData, @@ -231,7 +230,7 @@ impl CommonHeader { } impl Bars<'_> { - pub fn iter(&self) -> impl Iterator + '_ { + pub fn iter(&self) -> impl Iterator> + use<'_> { struct BarsIterator<'a> { bars: &'a [AtomicU32], pos: usize, diff --git a/src/kernel/pcie/init.rs b/src/kernel/pcie/init.rs index 4c183bc5..74a490b4 100644 --- a/src/kernel/pcie/init.rs +++ b/src/kernel/pcie/init.rs @@ -10,6 +10,7 @@ use super::error::PciError; use crate::kernel::mem::PhysAccess as _; use crate::kernel::pcie::device::PciMemoryAllocator; +#[allow(unused)] #[derive(Clone)] struct AcpiHandlerImpl; @@ -34,7 +35,6 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(target_arch = "x86_64")] { use acpi::{AcpiTables, PciConfigRegions}; - use eonix_mm::address::PAddr; let acpi_tables = unsafe { // SAFETY: Our impl should be correct. @@ -69,7 +69,6 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { use eonix_hal::arch_exported::fdt::FDT; - use eonix_mm::address::PRange; use crate::kernel::constants::{EINVAL, EIO, ENOENT}; @@ -88,7 +87,7 @@ pub fn init_pcie() -> Result<(), PciError> { let size = u64::from_be_bytes(entry[20..28].try_into().unwrap()); println_trace!( - "trace_pci", + feat: "trace_pci", "PCIe range: PCI address = {:#x}, CPU address = {:#x}, size = {:#x}", pci_address, cpu_address, diff --git a/src/kernel/syscall.rs b/src/kernel/syscall.rs index 78ddcd1c..d06c5d88 100644 --- a/src/kernel/syscall.rs +++ b/src/kernel/syscall.rs @@ -1,11 +1,17 @@ -use super::task::ThreadAlloc; -use crate::kernel::task::Thread; use alloc::boxed::Box; -use core::{future::Future, marker::PhantomData, ops::Deref, pin::Pin}; +use core::future::Future; +use core::marker::PhantomData; +use core::ops::Deref; +use core::pin::Pin; + +use eonix_hal::extern_symbol_addr; use eonix_mm::address::{Addr, VAddr}; use eonix_sync::LazyLock; use posix_types::ctypes::PtrT; +use super::task::ThreadAlloc; +use crate::kernel::task::Thread; + pub mod file_rw; pub mod mm; pub mod net; @@ -280,12 +286,6 @@ impl core::fmt::Debug for UserMut { } static SYSCALL_HANDLERS: LazyLock<[Option; MAX_SYSCALL_NO]> = LazyLock::new(|| { - extern "C" { - // SAFETY: `SYSCALL_HANDLERS` is defined in linker script. - fn RAW_SYSCALL_HANDLERS(); - fn RAW_SYSCALL_HANDLERS_SIZE(); - } - // DO NOT TOUCH THESE FUNCTIONS!!! // THEY ARE USED FOR KEEPING THE OBJECTS NOT STRIPPED BY THE LINKER!!! file_rw::keep_alive(); @@ -294,15 +294,14 @@ static SYSCALL_HANDLERS: LazyLock<[Option; MAX_SYSCALL_NO]> = La procops::keep_alive(); sysinfo::keep_alive(); - let raw_handlers_addr = RAW_SYSCALL_HANDLERS as *const (); - let raw_handlers_size_byte = RAW_SYSCALL_HANDLERS_SIZE as usize; + let raw_handlers_size_byte = extern_symbol_addr!(RAW_SYSCALL_HANDLERS_SIZE); assert!(raw_handlers_size_byte % size_of::() == 0); let raw_handlers_count = raw_handlers_size_byte / size_of::(); let raw_handlers = unsafe { core::slice::from_raw_parts( - raw_handlers_addr as *const RawSyscallHandler, + extern_symbol_addr!(RAW_SYSCALL_HANDLERS, RawSyscallHandler), raw_handlers_count, ) }; diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 3fe6fe97..b0966046 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -11,6 +11,7 @@ mod thread; mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; +use eonix_hal::symbol_addr; pub use futex::{futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, RobustListHead}; pub use kernel_stack::KernelStack; pub use loader::ProgramLoader; @@ -185,7 +186,7 @@ where trap_ctx.set_user_mode(false); trap_ctx.set_interrupt_enabled(true); let _ = trap_ctx.set_user_call_frame( - execute:: as usize, + symbol_addr!(execute::), Some(sp.addr().get()), None, &[(&raw mut future) as usize, output.get() as usize], diff --git a/src/kernel/task/process.rs b/src/kernel/task/process.rs index 421e4b8b..3eff5949 100644 --- a/src/kernel/task/process.rs +++ b/src/kernel/task/process.rs @@ -1,34 +1,30 @@ -use super::{ - process_group::ProcessGroupBuilder, signal::RaiseResult, thread::ThreadBuilder, ProcessGroup, - ProcessList, Session, Thread, -}; -use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; -use crate::kernel::task::{CloneArgs, CloneFlags}; -use crate::rcu::call_rcu; -use crate::{ - kernel::mem::MMList, - prelude::*, - rcu::{RCUPointer, RCUReadGuard}, - sync::CondVar, -}; -use alloc::{ - collections::{btree_map::BTreeMap, vec_deque::VecDeque}, - sync::{Arc, Weak}, -}; +use alloc::collections::btree_map::BTreeMap; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::{Arc, Weak}; use core::sync::atomic::{AtomicU32, Ordering}; -use eonix_mm::address::VAddr; + use eonix_sync::{ AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, SpinGuard, UnlockableGuard as _, UnlockedGuard as _, }; use pointers::BorrowedArc; use posix_types::constants::{ - CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_PGID, P_PIDFD, + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_ALL, P_PGID, P_PID, P_PIDFD, }; -use posix_types::constants::{P_ALL, P_PID}; use posix_types::signal::Signal; use posix_types::SIGNAL_COREDUMP; +use super::process_group::ProcessGroupBuilder; +use super::signal::RaiseResult; +use super::thread::ThreadBuilder; +use super::{ProcessGroup, ProcessList, Session, Thread}; +use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; +use crate::kernel::mem::MMList; +use crate::kernel::task::{CloneArgs, CloneFlags}; +use crate::prelude::*; +use crate::rcu::{call_rcu, RCUPointer, RCUReadGuard}; +use crate::sync::CondVar; + pub struct ProcessBuilder { mm_list: Option, exit_signal: Option, @@ -51,8 +47,6 @@ pub struct Process { pub exit_signal: Option, - pub shm_areas: Spin>, - /// Parent process /// /// `parent` must be valid during the whole life of the process. @@ -256,7 +250,6 @@ impl ProcessBuilder { pid: self.pid.expect("should set pid before building"), wait_list: WaitList::new(), mm_list, - shm_areas: Spin::new(BTreeMap::new()), exit_signal: self.exit_signal, parent: RCUPointer::empty(), pgroup: RCUPointer::empty(), @@ -522,17 +515,17 @@ impl Process { } /// Provide RCU locked (maybe inconsistent) access to the session. - pub fn session_rcu(&self) -> RCUReadGuard<'_, BorrowedArc> { + pub fn session_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<'_, Session>> { self.session.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the process group. - pub fn pgroup_rcu(&self) -> RCUReadGuard<'_, BorrowedArc> { + pub fn pgroup_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<'_, ProcessGroup>> { self.pgroup.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the parent process. - pub fn parent_rcu(&self) -> Option>> { + pub fn parent_rcu(&self) -> Option>> { self.parent.load() } @@ -569,7 +562,7 @@ impl WaitList { self.cv_wait_procs.notify_all(); } - pub fn drain_exited(&self) -> DrainExited { + pub fn drain_exited(&self) -> DrainExited<'_> { DrainExited { wait_procs: self.wait_procs.lock(), } @@ -578,7 +571,12 @@ impl WaitList { /// # Safety /// Locks `ProcessList` and `WaitList` at the same time. When `wait` is called, /// releases the lock on `ProcessList` and `WaitList` and waits on `cv_wait_procs`. - pub async fn entry(&self, wait_id: WaitId, want_stop: bool, want_continue: bool) -> Entry { + pub async fn entry( + &self, + wait_id: WaitId, + want_stop: bool, + want_continue: bool, + ) -> Entry<'_, '_, '_> { Entry { process_list: ProcessList::get().read().await, wait_procs: self.wait_procs.lock(), diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index 22760de9..a401f4f7 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -27,6 +27,8 @@ use crate::path::Path; use crate::prelude::*; use crate::rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}; +// TODO: Implement slab reclaim +#[allow(unused)] const D_INVALID: u8 = 0; const D_REGULAR: u8 = 1; const D_DIRECTORY: u8 = 2; @@ -159,7 +161,7 @@ impl Dentry { && &***self.name() == &***other.name() } - pub fn name(&self) -> RCUReadGuard>> { + pub fn name(&self) -> RCUReadGuard<'_, BorrowedArc<'_, Arc<[u8]>>> { self.name.load().expect("Dentry has no name") } @@ -167,7 +169,7 @@ impl Dentry { (***self.name()).clone() } - pub fn parent<'a>(&self) -> RCUReadGuard<'a, BorrowedArc> { + pub fn parent<'a>(&self) -> RCUReadGuard<'a, BorrowedArc<'_, Dentry>> { self.parent.load().expect("Dentry has no parent") } diff --git a/src/kernel/vfs/types/device_id.rs b/src/kernel/vfs/types/device_id.rs index cf3ea886..6dd128ee 100644 --- a/src/kernel/vfs/types/device_id.rs +++ b/src/kernel/vfs/types/device_id.rs @@ -10,17 +10,6 @@ impl DeviceId { pub const fn new(major: u16, minor: u16) -> Self { Self { major, minor } } - - pub const fn from_raw(raw: u32) -> Self { - Self { - major: (raw >> 16) as u16, - minor: (raw & 0xFFFF) as u16, - } - } - - pub const fn to_raw(self) -> u32 { - ((self.major as u32) << 16) | (self.minor as u32) - } } impl Debug for DeviceId { diff --git a/src/lib.rs b/src/lib.rs index 8457169c..4f7fb262 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,7 @@ use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use eonix_hal::arch_exported::bootstrap::shutdown; use eonix_hal::context::TaskContext; use eonix_hal::processor::{halt, CPU, CPU_COUNT}; +use eonix_hal::symbol_addr; use eonix_hal::traits::context::RawTaskContext; use eonix_hal::traits::trap::IrqState; use eonix_hal::trap::disable_irqs_save; @@ -136,7 +137,7 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { bottom }; ctx.set_interrupt_enabled(true); - ctx.set_program_counter(standard_main as usize); + ctx.set_program_counter(symbol_addr!(standard_main)); ctx.set_stack_pointer(stack_bottom); unsafe { @@ -162,7 +163,7 @@ fn kernel_ap_main(_stack_range: PRange) -> ! { bottom }; ctx.set_interrupt_enabled(true); - ctx.set_program_counter(standard_main as usize); + ctx.set_program_counter(symbol_addr!(standard_main)); ctx.set_stack_pointer(stack_bottom); unsafe { diff --git a/src/path.rs b/src/path.rs index b342ef5f..47b9a4b6 100644 --- a/src/path.rs +++ b/src/path.rs @@ -1,6 +1,8 @@ -use crate::{kernel::constants::ENOENT, prelude::*}; use core::fmt::{self, Debug, Formatter}; +use crate::kernel::constants::ENOENT; +use crate::prelude::*; + #[repr(transparent)] pub struct Path { all: [u8], @@ -23,7 +25,7 @@ impl Path { self.all.starts_with(&['/' as u8]) } - pub fn iter(&self) -> PathIterator { + pub fn iter(&self) -> PathIterator<'_> { PathIterator::new(&self.all) } } diff --git a/src/sync/arcswap.rs b/src/sync/arcswap.rs index fb8219b2..7421659f 100644 --- a/src/sync/arcswap.rs +++ b/src/sync/arcswap.rs @@ -1,9 +1,8 @@ use alloc::sync::Arc; -use core::{ - fmt::{self, Debug, Formatter}, - ptr::NonNull, - sync::atomic::{AtomicPtr, Ordering}, -}; +use core::fmt::{self, Debug, Formatter}; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicPtr, Ordering}; + use pointers::BorrowedArc; unsafe impl Send for ArcSwap where T: Send + Sync {} @@ -33,7 +32,7 @@ impl ArcSwap { } } - pub fn borrow(&self) -> BorrowedArc { + pub fn borrow(&self) -> BorrowedArc<'_, T> { unsafe { BorrowedArc::from_raw( NonNull::new(self.pointer.load(Ordering::Acquire)) From bc84f0be804af2416e0b37a2cbf565f01cd13556 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 18 Jan 2026 02:56:56 +0800 Subject: [PATCH 013/153] user, init: update riscv64 init script Make print message prettier. Also panic when we have an error. Signed-off-by: greatbridf --- user-programs/init_script_riscv64.sh | 88 ++++++++++++++++------------ 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/user-programs/init_script_riscv64.sh b/user-programs/init_script_riscv64.sh index 52b2628c..b5ce95d7 100644 --- a/user-programs/init_script_riscv64.sh +++ b/user-programs/init_script_riscv64.sh @@ -1,60 +1,75 @@ #!/mnt/busybox sh BUSYBOX=/mnt/busybox +TERMINAL=/dev/ttyS0 +VERBOSE= -freeze() { - echo "an error occurred while executing '''$@''', freezing..." >&2 +error() { + printf "\033[91merror: \033[0m%s\n" "$1" >&2 +} + +warn() { + printf "\033[93mwarn : \033[0m%s\n" "$1" >&2 +} + +info() { + printf "\033[92minfo : \033[0m%s\n" "$1" >&2 +} + +die() { + error "$1" && freeze +} +freeze() { + info "freezing..." >&2 while true; do - true + : done + + exit 1 } -do_or_freeze() { - if $@; then - return - fi +unrecoverable() { + die "unrecoverable error occurred. check the message above." +} - freeze $@ +busybox() { + $BUSYBOX "$@" } -do_or_freeze $BUSYBOX mkdir -p /dev +trap unrecoverable EXIT + +set -euo pipefail -do_or_freeze $BUSYBOX mknod -m 666 /dev/console c 5 1 -do_or_freeze $BUSYBOX mknod -m 666 /dev/null c 1 3 -do_or_freeze $BUSYBOX mknod -m 666 /dev/zero c 1 5 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vda b 8 0 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vda1 b 8 1 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vdb b 8 16 -do_or_freeze $BUSYBOX mknod -m 666 /dev/ttyS0 c 4 64 -do_or_freeze $BUSYBOX mknod -m 666 /dev/ttyS1 c 4 65 +if [ -n "$VERBOSE" ]; then + set -x +fi -echo -n -e "deploying busybox... " >&2 +busybox mkdir -p /dev -do_or_freeze $BUSYBOX mkdir -p /bin -do_or_freeze $BUSYBOX --install -s /bin -do_or_freeze $BUSYBOX mkdir -p /lib +busybox mknod -m 666 /dev/console c 5 1 +busybox mknod -m 666 /dev/null c 1 3 +busybox mknod -m 666 /dev/zero c 1 5 +busybox mknod -m 666 /dev/vda b 8 0 +busybox mknod -m 666 /dev/vda1 b 8 1 +busybox mknod -m 666 /dev/vdb b 8 16 +busybox mknod -m 666 /dev/ttyS0 c 4 64 +busybox mknod -m 666 /dev/ttyS1 c 4 65 -export PATH="/bin" +info "deploying busybox..." -echo ok >&2 +busybox mkdir -p /bin /lib +busybox --install -s /bin -do_or_freeze mkdir -p /etc /root /proc -do_or_freeze mount -t procfs proc proc +info "done" -# Check if the device /dev/vdb is available and can be read -if dd if=/dev/vdb of=/dev/null bs=512 count=1; then - echo -n -e "Mounting the ext4 image... " >&2 - do_or_freeze mkdir -p /mnt1 - do_or_freeze mount -t ext4 /dev/vdb /mnt1 - echo ok >&2 -fi +export PATH="/bin" -cp /mnt/ld-musl-i386.so.1 /lib/ld-musl-i386.so.1 -ln -s /lib/ld-musl-i386.so.1 /lib/libc.so +mkdir -p /etc /root /proc +mount -t procfs proc proc cat > /etc/passwd < /etc/group < /dev/ttyS0 2> /dev/ttyS0 +# shellcheck disable=SC2094 +exec sh -l < "$TERMINAL" > "$TERMINAL" 2> "$TERMINAL" # We don't have a working init yet, so we use busybox sh directly for now. # exec /mnt/init /bin/sh -c 'exec sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0' From 47d890648a7895eb620335d4f233e8d1ad160938 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 18 Jan 2026 14:33:38 +0800 Subject: [PATCH 014/153] chore: remove and ignore vscode settings from svc Signed-off-by: greatbridf --- .gitignore | 1 + .vscode/settings.json | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index fbc2a9b1..4684b698 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ build/ .idea/ +.vscode/settings.json test/ diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 10b4a8b4..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "makefile.configureOnOpen": false, - "editor.formatOnSave": true, -} From b6d54d6a0ff15b96273ddc499670620b2baceefe Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 18 Jan 2026 21:35:39 +0800 Subject: [PATCH 015/153] mm, proc: add an exited thread reaper - thd.exit() will set thd.dead and send it to the reaper - delay the release of process mm until we reap it - extract futex logic out of exit and exec routine Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 30 +++---- src/kernel/syscall/procops.rs | 26 ++---- src/kernel/task.rs | 5 +- src/kernel/task/futex.rs | 53 +++++++++--- src/kernel/task/process_list.rs | 144 ++++++++++++++++++++------------ src/kernel/task/signal.rs | 20 ++--- src/kernel/task/thread.rs | 49 ++++++++--- src/lib.rs | 23 +---- 8 files changed, 209 insertions(+), 141 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 5221c73b..f073025b 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -398,20 +398,6 @@ impl MMList { assert_ne!(old_user_count, 0); } - /// Deactivate `self` and activate `to` with root page table changed only once. - /// This might reduce the overhead of switching page tables twice. - #[allow(dead_code)] - pub fn switch(&self, to: &Self) { - self.user_count.fetch_add(1, Ordering::Acquire); - - let root_page_table = self.root_page_table.load(Ordering::Relaxed); - assert_ne!(root_page_table, 0); - set_root_page_table_pfn(PFN::from(PAddr::from(root_page_table))); - - let old_user_count = to.user_count.fetch_sub(1, Ordering::Release); - assert_ne!(old_user_count, 0); - } - /// Replace the current page table with a new one. /// /// # Safety @@ -454,10 +440,24 @@ impl MMList { // TODO: Check whether we should wake someone up if they've been put // to sleep when calling `vfork`. - self.inner + let old_mm = self + .inner .swap(new.map(|new_mm| new_mm.inner.swap(None)).flatten()); eonix_preempt::enable(); + + // This could take long... + drop(old_mm); + } + + pub fn release(&self) { + let old_mm = self.inner.swap(None); + let old_table = self.root_page_table.swap(0, Ordering::Relaxed); + + // TODO: Remove this completely... + // XXX: `ArcSwap` is broken and never safe to use. Check `replace` above. + assert_ne!(old_table, 0, "Already released?"); + assert!(old_mm.is_some(), "Already released?"); } /// No need to do invalidation manually, `PageTable` already does it. diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 1359d0ab..3e815f25 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -22,8 +22,8 @@ use crate::kernel::constants::{ use crate::kernel::mem::PageBuffer; use crate::kernel::syscall::{User, UserMut}; use crate::kernel::task::{ - do_clone, futex_wait, futex_wake, parse_futexop, yield_now, CloneArgs, FutexFlags, FutexOp, - ProcessList, ProgramLoader, RobustListHead, SignalAction, Thread, WaitId, WaitType, + do_clone, futex_exec, futex_wait, futex_wake, parse_futexop, yield_now, CloneArgs, FutexFlags, + FutexOp, ProcessList, ProgramLoader, RobustListHead, SignalAction, Thread, WaitId, WaitType, }; use crate::kernel::timer::sleep; use crate::kernel::user::{UserBuffer, UserPointer, UserPointerMut, UserString}; @@ -213,10 +213,7 @@ async fn execve(exec: User, argv: User, envp: User) -> KResult, argv: User, envp: User) -> KResult SyscallNoReturn { - let mut procs = ProcessList::get().write().await; - - unsafe { - procs - .do_exit(&thread, WaitType::Exited(status), false) - .await; - } + thread.exit(WaitType::Exited(status)); SyscallNoReturn } #[eonix_macros::define_syscall(SYS_EXIT_GROUP)] async fn exit_group(status: u32) -> SyscallNoReturn { - let mut procs = ProcessList::get().write().await; - - unsafe { - procs.do_exit(&thread, WaitType::Exited(status), true).await; - } + // XXX: Send SIGKILL to our sibling threads. + thread.exit(WaitType::Exited(status)); SyscallNoReturn } @@ -856,7 +844,7 @@ async fn rt_sigreturn() -> KResult { "`rt_sigreturn` failed in thread {} with error {err}!", thread.tid ); - thread.force_kill(Signal::SIGSEGV).await; + thread.force_kill(Signal::SIGSEGV); return Err(err); } diff --git a/src/kernel/task.rs b/src/kernel/task.rs index b0966046..6505666c 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -12,7 +12,10 @@ mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; use eonix_hal::symbol_addr; -pub use futex::{futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, RobustListHead}; +pub use futex::{ + futex_exec, futex_exit, futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, + RobustListHead, +}; pub use kernel_stack::KernelStack; pub use loader::ProgramLoader; pub use process::{alloc_pid, Process, ProcessBuilder, WaitId, WaitObject, WaitType}; diff --git a/src/kernel/task/futex.rs b/src/kernel/task/futex.rs index a04d7091..4dd57615 100644 --- a/src/kernel/task/futex.rs +++ b/src/kernel/task/futex.rs @@ -1,19 +1,17 @@ -use core::pin::pin; - use alloc::sync::Arc; use alloc::vec::Vec; +use core::pin::pin; + use bitflags::bitflags; +use eonix_mm::address::Addr; use eonix_sync::{LazyLock, Mutex, MutexGuard, WaitList}; use intrusive_collections::{intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink}; -use crate::{ - kernel::{ - constants::{EAGAIN, EINVAL}, - syscall::User, - user::UserPointer, - }, - prelude::KResult, -}; +use super::Thread; +use crate::kernel::constants::{EAGAIN, EINVAL}; +use crate::kernel::syscall::User; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::prelude::KResult; #[derive(PartialEq, Debug, Clone, Copy)] #[repr(u32)] @@ -318,3 +316,38 @@ impl RobustListHead { Ok(()) } } + +async fn do_futex_exit(thread: &Thread) -> KResult<()> { + if let Some(clear_ctid) = thread.get_clear_ctid() { + UserPointerMut::new(clear_ctid)?.write(0u32)?; + + futex_wake(clear_ctid.addr(), None, 1).await?; + } + + if let Some(robust_list) = thread.get_robust_list() { + robust_list.wake_all().await?; + } + + Ok(()) +} + +pub async fn futex_exit(thread: &Thread) { + // We don't care about any error happened inside. + // If they've set up a wrong pointer, good luck to them... + let _ = do_futex_exit(thread); +} + +async fn do_futex_exec(thread: &Thread) -> KResult<()> { + if let Some(robust_list) = thread.get_robust_list() { + robust_list.wake_all().await?; + thread.set_robust_list(None); + } + + Ok(()) +} + +pub async fn futex_exec(thread: &Thread) { + // We don't care about any error happened inside. + // If they've set up a wrong pointer, good luck to them... + let _ = do_futex_exec(thread); +} diff --git a/src/kernel/task/process_list.rs b/src/kernel/task/process_list.rs index af073e84..c676d22e 100644 --- a/src/kernel/task/process_list.rs +++ b/src/kernel/task/process_list.rs @@ -1,16 +1,17 @@ +use alloc::collections::btree_map::BTreeMap; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::{Arc, Weak}; +use core::pin::pin; use core::sync::atomic::Ordering; -use super::{Process, ProcessGroup, Session, Thread, WaitObject, WaitType}; -use crate::{ - kernel::{task::futex_wake, user::UserPointerMut}, - rcu::rcu_sync, -}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use eonix_runtime::scheduler::RUNTIME; +use eonix_sync::{AsProof as _, AsProofMut as _, RwLock, Spin, WaitList}; + +use super::loader::LoadInfo; +use super::{ + alloc_pid, Process, ProcessBuilder, ProcessGroup, Session, Thread, ThreadBuilder, WaitObject, }; -use eonix_mm::address::Addr; -use eonix_sync::{AsProof as _, AsProofMut as _, RwLock}; +use crate::rcu::rcu_sync; pub struct ProcessList { /// The init process. @@ -78,7 +79,7 @@ impl ProcessList { } } - pub fn set_init_process(&mut self, init: Arc) { + fn set_init_process(&mut self, init: Arc) { let old_init = self.init.replace(init); assert!(old_init.is_none(), "Init process already set"); } @@ -103,45 +104,66 @@ impl ProcessList { self.sessions.get(&sid).and_then(Weak::upgrade) } - /// Make the process a zombie and notify the parent. - /// # Safety - /// This function will destroy the process and all its threads. - /// It is the caller's responsibility to ensure that the process is not - /// running or will not run after this function is called. - pub async unsafe fn do_exit( - &mut self, - thread: &Thread, - exit_status: WaitType, - is_exiting_group: bool, - ) { - let process = thread.process.clone(); - - if process.pid == 1 { - panic!("init exited"); - } + pub async fn sys_init(load_info: LoadInfo) { + let thread_builder = ThreadBuilder::new() + .name(Arc::from(&b"busybox"[..])) + .entry(load_info.entry_ip, load_info.sp); - let inner = process.inner.access_mut(self.prove_mut()); + let mut process_list = ProcessList::get().write().await; + let (thread, process) = ProcessBuilder::new() + .pid(alloc_pid()) + .mm_list(load_info.mm_list) + .thread_builder(thread_builder) + .build(&mut process_list); - thread.dead.store(true, Ordering::SeqCst); + process_list.set_init_process(process); - if is_exiting_group { - // TODO: Send SIGKILL to all threads. - // todo!() - } + // TODO!!!: Remove this. + thread.files.open_console(); - if thread.tid != process.pid { - self.threads.remove(&thread.tid); - inner.threads.remove(&thread.tid).unwrap(); - } + RUNTIME.spawn(Reaper::daemon()); + RUNTIME.spawn(thread.run()); + } + + pub fn send_to_reaper(thread: Arc) { + GLOBAL_REAPER.reap_list.lock().push_back(thread); + GLOBAL_REAPER.wait.notify_one(); + } +} + +struct Reaper { + reap_list: Spin>>, + wait: WaitList, +} + +static GLOBAL_REAPER: Reaper = Reaper { + reap_list: Spin::new(VecDeque::new()), + wait: WaitList::new(), +}; - if let Some(clear_ctid) = thread.get_clear_ctid() { - let _ = UserPointerMut::new(clear_ctid).unwrap().write(0u32); +impl Reaper { + async fn reap(&self, thread: Arc) { + let exit_status = thread + .exit_status + .lock() + .take() + .expect("Exited thread with no exit status"); - let _ = futex_wake(clear_ctid.addr(), None, 1).await; + let process = &thread.process; + + if process.pid == 1 && thread.tid == process.pid { + panic!("init exited"); } - if let Some(robust_list) = thread.get_robust_list() { - let _ = robust_list.wake_all().await; + let mut procs = ProcessList::get().write().await; + + let inner = process.inner.access_mut(procs.prove_mut()); + + thread.dead.store(true, Ordering::SeqCst); + + if thread.tid != process.pid { + procs.threads.remove(&thread.tid); + inner.threads.remove(&thread.tid).unwrap(); } // main thread exit @@ -151,48 +173,62 @@ impl ProcessList { thread.files.close_all().await; // If we are the session leader, we should drop the control terminal. - if process.session(self.prove()).sid == process.pid { - if let Some(terminal) = process.session(self.prove()).drop_control_terminal().await + if process.session(procs.prove()).sid == process.pid { + if let Some(terminal) = process.session(procs.prove()).drop_control_terminal().await { terminal.drop_session().await; } } // Release the MMList as well as the page table. - unsafe { - // SAFETY: We are exiting the process, so no one might be using it. - process.mm_list.replace(None); - } + process.mm_list.release(); // Make children orphans (adopted by init) { - let init = self.init_process(); + let init = procs.init_process(); inner.children.retain(|_, child| { let child = child.upgrade().unwrap(); // SAFETY: `child.parent` must be ourself. So we don't need to free it. unsafe { child.parent.swap(Some(init.clone())) }; - init.add_child(&child, self.prove_mut()); + init.add_child(&child, procs.prove_mut()); false }); } - let mut init_notify = self.init_process().notify_batch(); + let mut init_notify = procs.init_process().notify_batch(); process .wait_list .drain_exited() .into_iter() .for_each(|item| init_notify.notify(item)); - init_notify.finish(self.prove()); + init_notify.finish(procs.prove()); - process.parent(self.prove()).notify( + process.parent(procs.prove()).notify( process.exit_signal, WaitObject { pid: process.pid, code: exit_status, }, - self.prove(), + procs.prove(), ); } } + + async fn daemon() { + let me = &GLOBAL_REAPER; + + loop { + let mut wait = pin!(me.wait.prepare_to_wait()); + wait.as_mut().add_to_wait_list(); + + let thd_to_reap = me.reap_list.lock().pop_front(); + if let Some(thd_to_reap) = thd_to_reap { + me.reap(thd_to_reap).await; + continue; + } + + wait.await; + } + } } diff --git a/src/kernel/task/signal.rs b/src/kernel/task/signal.rs index d9970cad..0a7b580d 100644 --- a/src/kernel/task/signal.rs +++ b/src/kernel/task/signal.rs @@ -1,11 +1,10 @@ mod signal_action; -use super::{ProcessList, Thread, WaitObject, WaitType}; -use crate::kernel::constants::{EFAULT, EINVAL}; -use crate::{kernel::user::UserPointer, prelude::*}; use alloc::collections::binary_heap::BinaryHeap; use alloc::sync::Arc; -use core::{cmp::Reverse, task::Waker}; +use core::cmp::Reverse; +use core::task::Waker; + use eonix_hal::fpu::FpuState; use eonix_hal::traits::trap::RawTrapContext; use eonix_hal::trap::TrapContext; @@ -14,9 +13,13 @@ use eonix_sync::AsProof as _; use intrusive_collections::UnsafeRef; use posix_types::signal::{SigSet, Signal}; use posix_types::{SIGNAL_IGNORE, SIGNAL_NOW, SIGNAL_STOP}; +pub use signal_action::SignalAction; use signal_action::SignalActionList; -pub use signal_action::SignalAction; +use super::{ProcessList, Thread, WaitObject, WaitType}; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::user::UserPointer; +use crate::prelude::*; pub(self) const SAVED_DATA_SIZE: usize = size_of::() + size_of::() + size_of::(); @@ -168,10 +171,7 @@ impl SignalList { pub async fn handle(&self, trap_ctx: &mut TrapContext, fpu_state: &mut FpuState) { loop { let signal = { - let signal = match self.inner.lock().pop() { - Some(signal) => signal, - None => return, - }; + let Some(signal) = self.inner.lock().pop() else { return }; let handler = self.inner.lock().actions.get(signal); if let SignalAction::SimpleHandler { mask, .. } = &handler { @@ -246,7 +246,7 @@ impl SignalList { } signal => { // Default to terminate the thread. - Thread::current().force_kill(signal).await; + Thread::current().force_kill(signal); return; } } diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 77e8e618..7e005875 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -24,8 +24,7 @@ use super::{stackful, Process, ProcessList, WaitType}; use crate::kernel::interrupt::default_irq_handler; use crate::kernel::syscall::{syscall_handlers, SyscallHandler, User, UserMut}; use crate::kernel::task::clone::CloneArgs; -use crate::kernel::task::futex::RobustListHead; -use crate::kernel::task::CloneFlags; +use crate::kernel::task::{futex_exit, CloneFlags, RobustListHead}; use crate::kernel::timer::{should_reschedule, timer_interrupt}; use crate::kernel::user::{UserPointer, UserPointerMut}; use crate::kernel::vfs::filearray::FileArray; @@ -83,6 +82,7 @@ pub struct Thread { pub fpu_state: AtomicUniqueRefCell, pub dead: AtomicBool, + pub exit_status: Spin>, inner: Spin, } @@ -240,6 +240,7 @@ impl ThreadBuilder { trap_ctx: AtomicUniqueRefCell::new(trap_ctx), fpu_state: AtomicUniqueRefCell::new(fpu_state), dead: AtomicBool::new(false), + exit_status: Spin::new(None), inner: Spin::new(ThreadInner { name, tls: self.tls, @@ -331,18 +332,26 @@ impl Thread { } } - pub async fn force_kill(&self, signal: Signal) { - let mut proc_list = ProcessList::get().write().await; - unsafe { - // SAFETY: Preemption is disabled. - proc_list - .do_exit(self, WaitType::Signaled(signal), false) - .await; + pub fn exit(&self, exit_status: WaitType) { + { + let mut self_status = self.exit_status.lock(); + if self_status.is_some() { + // Someone has got here before us. + return; + } + + *self_status = Some(exit_status); } + + self.dead.store(true, Ordering::Release); + } + + pub fn force_kill(&self, signal: Signal) { + self.exit(WaitType::Signaled(signal)); } pub fn is_dead(&self) -> bool { - self.dead.load(Ordering::SeqCst) + self.dead.load(Ordering::Acquire) } async fn real_run(&self) { @@ -385,6 +394,10 @@ impl Thread { error_code, address: addr, }) => { + if self.is_dead() { + return; + } + let mms = &self.process.mm_list; if let Err(signal) = mms.handle_user_page_fault(addr, error_code).await { self.signal_list.raise(signal); @@ -407,6 +420,10 @@ impl Thread { } } TrapType::Syscall { no, args } => { + if self.is_dead() { + return; + } + if let Some(retval) = self.handle_syscall(thd_alloc, no, args).await { let mut trap_ctx = self.trap_ctx.borrow(); trap_ctx.set_user_return_value(retval); @@ -447,7 +464,17 @@ impl Thread { } pub fn run(self: Arc) -> impl Future + Send + 'static { - async move { self.contexted(stackful(self.real_run())).await } + async move { + self.contexted(async { + stackful(self.real_run()).await; + + futex_exit(&self).await; + }) + .await; + + assert!(self.is_dead(), "`real_run` returned before the thread die?"); + ProcessList::send_to_reaper(self); + } } } diff --git a/src/lib.rs b/src/lib.rs index 4f7fb262..2e28db24 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,6 @@ mod rcu; mod sync; use alloc::ffi::CString; -use alloc::sync::Arc; use core::hint::spin_loop; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -46,7 +45,7 @@ use eonix_mm::address::PRange; use eonix_runtime::executor::Stack; use eonix_runtime::scheduler::RUNTIME; use kernel::mem::GlobalPageAlloc; -use kernel::task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}; +use kernel::task::{KernelStack, ProcessList, ProgramLoader}; use kernel::vfs::dentry::Dentry; use kernel::vfs::mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}; use kernel::vfs::types::Permission; @@ -56,8 +55,6 @@ use kernel_init::setup_memory; use path::Path; use prelude::*; -use crate::kernel::task::alloc_pid; - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] fn do_panic() -> ! { #[cfg(target_arch = "riscv64")] @@ -276,21 +273,5 @@ async fn init_process(early_kstack: PRange) { .expect("Failed to load init program") }; - let thread_builder = ThreadBuilder::new() - .name(Arc::from(&b"busybox"[..])) - .entry(load_info.entry_ip, load_info.sp); - - let mut process_list = ProcessList::get().write().await; - let (thread, process) = ProcessBuilder::new() - .pid(alloc_pid()) - .mm_list(load_info.mm_list) - .thread_builder(thread_builder) - .build(&mut process_list); - - process_list.set_init_process(process); - - // TODO!!!: Remove this. - thread.files.open_console(); - - RUNTIME.spawn(thread.run()); + ProcessList::sys_init(load_info).await; } From 997edb05a2f286c97e49cf155eca7270fed79e05 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 21 Jan 2026 00:41:58 +0800 Subject: [PATCH 016/153] proc: rewrite process list organization - Use intrusive lists to store and organize the process hierarchy. - Remove `FileArray::open_console()`. Do it in the init script instead. - Fix open logic: acquire controlling terminals only if O_NOCTTY is not set. Put this into TerminalFile::open(). - Send SIGHUP and then SIGCONT to foreground pgroup procs when the controlling terminal is dropped. - Set the controlling terminal of sessions in Terminal. - Limit max line width to 80. Format some codes. Signed-off-by: greatbridf --- .rustfmt.toml | 4 +- Cargo.toml | 4 +- crates/posix_types/src/open.rs | 6 +- src/kernel/chardev.rs | 84 ++++----- src/kernel/syscall/file_rw.rs | 144 +++++++++++--- src/kernel/task/process.rs | 270 ++++++++++++++++++--------- src/kernel/task/process_group.rs | 146 +++++++++------ src/kernel/task/process_list.rs | 151 ++++++++------- src/kernel/task/session.rs | 187 ++++++++++++------- src/kernel/task/thread.rs | 106 +++++++++-- src/kernel/terminal.rs | 156 ++++++++++------ src/kernel/vfs/file/terminal_file.rs | 68 +++++-- src/kernel/vfs/filearray.rs | 88 ++++----- user-programs/init_script_riscv64.sh | 6 +- 14 files changed, 927 insertions(+), 493 deletions(-) diff --git a/.rustfmt.toml b/.rustfmt.toml index 17b2bbc5..85b1cfc7 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,4 +1,4 @@ -max_width = 100 +max_width = 80 hard_tabs = false tab_spaces = 4 newline_style = "Auto" @@ -14,7 +14,7 @@ single_line_if_else_max_width = 60 single_line_let_else_max_width = 60 wrap_comments = false format_code_in_doc_comments = false -doc_comment_code_block_width = 100 +doc_comment_code_block_width = 80 comment_width = 80 normalize_comments = false normalize_doc_attributes = false diff --git a/Cargo.toml b/Cargo.toml index 4fcb6f70..214e5941 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,8 +27,10 @@ pointers = { path = "./crates/pointers" } posix_types = { path = "./crates/posix_types" } slab_allocator = { path = "./crates/slab_allocator" } +intrusive-collections = { version = "0.9.8", features = [ + "nightly", +], git = "https://github.com/greatbridf/intrusive-rs" } bitflags = "2.6.0" -intrusive-collections = { version = "0.9.8", git = "https://github.com/greatbridf/intrusive-rs" } itertools = { version = "0.13.0", default-features = false } acpi = "5.2.0" align_ext = "0.1.0" diff --git a/crates/posix_types/src/open.rs b/crates/posix_types/src/open.rs index 758ea331..7135e5b8 100644 --- a/crates/posix_types/src/open.rs +++ b/crates/posix_types/src/open.rs @@ -11,6 +11,8 @@ bitflags! { const O_CREAT = 0x40; /// Exclusive access, fail if file exists const O_EXCL = 0x80; + /// Don't set controlling terminal. + const O_NOCTTY = 0x100; /// Truncate file to zero length if it exists const O_TRUNC = 0x200; /// Open file in append mode @@ -116,6 +118,8 @@ impl AtFlags { } pub fn statx_default_sync(&self) -> bool { - !self.intersects(AtFlags::AT_STATX_FORCE_SYNC | AtFlags::AT_STATX_DONT_SYNC) + !self.intersects( + AtFlags::AT_STATX_FORCE_SYNC | AtFlags::AT_STATX_DONT_SYNC, + ) } } diff --git a/src/kernel/chardev.rs b/src/kernel/chardev.rs index 4e01d83a..e4a6e1b3 100644 --- a/src/kernel/chardev.rs +++ b/src/kernel/chardev.rs @@ -1,22 +1,18 @@ -use super::{ - console::get_console, - constants::{EEXIST, EIO}, - task::{block_on, ProcessList, Thread}, - terminal::Terminal, - vfs::{types::DeviceId, File, FileType, TerminalFile}, -}; -use crate::{ - io::{Buffer, Stream, StreamRead}, - prelude::*, -}; -use alloc::{ - boxed::Box, - collections::btree_map::{BTreeMap, Entry}, - sync::Arc, -}; -use eonix_sync::AsProof as _; +use alloc::boxed::Box; +use alloc::collections::btree_map::{BTreeMap, Entry}; +use alloc::sync::Arc; + use posix_types::open::OpenFlags; +use super::console::get_console; +use super::constants::{EEXIST, EIO}; +use super::task::{block_on, Thread}; +use super::terminal::Terminal; +use super::vfs::types::DeviceId; +use super::vfs::{File, FileType, TerminalFile}; +use crate::io::{Buffer, Stream, StreamRead}; +use crate::prelude::*; + pub trait VirtualCharDevice: Send + Sync { fn read(&self, buffer: &mut dyn Buffer) -> KResult; fn write(&self, stream: &mut dyn Stream) -> KResult; @@ -33,12 +29,15 @@ pub struct CharDevice { device: CharDeviceType, } -static CHAR_DEVICES: Spin>> = Spin::new(BTreeMap::new()); +static CHAR_DEVICES: Spin>> = + Spin::new(BTreeMap::new()); impl CharDevice { pub fn read(&self, buffer: &mut dyn Buffer) -> KResult { match &self.device { - CharDeviceType::Terminal(terminal) => block_on(terminal.read(buffer)), + CharDeviceType::Terminal(terminal) => { + block_on(terminal.read(buffer)) + } CharDeviceType::Virtual(device) => device.read(buffer), } } @@ -46,10 +45,12 @@ impl CharDevice { pub fn write(&self, stream: &mut dyn Stream) -> KResult { match &self.device { CharDeviceType::Virtual(device) => device.write(stream), - CharDeviceType::Terminal(terminal) => stream.read_till_end(&mut [0; 128], |data| { - terminal.write(data); - Ok(()) - }), + CharDeviceType::Terminal(terminal) => { + stream.read_till_end(&mut [0; 128], |data| { + terminal.write(data); + Ok(()) + }) + } } } @@ -57,7 +58,11 @@ impl CharDevice { CHAR_DEVICES.lock().get(&devid).cloned() } - pub fn register(devid: DeviceId, name: Arc, device: CharDeviceType) -> KResult<()> { + pub fn register( + devid: DeviceId, + name: Arc, + device: CharDeviceType, + ) -> KResult<()> { match CHAR_DEVICES.lock().entry(devid) { Entry::Vacant(entry) => { entry.insert(Arc::new(CharDevice { name, device })); @@ -67,26 +72,21 @@ impl CharDevice { } } - pub fn open(self: &Arc, flags: OpenFlags) -> KResult { - Ok(match &self.device { + pub async fn open( + self: &Arc, + thread: &Thread, + flags: OpenFlags, + ) -> KResult { + let file = match &self.device { + CharDeviceType::Virtual(_) => { + File::new(flags, FileType::CharDev(self.clone())) + } CharDeviceType::Terminal(terminal) => { - let procs = block_on(ProcessList::get().read()); - let current = Thread::current(); - let session = current.process.session(procs.prove()); - // We only set the control terminal if the process is the session leader. - if session.sid == Thread::current().process.pid { - // Silently fail if we can't set the control terminal. - dont_check!(block_on(session.set_control_terminal( - &terminal, - false, - procs.prove() - ))); - } - - TerminalFile::new(terminal.clone(), flags) + TerminalFile::open(thread, terminal, flags).await } - CharDeviceType::Virtual(_) => File::new(flags, FileType::CharDev(self.clone())), - }) + }; + + Ok(file) } } diff --git a/src/kernel/syscall/file_rw.rs b/src/kernel/syscall/file_rw.rs index 93a543d7..8ac9c22a 100644 --- a/src/kernel/syscall/file_rw.rs +++ b/src/kernel/syscall/file_rw.rs @@ -12,7 +12,8 @@ use posix_types::syscall_no::*; use super::{FromSyscallArg, User}; use crate::io::{Buffer, BufferFill, IntoStream}; use crate::kernel::constants::{ - EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, SEEK_SET, + EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, + SEEK_SET, }; use crate::kernel::syscall::UserMut; use crate::kernel::task::Thread; @@ -61,7 +62,13 @@ async fn dentry_from( let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; let dir_dentry = dir_file.as_path().ok_or(ENOTDIR)?; - Dentry::open_at(&thread.fs_context, dir_dentry, path, follow_symlink).await + Dentry::open_at( + &thread.fs_context, + dir_dentry, + path, + follow_symlink, + ) + .await } } } @@ -79,7 +86,12 @@ async fn read(fd: FD, buffer: UserMut, bufsize: usize) -> KResult { } #[eonix_macros::define_syscall(SYS_PREAD64)] -async fn pread64(fd: FD, buffer: UserMut, bufsize: usize, offset: usize) -> KResult { +async fn pread64( + fd: FD, + buffer: UserMut, + bufsize: usize, + offset: usize, +) -> KResult { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -104,7 +116,12 @@ async fn write(fd: FD, buffer: User, count: usize) -> KResult { } #[eonix_macros::define_syscall(SYS_PWRITE64)] -async fn pwrite64(fd: FD, buffer: User, count: usize, offset: usize) -> KResult { +async fn pwrite64( + fd: FD, + buffer: User, + count: usize, + offset: usize, +) -> KResult { let buffer = CheckedUserPointer::new(buffer, count)?; let mut stream = buffer.into_stream(); @@ -117,11 +134,17 @@ async fn pwrite64(fd: FD, buffer: User, count: usize, offset: usize) -> KRes } #[eonix_macros::define_syscall(SYS_OPENAT)] -async fn openat(dirfd: FD, pathname: User, flags: OpenFlags, mode: Mode) -> KResult { - let dentry = dentry_from(thread, dirfd, pathname, flags.follow_symlink()).await?; +async fn openat( + dirfd: FD, + pathname: User, + flags: OpenFlags, + mode: Mode, +) -> KResult { + let dentry = + dentry_from(thread, dirfd, pathname, flags.follow_symlink()).await?; let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - thread.files.open(&dentry, flags, perm).await + thread.files.open(thread, &dentry, flags, perm).await } #[cfg(target_arch = "x86_64")] @@ -156,7 +179,8 @@ async fn dup3(old_fd: FD, new_fd: FD, flags: OpenFlags) -> KResult { #[eonix_macros::define_syscall(SYS_PIPE2)] async fn pipe2(pipe_fd: UserMut<[FD; 2]>, flags: OpenFlags) -> KResult<()> { - let mut buffer = UserBuffer::new(pipe_fd.cast(), core::mem::size_of::<[FD; 2]>())?; + let mut buffer = + UserBuffer::new(pipe_fd.cast(), core::mem::size_of::<[FD; 2]>())?; let (read_fd, write_fd) = thread.files.pipe(flags)?; buffer.copy(&[read_fd, write_fd])?.ok_or(EFAULT) @@ -170,7 +194,11 @@ async fn pipe(pipe_fd: UserMut<[FD; 2]>) -> KResult<()> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_GETDENTS)] -async fn getdents(fd: FD, buffer: UserMut, bufsize: usize) -> KResult { +async fn getdents( + fd: FD, + buffer: UserMut, + bufsize: usize, +) -> KResult { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -184,7 +212,11 @@ async fn getdents(fd: FD, buffer: UserMut, bufsize: usize) -> KResult } #[eonix_macros::define_syscall(SYS_GETDENTS64)] -async fn getdents64(fd: FD, buffer: UserMut, bufsize: usize) -> KResult { +async fn getdents64( + fd: FD, + buffer: UserMut, + bufsize: usize, +) -> KResult { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -230,7 +262,8 @@ async fn newfstatat( )] #[cfg_attr(target_arch = "x86_64", eonix_macros::define_syscall(SYS_FSTAT64))] async fn newfstat(fd: FD, statbuf: UserMut) -> KResult<()> { - sys_newfstatat(thread, fd, User::null(), statbuf, AtFlags::AT_EMPTY_PATH).await + sys_newfstatat(thread, fd, User::null(), statbuf, AtFlags::AT_EMPTY_PATH) + .await } #[eonix_macros::define_syscall(SYS_STATX)] @@ -307,7 +340,11 @@ async fn unlink(pathname: User) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_SYMLINKAT)] -async fn symlinkat(target: User, dirfd: FD, linkpath: User) -> KResult<()> { +async fn symlinkat( + target: User, + dirfd: FD, + linkpath: User, +) -> KResult<()> { let target = UserString::new(target)?; let dentry = dentry_from(thread, dirfd, linkpath, false).await?; @@ -341,7 +378,12 @@ impl UserDeviceId { } #[eonix_macros::define_syscall(SYS_MKNODAT)] -async fn mknodat(dirfd: FD, pathname: User, mut mode: Mode, dev: UserDeviceId) -> KResult<()> { +async fn mknodat( + dirfd: FD, + pathname: User, + mut mode: Mode, + dev: UserDeviceId, +) -> KResult<()> { if !mode.is_blk() && !mode.is_chr() { return Err(EINVAL); } @@ -354,7 +396,11 @@ async fn mknodat(dirfd: FD, pathname: User, mut mode: Mode, dev: UserDeviceI #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKNOD)] -async fn mknod(pathname: User, mode: Mode, dev: UserDeviceId) -> KResult<()> { +async fn mknod( + pathname: User, + mode: Mode, + dev: UserDeviceId, +) -> KResult<()> { sys_mknodat(thread, FD::AT_FDCWD, pathname, mode, dev).await } @@ -373,11 +419,20 @@ async fn readlinkat( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_READLINK)] -async fn readlink(pathname: User, buffer: UserMut, bufsize: usize) -> KResult { +async fn readlink( + pathname: User, + buffer: UserMut, + bufsize: usize, +) -> KResult { sys_readlinkat(thread, FD::AT_FDCWD, pathname, buffer, bufsize).await } -async fn do_lseek(thread: &Thread, fd: FD, offset: u64, whence: u32) -> KResult { +async fn do_lseek( + thread: &Thread, + fd: FD, + offset: u64, + whence: u32, +) -> KResult { let file = thread.files.get(fd).ok_or(EBADF)?; Ok(match whence { @@ -403,7 +458,8 @@ async fn llseek( result: UserMut, whence: u32, ) -> KResult<()> { - let mut result = UserBuffer::new(result.cast(), core::mem::size_of::())?; + let mut result = + UserBuffer::new(result.cast(), core::mem::size_of::())?; let offset = ((offset_high as u64) << 32) | (offset_low as u64); let new_offset = do_lseek(thread, fd, offset, whence).await?; @@ -434,9 +490,10 @@ async fn readv(fd: FD, iov_user: User, iovcnt: u32) -> KResult { Ok(IoVec { len: Long::ZERO, .. }) => None, - Ok(IoVec { base, len }) => { - Some(UserBuffer::new(UserMut::with_addr(base.addr()), len.get())) - } + Ok(IoVec { base, len }) => Some(UserBuffer::new( + UserMut::with_addr(base.addr()), + len.get(), + )), }) .collect::>>()?; @@ -471,8 +528,11 @@ async fn writev(fd: FD, iov_user: User, iovcnt: u32) -> KResult { len: Long::ZERO, .. }) => None, Ok(IoVec { base, len }) => Some( - CheckedUserPointer::new(User::with_addr(base.addr()), len.get()) - .map(|ptr| ptr.into_stream()), + CheckedUserPointer::new( + User::with_addr(base.addr()), + len.get(), + ) + .map(|ptr| ptr.into_stream()), ), }) .collect::>>()?; @@ -491,7 +551,12 @@ async fn writev(fd: FD, iov_user: User, iovcnt: u32) -> KResult { } #[eonix_macros::define_syscall(SYS_FACCESSAT)] -async fn faccessat(dirfd: FD, pathname: User, _mode: u32, flags: AtFlags) -> KResult<()> { +async fn faccessat( + dirfd: FD, + pathname: User, + _mode: u32, + flags: AtFlags, +) -> KResult<()> { let dentry = if flags.at_empty_path() { let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() @@ -522,7 +587,12 @@ async fn access(pathname: User, mode: u32) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_SENDFILE64)] -async fn sendfile64(out_fd: FD, in_fd: FD, offset: UserMut, count: usize) -> KResult { +async fn sendfile64( + out_fd: FD, + in_fd: FD, + offset: UserMut, + count: usize, +) -> KResult { let in_file = thread.files.get(in_fd).ok_or(EBADF)?; let out_file = thread.files.get(out_fd).ok_or(EBADF)?; @@ -627,7 +697,11 @@ async fn pselect6( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_POLL)] -async fn poll(fds: UserMut, nfds: u32, timeout: u32) -> KResult { +async fn poll( + fds: UserMut, + nfds: u32, + timeout: u32, +) -> KResult { do_poll(thread, fds, nfds, timeout).await } @@ -639,7 +713,8 @@ async fn fchownat( gid: u32, flags: AtFlags, ) -> KResult<()> { - let dentry = dentry_from(thread, dirfd, pathname, !flags.no_follow()).await?; + let dentry = + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await?; if !dentry.is_valid() { return Err(ENOENT); } @@ -648,7 +723,12 @@ async fn fchownat( } #[eonix_macros::define_syscall(SYS_FCHMODAT)] -async fn fchmodat(dirfd: FD, pathname: User, mode: Mode, flags: AtFlags) -> KResult<()> { +async fn fchmodat( + dirfd: FD, + pathname: User, + mode: Mode, + flags: AtFlags, +) -> KResult<()> { let dentry = if flags.at_empty_path() { let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() @@ -709,12 +789,16 @@ async fn renameat2( let flags = RenameFlags::from_bits(flags).ok_or(EINVAL)?; // The two flags RENAME_NOREPLACE and RENAME_EXCHANGE are mutually exclusive. - if flags.contains(RenameFlags::RENAME_NOREPLACE | RenameFlags::RENAME_EXCHANGE) { + if flags + .contains(RenameFlags::RENAME_NOREPLACE | RenameFlags::RENAME_EXCHANGE) + { Err(EINVAL)?; } - let old_dentry = dentry_from(thread, old_dirfd, old_pathname, false).await?; - let new_dentry = dentry_from(thread, new_dirfd, new_pathname, false).await?; + let old_dentry = + dentry_from(thread, old_dirfd, old_pathname, false).await?; + let new_dentry = + dentry_from(thread, new_dirfd, new_pathname, false).await?; old_dentry.rename(&new_dentry, flags).await } diff --git a/src/kernel/task/process.rs b/src/kernel/task/process.rs index 3eff5949..1385235d 100644 --- a/src/kernel/task/process.rs +++ b/src/kernel/task/process.rs @@ -1,22 +1,24 @@ -use alloc::collections::btree_map::BTreeMap; use alloc::collections::vec_deque::VecDeque; -use alloc::sync::{Arc, Weak}; +use alloc::sync::Arc; use core::sync::atomic::{AtomicU32, Ordering}; use eonix_sync::{ - AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, SpinGuard, - UnlockableGuard as _, UnlockedGuard as _, + AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, + SpinGuard, UnlockableGuard as _, UnlockedGuard as _, +}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; use pointers::BorrowedArc; use posix_types::constants::{ - CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_ALL, P_PGID, P_PID, P_PIDFD, + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_ALL, + P_PGID, P_PID, P_PIDFD, }; use posix_types::signal::Signal; use posix_types::SIGNAL_COREDUMP; -use super::process_group::ProcessGroupBuilder; use super::signal::RaiseResult; -use super::thread::ThreadBuilder; +use super::thread::{ProcessThreads, ThreadBuilder}; use super::{ProcessGroup, ProcessList, Session, Thread}; use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; use crate::kernel::mem::MMList; @@ -35,7 +37,6 @@ pub struct ProcessBuilder { pid: Option, } -#[derive(Debug)] pub struct Process { /// Process id /// @@ -66,14 +67,55 @@ pub struct Process { /// The only case where it may be `None` is when the process is kernel thread. pub(super) session: RCUPointer, - /// All things related to the process list. - pub(super) inner: Locked, + pub children: Locked, ProcessList>, + pub threads: Locked, ProcessList>, + + all_procs_link: RBTreeAtomicLink, + group_procs_link: RBTreeAtomicLink, + siblings_link: RBTreeAtomicLink, } -#[derive(Debug)] -pub(super) struct ProcessInner { - pub(super) children: BTreeMap>, - pub(super) threads: BTreeMap>, +intrusive_adapter!(pub AllProcs = Arc: Process { + all_procs_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub GroupProcs = Arc: Process { + group_procs_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub ProcessChildren = Arc: Process { + siblings_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllProcs { + type Key = u32; + + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.pid + } +} + +impl KeyAdapter<'_> for GroupProcs { + type Key = u32; + + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.pid + } +} + +impl KeyAdapter<'_> for ProcessChildren { + type Key = u32; + + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.pid + } } #[derive(Debug)] @@ -148,7 +190,9 @@ impl WaitType { pub fn to_wstatus(self) -> u32 { match self { WaitType::Exited(status) => (status & 0xff) << 8, - WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => signal.into_raw() | 0x80, + WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => { + signal.into_raw() | 0x80 + } WaitType::Signaled(signal) => signal.into_raw(), WaitType::Stopped(signal) => 0x7f | (signal.into_raw() << 8), WaitType::Continued => 0xffff, @@ -159,7 +203,9 @@ impl WaitType { // TODO: CLD_TRAPPED match self { WaitType::Exited(status) => (status, CLD_EXITED), - WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => (signal.into_raw(), CLD_DUMPED), + WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => { + (signal.into_raw(), CLD_DUMPED) + } WaitType::Signaled(signal) => (signal.into_raw(), CLD_KILLED), WaitType::Stopped(signal) => (signal.into_raw(), CLD_STOPPED), WaitType::Continued => (Signal::SIGCONT.into_raw(), CLD_CONTINUED), @@ -194,7 +240,11 @@ impl ProcessBuilder { } } - pub async fn clone_from(mut self, process: Arc, clone_args: &CloneArgs) -> Self { + pub async fn clone_from( + mut self, + process: Arc, + clone_args: &CloneArgs, + ) -> Self { let mm_list = if clone_args.flags.contains(CloneFlags::CLONE_VM) { process.mm_list.new_shared().await } else { @@ -243,7 +293,10 @@ impl ProcessBuilder { self } - pub fn build(self, process_list: &mut ProcessList) -> (Arc, Arc) { + pub fn build( + self, + process_list: &mut ProcessList, + ) -> (Arc, Arc) { let mm_list = self.mm_list.unwrap_or_else(|| MMList::new()); let process = Arc::new(Process { @@ -254,18 +307,23 @@ impl ProcessBuilder { parent: RCUPointer::empty(), pgroup: RCUPointer::empty(), session: RCUPointer::empty(), - inner: Locked::new( - ProcessInner { - children: BTreeMap::new(), - threads: BTreeMap::new(), - }, + children: Locked::new( + RBTree::new(ProcessChildren::NEW), + process_list, + ), + threads: Locked::new( + RBTree::new(ProcessThreads::NEW), process_list, ), + all_procs_link: RBTreeAtomicLink::new(), + group_procs_link: RBTreeAtomicLink::new(), + siblings_link: RBTreeAtomicLink::new(), }); process_list.add_process(&process); - let thread_builder = self.thread_builder.expect("Thread builder is not set"); + let thread_builder = + self.thread_builder.expect("Thread builder is not set"); let thread = thread_builder .process(process.clone()) .tid(process.pid) @@ -281,10 +339,7 @@ impl ProcessBuilder { pgroup.add_member(&process, process_list.prove_mut()); pgroup } - None => ProcessGroupBuilder::new() - .leader(&process) - .session(session.clone()) - .build(process_list), + None => ProcessGroup::new(&process, &session, process_list), }; if let Some(parent) = &self.parent { @@ -304,30 +359,30 @@ impl ProcessBuilder { impl Process { pub fn raise(&self, signal: Signal, procs: Proof<'_, ProcessList>) { - let inner = self.inner.access(procs); - for thread in inner.threads.values().map(|t| t.upgrade().unwrap()) { + let threads = self.threads.access(procs); + for thread in threads.iter() { if let RaiseResult::Finished = thread.raise(signal) { break; } } } - pub(super) fn add_child(&self, child: &Arc, procs: ProofMut<'_, ProcessList>) { - assert!(self - .inner - .access_mut(procs) - .children - .insert(child.pid, Arc::downgrade(child)) - .is_none()); + pub fn add_child( + &self, + child: &Arc, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_procs_link.is_linked(), "Dead process"); + self.children.access_mut(procs).insert(child.clone()); } - pub(super) fn add_thread(&self, thread: &Arc, procs: ProofMut<'_, ProcessList>) { - assert!(self - .inner - .access_mut(procs) - .threads - .insert(thread.tid, Arc::downgrade(thread)) - .is_none()); + pub fn add_thread( + &self, + thread: &Arc, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_procs_link.is_linked(), "Dead process"); + self.threads.access_mut(procs).insert(thread.clone()); } pub async fn wait( @@ -354,12 +409,7 @@ impl Process { break object; } - if self - .inner - .access(waits.process_list.prove()) - .children - .is_empty() - { + if self.children.access(waits.process_list.prove()).is_empty() { return Err(ECHILD); } @@ -375,12 +425,12 @@ impl Process { Ok(Some(wait_object)) } else { let mut procs = ProcessList::get().write().await; - procs.remove_process(wait_object.pid).await; + procs.remove_process(wait_object.pid); assert!(self - .inner - .access_mut(procs.prove_mut()) .children - .remove(&wait_object.pid) + .access_mut(procs.prove_mut()) + .find_mut(&wait_object.pid) + .remove() .is_some()); Ok(Some(wait_object)) @@ -396,15 +446,17 @@ impl Process { if process_list.try_find_session(self.pid).is_some() { return Err(EPERM); } + + self.pgroup(process_list.prove()) + .remove_member(self, &mut process_list); + let session = Session::new(self, &mut process_list); - let pgroup = ProcessGroupBuilder::new() - .leader(self) - .session(session.clone()) - .build(&mut process_list); + let pgroup = ProcessGroup::new(self, &session, &mut process_list); - let old_session = unsafe { self.session.swap(Some(session.clone())) }.unwrap(); - let old_pgroup = unsafe { self.pgroup.swap(Some(pgroup.clone())) }.unwrap(); - old_pgroup.remove_member(self.pid, process_list.prove_mut()); + let old_session = + unsafe { self.session.swap(Some(session.clone())) }.unwrap(); + let old_pgroup = + unsafe { self.pgroup.swap(Some(pgroup.clone())) }.unwrap(); call_rcu(move || { drop(old_session); @@ -417,47 +469,56 @@ impl Process { /// Set the process group id of the process to `pgid`. /// /// This function does the actual work. - fn do_setpgid(self: &Arc, pgid: u32, procs: &mut ProcessList) -> KResult<()> { + fn do_setpgid( + self: &Arc, + pgid: u32, + procs: &mut ProcessList, + ) -> KResult<()> { // SAFETY: We are holding the process list lock. let session = unsafe { self.session.load_locked().unwrap() }; - let pgroup = unsafe { self.pgroup.load_locked().unwrap() }; // Changing the process group of a session leader is not allowed. if session.sid == self.pid { return Err(EPERM); } - let new_pgroup = if let Some(new_pgroup) = procs.try_find_pgroup(pgid) { + let cur_pgroup = self.pgroup(procs.prove()).clone(); + let existing_pgroup = procs.try_find_pgroup(pgid); + + if let Some(new_pgroup) = &existing_pgroup { // Move us to an existing process group. // Check that the two groups are in the same session. - if new_pgroup.session.upgrade().unwrap().sid != session.sid { + if new_pgroup.session.sid != session.sid { return Err(EPERM); } // If we are already in the process group, we are done. - if new_pgroup.pgid == pgroup.pgid { + if new_pgroup.pgid == cur_pgroup.pgid { return Ok(()); } - - new_pgroup.add_member(self, procs.prove_mut()); - - new_pgroup } else { // Create a new process group only if `pgid` matches our `pid`. if pgid != self.pid { return Err(EPERM); } + } - ProcessGroupBuilder::new() - .leader(self) - .session(session.clone()) - .build(procs) - }; + // Permission checks done. Let's do the actual work. + cur_pgroup.remove_member(self, procs); - pgroup.remove_member(self.pid, procs.prove_mut()); + let new_pgroup; + if let Some(pgroup) = existing_pgroup { + pgroup.add_member(self, procs.prove_mut()); + new_pgroup = pgroup; + } else { + new_pgroup = ProcessGroup::new(self, &session, procs); + } - let old_pgroup = unsafe { self.pgroup.swap(Some(new_pgroup)) }.unwrap(); - call_rcu(move || drop(old_pgroup)); + unsafe { + // SAFETY: `cur_pgroup` held above. + self.pgroup.swap(Some(new_pgroup)); + } + call_rcu(move || drop(cur_pgroup)); Ok(()) } @@ -475,15 +536,14 @@ impl Process { let child = { // If `pid` refers to one of our children, the thread leaders must be // in out children list. - let children = &self.inner.access(procs.prove()).children; - let child = { - let child = children.get(&pid); - child.and_then(Weak::upgrade).ok_or(ESRCH)? - }; + let children = self.children.access(procs.prove()); + let child = children.find(&pid).clone_pointer().ok_or(ESRCH)?; // Changing the process group of a child is only allowed // if we are in the same session. - if child.session(procs.prove()).sid != self.session(procs.prove()).sid { + if child.session(procs.prove()).sid + != self.session(procs.prove()).sid + { return Err(EPERM); } @@ -497,19 +557,28 @@ impl Process { } /// Provide locked (consistent) access to the session. - pub fn session<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, Session> { + pub fn session<'r>( + &'r self, + _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, Session> { // SAFETY: We are holding the process list lock. unsafe { self.session.load_locked() }.unwrap() } /// Provide locked (consistent) access to the process group. - pub fn pgroup<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, ProcessGroup> { + pub fn pgroup<'r>( + &'r self, + _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, ProcessGroup> { // SAFETY: We are holding the process list lock. unsafe { self.pgroup.load_locked() }.unwrap() } /// Provide locked (consistent) access to the parent process. - pub fn parent<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, Process> { + pub fn parent<'r>( + &'r self, + _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, Process> { // SAFETY: We are holding the process list lock. unsafe { self.parent.load_locked() }.unwrap() } @@ -520,16 +589,25 @@ impl Process { } /// Provide RCU locked (maybe inconsistent) access to the process group. - pub fn pgroup_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<'_, ProcessGroup>> { + pub fn pgroup_rcu( + &self, + ) -> RCUReadGuard<'_, BorrowedArc<'_, ProcessGroup>> { self.pgroup.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the parent process. - pub fn parent_rcu(&self) -> Option>> { + pub fn parent_rcu( + &self, + ) -> Option>> { self.parent.load() } - pub fn notify(&self, signal: Option, wait: WaitObject, procs: Proof<'_, ProcessList>) { + pub fn notify( + &self, + signal: Option, + wait: WaitObject, + procs: Proof<'_, ProcessList>, + ) { self.wait_list.notify(wait); if let Some(signal) = signal { @@ -607,8 +685,11 @@ impl Entry<'_, '_, '_> { WaitId::Any => true, WaitId::Pid(pid) => item.pid == pid, WaitId::Pgid(pgid) => { - if let Some(process) = self.process_list.try_find_process(item.pid) { - return process.pgroup(self.process_list.prove()).pgid == pgid; + if let Some(process) = + self.process_list.try_find_process(item.pid) + { + return process.pgroup(self.process_list.prove()).pgid + == pgid; } false } @@ -622,7 +703,10 @@ impl Entry<'_, '_, '_> { } } - pub fn wait(self, no_block: bool) -> impl core::future::Future> + Send { + pub fn wait( + self, + no_block: bool, + ) -> impl core::future::Future> + Send { let wait_procs = self.wait_procs.unlock(); async move { diff --git a/src/kernel/task/process_group.rs b/src/kernel/task/process_group.rs index 137c5191..8c708b5c 100644 --- a/src/kernel/task/process_group.rs +++ b/src/kernel/task/process_group.rs @@ -1,87 +1,121 @@ -use super::{Process, ProcessList, Session}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use alloc::sync::{Arc, Weak}; + +use eonix_sync::{AsProofMut, Locked, Proof, ProofMut}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; -use eonix_sync::{Locked, Proof, ProofMut}; use posix_types::signal::Signal; -pub struct ProcessGroupBuilder { - pgid: Option, - leader: Option>, - session: Option>, -} +use super::process::GroupProcs; +use super::{Process, ProcessList, Session}; -#[derive(Debug)] pub struct ProcessGroup { pub pgid: u32, - pub _leader: Weak, - pub session: Weak, + pub leader: Weak, + pub session: Arc, - pub processes: Locked>, ProcessList>, + pub procs: Locked, ProcessList>, + + all_groups_link: RBTreeAtomicLink, + session_groups_link: RBTreeAtomicLink, } -impl ProcessGroupBuilder { - pub const fn new() -> Self { - Self { - pgid: None, - leader: None, - session: None, - } - } +intrusive_adapter!(pub AllGroups = Arc: ProcessGroup { + all_groups_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub SessionGroups = Arc: ProcessGroup { + session_groups_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllGroups { + type Key = u32; - pub fn leader(mut self, leader: &Arc) -> Self { - self.pgid = Some(leader.pid); - self.leader = Some(Arc::downgrade(leader)); - self + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.pgid } +} + +impl KeyAdapter<'_> for SessionGroups { + type Key = u32; - pub fn session(mut self, session: Arc) -> Self { - self.session = Some(session); - self + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.pgid } +} - pub fn build(self, process_list: &mut ProcessList) -> Arc { - let pgid = self.pgid.expect("PGID is not set"); - let leader = self.leader.expect("Leader is not set"); - let session = self.session.expect("Session is not set"); +impl ProcessGroup { + /// Create a pgroup and add it to the global pgroup list. + /// Add the pgroup to the session. + /// + /// # Panics + /// Panics if `leader` is already in some pgroup. + pub fn new( + leader: &Arc, + session: &Arc, + procs: &mut ProcessList, + ) -> Arc { + let pgid = leader.pid; + let pgroup_procs = { + let mut list = RBTree::new(GroupProcs::new()); + list.insert(leader.clone()); + list + }; let pgroup = Arc::new(ProcessGroup { pgid, - session: Arc::downgrade(&session), - processes: Locked::new(BTreeMap::from([(pgid, leader.clone())]), process_list), - _leader: leader, + session: session.clone(), + procs: Locked::new(pgroup_procs, procs), + leader: Arc::downgrade(leader), + all_groups_link: RBTreeAtomicLink::new(), + session_groups_link: RBTreeAtomicLink::new(), }); - process_list.add_pgroup(&pgroup); - session.add_member(process_list, &pgroup); + procs.add_pgroup(&pgroup); + session.add_member(&pgroup, procs.prove_mut()); pgroup } -} -impl ProcessGroup { - pub(super) fn add_member(&self, process: &Arc, procs: ProofMut<'_, ProcessList>) { - assert!(self - .processes - .access_mut(procs) - .insert(process.pid, Arc::downgrade(process)) - .is_none()); + /// Add `process` to the pgroup. + /// + /// # Panics + /// Panics if `process` is already in some pgroup or the pgroup is dead. + pub fn add_member( + &self, + process: &Arc, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_groups_link.is_linked(), "Dead pgroup"); + self.procs.access_mut(procs).insert(process.clone()); } - pub(super) fn remove_member(&self, pid: u32, procs: ProofMut<'_, ProcessList>) { - let processes = self.processes.access_mut(procs); - assert!(processes.remove(&pid).is_some()); - if processes.is_empty() { - self.session - .upgrade() - .unwrap() - .remove_member(self.pgid, procs); + pub fn remove_member( + self: &Arc, + process: &Arc, + procs: &mut ProcessList, + ) { + let members = self.procs.access_mut(procs.prove_mut()); + assert!( + members.find_mut(&process.pid).remove().is_some(), + "Not a member" + ); + + if !members.is_empty() { + return; } + + self.session.remove_member(self, procs); + procs.remove_pgroup(self); } pub fn raise(&self, signal: Signal, procs: Proof<'_, ProcessList>) { - let processes = self.processes.access(procs); - for process in processes.values().map(|p| p.upgrade().unwrap()) { + let members = self.procs.access(procs); + for process in members.iter() { process.raise(signal, procs); } } diff --git a/src/kernel/task/process_list.rs b/src/kernel/task/process_list.rs index c676d22e..f3371f25 100644 --- a/src/kernel/task/process_list.rs +++ b/src/kernel/task/process_list.rs @@ -1,37 +1,41 @@ -use alloc::collections::btree_map::BTreeMap; use alloc::collections::vec_deque::VecDeque; -use alloc::sync::{Arc, Weak}; +use alloc::sync::Arc; use core::pin::pin; -use core::sync::atomic::Ordering; use eonix_runtime::scheduler::RUNTIME; use eonix_sync::{AsProof as _, AsProofMut as _, RwLock, Spin, WaitList}; +use intrusive_collections::RBTree; use super::loader::LoadInfo; +use super::process::AllProcs; +use super::process_group::AllGroups; +use super::session::AllSessions; +use super::thread::AllThreads; use super::{ - alloc_pid, Process, ProcessBuilder, ProcessGroup, Session, Thread, ThreadBuilder, WaitObject, + alloc_pid, Process, ProcessBuilder, ProcessGroup, Session, Thread, + ThreadBuilder, WaitObject, }; -use crate::rcu::rcu_sync; +use crate::rcu::call_rcu; pub struct ProcessList { /// The init process. init: Option>, /// All threads. - threads: BTreeMap>, + threads: RBTree, /// All processes. - processes: BTreeMap>, + procs: RBTree, /// All process groups. - pgroups: BTreeMap>, + pgroups: RBTree, /// All sessions. - sessions: BTreeMap>, + sessions: RBTree, } static GLOBAL_PROC_LIST: RwLock = RwLock::new(ProcessList { init: None, - threads: BTreeMap::new(), - processes: BTreeMap::new(), - pgroups: BTreeMap::new(), - sessions: BTreeMap::new(), + threads: RBTree::new(AllThreads::NEW), + procs: RBTree::new(AllProcs::NEW), + pgroups: RBTree::new(AllGroups::NEW), + sessions: RBTree::new(AllSessions::NEW), }); impl ProcessList { @@ -40,43 +44,64 @@ impl ProcessList { } pub fn add_session(&mut self, session: &Arc) { - self.sessions.insert(session.sid, Arc::downgrade(session)); + self.sessions.insert(session.clone()); } pub fn add_pgroup(&mut self, pgroup: &Arc) { - self.pgroups.insert(pgroup.pgid, Arc::downgrade(pgroup)); + self.pgroups.insert(pgroup.clone()); } pub fn add_process(&mut self, process: &Arc) { - self.processes.insert(process.pid, Arc::downgrade(process)); + self.procs.insert(process.clone()); } pub fn add_thread(&mut self, thread: &Arc) { - self.threads.insert(thread.tid, thread.clone()); + self.threads.insert(thread.clone()); } - pub async fn remove_process(&mut self, pid: u32) { + pub fn remove_process(&mut self, pid: u32) { // Thread group leader has the same tid as the pid. - if let Some(thread) = self.threads.remove(&pid) { - self.processes.remove(&pid); - - // SAFETY: We wait until all references are dropped below with `rcu_sync()`. - let session = unsafe { thread.process.session.swap(None) }.unwrap(); - let pgroup = unsafe { thread.process.pgroup.swap(None) }.unwrap(); - let _parent = unsafe { thread.process.parent.swap(None) }.unwrap(); - pgroup.remove_member(pid, self.prove_mut()); - rcu_sync().await; - - if Arc::strong_count(&pgroup) == 1 { - self.pgroups.remove(&pgroup.pgid); - } + let Some(_) = self.threads.find_mut(&pid).remove() else { + panic!("Thread {} not found", pid); + }; - if Arc::strong_count(&session) == 1 { - self.sessions.remove(&session.sid); - } - } else { + let Some(proc) = self.procs.find_mut(&pid).remove() else { panic!("Process {} not found", pid); - } + }; + + // SAFETY: `call_rcu` below. + let session = unsafe { proc.session.swap(None) }.unwrap(); + let pgroup = unsafe { proc.pgroup.swap(None) }.unwrap(); + let parent = unsafe { proc.parent.swap(None) }.unwrap(); + + pgroup.remove_member(&proc, self); + + call_rcu(move || { + drop(session); + drop(pgroup); + drop(parent); + }); + } + + pub fn remove_thread(&mut self, thread: &Arc) { + assert!( + self.threads.find_mut(&thread.tid).remove().is_some(), + "Double remove" + ); + } + + pub fn remove_session(&mut self, session: &Arc) { + assert!( + self.sessions.find_mut(&session.sid).remove().is_some(), + "Double remove" + ); + } + + pub fn remove_pgroup(&mut self, pgroup: &Arc) { + assert!( + self.pgroups.find_mut(&pgroup.pgid).remove().is_some(), + "Double remove" + ); } fn set_init_process(&mut self, init: Arc) { @@ -88,20 +113,20 @@ impl ProcessList { self.init.as_ref().unwrap() } - pub fn try_find_thread(&self, tid: u32) -> Option<&Arc> { - self.threads.get(&tid) + pub fn try_find_thread(&self, tid: u32) -> Option> { + self.threads.find(&tid).clone_pointer() } pub fn try_find_process(&self, pid: u32) -> Option> { - self.processes.get(&pid).and_then(Weak::upgrade) + self.procs.find(&pid).clone_pointer() } pub fn try_find_pgroup(&self, pgid: u32) -> Option> { - self.pgroups.get(&pgid).and_then(Weak::upgrade) + self.pgroups.find(&pgid).clone_pointer() } pub fn try_find_session(&self, sid: u32) -> Option> { - self.sessions.get(&sid).and_then(Weak::upgrade) + self.sessions.find(&sid).clone_pointer() } pub async fn sys_init(load_info: LoadInfo) { @@ -118,9 +143,6 @@ impl ProcessList { process_list.set_init_process(process); - // TODO!!!: Remove this. - thread.files.open_console(); - RUNTIME.spawn(Reaper::daemon()); RUNTIME.spawn(thread.run()); } @@ -152,18 +174,19 @@ impl Reaper { let process = &thread.process; if process.pid == 1 && thread.tid == process.pid { - panic!("init exited"); + panic!("init exited: {}", alloc_pid()); } let mut procs = ProcessList::get().write().await; - let inner = process.inner.access_mut(procs.prove_mut()); - - thread.dead.store(true, Ordering::SeqCst); - if thread.tid != process.pid { - procs.threads.remove(&thread.tid); - inner.threads.remove(&thread.tid).unwrap(); + let threads = process.threads.access_mut(procs.prove_mut()); + assert!( + threads.find_mut(&thread.tid).remove().is_some(), + "Thread gone?" + ); + + procs.remove_thread(&thread); } // main thread exit @@ -172,11 +195,11 @@ impl Reaper { thread.files.close_all().await; + let session = process.session(procs.prove()).clone(); // If we are the session leader, we should drop the control terminal. - if process.session(procs.prove()).sid == process.pid { - if let Some(terminal) = process.session(procs.prove()).drop_control_terminal().await - { - terminal.drop_session().await; + if session.sid == process.pid { + if let Some(terminal) = session.control_terminal() { + terminal.drop_session(procs.prove()); } } @@ -184,16 +207,14 @@ impl Reaper { process.mm_list.release(); // Make children orphans (adopted by init) - { - let init = procs.init_process(); - inner.children.retain(|_, child| { - let child = child.upgrade().unwrap(); - // SAFETY: `child.parent` must be ourself. So we don't need to free it. - unsafe { child.parent.swap(Some(init.clone())) }; - init.add_child(&child, procs.prove_mut()); - - false - }); + let init = procs.init_process(); + let children = process.children.access_mut(procs.prove_mut()); + for child in children.take() { + // XXX: May buggy. Check here again. + // SAFETY: `child.parent` must be ourself. + // So we don't need to free it. + unsafe { child.parent.swap(Some(init.clone())) }; + init.add_child(&child, procs.prove_mut()); } let mut init_notify = procs.init_process().notify_batch(); diff --git a/src/kernel/task/session.rs b/src/kernel/task/session.rs index a7b57afd..899aa395 100644 --- a/src/kernel/task/session.rs +++ b/src/kernel/task/session.rs @@ -1,117 +1,170 @@ -use super::{Process, ProcessGroup, ProcessList, Thread}; -use crate::kernel::constants::EPERM; -use crate::{kernel::Terminal, prelude::*}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use alloc::sync::{Arc, Weak}; + +use eonix_sync::{AsProof as _, AsProofMut, Locked, Proof, ProofMut}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; -use eonix_sync::{AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLock}; use posix_types::signal::Signal; -#[derive(Debug)] +use super::process_group::SessionGroups; +use super::{Process, ProcessGroup, ProcessList}; +use crate::kernel::constants::EPERM; +use crate::kernel::Terminal; +use crate::prelude::*; + struct SessionJobControl { - /// Foreground process group - foreground: Weak, + foreground: Option>, control_terminal: Option>, } -#[allow(dead_code)] -#[derive(Debug)] pub struct Session { pub sid: u32, pub leader: Weak, - job_control: RwLock, + job_control: Spin, + groups: Locked, ProcessList>, + all_sessions_link: RBTreeAtomicLink, +} - groups: Locked>, ProcessList>, +intrusive_adapter!(pub AllSessions = Arc: Session { + all_sessions_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllSessions { + type Key = u32; + + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.sid + } } impl Session { /// Create a session and add it to the global session list. - pub fn new(leader: &Arc, process_list: &mut ProcessList) -> Arc { + pub fn new(leader: &Arc, proclist: &mut ProcessList) -> Arc { let session = Arc::new(Self { sid: leader.pid, leader: Arc::downgrade(leader), - job_control: RwLock::new(SessionJobControl { - foreground: Weak::new(), + job_control: Spin::new(SessionJobControl { + foreground: None, control_terminal: None, }), - groups: Locked::new( - BTreeMap::new(), - // SAFETY: `procs` must be the global process list, which won't be moved. - process_list, - ), + groups: Locked::new(RBTree::new(SessionGroups::NEW), proclist), + all_sessions_link: RBTreeAtomicLink::new(), }); - process_list.add_session(&session); + proclist.add_session(&session); session } - pub(super) fn add_member(&self, procs: &mut ProcessList, pgroup: &Arc) { - let groups = self.groups.access_mut(procs.prove_mut()); - let old = groups.insert(pgroup.pgid, Arc::downgrade(pgroup)); - assert!(old.is_none(), "Process group already exists"); + pub fn add_member( + &self, + pgroup: &Arc, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_sessions_link.is_linked(), "Dead session"); + self.groups.access_mut(procs).insert(pgroup.clone()); } - pub(super) fn remove_member(&self, pgid: u32, procs: ProofMut<'_, ProcessList>) { - assert!(self.groups.access_mut(procs).remove(&pgid).is_some()); + pub fn remove_member( + self: &Arc, + pgroup: &Arc, + procs: &mut ProcessList, + ) { + let members = self.groups.access_mut(procs.prove_mut()); + assert!( + members.find_mut(&pgroup.pgid).remove().is_some(), + "Not a member" + ); + + if let Some(fg_pgroup) = self.foreground_pgroup() { + if fg_pgroup.pgid == pgroup.pgid { + let _ = self.set_foreground_pgroup(None); + } + } + + if !members.is_empty() { + return; + } + + // Recycle dead session. + procs.remove_session(self); } - pub async fn foreground(&self) -> Option> { - self.job_control.read().await.foreground.upgrade() + pub fn leader(&self) -> Option> { + self.leader.upgrade() + } + + pub fn foreground_pgroup(&self) -> Option> { + self.job_control.lock().foreground.clone() + } + + pub fn control_terminal(&self) -> Option> { + self.job_control.lock().control_terminal.clone() } /// Set the foreground process group identified by `pgid`. /// The process group must belong to the session. - pub async fn set_foreground_pgid( + pub fn set_foreground_pgroup( &self, - pgid: u32, - procs: Proof<'_, ProcessList>, + pgroup: Option<&Arc>, ) -> KResult<()> { - if let Some(group) = self.groups.access(procs).get(&pgid) { - self.job_control.write().await.foreground = group.clone(); - Ok(()) - } else { - // TODO: Check if the process group refers to an existing process group. - // That's not a problem though, the operation will fail anyway. - Err(EPERM) + if let Some(pgroup) = pgroup { + if pgroup.session.sid != self.sid { + return Err(EPERM); + } } + + self.job_control.lock().foreground = pgroup.cloned(); + Ok(()) } - /// Only session leaders can set the control terminal. - /// Make sure we've checked that before calling this function. - pub async fn set_control_terminal( + /// Set our controlling terminal to `terminal`. Only meant to be called by + /// the session leader. The pgroup that the session leader is in becomes the + /// new foreground pgroup. + /// + /// # Panics + /// Panics if we have a controlling terminal already + /// or the session leader is gone. + pub fn _set_control_terminal( self: &Arc, terminal: &Arc, - forced: bool, procs: Proof<'_, ProcessList>, - ) -> KResult<()> { - let mut job_control = self.job_control.write().await; - if let Some(_) = job_control.control_terminal.as_ref() { - if let Some(session) = terminal.session().await.as_ref() { - if session.sid == self.sid { - return Ok(()); - } - } - return Err(EPERM); - } - terminal.set_session(self, forced).await?; + ) { + let mut job_control = self.job_control.lock(); + let leader = self.leader().expect("Leader is gone?"); + + assert!( + job_control.control_terminal.is_none(), + "We have a controlling terminal already" + ); + job_control.control_terminal = Some(terminal.clone()); - job_control.foreground = Arc::downgrade(&Thread::current().process.pgroup(procs)); - Ok(()) + job_control.foreground = Some(leader.pgroup(procs).clone()); } /// Drop the control terminal reference inside the session. - /// DO NOT TOUCH THE TERMINAL'S SESSION FIELD. - pub async fn drop_control_terminal(&self) -> Option> { - let mut inner = self.job_control.write().await; - inner.foreground = Weak::new(); - inner.control_terminal.take() + /// Send SIGHUP and then SIGCONT to our foreground pgroup. + pub fn _drop_control_terminal(&self, procs: Proof<'_, ProcessList>) { + let foreground = { + let mut inner = self.job_control.lock(); + inner.control_terminal = None; + inner.foreground.take() + }; + + if let Some(foreground) = foreground { + foreground.raise(Signal::SIGHUP, procs); + foreground.raise(Signal::SIGCHLD, procs); + } } pub async fn raise_foreground(&self, signal: Signal) { - if let Some(fg) = self.foreground().await { - let procs = ProcessList::get().read().await; - fg.raise(signal, procs.prove()); - } + let Some(fg) = self.foreground_pgroup() else { + return; + }; + + let procs = ProcessList::get().read().await; + fg.raise(signal, procs.prove()); } } diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 7e005875..76c56dcc 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -14,6 +14,7 @@ use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::AsProofMut as _; +use intrusive_collections::{intrusive_adapter, KeyAdapter, RBTreeAtomicLink}; use pointers::BorrowedArc; use posix_types::signal::Signal; use stalloc::UnsafeStalloc; @@ -84,9 +85,44 @@ pub struct Thread { pub dead: AtomicBool, pub exit_status: Spin>, + /// Link in the global thread list. + all_threads_link: RBTreeAtomicLink, + + /// Link in the process's thread list. + process_threads_link: RBTreeAtomicLink, + inner: Spin, } +intrusive_adapter!(pub AllThreads = Arc: Thread { + all_threads_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub ProcessThreads = Arc: Thread { + process_threads_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllThreads { + type Key = u32; + + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.tid + } +} + +impl KeyAdapter<'_> for ProcessThreads { + type Key = u32; + + fn get_key( + &self, + value: &'_ ::Value, + ) -> Self::Key { + value.tid + } +} + impl ThreadBuilder { pub fn new() -> Self { Self { @@ -139,12 +175,18 @@ impl ThreadBuilder { self } - pub fn set_child_tid(mut self, set_child_tid: Option>) -> Self { + pub fn set_child_tid( + mut self, + set_child_tid: Option>, + ) -> Self { self.set_child_tid = set_child_tid; self } - pub fn clear_child_tid(mut self, clear_child_tid: Option>) -> Self { + pub fn clear_child_tid( + mut self, + clear_child_tid: Option>, + ) -> Self { self.clear_child_tid = clear_child_tid; self } @@ -171,7 +213,11 @@ impl ThreadBuilder { } /// Clone the thread from another thread. - pub fn clone_from(self, thread: &Thread, clone_args: &CloneArgs) -> KResult { + pub fn clone_from( + self, + thread: &Thread, + clone_args: &CloneArgs, + ) -> KResult { let inner = thread.inner.lock(); let mut trap_ctx = thread.trap_ctx.borrow().clone(); @@ -199,11 +245,12 @@ impl ThreadBuilder { FileArray::new_cloned(&thread.files) }; - let signal_list = if clone_args.flags.contains(CloneFlags::CLONE_SIGHAND) { - SignalList::new_shared(&thread.signal_list) - } else { - SignalList::new_cloned(&thread.signal_list) - }; + let signal_list = + if clone_args.flags.contains(CloneFlags::CLONE_SIGHAND) { + SignalList::new_shared(&thread.signal_list) + } else { + SignalList::new_cloned(&thread.signal_list) + }; Ok(self .files(files) @@ -241,6 +288,8 @@ impl ThreadBuilder { fpu_state: AtomicUniqueRefCell::new(fpu_state), dead: AtomicBool::new(false), exit_status: Spin::new(None), + all_threads_link: RBTreeAtomicLink::new(), + process_threads_link: RBTreeAtomicLink::new(), inner: Spin::new(ThreadInner { name, tls: self.tls, @@ -281,7 +330,10 @@ impl Thread { Ok(()) } - pub fn set_robust_list(&self, robust_list_address: Option>) { + pub fn set_robust_list( + &self, + robust_list_address: Option>, + ) { self.inner.lock().robust_list_address = robust_list_address; } @@ -371,7 +423,10 @@ impl Thread { while !self.is_dead() { if self.signal_list.has_pending_signal() { self.signal_list - .handle(&mut self.trap_ctx.borrow(), &mut self.fpu_state.borrow()) + .handle( + &mut self.trap_ctx.borrow(), + &mut self.fpu_state.borrow(), + ) .await; } @@ -399,7 +454,9 @@ impl Thread { } let mms = &self.process.mm_list; - if let Err(signal) = mms.handle_user_page_fault(addr, error_code).await { + if let Err(signal) = + mms.handle_user_page_fault(addr, error_code).await + { self.signal_list.raise(signal); } } @@ -409,8 +466,12 @@ impl Thread { TrapType::Fault(Fault::InvalidOp) => { self.signal_list.raise(Signal::SIGILL); } - TrapType::Fault(Fault::Unknown(_)) => unimplemented!("Unhandled fault"), - TrapType::Breakpoint => unimplemented!("Breakpoint in user space"), + TrapType::Fault(Fault::Unknown(_)) => { + unimplemented!("Unhandled fault") + } + TrapType::Breakpoint => { + unimplemented!("Breakpoint in user space") + } TrapType::Irq { callback } => callback(default_irq_handler), TrapType::Timer { callback } => { callback(timer_interrupt); @@ -424,11 +485,16 @@ impl Thread { return; } - if let Some(retval) = self.handle_syscall(thd_alloc, no, args).await { + if let Some(retval) = + self.handle_syscall(thd_alloc, no, args).await + { let mut trap_ctx = self.trap_ctx.borrow(); trap_ctx.set_user_return_value(retval); - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] + #[cfg(any( + target_arch = "riscv64", + target_arch = "loongarch64" + ))] { let pc = trap_ctx.get_program_counter(); trap_ctx.set_program_counter(pc + 4); @@ -472,7 +538,10 @@ impl Thread { }) .await; - assert!(self.is_dead(), "`real_run` returned before the thread die?"); + assert!( + self.is_dead(), + "`real_run` returned before the thread die?" + ); ProcessList::send_to_reaper(self); } } @@ -499,7 +568,10 @@ pub async fn yield_now() { impl Future for Yield { type Output = (); - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + fn poll( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll { if self.as_mut().yielded { Poll::Ready(()) } else { diff --git a/src/kernel/terminal.rs b/src/kernel/terminal.rs index 734655b7..ddc3cc1f 100644 --- a/src/kernel/terminal.rs +++ b/src/kernel/terminal.rs @@ -1,18 +1,19 @@ -use super::{ - task::{ProcessList, Session, Thread}, - user::{UserPointer, UserPointerMut}, -}; -use crate::kernel::constants::{EINTR, ENOTTY, EPERM}; -use crate::{io::Buffer, prelude::*, sync::CondVar}; -use alloc::{ - collections::vec_deque::VecDeque, - sync::{Arc, Weak}, -}; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::{Arc, Weak}; + use bitflags::bitflags; use eonix_log::ConsoleWrite; -use eonix_sync::{AsProof as _, Mutex}; +use eonix_sync::{Mutex, Proof}; use posix_types::signal::Signal; +use super::constants::ESRCH; +use super::task::{ProcessList, Session, Thread}; +use super::user::{UserPointer, UserPointerMut}; +use crate::io::Buffer; +use crate::kernel::constants::{EINTR, ENOTTY, EPERM}; +use crate::prelude::*; +use crate::sync::CondVar; + const BUFFER_SIZE: usize = 4096; const NCCS: usize = 19; @@ -351,12 +352,12 @@ pub trait TerminalDevice: Send + Sync { struct TerminalInner { termio: Termios, - session: Weak, buffer: VecDeque, } pub struct Terminal { inner: Mutex, + session: Spin>, device: Arc, cv: CondVar, } @@ -400,9 +401,9 @@ impl Terminal { Arc::new(Self { inner: Mutex::new(TerminalInner { termio: Termios::new_standard(), - session: Weak::new(), buffer: VecDeque::with_capacity(BUFFER_SIZE), }), + session: Spin::new(Weak::new()), cv: CondVar::new(), device, }) @@ -447,15 +448,21 @@ impl Terminal { } async fn signal(&self, inner: &mut TerminalInner, signal: Signal) { - if let Some(session) = inner.session.upgrade() { + if let Some(session) = self.session() { session.raise_foreground(signal).await; } + if !inner.termio.noflsh() { self.clear_read_buffer(inner); } } - async fn echo_and_signal(&self, inner: &mut TerminalInner, ch: u8, signal: Signal) { + async fn echo_and_signal( + &self, + inner: &mut TerminalInner, + ch: u8, + signal: Signal, + ) { self.echo_char(inner, ch); self.signal(inner, signal).await; } @@ -481,13 +488,19 @@ impl Terminal { match ch { 0xff => {} ch if ch == inner.termio.vintr() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGINT).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGINT) + .await } ch if ch == inner.termio.vquit() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGQUIT).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGQUIT) + .await } ch if ch == inner.termio.vsusp() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGTSTP).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGTSTP) + .await } _ => {} } @@ -517,8 +530,12 @@ impl Terminal { match ch { b'\r' if inner.termio.igncr() => {} - b'\r' if inner.termio.icrnl() => return self.do_commit_char(&mut inner, b'\n'), - b'\n' if inner.termio.inlcr() => return self.do_commit_char(&mut inner, b'\r'), + b'\r' if inner.termio.icrnl() => { + return self.do_commit_char(&mut inner, b'\n') + } + b'\n' if inner.termio.inlcr() => { + return self.do_commit_char(&mut inner, b'\r') + } _ => self.do_commit_char(&mut inner, ch), } } @@ -589,26 +606,30 @@ impl Terminal { pub async fn ioctl(&self, request: TerminalIORequest<'_>) -> KResult<()> { match request { TerminalIORequest::GetProcessGroup(pgid_pointer) => { - if let Some(session) = self.inner.lock().await.session.upgrade() { - if let Some(pgroup) = session.foreground().await { - return pgid_pointer.write(pgroup.pgid); - } - } + let Some(session) = self.session() else { + return Err(ENOTTY); + }; + + let Some(pgroup) = session.foreground_pgroup() else { + return Err(ENOTTY); + }; - Err(ENOTTY) + pgid_pointer.write(pgroup.pgid) } TerminalIORequest::SetProcessGroup(pgid) => { let pgid = pgid.read()?; let procs = ProcessList::get().read().await; - let inner = self.inner.lock().await; - let session = inner.session.upgrade(); + let Some(session) = self.session() else { + return Err(ENOTTY); + }; - if let Some(session) = session { - session.set_foreground_pgid(pgid, procs.prove()).await - } else { - Err(ENOTTY) - } + let Some(pgroup) = procs.try_find_pgroup(pgid) else { + return Err(ESRCH); + }; + + session.set_foreground_pgroup(Some(&pgroup))?; + Ok(()) } TerminalIORequest::GetWindowSize(ptr) => { // TODO: Get the actual window size @@ -630,9 +651,12 @@ impl Terminal { let mut inner = self.inner.lock().await; // TODO: We ignore unknown bits for now. - inner.termio.iflag = TermioIFlags::from_bits_truncate(user_termios.iflag as u16); - inner.termio.oflag = TermioOFlags::from_bits_truncate(user_termios.oflag as u16); - inner.termio.lflag = TermioLFlags::from_bits_truncate(user_termios.lflag as u16); + inner.termio.iflag = + TermioIFlags::from_bits_truncate(user_termios.iflag as u16); + inner.termio.oflag = + TermioOFlags::from_bits_truncate(user_termios.oflag as u16); + inner.termio.lflag = + TermioLFlags::from_bits_truncate(user_termios.lflag as u16); inner.termio.cflag = user_termios.cflag; inner.termio.line = user_termios.line; inner.termio.cc = user_termios.cc; @@ -642,30 +666,52 @@ impl Terminal { } } - /// Assign the `session` to this terminal. Drop the previous session if `forced` is true. - pub async fn set_session(&self, session: &Arc, forced: bool) -> KResult<()> { - let mut inner = self.inner.lock().await; - if let Some(session) = inner.session.upgrade() { + pub fn session(&self) -> Option> { + self.session.lock().upgrade() + } + + /// Drop our current controlled session. The old session lose its controlling + /// terminal and all processes in it will receive a SIGHUP and then SIGCONT. + pub fn drop_session(&self, procs: Proof<'_, ProcessList>) { + let session = + core::mem::replace(&mut *self.session.lock(), Weak::new()); + let Some(old_session) = session.upgrade() else { + return; + }; + + old_session._drop_control_terminal(procs); + } + + /// Assign the `session` to this terminal. + /// Drop the previous session if `forced` is true. + pub async fn set_session( + self: &Arc, + session: &Arc, + forced: bool, + procs: Proof<'_, ProcessList>, + ) -> KResult<()> { + let mut cur_session = self.session.lock(); + + // XXX: Holding spinlock for too long? + if let Some(old_session) = cur_session.upgrade() { + if old_session.sid == session.sid { + return Ok(()); + } + if !forced { - Err(EPERM) - } else { - session.drop_control_terminal().await; - inner.session = Arc::downgrade(&session); - Ok(()) + return Err(EPERM); } - } else { - // Sessions should set their `control_terminal` field. - inner.session = Arc::downgrade(&session); - Ok(()) + + // TODO: Check whether the caller has the CAP_SYS_ADMIN capability. + + // We've stolen the terminal from the old session. + old_session._drop_control_terminal(procs); } - } - pub async fn drop_session(&self) { - self.inner.lock().await.session = Weak::new(); - } + *cur_session = Arc::downgrade(session); + session._set_control_terminal(self, procs); - pub async fn session(&self) -> Option> { - self.inner.lock().await.session.upgrade() + Ok(()) } } diff --git a/src/kernel/vfs/file/terminal_file.rs b/src/kernel/vfs/file/terminal_file.rs index f318c5b2..04a022b5 100644 --- a/src/kernel/vfs/file/terminal_file.rs +++ b/src/kernel/vfs/file/terminal_file.rs @@ -1,24 +1,46 @@ -use super::{File, FileType, PollEvent}; -use crate::{ - io::{Buffer, Stream, StreamRead}, - kernel::{ - constants::{EINVAL, TCGETS, TCSETS, TIOCGPGRP, TIOCGWINSZ, TIOCSPGRP}, - terminal::TerminalIORequest, - user::{UserPointer, UserPointerMut}, - Terminal, - }, - prelude::KResult, -}; use alloc::sync::Arc; + +use eonix_sync::AsProof; use posix_types::open::OpenFlags; +use super::{File, FileType, PollEvent}; +use crate::io::{Buffer, Stream, StreamRead}; +use crate::kernel::constants::{ + EINVAL, TCGETS, TCSETS, TIOCGPGRP, TIOCGWINSZ, TIOCSPGRP, +}; +use crate::kernel::task::{ProcessList, Thread}; +use crate::kernel::terminal::TerminalIORequest; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::Terminal; +use crate::prelude::KResult; + pub struct TerminalFile { terminal: Arc, } impl TerminalFile { - pub fn new(tty: Arc, flags: OpenFlags) -> File { - File::new(flags, FileType::Terminal(TerminalFile { terminal: tty })) + pub async fn open( + thread: &Thread, + terminal: &Arc, + flags: OpenFlags, + ) -> File { + let set_control_tty = !flags.contains(OpenFlags::O_NOCTTY); + + let procs = ProcessList::get().read().await; + let session = thread.process.session(procs.prove()); + + // We only set the control terminal if the process is the session leader. + if set_control_tty && session.sid == thread.process.pid { + // Silently fail if we can't set the control terminal. + let _ = terminal.set_session(&session, false, procs.prove()).await; + } + + File::new( + flags, + FileType::Terminal(TerminalFile { + terminal: terminal.clone(), + }), + ) } pub async fn read(&self, buffer: &mut dyn Buffer) -> KResult { @@ -43,11 +65,21 @@ impl TerminalFile { pub async fn ioctl(&self, request: usize, arg3: usize) -> KResult<()> { self.terminal .ioctl(match request as u32 { - TCGETS => TerminalIORequest::GetTermios(UserPointerMut::with_addr(arg3)?), - TCSETS => TerminalIORequest::SetTermios(UserPointer::with_addr(arg3)?), - TIOCGPGRP => TerminalIORequest::GetProcessGroup(UserPointerMut::with_addr(arg3)?), - TIOCSPGRP => TerminalIORequest::SetProcessGroup(UserPointer::with_addr(arg3)?), - TIOCGWINSZ => TerminalIORequest::GetWindowSize(UserPointerMut::with_addr(arg3)?), + TCGETS => TerminalIORequest::GetTermios( + UserPointerMut::with_addr(arg3)?, + ), + TCSETS => { + TerminalIORequest::SetTermios(UserPointer::with_addr(arg3)?) + } + TIOCGPGRP => TerminalIORequest::GetProcessGroup( + UserPointerMut::with_addr(arg3)?, + ), + TIOCSPGRP => TerminalIORequest::SetProcessGroup( + UserPointer::with_addr(arg3)?, + ), + TIOCGWINSZ => TerminalIORequest::GetWindowSize( + UserPointerMut::with_addr(arg3)?, + ), _ => return Err(EINVAL), }) .await diff --git a/src/kernel/vfs/filearray.rs b/src/kernel/vfs/filearray.rs index 609d969c..c0b6a49e 100644 --- a/src/kernel/vfs/filearray.rs +++ b/src/kernel/vfs/filearray.rs @@ -1,19 +1,22 @@ use alloc::sync::Arc; use intrusive_collections::rbtree::Entry; -use intrusive_collections::{intrusive_adapter, Bound, KeyAdapter, RBTree, RBTreeAtomicLink}; +use intrusive_collections::{ + intrusive_adapter, Bound, KeyAdapter, RBTree, RBTreeAtomicLink, +}; use itertools::FoldWhile::{Continue, Done}; use itertools::Itertools; use posix_types::open::{FDFlags, OpenFlags}; use super::file::{File, InodeFile, Pipe}; use super::types::{Format, Permission}; -use super::{Spin, TerminalFile}; -use crate::kernel::console::get_console; +use super::Spin; use crate::kernel::constants::{ - EBADF, EISDIR, ENOTDIR, ENXIO, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, + EBADF, EISDIR, ENOTDIR, ENXIO, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, + F_SETFD, F_SETFL, }; use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; +use crate::kernel::task::Thread; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::CharDevice; use crate::prelude::*; @@ -80,7 +83,11 @@ impl FDAllocator { self.min_avail = FD(0); } - fn find_available(&mut self, from: FD, files: &RBTree) -> FD { + fn find_available( + &mut self, + from: FD, + files: &RBTree, + ) -> FD { files .range(Bound::Included(&from), Bound::Unbounded) .fold_while(from, |current, OpenFile { fd, .. }| { @@ -143,7 +150,8 @@ impl FileArray { let other_inner = other.inner.lock(); for file in other_inner.files.iter() { - let new_file = OpenFile::new(file.fd, file.flags, file.file.dup()); + let new_file = + OpenFile::new(file.fd, file.flags, file.file.dup()); new_files.insert(new_file); } (new_files, other_inner.fd_alloc.clone()) @@ -223,7 +231,12 @@ impl FileArray { /// Duplicates the file to a new file descriptor, returning the old file /// description to be dropped. - fn dup_to_no_close(&self, old_fd: FD, new_fd: FD, fd_flags: FDFlags) -> KResult> { + fn dup_to_no_close( + &self, + old_fd: FD, + new_fd: FD, + fd_flags: FDFlags, + ) -> KResult> { let mut inner = self.inner.lock(); let (files, fd_alloc) = inner.split_borrow(); @@ -240,7 +253,8 @@ impl FileArray { Entry::Occupied(mut entry) => { let mut file = entry.remove().unwrap(); file.flags = fd_flags; - let old_file = core::mem::replace(&mut file.file, new_file_data); + let old_file = + core::mem::replace(&mut file.file, new_file_data); entry.insert(file); @@ -249,8 +263,15 @@ impl FileArray { } } - pub async fn dup_to(&self, old_fd: FD, new_fd: FD, flags: OpenFlags) -> KResult { - if let Some(old_file) = self.dup_to_no_close(old_fd, new_fd, flags.as_fd_flags())? { + pub async fn dup_to( + &self, + old_fd: FD, + new_fd: FD, + flags: OpenFlags, + ) -> KResult { + if let Some(old_file) = + self.dup_to_no_close(old_fd, new_fd, flags.as_fd_flags())? + { old_file.close().await; } @@ -277,6 +298,7 @@ impl FileArray { pub async fn open( &self, + thread: &Thread, dentry: &Arc, flags: OpenFlags, perm: Permission, @@ -300,7 +322,7 @@ impl FileArray { let file = if inode.format == Format::CHR { let device = CharDevice::get(inode.devid()?).ok_or(ENXIO)?; - device.open(flags)? + device.open(thread, flags).await? } else { InodeFile::new(dentry.clone(), flags) }; @@ -323,7 +345,8 @@ impl FileArray { F_DUPFD | F_DUPFD_CLOEXEC => { let ofile = cursor.get().ok_or(EBADF)?; - let cloexec = cmd == F_DUPFD_CLOEXEC || ofile.flags.close_on_exec(); + let cloexec = + cmd == F_DUPFD_CLOEXEC || ofile.flags.close_on_exec(); let flags = cloexec .then_some(FDFlags::FD_CLOEXEC) .unwrap_or(FDFlags::empty()); @@ -342,7 +365,9 @@ impl FileArray { cursor.insert(ofile); 0 } - F_GETFL => cursor.get().ok_or(EBADF)?.file.get_flags().bits() as usize, + F_GETFL => { + cursor.get().ok_or(EBADF)?.file.get_flags().bits() as usize + } F_SETFL => { cursor .get() @@ -357,35 +382,6 @@ impl FileArray { Ok(ret) } - - /// Only used for init process. - pub fn open_console(&self) { - let mut inner = self.inner.lock(); - let (files, fd_alloc) = inner.split_borrow(); - - let (stdin, stdout, stderr) = ( - fd_alloc.next_fd(files), - fd_alloc.next_fd(files), - fd_alloc.next_fd(files), - ); - let console_terminal = get_console().expect("No console terminal"); - - inner.do_insert( - stdin, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - inner.do_insert( - stdout, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - inner.do_insert( - stderr, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - } } impl FileArrayInner { @@ -397,7 +393,9 @@ impl FileArrayInner { fn do_insert(&mut self, fd: FD, flags: FDFlags, file: File) { match self.files.entry(&fd) { Entry::Occupied(_) => { - panic!("File descriptor {fd:?} already exists in the file array."); + panic!( + "File descriptor {fd:?} already exists in the file array." + ); } Entry::Vacant(insert_cursor) => { insert_cursor.insert(OpenFile::new(fd, flags, file)); @@ -405,7 +403,9 @@ impl FileArrayInner { } } - fn split_borrow(&mut self) -> (&mut RBTree, &mut FDAllocator) { + fn split_borrow( + &mut self, + ) -> (&mut RBTree, &mut FDAllocator) { let Self { files, fd_alloc } = self; (files, fd_alloc) } diff --git a/user-programs/init_script_riscv64.sh b/user-programs/init_script_riscv64.sh index b5ce95d7..f67e2a27 100644 --- a/user-programs/init_script_riscv64.sh +++ b/user-programs/init_script_riscv64.sh @@ -56,6 +56,9 @@ busybox mknod -m 666 /dev/vdb b 8 16 busybox mknod -m 666 /dev/ttyS0 c 4 64 busybox mknod -m 666 /dev/ttyS1 c 4 65 +exec < "$TERMINAL" +exec > "$TERMINAL" 2>&1 + info "deploying busybox..." busybox mkdir -p /bin /lib @@ -106,8 +109,7 @@ int main() { } EOF -# shellcheck disable=SC2094 -exec sh -l < "$TERMINAL" > "$TERMINAL" 2> "$TERMINAL" +exec sh -l # We don't have a working init yet, so we use busybox sh directly for now. # exec /mnt/init /bin/sh -c 'exec sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0' From 2392ac19d2138a9abf307252a4adb78b18afa405 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 01:31:13 +0800 Subject: [PATCH 017/153] style: reformat the files related to next patches Reformat the files with new the format style to make the real changes clearer. Signed-off-by: greatbridf --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 50 +++++++++++++----- crates/eonix_hal/src/arch/riscv64/mm.rs | 52 ++++++++++++------- src/kernel_init.rs | 3 +- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index b2305f99..4e5afcfb 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -7,8 +7,12 @@ use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; use eonix_hal_traits::mm::Memory; use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, PageTable, PagingMode, TableAttribute, PTE as _}; -use eonix_mm::paging::{Folio, FrameAlloc, PageAccess, PageBlock, PAGE_SIZE, PFN}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PagingMode, TableAttribute, PTE as _, +}; +use eonix_mm::paging::{ + Folio, FrameAlloc, PageAccess, PageBlock, PAGE_SIZE, PFN, +}; use eonix_percpu::PercpuArea; use fdt::Fdt; use riscv::asm::sfence_vma_all; @@ -25,11 +29,13 @@ use super::time::set_next_timer; use crate::arch::cpu::CPU; use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; use crate::arch::mm::{ - ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, RawPageTableSv48, - GLOBAL_PAGE_TABLE, + ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, + RawPageTableSv48, GLOBAL_PAGE_TABLE, }; use crate::bootstrap::BootStrapData; -use crate::mm::{ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}; +use crate::mm::{ + ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, +}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -64,7 +70,8 @@ static PT1: BootPageTable = { BootPageTable(arr) }; -static BSP_PAGE_ALLOC: AtomicPtr> = AtomicPtr::new(core::ptr::null_mut()); +static BSP_PAGE_ALLOC: AtomicPtr> = + AtomicPtr::new(core::ptr::null_mut()); static AP_COUNT: AtomicUsize = AtomicUsize::new(0); static AP_STACK: AtomicUsize = AtomicUsize::new(0); @@ -130,11 +137,14 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { } let start = unsafe { - ((&BOOT_STACK_START) as *const &'static [u8; 4096 * 16]).read_volatile() as *const _ - as usize + ((&BOOT_STACK_START) as *const &'static [u8; 4096 * 16]).read_volatile() + as *const _ as usize }; let bootstrap_data = BootStrapData { - early_stack: PRange::new(PAddr::from(start), PAddr::from(start + 4096 * 16)), + early_stack: PRange::new( + PAddr::from(start), + PAddr::from(start + 4096 * 16), + ), allocator: Some(real_allocator), }; @@ -179,7 +189,11 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { sfence_vma_all(); unsafe { - core::ptr::write_bytes(KERNEL_BSS_START.addr() as *mut (), 0, BSS_LENGTH as usize); + core::ptr::write_bytes( + KERNEL_BSS_START.addr() as *mut (), + 0, + BSS_LENGTH as usize, + ); } unsafe { @@ -255,7 +269,8 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { stack_range }; - let old = BSP_PAGE_ALLOC.swap((&raw const *page_alloc) as *mut _, Ordering::Release); + let old = BSP_PAGE_ALLOC + .swap((&raw const *page_alloc) as *mut _, Ordering::Release); assert!(old.is_null()); while AP_STACK @@ -324,7 +339,12 @@ unsafe extern "C" fn _ap_start(hart_id: usize) { fn get_ap_stack() -> usize { while AP_SEM - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .compare_exchange_weak( + false, + true, + Ordering::Acquire, + Ordering::Relaxed, + ) .is_err() { core::hint::spin_loop(); @@ -344,12 +364,14 @@ fn get_ap_stack() -> usize { } fn ap_entry(hart_id: usize, stack_bottom: PAddr) -> ! { - let stack_range = PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); + let stack_range = + PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); { // SAFETY: Acquire all the work done by the BSP and other APs. let alloc = loop { - let alloc = BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); + let alloc = + BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); if !alloc.is_null() { break alloc; diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index f67646cf..45d44c6f 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -4,8 +4,8 @@ use core::ptr::NonNull; use eonix_hal_traits::mm::Memory; use eonix_mm::address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}; use eonix_mm::page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, + PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, + RawPageTable, TableAttribute, PTE, }; use eonix_mm::paging::{BasicFolio, Folio, PageAccess, PageBlock, PFN}; use eonix_sync_base::LazyLock; @@ -115,7 +115,9 @@ impl RawAttribute for PageAttribute64 { table_attr |= TableAttribute::PRESENT; } - if table_attr.contains(TableAttribute::PRESENT) && self.0 & (PA_R | PA_W | PA_X) != 0 { + if table_attr.contains(TableAttribute::PRESENT) + && self.0 & (PA_R | PA_W | PA_X) != 0 + { return None; } @@ -139,7 +141,9 @@ impl RawAttribute for PageAttribute64 { page_attr |= PageAttribute::PRESENT; } - if page_attr.contains(PageAttribute::PRESENT) && (self.0 & (PA_R | PA_W | PA_X) == 0) { + if page_attr.contains(PageAttribute::PRESENT) + && (self.0 & (PA_R | PA_W | PA_X) == 0) + { return None; } @@ -278,18 +282,22 @@ impl Memory for ArchMemory { let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)).chain( - Self::present_ram() - .filter(move |range| range.end() > paddr_after_kimage_aligned) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) + core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) + .chain( + Self::present_ram() + .filter(move |range| { + range.end() > paddr_after_kimage_aligned + }) + .map(move |range| { + if range.start() < paddr_after_kimage_aligned { + let (_, right) = + range.split_at(paddr_after_kimage_aligned); + right + } else { + range + } + }), + ) } } @@ -314,17 +322,21 @@ where let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)).chain( - self.filter(move |range| range.end() > paddr_after_kimage_aligned) + core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) + .chain( + self.filter(move |range| { + range.end() > paddr_after_kimage_aligned + }) .map(move |range| { if range.start() < paddr_after_kimage_aligned { - let (_, right) = range.split_at(paddr_after_kimage_aligned); + let (_, right) = + range.split_at(paddr_after_kimage_aligned); right } else { range } }), - ) + ) } } diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 65af41e4..64c220b2 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -19,7 +19,8 @@ fn setup_kernel_page_array(alloc: BasicPageAllocRef, count_pages: usize) { // Map kernel page array. const V_KERNEL_PAGE_ARRAY_START: VAddr = VAddr::from(0xffffff8040000000); - let range = VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages); + let range = + VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages); for pte in global_page_table.iter_kernel(range) { let attr = PageAttribute::PRESENT | PageAttribute::WRITE From 6a007864070aa997379256117584da67ec7d49d8 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 01:32:55 +0800 Subject: [PATCH 018/153] riscv64, hal: simplify ArchMemory::free_ram implementation We already have `FDT.present_ram().free_ram()`. Remove the impl in `ArchMemory` to avoid confusion. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/riscv64/mm.rs | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 45d44c6f..3b6df07c 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -274,30 +274,7 @@ impl Memory for ArchMemory { } fn free_ram() -> impl Iterator { - unsafe extern "C" { - fn __kernel_start(); - fn __kernel_end(); - } - - let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); - let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) - .chain( - Self::present_ram() - .filter(move |range| { - range.end() > paddr_after_kimage_aligned - }) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = - range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) + FDT.present_ram().free_ram() } } From 53ae1851d9da456532f55457b34be5cbcf65ad11 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 01:51:32 +0800 Subject: [PATCH 019/153] style: add helper macros to retrieve symbol constants - Add `extern_symbol_value` to retrieve far relative symbol values. - Get `BSS_LENGTH` and `__kernel_end` using `extern_symbol_addr`. - Get `_ap_start` using `extern_symbol_value`. Signed-off-by: greatbridf --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 28 +++++++------------ crates/eonix_hal/src/arch/riscv64/mm.rs | 8 ++---- crates/eonix_hal/src/lib.rs | 14 ++++++++++ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index 4e5afcfb..d317e448 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -36,6 +36,7 @@ use crate::bootstrap::BootStrapData; use crate::mm::{ ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, }; +use crate::{extern_symbol_addr, extern_symbol_value}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -156,10 +157,6 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { } } -unsafe extern "C" { - fn BSS_LENGTH(); -} - /// TODO: /// 对kernel image添加更细的控制，或者不加也行 fn setup_kernel_page_table(alloc: BasicPageAllocRef) { @@ -171,17 +168,17 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { let attr = PageAttribute::WRITE | PageAttribute::READ - | PageAttribute::EXECUTE | PageAttribute::GLOBAL | PageAttribute::PRESENT; const KERNEL_BSS_START: VAddr = VAddr::from(0xffffffff40000000); + let bss_length = extern_symbol_addr!(BSS_LENGTH); + // Map kernel BSS - let bss_range = VRange::from(KERNEL_BSS_START).grow(BSS_LENGTH as usize); + let bss_range = VRange::from(KERNEL_BSS_START).grow(bss_length); for pte in global_page_table.iter_kernel(bss_range) { let page = alloc.alloc().unwrap(); - let attr = attr.difference(PageAttribute::EXECUTE); pte.set(page.into_raw(), attr.into()); } @@ -192,7 +189,7 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { core::ptr::write_bytes( KERNEL_BSS_START.addr() as *mut (), 0, - BSS_LENGTH as usize, + bss_length, ); } @@ -247,15 +244,6 @@ fn setup_cpu(alloc: impl FrameAlloc, hart_id: usize) { percpu_area.register(cpu.cpuid()); } -fn get_ap_start_addr() -> usize { - unsafe extern "C" { - fn _ap_start(); - } - static AP_START_VALUE: &'static unsafe extern "C" fn() = - &(_ap_start as unsafe extern "C" fn()); - unsafe { (AP_START_VALUE as *const _ as *const usize).read_volatile() } -} - fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { let local_hart_id = CPU::local().cpuid(); let mut ap_count = 0; @@ -286,7 +274,11 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { } unsafe { - hart_start(hart_id, PhysicalAddress::new(get_ap_start_addr()), 0); + hart_start( + hart_id, + PhysicalAddress::new(extern_symbol_value!(_ap_start)), + 0, + ); } while AP_COUNT.load(Ordering::Acquire) == ap_count { diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 3b6df07c..6362cdca 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -16,6 +16,7 @@ use riscv::register::satp; use super::config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}; use super::fdt::{FdtExt, FDT}; use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::extern_symbol_addr; use crate::mm::BasicPageAlloc; const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); @@ -291,12 +292,9 @@ where T: PresentRam, { fn free_ram(self) -> impl Iterator { - unsafe extern "C" { - fn __kernel_start(); - fn __kernel_end(); - } + let kernel_end = extern_symbol_addr!(__kernel_end) - KIMAGE_OFFSET; + let kernel_end = PAddr::from(kernel_end).ceil(); - let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index d3bf7825..3f49a326 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -69,3 +69,17 @@ macro_rules! extern_symbol_addr { $crate::symbol_addr!($sym, $type) }}; } + +#[macro_export] +macro_rules! extern_symbol_value { + ($sym:ident) => {{ + unsafe extern "C" { + fn $sym(); + } + + static SYMBOL_ADDR: &'static unsafe extern "C" fn() = + &($sym as unsafe extern "C" fn()); + + unsafe { (SYMBOL_ADDR as *const _ as *const usize).read_volatile() } + }}; +} From a117be1530a10b0e3a56b597f088c66f5abc3060 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 01:53:22 +0800 Subject: [PATCH 020/153] style: reformat files Signed-off-by: greatbridf --- crates/eonix_hal/src/lib.rs | 4 +- src/kernel/task/signal/signal_action.rs | 52 +++++++++++++++---------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index 3f49a326..d8696994 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -11,7 +11,9 @@ pub mod mm; pub mod trap; pub mod fence { - pub use crate::arch::fence::{memory_barrier, read_memory_barrier, write_memory_barrier}; + pub use crate::arch::fence::{ + memory_barrier, read_memory_barrier, write_memory_barrier, + }; } pub mod fpu { diff --git a/src/kernel/task/signal/signal_action.rs b/src/kernel/task/signal/signal_action.rs index 708f9802..cbf81935 100644 --- a/src/kernel/task/signal/signal_action.rs +++ b/src/kernel/task/signal/signal_action.rs @@ -1,22 +1,24 @@ -use super::{KResult, SAVED_DATA_SIZE}; -use crate::{ - io::BufferFill as _, - kernel::{ - constants::{EFAULT, EINVAL}, - syscall::UserMut, - user::UserBuffer, - }, -}; -use alloc::{collections::btree_map::BTreeMap, sync::Arc}; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Arc; use core::arch::naked_asm; -use eonix_hal::{fpu::FpuState, traits::trap::RawTrapContext, trap::TrapContext}; + +use eonix_hal::fpu::FpuState; +use eonix_hal::traits::trap::RawTrapContext; +use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr}; use eonix_sync::Spin; -use posix_types::{ - ctypes::Long, - signal::{SigAction, SigActionHandler, SigActionRestorer, SigSet, Signal, TryFromSigAction}, - SIGNAL_NOW, +use posix_types::ctypes::Long; +use posix_types::signal::{ + SigAction, SigActionHandler, SigActionRestorer, SigSet, Signal, + TryFromSigAction, }; +use posix_types::SIGNAL_NOW; + +use super::{KResult, SAVED_DATA_SIZE}; +use crate::io::BufferFill as _; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::syscall::UserMut; +use crate::kernel::user::UserBuffer; #[cfg(target_arch = "x86_64")] #[unsafe(naked)] @@ -139,7 +141,9 @@ impl SignalAction { handler, restorer, .. } = self else { - unreachable!("Default and Ignore actions should not be handled here"); + unreachable!( + "Default and Ignore actions should not be handled here" + ); }; let current_sp = VAddr::from(trap_ctx.get_stack_pointer()); @@ -167,7 +171,9 @@ impl SignalAction { target_arch = "riscv64", target_arch = "loongarch64" )))] - compile_error!("`vdso_sigreturn` is not implemented for this architecture"); + compile_error!( + "`vdso_sigreturn` is not implemented for this architecture" + ); #[cfg(target_arch = "x86_64")] { @@ -178,19 +184,22 @@ impl SignalAction { unsafe { // SAFETY: To prevent the compiler from optimizing this into `la` instructions // and causing a linking error. - (VDSO_SIGRETURN_ADDR as *const _ as *const usize).read_volatile() + (VDSO_SIGRETURN_ADDR as *const _ as *const usize) + .read_volatile() } } #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - static VDSO_RT_SIGRETURN_ADDR: &'static unsafe extern "C" fn() = + static VDSO_RT_SIGRETURN_ADDR: + &'static unsafe extern "C" fn() = &(vdso_rt_sigreturn as unsafe extern "C" fn()); unsafe { // SAFETY: To prevent the compiler from optimizing this into `la` instructions // and causing a linking error. - (VDSO_RT_SIGRETURN_ADDR as *const _ as *const usize).read_volatile() + (VDSO_RT_SIGRETURN_ADDR as *const _ as *const usize) + .read_volatile() } } }; @@ -201,7 +210,8 @@ impl SignalAction { Some(return_address), &[Long::new_val(signal.into_raw() as _).get()], |vaddr, data| -> Result<(), u32> { - let mut buffer = UserBuffer::new(UserMut::new(vaddr), data.len())?; + let mut buffer = + UserBuffer::new(UserMut::new(vaddr), data.len())?; for ch in data.iter() { buffer.copy(&ch)?.ok_or(EFAULT)?; } From 4d272fe8b2d4a965c1db133febe3d5cda6dc77f4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 02:07:08 +0800 Subject: [PATCH 021/153] riscv64, linker: make sure vdso lies inside .data With current ldscript, linkers will put vdso data after `__kernel_end`, which is buggy since we use the symbol to indicate the end of our kernel image and newly allocated pages may overwrite those positions. Change by place the vdso inside REGION_DATA. Remove old VDSO memory region. Align .data section end to page size border. Add a helper macro to retrieve .vdso section symbol addresses. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/riscv64/link.x | 4 ++- crates/eonix_hal/src/arch/riscv64/memory.x | 1 - crates/eonix_hal/src/link.x.in | 4 +-- src/kernel/task/signal/signal_action.rs | 31 +++++++++------------- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/link.x b/crates/eonix_hal/src/arch/riscv64/link.x index e348e1be..a74f0d0d 100644 --- a/crates/eonix_hal/src/arch/riscv64/link.x +++ b/crates/eonix_hal/src/arch/riscv64/link.x @@ -81,10 +81,12 @@ INSERT AFTER .rodata; SECTIONS { .vdso ALIGN(0x1000) : ALIGN(0x1000) { + VDSO_START = ABSOLUTE(.); + KEEP(*(.vdso .vdso.*)); . = ALIGN(0x1000); - } > VDSO AT> RAM + } > REGION_DATA AT> RAM VDSO_PADDR = LOADADDR(.vdso); } diff --git a/crates/eonix_hal/src/arch/riscv64/memory.x b/crates/eonix_hal/src/arch/riscv64/memory.x index 0dc7c4ff..f2029c9a 100644 --- a/crates/eonix_hal/src/arch/riscv64/memory.x +++ b/crates/eonix_hal/src/arch/riscv64/memory.x @@ -3,7 +3,6 @@ ENTRY(_start) MEMORY { RAM : org = 0x0000000080200000, len = 8M - VDSO : org = 0x00007f0000000000, len = 4K KBSS : org = 0xffffffff40000000, len = 2M KIMAGE : org = 0xffffffff80200000, len = 8M } diff --git a/crates/eonix_hal/src/link.x.in b/crates/eonix_hal/src/link.x.in index 81c269c2..eaabdfda 100644 --- a/crates/eonix_hal/src/link.x.in +++ b/crates/eonix_hal/src/link.x.in @@ -18,7 +18,7 @@ SECTIONS { __srodata = .; *(.rodata .rodata.*); - + . = ALIGN(8); PROVIDE(__eh_frame = .); @@ -41,7 +41,7 @@ SECTIONS { } > REGION_DATA AT> LINK_REGION_DATA - .data.after : + .data.after : ALIGN(0x1000) { __data_after = .; } > REGION_DATA AT> LINK_REGION_DATA diff --git a/src/kernel/task/signal/signal_action.rs b/src/kernel/task/signal/signal_action.rs index cbf81935..18348c32 100644 --- a/src/kernel/task/signal/signal_action.rs +++ b/src/kernel/task/signal/signal_action.rs @@ -20,6 +20,16 @@ use crate::kernel::constants::{EFAULT, EINVAL}; use crate::kernel::syscall::UserMut; use crate::kernel::user::UserBuffer; +macro_rules! vdso_sym_addr { + ($sym:expr) => {{ + const VDSO_START_VADDR: VAddr = VAddr::from(0x7f00_0000_0000); + let vdso_link_start = eonix_hal::extern_symbol_addr!(VDSO_START); + + eonix_hal::symbol_addr!($sym) - vdso_link_start + + VDSO_START_VADDR.addr() + }}; +} + #[cfg(target_arch = "x86_64")] #[unsafe(naked)] #[unsafe(link_section = ".vdso.sigreturn")] @@ -178,29 +188,12 @@ impl SignalAction { #[cfg(target_arch = "x86_64")] { // TODO: Check and use `vdso_rt_sigreturn` for x86 as well. - static VDSO_SIGRETURN_ADDR: &'static unsafe extern "C" fn() = - &(vdso_rt_sigreturn as unsafe extern "C" fn()); - - unsafe { - // SAFETY: To prevent the compiler from optimizing this into `la` instructions - // and causing a linking error. - (VDSO_SIGRETURN_ADDR as *const _ as *const usize) - .read_volatile() - } + vdso_sym_addr!(vdso_rt_sigreturn) } #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - static VDSO_RT_SIGRETURN_ADDR: - &'static unsafe extern "C" fn() = - &(vdso_rt_sigreturn as unsafe extern "C" fn()); - - unsafe { - // SAFETY: To prevent the compiler from optimizing this into `la` instructions - // and causing a linking error. - (VDSO_RT_SIGRETURN_ADDR as *const _ as *const usize) - .read_volatile() - } + vdso_sym_addr!(vdso_rt_sigreturn) } }; From 5814c006a88ed2397b3e085e86e59c179f6cdcb2 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 03:08:28 +0800 Subject: [PATCH 022/153] riscv64: rewrite FDT and present free memory parsing Strip out the memory used by the kernel and FDT data out of free memory block returned by FDT. Closes: #54 ("Random kernel freezing on process creation / exiting") Fixes: 4351cf55739f3 ("partial work: fix riscv64 bootstrap") Signed-off-by: greatbridf --- .../eonix_hal/src/arch/riscv64/bootstrap.rs | 7 +- crates/eonix_hal/src/arch/riscv64/fdt.rs | 115 +++++++++++------- crates/eonix_hal/src/arch/riscv64/mm.rs | 36 +----- 3 files changed, 78 insertions(+), 80 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index d317e448..ccb32527 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -29,7 +29,7 @@ use super::time::set_next_timer; use crate::arch::cpu::CPU; use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; use crate::arch::mm::{ - ArchPagingMode, ArchPhysAccess, FreeRam, PageAccessImpl, PageAttribute64, + ArchPagingMode, ArchPhysAccess, PageAccessImpl, PageAttribute64, RawPageTableSv48, GLOBAL_PAGE_TABLE, }; use crate::bootstrap::BootStrapData; @@ -113,13 +113,12 @@ unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) { } pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { - let fdt = Fdt::from_ptr(ArchPhysAccess::as_ptr(dtb_addr).as_ptr()) - .expect("Failed to parse DTB from static memory."); + let fdt = unsafe { FdtExt::new(dtb_addr) }; let real_allocator = RefCell::new(BasicPageAlloc::new()); let alloc = BasicPageAllocRef::new(&real_allocator); - for range in fdt.present_ram().free_ram() { + for range in fdt.free_ram() { real_allocator.borrow_mut().add_range(range); } diff --git a/crates/eonix_hal/src/arch/riscv64/fdt.rs b/crates/eonix_hal/src/arch/riscv64/fdt.rs index 5efcc98d..908256c7 100644 --- a/crates/eonix_hal/src/arch/riscv64/fdt.rs +++ b/crates/eonix_hal/src/arch/riscv64/fdt.rs @@ -1,62 +1,95 @@ -use super::mm::{ArchPhysAccess, PresentRam}; -use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; -use core::sync::atomic::{AtomicPtr, Ordering}; -use eonix_mm::address::{PAddr, PRange, PhysAccess}; +use core::ops::Deref; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + +use eonix_mm::address::{Addr, AddrOps, PAddr, PRange, PhysAccess}; use eonix_sync_base::LazyLock; use fdt::Fdt; -static DTB_VIRT_PTR: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); -pub static FDT: LazyLock> = LazyLock::new(|| unsafe { - Fdt::from_ptr(DTB_VIRT_PTR.load(Ordering::Acquire)) - .expect("Failed to parse DTB from static memory.") +use super::mm::ArchPhysAccess; +use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::extern_symbol_addr; + +static DTB_PADDR: AtomicUsize = AtomicUsize::new(0); +pub static FDT: LazyLock = LazyLock::new(|| unsafe { + FdtExt::new(PAddr::from_val(DTB_PADDR.load(Ordering::Relaxed))) }); -pub trait FdtExt { - fn harts(&self) -> impl Iterator; +pub struct FdtExt { + fdt: Fdt<'static>, + range: PRange, +} - fn hart_count(&self) -> usize { - self.harts().count() - } +impl FdtExt { + /// # Safety + /// The caller MUST ensure that [`addr`] points to valid FDT. + pub unsafe fn new(addr: PAddr) -> Self { + let fdt = unsafe { + Fdt::from_ptr(ArchPhysAccess::as_ptr(addr).as_ptr()) + .expect("Failed to parse DTB from static memory.") + }; - fn present_ram(&self) -> impl Iterator; -} + Self { + range: PRange::from(addr).grow(fdt.total_size()), + fdt, + } + } -impl FdtExt for Fdt<'_> { - fn harts(&self) -> impl Iterator { + pub fn harts(&self) -> impl Iterator { self.cpus().map(|cpu| cpu.ids().all()).flatten() } - fn present_ram(&self) -> impl Iterator + PresentRam { - struct Present(I); - impl PresentRam for Present where I: Iterator {} - impl Iterator for Present - where - I: Iterator, - { - type Item = PRange; - - fn next(&mut self) -> Option { - self.0.next() - } - } + pub fn hart_count(&self) -> usize { + self.harts().count() + } + pub fn present_ram(&self) -> impl Iterator { let mut index = 0; - Present(core::iter::from_fn(move || { - self.memory() + + core::iter::from_fn(move || { + let item = self + .memory() .regions() .filter_map(|region| { - region.size.map(|len| { - PRange::from(PAddr::from(region.starting_address as usize)).grow(len) - }) + let start = PAddr::from(region.starting_address as usize); + Some(start).zip(region.size) }) - .skip(index) - .next() - .inspect(|_| index += 1) - })) + .map(|(start, len)| PRange::from(start).grow(len)) + .nth(index); + + index += 1; + item + }) + } + + pub fn free_ram(&self) -> impl Iterator { + let kernel_end = extern_symbol_addr!(__kernel_end) - KIMAGE_OFFSET; + let kernel_end = PAddr::from(kernel_end).ceil(); + + // TODO: move this to some platform-specific crate + self.present_ram().map(move |mut range| { + // Strip out parts before __kernel_end + if range.overlap_with(&PRange::from(kernel_end)) { + (_, range) = range.split_at(kernel_end); + } + + // Strip out part after the FDT + if range.overlap_with(&self.range) { + (range, _) = range.split_at(self.range.start()); + } + + range + }) + } +} + +impl Deref for FdtExt { + type Target = Fdt<'static>; + + fn deref(&self) -> &Self::Target { + &self.fdt } } pub unsafe fn init_dtb_and_fdt(dtb_paddr: PAddr) { - let dtb_virt_ptr = ArchPhysAccess::as_ptr(dtb_paddr); - DTB_VIRT_PTR.store(dtb_virt_ptr.as_ptr(), Ordering::Release); + DTB_PADDR.store(dtb_paddr.addr(), Ordering::Relaxed); } diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 6362cdca..7891f094 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -275,46 +275,12 @@ impl Memory for ArchMemory { } fn free_ram() -> impl Iterator { - FDT.present_ram().free_ram() + FDT.free_ram() } } pub type DefaultPagingMode = PagingModeSv48; -pub trait PresentRam: Iterator {} - -pub trait FreeRam: PresentRam { - fn free_ram(self) -> impl Iterator; -} - -impl FreeRam for T -where - T: PresentRam, -{ - fn free_ram(self) -> impl Iterator { - let kernel_end = extern_symbol_addr!(__kernel_end) - KIMAGE_OFFSET; - let kernel_end = PAddr::from(kernel_end).ceil(); - - let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)) - .chain( - self.filter(move |range| { - range.end() > paddr_after_kimage_aligned - }) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = - range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) - } -} - #[inline(always)] pub fn flush_tlb(vaddr: usize) { sfence_vma(0, vaddr); From 20b12dfa221a87471f2fe49a3f5e7730bce3c2dc Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 03:12:29 +0800 Subject: [PATCH 023/153] sysinit: pointee type should be u8 when using ptr::write_bytes Using *const (), no bytes are written to the position, which might result in uninitialized memory access. Fixes: ebd3d1224c01 ("change(x86): optimize bootstrap code, remove kinit.cpp") Fixes: 191877a3acd0 ("feat(hal): impl basic single hart bootstrap for riscv64") Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/riscv64/bootstrap.rs | 2 +- src/kernel_init.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index ccb32527..b0c235aa 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -186,7 +186,7 @@ fn setup_kernel_page_table(alloc: BasicPageAllocRef) { unsafe { core::ptr::write_bytes( - KERNEL_BSS_START.addr() as *mut (), + KERNEL_BSS_START.addr() as *mut u8, 0, bss_length, ); diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 64c220b2..2259f6cf 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -41,7 +41,7 @@ fn setup_kernel_page_array(alloc: BasicPageAllocRef, count_pages: usize) { unsafe { // SAFETY: We've just mapped the area with sufficient length. core::ptr::write_bytes( - V_KERNEL_PAGE_ARRAY_START.addr() as *mut (), + V_KERNEL_PAGE_ARRAY_START.addr() as *mut u8, 0, count_pages * PAGE_SIZE, ); From 74c7e0a73628697e9a7fa8d06fa82091dcdf25b4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 03:21:52 +0800 Subject: [PATCH 024/153] style: reformat file Signed-off-by: greatbridf --- crates/eonix_hal/src/mm.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/eonix_hal/src/mm.rs b/crates/eonix_hal/src/mm.rs index c4b9bb74..c6c6a369 100644 --- a/crates/eonix_hal/src/mm.rs +++ b/crates/eonix_hal/src/mm.rs @@ -7,8 +7,8 @@ use eonix_mm::page_table::PageTableAlloc; use eonix_mm::paging::{BasicFolio, FrameAlloc, PAGE_SIZE, PFN}; pub use crate::arch::mm::{ - flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchMemory, - ArchPhysAccess, GLOBAL_PAGE_TABLE, + flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, + ArchMemory, ArchPhysAccess, GLOBAL_PAGE_TABLE, }; pub struct BasicPageAlloc { @@ -87,7 +87,8 @@ impl BasicPageAlloc { panic!("Page allocator is full"); } - self.ranges[tail] = Some(PRange::new(range.start().ceil(), range.end().floor())); + self.ranges[tail] = + Some(PRange::new(range.start().ceil(), range.end().floor())); } pub fn alloc(&mut self, order: u32) -> PFN { @@ -147,7 +148,10 @@ impl<'a> ScopedAllocator<'a> { } } - pub fn with_alloc<'b, 'r, O>(&'r self, func: impl FnOnce(&'b ScopedAllocator<'a>) -> O) -> O + pub fn with_alloc<'b, 'r, O>( + &'r self, + func: impl FnOnce(&'b ScopedAllocator<'a>) -> O, + ) -> O where 'a: 'b, 'r: 'b, From 6a0f7033a51c53f3985dfeed42b792b7a0e69e3d Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 03:24:11 +0800 Subject: [PATCH 025/153] hal, mm: alloc basic folios from low to high addr This can shorten qemu memory map. Signed-off-by: greatbridf --- crates/eonix_hal/src/mm.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/eonix_hal/src/mm.rs b/crates/eonix_hal/src/mm.rs index c6c6a369..ef006cb5 100644 --- a/crates/eonix_hal/src/mm.rs +++ b/crates/eonix_hal/src/mm.rs @@ -41,9 +41,8 @@ impl BasicPageAlloc { fn alloc_one(&mut self) -> PFN { assert_ne!(self.head, self.tail, "No free pages available"); let mut range = self.ranges[self.head].take().unwrap(); - range = range.shrink(PAGE_SIZE); - - let pfn = PFN::from(range.end()); + let pfn = PFN::from(range.start()); + range = PRange::new(range.start() + PAGE_SIZE, range.end()); if range.len() != 0 { self.ranges[self.head] = Some(range); From 3842078d05def0200d3db3bacfd53a61d974e5aa Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 14:01:16 +0800 Subject: [PATCH 026/153] script: refactor the build image script Rearrange code to pass args instead of by envs, no functionality changes. Signed-off-by: greatbridf --- Makefile.src | 4 +- script/build-img.sh | 58 ----------------- script/build-img/build-img | 125 +++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 60 deletions(-) delete mode 100644 script/build-img.sh create mode 100755 script/build-img/build-img diff --git a/Makefile.src b/Makefile.src index ab13f5b8..1c8ecb2a 100644 --- a/Makefile.src +++ b/Makefile.src @@ -173,8 +173,8 @@ $(BINARY_DIR)/eonix_kernel: $(KERNEL_DEPS) build/kernel.sym: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- --only-keep-debug build/kernel.sym -build/fs-%.img: user-programs/init_script_%.sh script/build-img.sh $(USER_PROGRAMS) - ARCH=$* OUTPUT=$@ sh script/build-img.sh +build/fs-%.img: user-programs/init_script_%.sh script/build-img/build-img $(USER_PROGRAMS) + script/build-img/build-img -a $* -o $@ build/mbr.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary -j .mbr build/mbr.bin diff --git a/script/build-img.sh b/script/build-img.sh deleted file mode 100644 index 6a83638f..00000000 --- a/script/build-img.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh - -OS=`uname -s` - -if sudo --version > /dev/null 2>&1; then - SUDO=sudo -fi - -if [ "$OUTPUT" = "" ]; then - OUTPUT="build/fs-$ARCH.img" -fi - -if [ "$ARCH" = "" ]; then - echo "ARCH is not set, exiting..." >&2 - exit 1 -fi - -dd if=/dev/zero of="$OUTPUT" bs=`expr 1024 \* 1024` count=1020 -mkfs.fat -n SYSTEM "$OUTPUT" - -if [ "$OS" = "Darwin" ]; then - SUDO='' - hdiutil detach build/mnt > /dev/null 2>&1 || true - hdiutil attach "$OUTPUT" -mountpoint build/mnt -else - mkdir -p build/mnt - $SUDO losetup -P /dev/loop2 "$OUTPUT" - $SUDO mount /dev/loop2 build/mnt -fi - -if [ "$ARCH" = "x86_64" ]; then - $SUDO cp ./user-programs/init.out build/mnt/init - $SUDO cp ./user-programs/int.out build/mnt/int - $SUDO cp ./user-programs/dynamic_test build/mnt/dynamic_test - $SUDO cp ./user-programs/busybox build/mnt/busybox - $SUDO cp ./user-programs/busybox-minimal build/mnt/busybox_ - $SUDO cp ./user-programs/ld-musl-i386.so.1 build/mnt/ld-musl-i386.so.1 - $SUDO cp ./user-programs/pthread_test build/mnt/pthread_test - $SUDO cp ./user-programs/init_script_x86_64.sh build/mnt/initsh -elif [ "$ARCH" = "riscv64" ]; then - $SUDO cp ./user-programs/busybox.static build/mnt/busybox - $SUDO cp ./user-programs/init_script_riscv64.sh build/mnt/initsh -elif [ "$ARCH" = "loongarch64" ]; then - $SUDO cp ./user-programs/busybox.la64 build/mnt/busybox - $SUDO cp ./user-programs/init_script_loongarch64.sh build/mnt/initsh -fi - -# Add your custom files here - - -# End of custom files - -if [ "$OS" = "Darwin" ]; then - hdiutil detach build/mnt -else - $SUDO losetup -d /dev/loop2 - $SUDO umount build/mnt -fi diff --git a/script/build-img/build-img b/script/build-img/build-img new file mode 100755 index 00000000..4b203968 --- /dev/null +++ b/script/build-img/build-img @@ -0,0 +1,125 @@ +#!/bin/sh + +KB=1024 +MB=$((1024 * KB)) + +OS=$(uname -s) +SUDO=$(which sudo || :) + +ARCH= +IMAGE= +MOUNTPOINT="$(mktemp -d)" + +die() { + echo "error: $1" >&2 + exit 1 +} + +usage() { + cat >&2 < -o [-h] + +Options + -a The arch building image for [available: riscv64 x86 loongarch64] + -o Output image path + -h Show help message +EOF + + exit "$1" +} + +sudo() { + "$SUDO" "$@" +} + +ensure_mount_macos() { + hdiutil detach "$MOUNTPOINT" > /dev/null 2>&1 || : + hdiutil attach "$IMAGE" -mountpoint "$MOUNTPOINT" +} + +ensure_mount_linux() { + DEV=$(sudo losetup -f "$IMAGE" --show) + sudo mount "$DEV" "$MOUNTPOINT" +} + +ensure_mount() { + if [ "$OS" = Darwin ]; then + ensure_mount_macos + return + fi + + ensure_mount_linux +} + +copy_to_image() { + _prefix=sudo + [ "$OS" = Darwin ] && _prefix= + + $_prefix cp "$1" "$MOUNTPOINT/$2" +} + +unmount_macos() { + hdiutil detach "$MOUNTPOINT" +} + +unmount_linux() { + sudo umount "$MOUNTPOINT" + sudo losetup -d "$DEV" +} + +cleanup() { + case "$OS" in + Darwin) + unmount_macos + ;; + *) + unmount_linux + ;; + esac +} + +set -eu + +while getopts "a:o:h" opt; do + case "$opt" in + a) ARCH="$OPTARG";; + o) IMAGE="$OPTARG";; + h) usage 0;; + ?) usage 1;; + esac +done + +shift $((OPTIND - 1)) + +[ -z "$ARCH" ] && die "ARCH is not set" +[ -z "$IMAGE" ] && die "output image path is not set" + +echo "Build image with ARCH=$ARCH OUTPUT=$IMAGE MOUNTPOINT=$MOUNTPOINT" + +dd if=/dev/zero of="$IMAGE" bs=$((1 * MB)) count=$((1024 - 4)) +mkfs.fat -n SYSTEM "$IMAGE" + +ensure_mount +trap cleanup EXIT + +if [ "$ARCH" = "x86_64" ]; then + copy_to_image ./user-programs/init.out init + copy_to_image ./user-programs/int.out int + copy_to_image ./user-programs/dynamic_test dynamic_test + copy_to_image ./user-programs/busybox busybox + copy_to_image ./user-programs/busybox-minimal busybox_ + copy_to_image ./user-programs/ld-musl-i386.so.1 ld-musl-i386.so.1 + copy_to_image ./user-programs/pthread_test pthread_test + copy_to_image ./user-programs/init_script_x86_64.sh initsh +elif [ "$ARCH" = "riscv64" ]; then + copy_to_image ./user-programs/busybox.static busybox + copy_to_image ./user-programs/init_script_riscv64.sh initsh +elif [ "$ARCH" = "loongarch64" ]; then + copy_to_image ./user-programs/busybox.la64 busybox + copy_to_image ./user-programs/init_script_loongarch64.sh initsh +fi + +# Add your custom files here + + +# End of custom files From c7c5cdac664168e2946b453708a1408729f72b2f Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 16:02:25 +0800 Subject: [PATCH 027/153] test: rework test script and github workflow Signed-off-by: greatbridf --- .github/workflows/test-build.yaml | 1 + script/test.sh | 74 +++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index f6063ab3..23f2d7ed 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -30,6 +30,7 @@ jobs: - name: Setup QEMU run: | + sudo apt-get update sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm - name: Configure diff --git a/script/test.sh b/script/test.sh index 01097bc3..81cdb9c8 100644 --- a/script/test.sh +++ b/script/test.sh @@ -1,13 +1,71 @@ -#!/bin/sh +#!/bin/bash +LOGFILE="build/test-$$.log" SUCCESS_MSG="###$RANDOM :SuCCeSS: $RANDOM###" -if [ "$ARCH" = "" ]; then - echo "Error: ARCH environment variable is not set." >&2 +die() { + echo "error: $1" >&2 exit 1 -fi +} + +runcmd() { + printf -v _cmd "%q" "$*" + + cat < /dev/null && echo TEST\ $$\ WITH\ ARCH=$ARCH\ PASSED +$(runcmd ls -la) + +$(wait_str proc) + +$(runcmd echo \""$SUCCESS_MSG"\") + +$(wait_str "\n$SUCCESS_MSG") + +$(runcmd poweroff) + +$(wait_init_exit) + +EOF + +status=$? + +echo + +# shellcheck disable=SC2181 +if [ $status -ne 0 ]; then + echo "=== Test $$ with ARCH=$ARCH failed" +else + echo "=== Test $$ with ARCH=$ARCH passed" +fi From bcc41588712a2d76e16bc5680fd88813365c1c3d Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 24 Jan 2026 16:10:21 +0800 Subject: [PATCH 028/153] ci: install expect and update printf usage - printf -v is non-standard, use var=$(printf) instead. - expect is not installed in the runner. install it manually Signed-off-by: greatbridf --- .github/workflows/test-build.yaml | 2 +- script/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index 23f2d7ed..0cd9ae87 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -31,7 +31,7 @@ jobs: - name: Setup QEMU run: | sudo apt-get update - sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm + sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm expect - name: Configure run: ./configure diff --git a/script/test.sh b/script/test.sh index 81cdb9c8..cc5bde7e 100644 --- a/script/test.sh +++ b/script/test.sh @@ -9,7 +9,7 @@ die() { } runcmd() { - printf -v _cmd "%q" "$*" + _cmd=$(printf "%q" "$*") cat < Date: Sat, 24 Jan 2026 16:46:38 +0800 Subject: [PATCH 029/153] script, build-img: add script folder for each platforms Add PLATFORM in Makefile to indicate the actual system setup. Put build image script of each platforms to dedicated folders. This makes creating test setups easier. Signed-off-by: greatbridf --- .github/workflows/test-build.yaml | 6 +- Makefile.src | 34 +++++----- script/build-img/build-img | 63 +++++-------------- script/build-img/lib/lib.sh | 39 ++++++++++++ .../platform/loongarch64-virt/10-base.sh | 5 ++ .../platform/riscv64-virt/10-base.sh | 5 ++ .../platform/riscv64-virt/20-tests.sh | 21 +++++++ .../platform/x86_64-acpi_compat/10-base.sh | 7 +++ .../platform/x86_64-acpi_compat/20-tests.sh | 7 +++ ...h64.sh => init_script_loongarch64-virt.sh} | 0 ...riscv64.sh => init_script_riscv64-virt.sh} | 0 ...4.sh => init_script_x86_64-acpi_compat.sh} | 0 12 files changed, 122 insertions(+), 65 deletions(-) create mode 100644 script/build-img/lib/lib.sh create mode 100644 script/build-img/platform/loongarch64-virt/10-base.sh create mode 100644 script/build-img/platform/riscv64-virt/10-base.sh create mode 100644 script/build-img/platform/riscv64-virt/20-tests.sh create mode 100644 script/build-img/platform/x86_64-acpi_compat/10-base.sh create mode 100644 script/build-img/platform/x86_64-acpi_compat/20-tests.sh rename user-programs/{init_script_loongarch64.sh => init_script_loongarch64-virt.sh} (100%) rename user-programs/{init_script_riscv64.sh => init_script_riscv64-virt.sh} (100%) rename user-programs/{init_script_x86_64.sh => init_script_x86_64-acpi_compat.sh} (100%) diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index 0cd9ae87..61289e57 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -19,8 +19,10 @@ jobs: include: - arch: riscv64 target: riscv64gc-unknown-none-elf + platform: virt - arch: loongarch64 target: loongarch64-unknown-none-softfloat + platform: virt steps: - name: Checkout code uses: actions/checkout@v4 @@ -39,7 +41,7 @@ jobs: - name: Run build for ${{ matrix.arch }} targets run: | make build ARCH=${{ matrix.arch }} MODE=release - zstd -k build/boot-${{ matrix.arch }}.img + zstd -k build/boot-${{ matrix.arch }}-${{ matrix.platform }}.img - name: Upload build artifacts uses: actions/upload-artifact@v4 @@ -47,7 +49,7 @@ jobs: name: eonix-kernel-and-image-${{ matrix.arch }} path: | build/${{ matrix.target }}/release/eonix_kernel - build/boot-${{ matrix.arch }}.img.zst + build/boot-${{ matrix.arch }}-${{ matrix.platform }}.img.zst - name: Test run for ${{ matrix.arch }} run: | diff --git a/Makefile.src b/Makefile.src index 1c8ecb2a..eca1eda4 100644 --- a/Makefile.src +++ b/Makefile.src @@ -54,6 +54,8 @@ endif ifeq ($(ARCH),riscv64) +PLATFORM ?= virt + BINARY_DIR_BASE := build/riscv64gc-unknown-none-elf BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -61,7 +63,7 @@ QEMU_ARGS += \ -machine virt -kernel $(BINARY_DIR)/eonix_kernel \ -device virtio-blk-device,drive=disk0,bus=virtio-mmio-bus.0 \ -device virtio-net-device,netdev=mynet0 \ - -drive id=disk0,file=build/boot-riscv64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-riscv64-virt.img,format=raw,if=none \ -netdev user,id=mynet0 \ -rtc base=utc @@ -73,11 +75,10 @@ endif CARGO_FLAGS += --target riscv64gc-unknown-none-elf -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-riscv64.img - else ifeq ($(ARCH),loongarch64) +PLATFORM ?= virt + BINARY_DIR_BASE := build/loongarch64-unknown-none-softfloat BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -85,7 +86,7 @@ QEMU_ARGS += \ -machine virt -kernel $(BINARY_DIR)/eonix_kernel -m 1G \ -device virtio-blk-pci,drive=disk0 \ -device virtio-net-pci,netdev=mynet0 \ - -drive id=disk0,file=build/boot-loongarch64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-loongarch64-virt.img,format=raw,if=none \ -netdev user,id=mynet0,hostfwd=tcp::5555-:5555,hostfwd=udp::5555-:5555 \ -rtc base=utc @@ -97,11 +98,10 @@ endif CARGO_FLAGS += --target loongarch64-unknown-none-softfloat -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-loongarch64.img - else ifeq ($(ARCH),x86_64) +PLATFORM ?= acpi_compat + BINARY_DIR_BASE := build/x86_64-unknown-none BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -110,7 +110,7 @@ QEMU_ARGS += \ -device ahci,id=ahci \ -device ide-hd,drive=disk0,bus=ahci.0 \ -device e1000e,netdev=mynet0 \ - -drive id=disk0,file=build/boot-x86_64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-x86_64-acpi_compat.img,format=raw,if=none \ -netdev user,id=mynet0 ifneq ($(IMG),) @@ -121,11 +121,13 @@ endif CARGO_FLAGS += --target x86_64-unknown-none.json -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-x86_64.img - endif +TARGET := $(ARCH)-$(PLATFORM) + +.PHONY: build +build: $(BINARY_DIR)/eonix_kernel build/boot-$(TARGET).img + .PHONY: run run: build build/kernel.sym $(QEMU) $(QEMU_ARGS) -display none -serial mon:stdio @@ -174,7 +176,7 @@ build/kernel.sym: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- --only-keep-debug build/kernel.sym build/fs-%.img: user-programs/init_script_%.sh script/build-img/build-img $(USER_PROGRAMS) - script/build-img/build-img -a $* -o $@ + script/build-img/build-img -t $* -o $@ build/mbr.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary -j .mbr build/mbr.bin @@ -186,7 +188,7 @@ build/kernel.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary --strip-debug \ -R .mbr -R .stage1 build/kernel.bin -build/boot-x86_64.img: build/fs-x86_64.img build/mbr.bin build/stage1.bin build/kernel.bin +build/boot-x86_64-acpi_compat.img: build/fs-x86_64-acpi_compat.img build/mbr.bin build/stage1.bin build/kernel.bin dd if=build/mbr.bin of=$@ bs=512 count=1 conv=notrunc 2> /dev/null dd if=build/stage1.bin of=$@ bs=512 seek=1 conv=notrunc 2> /dev/null dd if=build/kernel.bin of=$@ bs=4096 seek=1 conv=notrunc 2> /dev/null @@ -195,13 +197,13 @@ build/boot-x86_64.img: build/fs-x86_64.img build/mbr.bin build/stage1.bin build/ sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ | $(FDISK) $@ 2> /dev/null > /dev/null -build/boot-riscv64.img: build/fs-riscv64.img +build/boot-riscv64-virt.img: build/fs-riscv64-virt.img dd if=$< of=$@ bs=$(shell expr 4 \* 1024 \* 1024) \ seek=1 conv=notrunc 2> /dev/null sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ | $(FDISK) $@ 2> /dev/null > /dev/null -build/boot-loongarch64.img: build/fs-loongarch64.img +build/boot-loongarch64-virt.img: build/fs-loongarch64-virt.img dd if=$< of=$@ bs=$(shell expr 4 \* 1024 \* 1024) \ seek=1 conv=notrunc 2> /dev/null sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ diff --git a/script/build-img/build-img b/script/build-img/build-img index 4b203968..638866af 100755 --- a/script/build-img/build-img +++ b/script/build-img/build-img @@ -1,26 +1,24 @@ #!/bin/sh +# shellcheck source=./lib/lib.sh + +. "$(dirname "$(realpath "$0")")/lib/lib.sh" + +BASE_DIR="$(dirname "$(realpath "$0")")" KB=1024 MB=$((1024 * KB)) -OS=$(uname -s) -SUDO=$(which sudo || :) - -ARCH= +TARGET= IMAGE= MOUNTPOINT="$(mktemp -d)" -die() { - echo "error: $1" >&2 - exit 1 -} - usage() { cat >&2 < -o [-h] +Usage: $0 -t -o [-h] Options - -a The arch building image for [available: riscv64 x86 loongarch64] + -t The target building image for + [available: riscv64 x86 loongarch64] -o Output image path -h Show help message EOF @@ -28,10 +26,6 @@ EOF exit "$1" } -sudo() { - "$SUDO" "$@" -} - ensure_mount_macos() { hdiutil detach "$MOUNTPOINT" > /dev/null 2>&1 || : hdiutil attach "$IMAGE" -mountpoint "$MOUNTPOINT" @@ -51,13 +45,6 @@ ensure_mount() { ensure_mount_linux } -copy_to_image() { - _prefix=sudo - [ "$OS" = Darwin ] && _prefix= - - $_prefix cp "$1" "$MOUNTPOINT/$2" -} - unmount_macos() { hdiutil detach "$MOUNTPOINT" } @@ -80,9 +67,9 @@ cleanup() { set -eu -while getopts "a:o:h" opt; do +while getopts "t:o:h" opt; do case "$opt" in - a) ARCH="$OPTARG";; + t) TARGET="$OPTARG";; o) IMAGE="$OPTARG";; h) usage 0;; ?) usage 1;; @@ -91,10 +78,10 @@ done shift $((OPTIND - 1)) -[ -z "$ARCH" ] && die "ARCH is not set" +[ -z "$TARGET" ] && die "ARCH is not set" [ -z "$IMAGE" ] && die "output image path is not set" -echo "Build image with ARCH=$ARCH OUTPUT=$IMAGE MOUNTPOINT=$MOUNTPOINT" +echo "Build image with ARCH=$TARGET OUTPUT=$IMAGE MOUNTPOINT=$MOUNTPOINT" dd if=/dev/zero of="$IMAGE" bs=$((1 * MB)) count=$((1024 - 4)) mkfs.fat -n SYSTEM "$IMAGE" @@ -102,24 +89,6 @@ mkfs.fat -n SYSTEM "$IMAGE" ensure_mount trap cleanup EXIT -if [ "$ARCH" = "x86_64" ]; then - copy_to_image ./user-programs/init.out init - copy_to_image ./user-programs/int.out int - copy_to_image ./user-programs/dynamic_test dynamic_test - copy_to_image ./user-programs/busybox busybox - copy_to_image ./user-programs/busybox-minimal busybox_ - copy_to_image ./user-programs/ld-musl-i386.so.1 ld-musl-i386.so.1 - copy_to_image ./user-programs/pthread_test pthread_test - copy_to_image ./user-programs/init_script_x86_64.sh initsh -elif [ "$ARCH" = "riscv64" ]; then - copy_to_image ./user-programs/busybox.static busybox - copy_to_image ./user-programs/init_script_riscv64.sh initsh -elif [ "$ARCH" = "loongarch64" ]; then - copy_to_image ./user-programs/busybox.la64 busybox - copy_to_image ./user-programs/init_script_loongarch64.sh initsh -fi - -# Add your custom files here - - -# End of custom files +for name in $(iter_files_sorted "$BASE_DIR/platform/$TARGET"); do + ( . "$name" ) +done diff --git a/script/build-img/lib/lib.sh b/script/build-img/lib/lib.sh new file mode 100644 index 00000000..5cc6c036 --- /dev/null +++ b/script/build-img/lib/lib.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +OS=$(uname -s) +SUDO=$(which sudo || :) + +info() { + echo "info : $1" +} + +warn() { + echo "warn : $1" >&2 +} + +error() { + echo "fatal: $1" >&2 +} + +die() { + error "$1" && exit 1 +} + +sudo() { + "$SUDO" "$@" +} + +copy_to_image() { + _prefix=sudo + [ "$OS" = Darwin ] && _prefix= + + $_prefix cp "$1" "$MOUNTPOINT/$2" +} + +iter_files() { + find -L "$1" -maxdepth 1 -type f +} + +iter_files_sorted() { + iter_files "$@" | sort +} diff --git a/script/build-img/platform/loongarch64-virt/10-base.sh b/script/build-img/platform/loongarch64-virt/10-base.sh new file mode 100644 index 00000000..d0f21dbd --- /dev/null +++ b/script/build-img/platform/loongarch64-virt/10-base.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/busybox.la64 busybox +copy_to_image user-programs/init_script_loongarch64-virt.sh initsh diff --git a/script/build-img/platform/riscv64-virt/10-base.sh b/script/build-img/platform/riscv64-virt/10-base.sh new file mode 100644 index 00000000..efb32cf7 --- /dev/null +++ b/script/build-img/platform/riscv64-virt/10-base.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/busybox.static busybox +copy_to_image user-programs/init_script_riscv64-virt.sh initsh diff --git a/script/build-img/platform/riscv64-virt/20-tests.sh b/script/build-img/platform/riscv64-virt/20-tests.sh new file mode 100644 index 00000000..1d9e0d71 --- /dev/null +++ b/script/build-img/platform/riscv64-virt/20-tests.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +name="$(mktemp)" + +cat > "$name" < Date: Sun, 25 Jan 2026 01:18:05 +0800 Subject: [PATCH 030/153] style, hal: rename x86 trap.rs to trap/mod.rs No functional change. Signed-off-by: greatbridf --- .../src/arch/x86_64/{trap.rs => trap/mod.rs} | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) rename crates/eonix_hal/src/arch/x86_64/{trap.rs => trap/mod.rs} (97%) diff --git a/crates/eonix_hal/src/arch/x86_64/trap.rs b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs similarity index 97% rename from crates/eonix_hal/src/arch/x86_64/trap.rs rename to crates/eonix_hal/src/arch/x86_64/trap/mod.rs index 6a47a753..20f1472f 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs @@ -1,14 +1,13 @@ mod trap_context; -use super::context::TaskContext; use core::arch::{asm, global_asm, naked_asm}; -use eonix_hal_traits::{ - context::RawTaskContext, - trap::{IrqState as IrqStateTrait, TrapReturn}, -}; +use eonix_hal_traits::context::RawTaskContext; +use eonix_hal_traits::trap::{IrqState as IrqStateTrait, TrapReturn}; pub use trap_context::TrapContext; +use super::context::TaskContext; + unsafe extern "C" { fn _default_trap_handler(trap_context: &mut TrapContext); pub fn trap_stubs_start(); @@ -147,7 +146,7 @@ global_asm!( build_isr_no_err %i .set i, i+1 .endr - + .globl _raw_trap_entry .type _raw_trap_entry @function _raw_trap_entry: @@ -157,10 +156,10 @@ global_asm!( .cfi_offset %rsp, 0x10 cfi_all_same_value - + sub $0x78, %rsp .cfi_def_cfa_offset CS - + mov %rax, RAX(%rsp) .cfi_rel_offset %rax, RAX mov %rbx, RBX(%rsp) @@ -191,24 +190,24 @@ global_asm!( .cfi_rel_offset %r15, R15 mov %rbp, RBP(%rsp) .cfi_rel_offset %rbp, RBP - + mov INT_NO(%rsp), %rcx sub ${trap_stubs_start}, %rcx shr $3, %rcx mov %rcx, INT_NO(%rsp) - + cmpq $0x08, CS(%rsp) je 2f swapgs - + 2: mov %gs:0, %rcx add ${handler}, %rcx mov (%rcx), %rcx - + jmp *%rcx .cfi_endproc - + _raw_trap_return: .cfi_startproc .cfi_def_cfa %rsp, CS @@ -228,7 +227,7 @@ global_asm!( .cfi_rel_offset %r15, R15 .cfi_rel_offset %rbp, RBP .cfi_rel_offset %rsp, RSP - + mov RAX(%rsp), %rax .cfi_restore %rax mov RBX(%rsp), %rbx @@ -259,16 +258,16 @@ global_asm!( .cfi_restore %r15 mov RBP(%rsp), %rbp .cfi_restore %rbp - + cmpq $0x08, CS(%rsp) je 2f swapgs - + 2: lea RIP(%rsp), %rsp .cfi_def_cfa %rsp, 0x08 .cfi_offset %rsp, 0x10 - + iretq .cfi_endproc ", From 2c49a12332edc24dae2d74e5ca05055703cb7c90 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 26 Jan 2026 02:00:49 +0800 Subject: [PATCH 031/153] chore: remove the specific target from config.toml Instead, add an example vscode settings file for IDE use. Signed-off-by: greatbridf --- .cargo/config.toml | 1 - .vscode/settings.example.json | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.example.json diff --git a/.cargo/config.toml b/.cargo/config.toml index 9c7ba798..12e65525 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,5 +1,4 @@ [build] -target = "riscv64gc-unknown-none-elf" target-dir = 'build' rustflags = ["-C", "force-unwind-tables"] diff --git a/.vscode/settings.example.json b/.vscode/settings.example.json new file mode 100644 index 00000000..fbafa867 --- /dev/null +++ b/.vscode/settings.example.json @@ -0,0 +1,10 @@ +{ + "editor.formatOnSave": true, + // Uncomment and set the target you'd like to use for analysis. + // + // Available targets: + // - x86_64-unknown-none.json + // - riscv64gc-unknown-none-elf + // - loongarch64-unknown-none-softfloat + // "rust-analyzer.cargo.target": "x86_64-unknown-none.json", +} From a5f6349c0dd450d23319cc95db321aed8e2d4207 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 26 Jan 2026 02:05:28 +0800 Subject: [PATCH 032/153] target: update x86 json to support new rust nightly Change the type of target width items to numbers according to the specifications. Signed-off-by: greatbridf --- x86_64-unknown-none.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x86_64-unknown-none.json b/x86_64-unknown-none.json index 77e4d1dd..a66b07ba 100644 --- a/x86_64-unknown-none.json +++ b/x86_64-unknown-none.json @@ -3,8 +3,8 @@ "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", "arch": "x86_64", "target-endian": "little", - "target-pointer-width": "64", - "target-c-int-width": "32", + "target-pointer-width": 64, + "target-c-int-width": 32, "os": "none", "executables": true, "linker-flavor": "ld.lld", From 7d47e631da4c00900564d58fec1c360ac37391a5 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 00:07:48 +0800 Subject: [PATCH 033/153] feat: add kernel config framework, rework panicking Support defining and specifying configs in build.rs. Add arch_has_stacktrace and arch_has_shutdown to control behavior on panicking. Move the codes into panic.rs entirely. TODO: How can we add dependencies based on arch configs in a more elegant way? Signed-off-by: greatbridf --- .rustfmt.toml | 4 +-- Cargo.toml | 13 ++++++++-- build.rs | 65 +++++++++++++++++++++++++++++++++++++++++++++- src/lib.rs | 56 +++++++++------------------------------- src/panic.rs | 71 ++++++++++++++++++++++++++++++++++++++++++--------- 5 files changed, 148 insertions(+), 61 deletions(-) diff --git a/.rustfmt.toml b/.rustfmt.toml index 85b1cfc7..1075fd32 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -42,13 +42,13 @@ binop_separator = "Front" remove_nested_parens = true combine_control_expr = true short_array_element_width_threshold = 10 -overflow_delimited_expr = false +overflow_delimited_expr = true struct_field_align_threshold = 0 enum_discrim_align_threshold = 0 match_arm_blocks = true match_arm_leading_pipes = "Never" force_multiline_blocks = false -fn_params_layout = "Tall" +fn_params_layout = "Compressed" brace_style = "SameLineWhere" control_brace_style = "AlwaysSameLine" trailing_semicolon = true diff --git a/Cargo.toml b/Cargo.toml index 214e5941..f5df1fa1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,10 +47,16 @@ futures = { version = "0.3.31", features = [ static_assertions = "1.1.0" cfg-if = "1.0.4" -[target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] -virtio-drivers = { version = "0.11.0" } +[target.'cfg(target_arch = "x86_64")'.dependencies] +unwinding = { version = "0.2.8", default-features = false, features = [ + "unwinder", + "fde-static", + "personality", + "panic", +] } [target.'cfg(target_arch = "riscv64")'.dependencies] +virtio-drivers = { version = "0.11.0" } unwinding = { version = "0.2.8", default-features = false, features = [ "unwinder", "fde-static", @@ -58,6 +64,9 @@ unwinding = { version = "0.2.8", default-features = false, features = [ "panic", ] } +[target.'cfg(target_arch = "loongarch64")'.dependencies] +virtio-drivers = { version = "0.11.0" } + [features] default = [] trace_pci = [] diff --git a/build.rs b/build.rs index 1c9d2c3d..68321dc9 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,69 @@ +const ARCH_CONFIGS: &[&str] = &["has_stacktrace", "has_shutdown"]; + +// Define configs + +fn define_config(config: &str) { + println!("cargo::rustc-check-cfg=cfg({config})"); +} + +fn define_arch_config(config: &str) { + define_config(&format!("arch_{config}")); +} + +fn define_arch_configs() { + for config in ARCH_CONFIGS { + define_arch_config(config); + } +} + +fn define_configs() { + define_arch_configs(); +} + +// Set configs + +fn set_config(config: &str) { + println!("cargo::rustc-cfg={config}"); +} + +fn set_arch_config(config: &str) { + set_config(&format!("arch_{config}")); +} + +fn set_arch_configs_x86() { + set_arch_config("has_stacktrace"); +} + +fn set_arch_configs_riscv64() { + set_arch_config("has_stacktrace"); + set_arch_config("has_shutdown"); +} + +fn set_arch_configs_loongarch64() { + set_arch_config("has_shutdown"); +} + +fn set_arch_configs() { + match std::env::var("CARGO_CFG_TARGET_ARCH").as_deref().unwrap() { + "x86_64" => set_arch_configs_x86(), + "riscv64" => set_arch_configs_riscv64(), + "loongarch64" => set_arch_configs_loongarch64(), + arch => panic!("Unsupported architecture: {}", arch), + } +} + +fn set_configs() { + set_arch_configs(); +} + fn main() -> Result<(), Box> { + define_configs(); + set_configs(); + println!("cargo:rustc-link-arg=-T{}", "link.x"); - if let Ok(extra_link_args) = std::env::var("DEP_EONIX_HAL_EXTRA_LINK_ARGS") { + + if let Ok(extra_link_args) = std::env::var("DEP_EONIX_HAL_EXTRA_LINK_ARGS") + { for arg in extra_link_args.split_whitespace() { println!("cargo:rustc-link-arg={}", arg); } diff --git a/src/lib.rs b/src/lib.rs index 2e28db24..f49e543e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,9 +13,6 @@ extern crate alloc; #[macro_use] extern crate static_assertions; -#[cfg(any(target_arch = "riscv64", target_arch = "x86_64"))] -extern crate unwinding; - mod driver; mod fs; mod hash; @@ -23,7 +20,6 @@ mod io; mod kernel; mod kernel_init; mod net; -#[cfg(any(target_arch = "riscv64", target_arch = "x86_64"))] mod panic; mod path; mod prelude; @@ -47,7 +43,9 @@ use eonix_runtime::scheduler::RUNTIME; use kernel::mem::GlobalPageAlloc; use kernel::task::{KernelStack, ProcessList, ProgramLoader}; use kernel::vfs::dentry::Dentry; -use kernel::vfs::mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}; +use kernel::vfs::mount::{ + do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY, +}; use kernel::vfs::types::Permission; use kernel::vfs::FsContext; use kernel::CharDevice; @@ -55,40 +53,6 @@ use kernel_init::setup_memory; use path::Path; use prelude::*; -#[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] -fn do_panic() -> ! { - #[cfg(target_arch = "riscv64")] - panic::stack_trace(); - - shutdown(); -} - -#[cfg(not(any(target_arch = "riscv64", target_arch = "loongarch64")))] -fn do_panic() -> ! { - // Spin forever. - loop { - spin_loop(); - } -} - -#[panic_handler] -fn panic(info: &core::panic::PanicInfo) -> ! { - if let Some(location) = info.location() { - println_fatal!( - "panicked at {}:{}:{}", - location.file(), - location.line(), - location.column() - ); - } else { - println_fatal!("panicked at "); - } - println_fatal!(); - println_fatal!("{}", info.message()); - - do_panic() -} - static BSP_OK: AtomicBool = AtomicBool::new(false); static CPU_SHUTTING_DOWN: AtomicUsize = AtomicUsize::new(0); @@ -220,9 +184,10 @@ async fn init_process(early_kstack: PRange) { let load_info = { // mount fat32 /mnt directory let fs_context = FsContext::global(); - let mnt_dir = Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true) - .await - .unwrap(); + let mnt_dir = + Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true) + .await + .unwrap(); mnt_dir .mkdir(Permission::new(0o755)) @@ -239,12 +204,15 @@ async fn init_process(early_kstack: PRange) { .await .unwrap(); - let init_names = [&b"/init"[..], &b"/sbin/init"[..], &b"/mnt/initsh"[..]]; + let init_names = + [&b"/init"[..], &b"/sbin/init"[..], &b"/mnt/initsh"[..]]; let mut init_name = None; let mut init = None; for name in init_names { - if let Ok(dentry) = Dentry::open(fs_context, Path::new(name).unwrap(), true).await { + if let Ok(dentry) = + Dentry::open(fs_context, Path::new(name).unwrap(), true).await + { if dentry.is_valid() { init_name = Some(CString::new(name).unwrap()); init = Some(dentry); diff --git a/src/panic.rs b/src/panic.rs index 4a9ef92c..46409de2 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -1,16 +1,61 @@ -use core::ffi::c_void; +use core::panic::PanicInfo; use eonix_log::println_fatal; -use unwinding::abi::{ - UnwindContext, UnwindReasonCode, _Unwind_Backtrace, _Unwind_GetIP, _Unwind_GetRegionStart, -}; -pub fn stack_trace() { +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + if let Some(location) = info.location() { + println_fatal!( + "panicked at {}:{}:{}", + location.file(), + location.line(), + location.column() + ); + } else { + println_fatal!("panicked at "); + } + println_fatal!(); + println_fatal!("{}", info.message()); + + #[cfg(arch_has_stacktrace)] + stacktrace::print_stacktrace(); + + panic_forever(); +} + +fn panic_forever() -> ! { + #[cfg(arch_has_shutdown)] + { + eonix_hal::arch_exported::bootstrap::shutdown(); + } + #[cfg(not(arch_has_shutdown))] + { + // Spin forever + loop { + core::hint::spin_loop(); + } + } +} + +#[cfg(arch_has_stacktrace)] +mod stacktrace { + extern crate unwinding; + + use core::ffi::c_void; + + use eonix_log::println_fatal; + use unwinding::abi::{ + UnwindContext, UnwindReasonCode, _Unwind_Backtrace, _Unwind_GetIP, + _Unwind_GetRegionStart, + }; + struct CallbackData { counter: usize, } - extern "C" fn callback(unwind_ctx: &UnwindContext<'_>, arg: *mut c_void) -> UnwindReasonCode { + extern "C" fn unwind_print( + unwind_ctx: &UnwindContext<'_>, arg: *mut c_void, + ) -> UnwindReasonCode { let data = unsafe { &mut *(arg as *mut CallbackData) }; data.counter += 1; @@ -24,12 +69,14 @@ pub fn stack_trace() { UnwindReasonCode::NO_REASON } - println_fatal!("--------------8< CUT HERE 8<--------------"); - println_fatal!("Stacktrace:"); - println_fatal!(); + pub fn print_stacktrace() { + println_fatal!("--------------8< CUT HERE 8<--------------"); + println_fatal!("Stacktrace:"); + println_fatal!(); - let mut data = CallbackData { counter: 0 }; - _Unwind_Backtrace(callback, &raw mut data as *mut c_void); + let mut data = CallbackData { counter: 0 }; + _Unwind_Backtrace(unwind_print, &raw mut data as *mut c_void); - println_fatal!("--------------8< CUT HERE 8<--------------"); + println_fatal!("--------------8< CUT HERE 8<--------------"); + } } From 681a12177490531045a8769b028b2b76a3195ea8 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 00:25:07 +0800 Subject: [PATCH 034/153] style: reformat files No functional changes. Signed-off-by: greatbridf --- crates/eonix_hal/eonix_hal_traits/src/trap.rs | 13 +- .../src/arch/riscv64/trap/trap_context.rs | 66 ++++---- .../src/arch/x86_64/bootstrap/init.rs | 66 ++++---- crates/eonix_hal/src/arch/x86_64/context.rs | 3 +- crates/eonix_hal/src/arch/x86_64/cpu.rs | 12 +- crates/eonix_hal/src/arch/x86_64/gdt.rs | 4 +- crates/eonix_hal/src/arch/x86_64/interrupt.rs | 9 +- crates/eonix_hal/src/arch/x86_64/mm.rs | 35 ++-- .../src/arch/x86_64/trap/trap_context.rs | 18 +- src/kernel/syscall/procops.rs | 159 ++++++++++-------- src/kernel/task.rs | 30 +++- src/kernel/task/signal/signal_action.rs | 5 +- 12 files changed, 234 insertions(+), 186 deletions(-) diff --git a/crates/eonix_hal/eonix_hal_traits/src/trap.rs b/crates/eonix_hal/eonix_hal_traits/src/trap.rs index e51a9eb4..74ed9a28 100644 --- a/crates/eonix_hal/eonix_hal_traits/src/trap.rs +++ b/crates/eonix_hal/eonix_hal_traits/src/trap.rs @@ -1,7 +1,10 @@ -use crate::{context::RawTaskContext, fault::Fault}; use core::marker::PhantomData; + use eonix_mm::address::VAddr; +use crate::context::RawTaskContext; +use crate::fault::Fault; + /// A raw trap context. /// /// This should be implemented by the architecture-specific trap context @@ -30,12 +33,8 @@ pub trait RawTrapContext: Copy { fn set_user_return_value(&mut self, retval: usize); fn set_user_call_frame( - &mut self, - pc: usize, - sp: Option, - ra: Option, - args: &[usize], - write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, + &mut self, pc: usize, sp: Option, ra: Option, + args: &[usize], write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, ) -> Result<(), E>; } diff --git a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs index ab4ca9a2..027db567 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs @@ -1,19 +1,17 @@ -use crate::{arch::time::set_next_timer, processor::CPU}; -use core::{arch::asm, mem::offset_of}; -use eonix_hal_traits::{ - fault::{Fault, PageFaultErrorCode}, - trap::{RawTrapContext, TrapType}, -}; +use core::arch::asm; +use core::mem::offset_of; + +use eonix_hal_traits::fault::{Fault, PageFaultErrorCode}; +use eonix_hal_traits::trap::{RawTrapContext, TrapType}; use eonix_mm::address::VAddr; -use riscv::{ - interrupt::{Exception, Interrupt, Trap}, - register::{ - scause::{self, Scause}, - sstatus::{self, Sstatus, FS, SPP}, - stval, - }, - ExceptionNumber, InterruptNumber, -}; +use riscv::interrupt::{Exception, Interrupt, Trap}; +use riscv::register::scause::{self, Scause}; +use riscv::register::sstatus::{self, Sstatus, FS, SPP}; +use riscv::register::stval; +use riscv::{ExceptionNumber, InterruptNumber}; + +use crate::arch::time::set_next_timer; +use crate::processor::CPU; #[repr(C)] #[derive(Default, Clone, Copy)] @@ -151,10 +149,13 @@ impl RawTrapContext for TrapContext { Interrupt::SupervisorExternal => TrapType::Irq { callback: |handler| { let mut cpu = CPU::local(); - match cpu.as_mut().interrupt.plic.claim_interrupt() { + match cpu.as_mut().interrupt.plic.claim_interrupt() + { None => {} Some(irqno) => { - cpu.interrupt.plic.complete_interrupt(irqno); + cpu.interrupt + .plic + .complete_interrupt(irqno); handler(irqno); } } @@ -172,18 +173,25 @@ impl RawTrapContext for TrapContext { | Exception::InstructionFault | Exception::LoadFault | Exception::StoreFault - | Exception::StoreMisaligned => TrapType::Fault(Fault::BadAccess), - Exception::IllegalInstruction => TrapType::Fault(Fault::InvalidOp), + | Exception::StoreMisaligned => { + TrapType::Fault(Fault::BadAccess) + } + Exception::IllegalInstruction => { + TrapType::Fault(Fault::InvalidOp) + } Exception::UserEnvCall => TrapType::Syscall { no: self.syscall_no(), args: self.syscall_args(), }, exception @ (Exception::InstructionPageFault | Exception::LoadPageFault - | Exception::StorePageFault) => TrapType::Fault(Fault::PageFault { - error_code: self.get_page_fault_error_code(exception), - address: VAddr::from(self.stval), - }), + | Exception::StorePageFault) => { + TrapType::Fault(Fault::PageFault { + error_code: self + .get_page_fault_error_code(exception), + address: VAddr::from(self.stval), + }) + } // breakpoint and supervisor env call _ => TrapType::Fault(Fault::Unknown(e)), } @@ -239,12 +247,8 @@ impl RawTrapContext for TrapContext { } fn set_user_call_frame( - &mut self, - pc: usize, - sp: Option, - ra: Option, - args: &[usize], - _write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, + &mut self, pc: usize, sp: Option, ra: Option, + args: &[usize], _write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, ) -> Result<(), E> { self.set_program_counter(pc); @@ -275,7 +279,9 @@ impl RawTrapContext for TrapContext { impl TrapContext { /// TODO: get PageFaultErrorCode also need check pagetable - fn get_page_fault_error_code(&self, exception: Exception) -> PageFaultErrorCode { + fn get_page_fault_error_code( + &self, exception: Exception, + ) -> PageFaultErrorCode { let mut error_code = PageFaultErrorCode::empty(); match exception { diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs index a3159a25..f12a98f0 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs @@ -1,30 +1,29 @@ -use crate::{ - arch::{ - bootstrap::{EARLY_GDT_DESCRIPTOR, KERNEL_PML4}, - cpu::{wrmsr, CPU}, - io::Port8, - mm::{ArchPhysAccess, GLOBAL_PAGE_TABLE, V_KERNEL_BSS_START}, - }, - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}, -}; -use acpi::{platform::ProcessorState, AcpiHandler, AcpiTables, PhysicalMapping, PlatformInfo}; -use core::{ - alloc::Allocator, - arch::{asm, global_asm}, - cell::RefCell, - hint::spin_loop, - sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}, -}; +use core::alloc::Allocator; +use core::arch::{asm, global_asm}; +use core::cell::RefCell; +use core::hint::spin_loop; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +use acpi::platform::ProcessorState; +use acpi::{AcpiHandler, AcpiTables, PhysicalMapping, PlatformInfo}; use eonix_hal_traits::mm::Memory; -use eonix_mm::{ - address::{Addr as _, PAddr, PRange, PhysAccess, VRange}, - page_table::{PageAttribute, PagingMode, PTE as _}, - paging::{Page, PageAccess, PageAlloc, PAGE_SIZE}, -}; +use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VRange}; +use eonix_mm::page_table::{PageAttribute, PagingMode, PTE as _}; +use eonix_mm::paging::{Page, PageAccess, PageAlloc, PAGE_SIZE}; use eonix_percpu::PercpuArea; -static BSP_PAGE_ALLOC: AtomicPtr> = AtomicPtr::new(core::ptr::null_mut()); +use crate::arch::bootstrap::{EARLY_GDT_DESCRIPTOR, KERNEL_PML4}; +use crate::arch::cpu::{wrmsr, CPU}; +use crate::arch::io::Port8; +use crate::arch::mm::{ArchPhysAccess, GLOBAL_PAGE_TABLE, V_KERNEL_BSS_START}; +use crate::bootstrap::BootStrapData; +use crate::mm::{ + ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, + ScopedAllocator, +}; + +static BSP_PAGE_ALLOC: AtomicPtr> = + AtomicPtr::new(core::ptr::null_mut()); static AP_COUNT: AtomicUsize = AtomicUsize::new(0); static AP_STACK: AtomicUsize = AtomicUsize::new(0); @@ -188,9 +187,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { impl AcpiHandler for Handler { unsafe fn map_physical_region( - &self, - physical_address: usize, - size: usize, + &self, physical_address: usize, size: usize, ) -> PhysicalMapping { unsafe { PhysicalMapping::new( @@ -236,7 +233,8 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { }; // SAFETY: All the APs can see the allocator work done before this point. - let old = BSP_PAGE_ALLOC.swap((&raw const *page_alloc) as *mut _, Ordering::Release); + let old = BSP_PAGE_ALLOC + .swap((&raw const *page_alloc) as *mut _, Ordering::Release); assert!( old.is_null(), "BSP_PAGE_ALLOC should be null before we release it" @@ -305,7 +303,11 @@ pub extern "C" fn kernel_init() -> ! { unsafe { // SAFETY: We've just mapped the area with sufficient length. - core::ptr::write_bytes(V_KERNEL_BSS_START.addr() as *mut (), 0, BSS_LENGTH as usize); + core::ptr::write_bytes( + V_KERNEL_BSS_START.addr() as *mut (), + 0, + BSS_LENGTH as usize, + ); } setup_cpu(&alloc); @@ -329,12 +331,14 @@ pub extern "C" fn kernel_init() -> ! { } pub extern "C" fn ap_entry(stack_bottom: PAddr) -> ! { - let stack_range = PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); + let stack_range = + PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); { // SAFETY: Acquire all the work done by the BSP and other APs. let alloc = loop { - let alloc = BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); + let alloc = + BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); if !alloc.is_null() { break alloc; diff --git a/crates/eonix_hal/src/arch/x86_64/context.rs b/crates/eonix_hal/src/arch/x86_64/context.rs index b6c28bf1..f55ede84 100644 --- a/crates/eonix_hal/src/arch/x86_64/context.rs +++ b/crates/eonix_hal/src/arch/x86_64/context.rs @@ -1,4 +1,5 @@ use core::arch::naked_asm; + use eonix_hal_traits::context::RawTaskContext; /// Necessary hardware states of task for doing context switches. @@ -36,7 +37,7 @@ impl TaskContext { } #[unsafe(naked)] - unsafe extern "C" fn do_call() -> ! { + unsafe extern "C" fn do_call() { naked_asm!( "mov %r12, %rdi", "push %rbp", // NULL return address. diff --git a/crates/eonix_hal/src/arch/x86_64/cpu.rs b/crates/eonix_hal/src/arch/x86_64/cpu.rs index 26f4f1b6..03ca78d2 100644 --- a/crates/eonix_hal/src/arch/x86_64/cpu.rs +++ b/crates/eonix_hal/src/arch/x86_64/cpu.rs @@ -1,13 +1,15 @@ -use super::gdt::{GDTEntry, GDT}; -use super::interrupt::InterruptControl; -use super::trap::TrapContext; use core::arch::asm; use core::marker::PhantomPinned; use core::mem::size_of; use core::pin::Pin; + use eonix_preempt::PreemptGuard; use eonix_sync_base::LazyLock; +use super::gdt::{GDTEntry, GDT}; +use super::interrupt::InterruptControl; +use super::trap::TrapContext; + #[eonix_percpu::define_percpu] static LOCAL_CPU: LazyLock = LazyLock::new(CPU::new); @@ -55,7 +57,9 @@ pub struct CPU { impl UserTLS { /// # Return /// Returns the TLS descriptor and the index of the TLS segment. - pub fn new32(base: u32, limit: u32, is_limit_in_pages: bool) -> (Self, u32) { + pub fn new32( + base: u32, limit: u32, is_limit_in_pages: bool, + ) -> (Self, u32) { let flags = if is_limit_in_pages { 0xc } else { 0x4 }; ( diff --git a/crates/eonix_hal/src/arch/x86_64/gdt.rs b/crates/eonix_hal/src/arch/x86_64/gdt.rs index 9db30793..4543e3e0 100644 --- a/crates/eonix_hal/src/arch/x86_64/gdt.rs +++ b/crates/eonix_hal/src/arch/x86_64/gdt.rs @@ -1,5 +1,7 @@ +use core::arch::asm; +use core::marker::PhantomPinned; + use super::cpu::TSS; -use core::{arch::asm, marker::PhantomPinned}; #[repr(transparent)] #[derive(Debug, Clone, Copy)] diff --git a/crates/eonix_hal/src/arch/x86_64/interrupt.rs b/crates/eonix_hal/src/arch/x86_64/interrupt.rs index 0ed44237..6397e9ec 100644 --- a/crates/eonix_hal/src/arch/x86_64/interrupt.rs +++ b/crates/eonix_hal/src/arch/x86_64/interrupt.rs @@ -1,5 +1,9 @@ +use core::arch::asm; +use core::marker::PhantomPinned; +use core::pin::Pin; +use core::ptr::NonNull; + use crate::arch::cpu::rdmsr; -use core::{arch::asm, marker::PhantomPinned, pin::Pin, ptr::NonNull}; #[repr(C)] #[derive(Clone, Copy)] @@ -121,7 +125,8 @@ impl InterruptControl { let apic_base = rdmsr(0x1b); assert_eq!(apic_base & 0x800, 0x800, "LAPIC not enabled"); - let apic_base = ((apic_base & !0xfff) + 0xffffff00_00000000) as *mut u32; + let apic_base = + ((apic_base & !0xfff) + 0xffffff00_00000000) as *mut u32; APICRegs { // TODO: A better way to convert to physical address base: NonNull::new(apic_base).expect("Invalid APIC base"), diff --git a/crates/eonix_hal/src/arch/x86_64/mm.rs b/crates/eonix_hal/src/arch/x86_64/mm.rs index 9dcae396..a991cecc 100644 --- a/crates/eonix_hal/src/arch/x86_64/mm.rs +++ b/crates/eonix_hal/src/arch/x86_64/mm.rs @@ -1,15 +1,19 @@ -use crate::traits::mm::Memory; -use core::{arch::asm, marker::PhantomData, ptr::NonNull}; -use eonix_mm::{ - address::{Addr as _, AddrOps as _, PAddr, PRange, PhysAccess, VAddr}, - page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, - }, - paging::{NoAlloc, Page, PageBlock, PAGE_SIZE, PFN}, +use core::arch::asm; +use core::marker::PhantomData; +use core::ptr::NonNull; + +use eonix_mm::address::{ + Addr as _, AddrOps as _, PAddr, PRange, PhysAccess, VAddr, +}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, + RawPageTable, TableAttribute, PTE, }; +use eonix_mm::paging::{NoAlloc, Page, PageBlock, PAGE_SIZE, PFN}; use eonix_sync_base::LazyLock; +use crate::traits::mm::Memory; + pub const PA_P: u64 = 0x001; pub const PA_RW: u64 = 0x002; pub const PA_US: u64 = 0x004; @@ -32,12 +36,13 @@ pub const V_KERNEL_BSS_START: VAddr = VAddr::from(0xffffffffc0200000); const KERNEL_PML4_PFN: PFN = PFN::from_val(0x1000 >> 12); -pub static GLOBAL_PAGE_TABLE: LazyLock> = - LazyLock::new(|| unsafe { - Page::with_raw(KERNEL_PML4_PFN, |root_table_page| { - PageTable::with_root_table(root_table_page.clone()) - }) - }); +pub static GLOBAL_PAGE_TABLE: LazyLock< + PageTable, +> = LazyLock::new(|| unsafe { + Page::with_raw(KERNEL_PML4_PFN, |root_table_page| { + PageTable::with_root_table(root_table_page.clone()) + }) +}); #[repr(transparent)] pub struct PTE64(u64); diff --git a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs index 196a4ad3..176f2de1 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs @@ -1,11 +1,11 @@ -use crate::processor::CPU; use core::arch::asm; -use eonix_hal_traits::{ - fault::{Fault, PageFaultErrorCode}, - trap::{RawTrapContext, TrapType}, -}; + +use eonix_hal_traits::fault::{Fault, PageFaultErrorCode}; +use eonix_hal_traits::trap::{RawTrapContext, TrapType}; use eonix_mm::address::VAddr; +use crate::processor::CPU; + #[derive(Clone, Copy, Default)] #[repr(C, align(16))] pub struct TrapContext { @@ -169,12 +169,8 @@ impl RawTrapContext for TrapContext { } fn set_user_call_frame( - &mut self, - pc: usize, - sp: Option, - ra: Option, - args: &[usize], - write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, + &mut self, pc: usize, sp: Option, ra: Option, + args: &[usize], write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, ) -> Result<(), E> { self.set_program_counter(pc); diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 3e815f25..1a5e30ad 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -16,17 +16,21 @@ use posix_types::SIGNAL_NOW; use super::SyscallNoReturn; use crate::io::Buffer; use crate::kernel::constants::{ - CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, ENOSYS, ENOTDIR, - ERANGE, ESRCH, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, + CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, + ENOSYS, ENOTDIR, ERANGE, ESRCH, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, + SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, }; use crate::kernel::mem::PageBuffer; use crate::kernel::syscall::{User, UserMut}; use crate::kernel::task::{ - do_clone, futex_exec, futex_wait, futex_wake, parse_futexop, yield_now, CloneArgs, FutexFlags, - FutexOp, ProcessList, ProgramLoader, RobustListHead, SignalAction, Thread, WaitId, WaitType, + do_clone, futex_exec, futex_wait, futex_wake, parse_futexop, yield_now, + CloneArgs, FutexFlags, FutexOp, ProcessList, ProgramLoader, RobustListHead, + SignalAction, Thread, WaitId, WaitType, }; use crate::kernel::timer::sleep; -use crate::kernel::user::{UserBuffer, UserPointer, UserPointerMut, UserString}; +use crate::kernel::user::{ + UserBuffer, UserPointer, UserPointerMut, UserString, +}; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::types::Permission; use crate::kernel::vfs::{self}; @@ -49,7 +53,9 @@ bitflags! { } #[eonix_macros::define_syscall(SYS_NANOSLEEP)] -async fn nanosleep(req: User<(u32, u32)>, rem: UserMut<(u32, u32)>) -> KResult { +async fn nanosleep( + req: User<(u32, u32)>, rem: UserMut<(u32, u32)>, +) -> KResult { let req = UserPointer::new(req)?.read()?; let rem = if rem.is_null() { None @@ -57,7 +63,8 @@ async fn nanosleep(req: User<(u32, u32)>, rem: UserMut<(u32, u32)>) -> KResult, rem: UserMut<(u32, u32)>) -> KResult, - rem: UserMut<(u32, u32)>, + clock_id: u32, _flags: u32, req: User<(u32, u32)>, rem: UserMut<(u32, u32)>, ) -> KResult { if clock_id != CLOCK_REALTIME && clock_id != CLOCK_REALTIME_COARSE @@ -88,7 +92,8 @@ async fn clock_nanosleep( Some(UserPointerMut::new(rem)?) }; - let duration = Duration::from_secs(req.0 as u64) + Duration::from_nanos(req.1 as u64); + let duration = + Duration::from_secs(req.0 as u64) + Duration::from_nanos(req.1 as u64); sleep(duration).await; if let Some(rem) = rem { @@ -101,7 +106,8 @@ async fn clock_nanosleep( #[eonix_macros::define_syscall(SYS_UMASK)] async fn umask(raw_new_mask: u32) -> KResult { let new_mask = Permission::new(!raw_new_mask); - let old_mask = core::mem::replace(&mut *thread.fs_context.umask.lock(), new_mask); + let old_mask = + core::mem::replace(&mut *thread.fs_context.umask.lock(), new_mask); Ok(!old_mask.bits()) } @@ -147,7 +153,9 @@ async fn umount(source: User) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_MOUNT)] -async fn mount(source: User, target: User, fstype: User, flags: usize) -> KResult<()> { +async fn mount( + source: User, target: User, fstype: User, flags: usize, +) -> KResult<()> { let source = UserString::new(source)?; if source.as_cstr().to_str().unwrap() == "/dev/vda2" { return Ok(()); @@ -176,7 +184,9 @@ async fn mount(source: User, target: User, fstype: User, flags: usiz .await } -fn get_strings(mut ptr_strings: UserPointer<'_, PtrT>) -> KResult> { +fn get_strings( + mut ptr_strings: UserPointer<'_, PtrT>, +) -> KResult> { let mut strings = Vec::new(); loop { @@ -194,24 +204,34 @@ fn get_strings(mut ptr_strings: UserPointer<'_, PtrT>) -> KResult> } #[eonix_macros::define_syscall(SYS_EXECVE)] -async fn execve(exec: User, argv: User, envp: User) -> KResult { +async fn execve( + exec: User, argv: User, envp: User, +) -> KResult { let exec = UserString::new(exec)?; let exec = exec.as_cstr().to_owned(); let argv = get_strings(UserPointer::new(argv)?)?; let envp = get_strings(UserPointer::new(envp)?)?; - let dentry = Dentry::open(&thread.fs_context, Path::new(exec.as_bytes())?, true).await?; + let dentry = + Dentry::open(&thread.fs_context, Path::new(exec.as_bytes())?, true) + .await?; if !dentry.is_valid() { Err(ENOENT)?; } // TODO: When `execve` is called by one of the threads in a process, the other threads // should be terminated and `execve` is performed in the thread group leader. - let load_info = ProgramLoader::parse(&thread.fs_context, exec, dentry.clone(), argv, envp) - .await? - .load() - .await?; + let load_info = ProgramLoader::parse( + &thread.fs_context, + exec, + dentry.clone(), + argv, + envp, + ) + .await? + .load() + .await?; futex_exec(thread).await; @@ -257,10 +277,7 @@ enum WaitInfo { } async fn do_waitid( - thread: &Thread, - wait_id: WaitId, - info: WaitInfo, - options: u32, + thread: &Thread, wait_id: WaitId, info: WaitInfo, options: u32, rusage: UserMut, ) -> KResult { if !rusage.is_null() { @@ -300,7 +317,8 @@ async fn do_waitid( Ok(0) } WaitInfo::Status(status_ptr) => { - UserPointerMut::new(status_ptr)?.write(wait_object.code.to_wstatus())?; + UserPointerMut::new(status_ptr)? + .write(wait_object.code.to_wstatus())?; Ok(wait_object.pid) } WaitInfo::None => Ok(wait_object.pid), @@ -309,10 +327,7 @@ async fn do_waitid( #[eonix_macros::define_syscall(SYS_WAITID)] async fn waitid( - id_type: u32, - id: u32, - info: UserMut, - options: u32, + id_type: u32, id: u32, info: UserMut, options: u32, rusage: UserMut, ) -> KResult { let wait_id = WaitId::from_type_and_id(id_type, id)?; @@ -334,10 +349,7 @@ async fn waitid( #[eonix_macros::define_syscall(SYS_WAIT4)] async fn wait4( - wait_id: i32, - arg1: UserMut, - options: u32, - rusage: UserMut, + wait_id: i32, arg1: UserMut, options: u32, rusage: UserMut, ) -> KResult { let waitinfo = if arg1.is_null() { WaitInfo::None @@ -352,7 +364,9 @@ async fn wait4( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_WAITPID)] -async fn waitpid(waitpid: i32, arg1: UserMut, options: u32) -> KResult { +async fn waitpid( + waitpid: i32, arg1: UserMut, options: u32, +) -> KResult { sys_wait4(thread, waitpid, arg1, options, UserMut::null()).await } @@ -503,7 +517,9 @@ async fn set_tid_address(tidptr: UserMut) -> KResult { async fn prctl(option: u32, arg2: PtrT) -> KResult<()> { match option { PR_SET_NAME => { - let name = UserPointer::<[u8; 16]>::new(User::with_addr(arg2.addr()))?.read()?; + let name = + UserPointer::<[u8; 16]>::new(User::with_addr(arg2.addr()))? + .read()?; let len = name.iter().position(|&c| c == 0).unwrap_or(15); thread.set_name(name[..len].into()); Ok(()) @@ -511,8 +527,10 @@ async fn prctl(option: u32, arg2: PtrT) -> KResult<()> { PR_GET_NAME => { let name = thread.get_name(); let len = name.len().min(15); - let name: [u8; 16] = core::array::from_fn(|i| if i < len { name[i] } else { 0 }); - UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(arg2.addr()))?.write(name)?; + let name: [u8; 16] = + core::array::from_fn(|i| if i < len { name[i] } else { 0 }); + UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(arg2.addr()))? + .write(name)?; Ok(()) } _ => Err(EINVAL), @@ -572,10 +590,7 @@ async fn tgkill(tgid: u32, tid: u32, sig: u32) -> KResult<()> { #[eonix_macros::define_syscall(SYS_RT_SIGPROCMASK)] async fn rt_sigprocmask( - how: u32, - set: UserMut, - oldset: UserMut, - sigsetsize: usize, + how: u32, set: UserMut, oldset: UserMut, sigsetsize: usize, ) -> KResult<()> { if sigsetsize != size_of::() { return Err(EINVAL); @@ -607,9 +622,7 @@ async fn rt_sigprocmask( eonix_macros::define_syscall(SYS_RT_SIGTIMEDWAIT) )] async fn rt_sigtimedwait( - _uthese: User, - _uinfo: UserMut, - _uts: User, + _uthese: User, _uinfo: UserMut, _uts: User, ) -> KResult { // TODO Ok(0) @@ -617,9 +630,7 @@ async fn rt_sigtimedwait( #[eonix_macros::define_syscall(SYS_RT_SIGACTION)] async fn rt_sigaction( - signum: u32, - act: User, - oldact: UserMut, + signum: u32, act: User, oldact: UserMut, sigsetsize: usize, ) -> KResult<()> { let signal = Signal::try_from_raw(signum)?; @@ -649,9 +660,7 @@ async fn rt_sigaction( #[eonix_macros::define_syscall(SYS_PRLIMIT64)] async fn prlimit64( - pid: u32, - resource: u32, - new_limit: User, + pid: u32, resource: u32, new_limit: User, old_limit: UserMut, ) -> KResult<()> { if pid != 0 { @@ -764,13 +773,16 @@ async fn fork() -> KResult { #[cfg(not(target_arch = "loongarch64"))] #[eonix_macros::define_syscall(SYS_CLONE)] async fn clone( - clone_flags: usize, - new_sp: usize, - parent_tidptr: UserMut, - tls: PtrT, + clone_flags: usize, new_sp: usize, parent_tidptr: UserMut, tls: PtrT, child_tidptr: UserMut, ) -> KResult { - let clone_args = CloneArgs::for_clone(clone_flags, new_sp, child_tidptr, parent_tidptr, tls)?; + let clone_args = CloneArgs::for_clone( + clone_flags, + new_sp, + child_tidptr, + parent_tidptr, + tls, + )?; do_clone(thread, clone_args).await } @@ -778,24 +790,23 @@ async fn clone( #[cfg(target_arch = "loongarch64")] #[eonix_macros::define_syscall(SYS_CLONE)] async fn clone( - clone_flags: usize, - new_sp: usize, - parent_tidptr: UserMut, - child_tidptr: UserMut, - tls: usize, + clone_flags: usize, new_sp: usize, parent_tidptr: UserMut, + child_tidptr: UserMut, tls: usize, ) -> KResult { - let clone_args = CloneArgs::for_clone(clone_flags, new_sp, child_tidptr, parent_tidptr, tls)?; + let clone_args = CloneArgs::for_clone( + clone_flags, + new_sp, + child_tidptr, + parent_tidptr, + tls, + )?; do_clone(thread, clone_args).await } #[eonix_macros::define_syscall(SYS_FUTEX)] async fn futex( - uaddr: usize, - op: u32, - val: u32, - _time_out: usize, - _uaddr2: usize, + uaddr: usize, op: u32, val: u32, _time_out: usize, _uaddr2: usize, _val3: u32, ) -> KResult { let (futex_op, futex_flag) = parse_futexop(op)?; @@ -824,7 +835,9 @@ async fn futex( } #[eonix_macros::define_syscall(SYS_SET_ROBUST_LIST)] -async fn set_robust_list(head: User, len: usize) -> KResult<()> { +async fn set_robust_list( + head: User, len: usize, +) -> KResult<()> { if len != size_of::() { return Err(EINVAL); } @@ -875,7 +888,9 @@ async fn sigreturn() -> KResult { async fn arch_prctl(option: u32, addr: PtrT) -> KResult { match option { PR_SET_NAME => { - let name = UserPointer::<[u8; 16]>::new(User::with_addr(addr.addr()))?.read()?; + let name = + UserPointer::<[u8; 16]>::new(User::with_addr(addr.addr()))? + .read()?; let len = name.iter().position(|&c| c == 0).unwrap_or(15); thread.set_name(name[..len].into()); Ok(0) @@ -883,8 +898,10 @@ async fn arch_prctl(option: u32, addr: PtrT) -> KResult { PR_GET_NAME => { let name = thread.get_name(); let len = name.len().min(15); - let name: [u8; 16] = core::array::from_fn(|i| if i < len { name[i] } else { 0 }); - UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(addr.addr()))?.write(name)?; + let name: [u8; 16] = + core::array::from_fn(|i| if i < len { name[i] } else { 0 }); + UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(addr.addr()))? + .write(name)?; Ok(0) } _ => Err(EINVAL), diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 6505666c..6bbccb28 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -13,12 +13,14 @@ mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; use eonix_hal::symbol_addr; pub use futex::{ - futex_exec, futex_exit, futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, - RobustListHead, + futex_exec, futex_exit, futex_wait, futex_wake, parse_futexop, FutexFlags, + FutexOp, RobustListHead, }; pub use kernel_stack::KernelStack; pub use loader::ProgramLoader; -pub use process::{alloc_pid, Process, ProcessBuilder, WaitId, WaitObject, WaitType}; +pub use process::{ + alloc_pid, Process, ProcessBuilder, WaitId, WaitObject, WaitType, +}; pub use process_group::ProcessGroup; pub use process_list::ProcessList; pub use session::Session; @@ -101,12 +103,16 @@ where use eonix_runtime::task::Task; use thread::wait_for_wakeups; - use crate::kernel::interrupt::{default_fault_handler, default_irq_handler}; + use crate::kernel::interrupt::{ + default_fault_handler, default_irq_handler, + }; use crate::kernel::timer::{should_reschedule, timer_interrupt}; let stack = KernelStack::new(); - fn execute(mut future: Pin<&mut F>, output_ptr: NonNull>) -> ! + fn execute( + mut future: Pin<&mut F>, output_ptr: NonNull>, + ) -> ! where F: Future, { @@ -139,7 +145,9 @@ where match future.as_mut().poll(&mut cx) { Poll::Ready(output) => break output, Poll::Pending => { - assert_preempt_enabled!("Blocking in stackful futures is not allowed."); + assert_preempt_enabled!( + "Blocking in stackful futures is not allowed." + ); if Task::current().is_ready() { continue; @@ -203,7 +211,9 @@ where match trap_ctx.trap_type() { TrapType::Syscall { .. } => {} - TrapType::Fault(fault) => default_fault_handler(fault, &mut trap_ctx), + TrapType::Fault(fault) => { + default_fault_handler(fault, &mut trap_ctx) + } TrapType::Irq { callback } => callback(default_irq_handler), TrapType::Timer { callback } => { callback(timer_interrupt); @@ -220,10 +230,12 @@ where } #[cfg(target_arch = "riscv64")] - trap_ctx.set_program_counter(trap_ctx.get_program_counter() + 2); + trap_ctx + .set_program_counter(trap_ctx.get_program_counter() + 2); #[cfg(target_arch = "loongarch64")] - trap_ctx.set_program_counter(trap_ctx.get_program_counter() + 4); + trap_ctx + .set_program_counter(trap_ctx.get_program_counter() + 4); } } } diff --git a/src/kernel/task/signal/signal_action.rs b/src/kernel/task/signal/signal_action.rs index 18348c32..cb70ccf4 100644 --- a/src/kernel/task/signal/signal_action.rs +++ b/src/kernel/task/signal/signal_action.rs @@ -141,10 +141,7 @@ impl SignalActionList { impl SignalAction { /// # Might Sleep pub(super) fn handle( - self, - signal: Signal, - old_mask: SigSet, - trap_ctx: &mut TrapContext, + self, signal: Signal, old_mask: SigSet, trap_ctx: &mut TrapContext, fpu_state: &mut FpuState, ) -> KResult<()> { let SignalAction::SimpleHandler { From 1be77d9145b29b72c1fa985181fbdec8c0b6b8c4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 00:37:32 +0800 Subject: [PATCH 035/153] signal: remove await on thread.force_kill in x86 sigreturn Prepare for x86 build fix. Fixes: 59f044422cd4 ("mm, proc: add an exited thread reaper") Signed-off-by: greatbridf --- src/kernel/syscall/procops.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 1a5e30ad..83b14081 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -876,7 +876,7 @@ async fn sigreturn() -> KResult { "`sigreturn` failed in thread {} with error {err}!", thread.tid ); - thread.force_kill(Signal::SIGSEGV).await; + thread.force_kill(Signal::SIGSEGV); return Err(err); } From e567c689ce409cd6f29c70bf9853d49a3b1bf7d3 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 00:38:55 +0800 Subject: [PATCH 036/153] style: extract common import out in vdso rt_sigreturn No functional change. Signed-off-by: greatbridf --- src/kernel/task/signal/signal_action.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/kernel/task/signal/signal_action.rs b/src/kernel/task/signal/signal_action.rs index cb70ccf4..2ee6dec9 100644 --- a/src/kernel/task/signal/signal_action.rs +++ b/src/kernel/task/signal/signal_action.rs @@ -12,6 +12,7 @@ use posix_types::signal::{ SigAction, SigActionHandler, SigActionRestorer, SigSet, Signal, TryFromSigAction, }; +use posix_types::syscall_no::SYS_RT_SIGRETURN; use posix_types::SIGNAL_NOW; use super::{KResult, SAVED_DATA_SIZE}; @@ -57,21 +58,21 @@ unsafe extern "C" fn vdso_rt_sigreturn() { naked_asm!( "li a7, {sys_rt_sigreturn}", "ecall", - sys_rt_sigreturn = const posix_types::syscall_no::SYS_RT_SIGRETURN, + sys_rt_sigreturn = const SYS_RT_SIGRETURN, ); #[cfg(target_arch = "loongarch64")] naked_asm!( "li.d $a7, {sys_rt_sigreturn}", "syscall 0", - sys_rt_sigreturn = const posix_types::syscall_no::SYS_RT_SIGRETURN, + sys_rt_sigreturn = const SYS_RT_SIGRETURN, ); #[cfg(target_arch = "x86_64")] naked_asm!( "mov ${sys_rt_sigreturn}, %eax", "int $0x80", - sys_rt_sigreturn = const posix_types::syscall_no::SYS_RT_SIGRETURN, + sys_rt_sigreturn = const SYS_RT_SIGRETURN, options(att_syntax), ); } From 22ac59408ab525d1851524faf7881bc1c8e0c0eb Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 00:40:24 +0800 Subject: [PATCH 037/153] style: reformat x86 bootstrap code No functional changes. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/bootstrap.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs index f2637c50..dd4f1330 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs @@ -1,8 +1,9 @@ pub(crate) mod init; -use super::mm::{E820_MEM_MAP_DATA, PA_G, PA_NXE, PA_P, PA_PS, PA_RW}; use core::arch::{global_asm, naked_asm}; +use super::mm::{E820_MEM_MAP_DATA, PA_G, PA_NXE, PA_P, PA_PS, PA_RW}; + const KERNEL_IMAGE_PADDR: usize = 0x200000; const KERNEL_PML4: usize = 0x1000; const KERNEL_PDPT_PHYS_MAPPING: usize = 0x2000; @@ -171,7 +172,7 @@ global_asm!( .Lhalt: hlt jmp . - + # scratch %eax # return address should be of 2 bytes, and will be zero extended to 4 bytes .Lgo_32bit: @@ -462,7 +463,7 @@ global_asm!( .code64 2: jmp {start_64bit} - + .popsection "#, EARLY_GDT = sym EARLY_GDT, From 8240ec2b82f40925fe52fd0f3555cc92a41b88fb Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 00:42:50 +0800 Subject: [PATCH 038/153] x86, hal: support loading kernel image up to 14M Still use only one kernel page table: use huge TLBs for all but the last 2M. Map those in the bootstrap code with a loop and then PTEs for the rest. Theoretically, this way we can support up to kernel images with up to 1G in size. Keep the image small for now. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/bootstrap.rs | 31 ++++++++++++++++--- crates/eonix_hal/src/arch/x86_64/memory.x | 2 +- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs index dd4f1330..0ad57a49 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs @@ -409,17 +409,38 @@ global_asm!( mov ${KERNEL_PD_KIMAGE}, %esi call fill_pxe - # PDE 0xff0 + # PDE 0xfc0 ~ ? mov ${KERNEL_PD_KIMAGE}, %edi - lea 0xff0(%edi), %edi - mov ${KERNEL_PT_KIMAGE}, %esi # 0x104000 + mov ${KERNEL_IMAGE_PADDR}, %esi + lea 0xfc0(%edi), %edi + + mov ${KIMAGE_PAGES}, %ecx + shr $9, %ecx # 2M blocks + + or ${PA_PS}, %ebx + + 2: + test %ecx, %ecx + jz 2f + + call fill_pxe + lea 8(%edi), %edi + add $0x200000, %esi + dec %ecx + jmp 2b + + 2: + and $(~{PA_PS}), %ebx + push %esi + mov ${KERNEL_PT_KIMAGE}, %esi call fill_pxe # fill PT (kernel image) mov ${KERNEL_PT_KIMAGE}, %edi - mov ${KERNEL_IMAGE_PADDR}, %esi + pop %esi mov ${KIMAGE_PAGES}, %ecx + and $511, %ecx 2: call fill_pxe @@ -454,10 +475,12 @@ global_asm!( # %esi: page physical address # %edi: page x entry address fill_pxe: + push %eax lea (%ebx, %esi, 1), %eax mov %eax, (%edi) mov %edx, 4(%edi) + pop %eax ret .code64 diff --git a/crates/eonix_hal/src/arch/x86_64/memory.x b/crates/eonix_hal/src/arch/x86_64/memory.x index 63f05753..67fdb7b1 100644 --- a/crates/eonix_hal/src/arch/x86_64/memory.x +++ b/crates/eonix_hal/src/arch/x86_64/memory.x @@ -2,7 +2,7 @@ MEMORY { LOWMEM : org = 0x0000000000000000, len = 1M VDSO : org = 0x00007f0000000000, len = 4K KBSS : org = 0xffffffffc0200000, len = 2M - KIMAGE : org = 0xffffffffffc00000, len = 2M + KIMAGE : org = 0xffffffffff000000, len = 14M } REGION_ALIAS("REGION_TEXT", KIMAGE); From 7b45c831b5d7b315ff48567308aea09ffca99513 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 00:59:47 +0800 Subject: [PATCH 039/153] kernel: put kernel shutdown into `power` module - Add abstraction for kernel shutting down. - Use this in the kernel panic routine. Signed-off-by: greatbridf --- src/kernel.rs | 2 ++ src/kernel/power.rs | 46 +++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 26 +++---------------------- src/panic.rs | 18 +++--------------- 4 files changed, 54 insertions(+), 38 deletions(-) create mode 100644 src/kernel/power.rs diff --git a/src/kernel.rs b/src/kernel.rs index 28d69626..39e3f9d3 100644 --- a/src/kernel.rs +++ b/src/kernel.rs @@ -12,8 +12,10 @@ pub mod user; pub mod vfs; mod chardev; +mod power; mod terminal; #[allow(unused_imports)] pub use chardev::{CharDevice, CharDeviceType, VirtualCharDevice}; +pub use power::shutdown_system; pub use terminal::{Terminal, TerminalDevice}; diff --git a/src/kernel/power.rs b/src/kernel/power.rs new file mode 100644 index 00000000..7cbe3a08 --- /dev/null +++ b/src/kernel/power.rs @@ -0,0 +1,46 @@ +use core::sync::atomic::{AtomicUsize, Ordering}; + +use eonix_hal::processor::{halt, CPU, CPU_COUNT}; +use eonix_log::println_info; + +static CPU_SHUTTING_DOWN: AtomicUsize = AtomicUsize::new(0); + +fn shutdown() { + #[cfg(arch_has_shutdown)] + { + eonix_hal::arch_exported::bootstrap::shutdown(); + } + #[cfg(not(arch_has_shutdown))] + { + println_info!("Shutdown not supported on this architecture..."); + } +} + +pub fn shutdown_system_nowait() { + shutdown(); + + loop { + halt(); + } +} + +pub fn shutdown_system() -> ! { + // TODO: We don't have IPI system for now. + shutdown_system_nowait(); + + let cpu_count = CPU_COUNT.load(Ordering::Relaxed); + + if CPU_SHUTTING_DOWN.fetch_add(1, Ordering::AcqRel) + 1 == cpu_count { + println_info!("All CPUs are shutting down. Gracefully powering off..."); + shutdown(); + } else { + println_info!( + "CPU {} is shutting down. Waiting for other CPUs...", + CPU::local().cpuid() + ); + } + + loop { + halt(); + } +} diff --git a/src/lib.rs b/src/lib.rs index f49e543e..bce90900 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,11 +28,10 @@ mod sync; use alloc::ffi::CString; use core::hint::spin_loop; -use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicBool, Ordering}; -use eonix_hal::arch_exported::bootstrap::shutdown; use eonix_hal::context::TaskContext; -use eonix_hal::processor::{halt, CPU, CPU_COUNT}; +use eonix_hal::processor::CPU; use eonix_hal::symbol_addr; use eonix_hal::traits::context::RawTaskContext; use eonix_hal::traits::trap::IrqState; @@ -48,31 +47,12 @@ use kernel::vfs::mount::{ }; use kernel::vfs::types::Permission; use kernel::vfs::FsContext; -use kernel::CharDevice; +use kernel::{shutdown_system, CharDevice}; use kernel_init::setup_memory; use path::Path; use prelude::*; static BSP_OK: AtomicBool = AtomicBool::new(false); -static CPU_SHUTTING_DOWN: AtomicUsize = AtomicUsize::new(0); - -fn shutdown_system() -> ! { - let cpu_count = CPU_COUNT.load(Ordering::Relaxed); - - if CPU_SHUTTING_DOWN.fetch_add(1, Ordering::AcqRel) + 1 == cpu_count { - println_info!("All CPUs are shutting down. Gracefully powering off..."); - shutdown(); - } else { - println_info!( - "CPU {} is shutting down. Waiting for other CPUs...", - CPU::local().cpuid() - ); - - loop { - halt(); - } - } -} #[eonix_hal::main] fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { diff --git a/src/panic.rs b/src/panic.rs index 46409de2..75471169 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,6 +2,8 @@ use core::panic::PanicInfo; use eonix_log::println_fatal; +use crate::kernel::shutdown_system; + #[panic_handler] fn panic(info: &PanicInfo) -> ! { if let Some(location) = info.location() { @@ -20,21 +22,7 @@ fn panic(info: &PanicInfo) -> ! { #[cfg(arch_has_stacktrace)] stacktrace::print_stacktrace(); - panic_forever(); -} - -fn panic_forever() -> ! { - #[cfg(arch_has_shutdown)] - { - eonix_hal::arch_exported::bootstrap::shutdown(); - } - #[cfg(not(arch_has_shutdown))] - { - // Spin forever - loop { - core::hint::spin_loop(); - } - } + shutdown_system(); } #[cfg(arch_has_stacktrace)] From b1aa66e28fe19b833cd830aedbc61094f414ee49 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 01:00:50 +0800 Subject: [PATCH 040/153] x86: add shutdown support Perform the shutdown with some port IO in an emulator-specific method. Signed-off-by: greatbridf --- build.rs | 1 + crates/eonix_hal/src/arch/x86_64/bootstrap.rs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/build.rs b/build.rs index 68321dc9..cea0d5d6 100644 --- a/build.rs +++ b/build.rs @@ -32,6 +32,7 @@ fn set_arch_config(config: &str) { fn set_arch_configs_x86() { set_arch_config("has_stacktrace"); + set_arch_config("has_shutdown"); } fn set_arch_configs_riscv64() { diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs index 0ad57a49..87905d08 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs @@ -538,3 +538,14 @@ pub unsafe extern "C" fn start_64bit() { options(att_syntax) ) } + +pub fn shutdown() -> ! { + unsafe { + core::arch::asm!( + "mov $0x2000, %ax", + "mov $0x604, %dx", + "outw %ax, %dx", + options(att_syntax, nomem, noreturn) + ); + } +} From 0e16753d5ca0d2378f6c1ad2e6ef336d1ba3c5d3 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 01:03:26 +0800 Subject: [PATCH 041/153] x86: export symbols needed by vdso Put .vdso in .data memory area. Export `VDSO_START` symbol. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/link.x | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/eonix_hal/src/arch/x86_64/link.x b/crates/eonix_hal/src/arch/x86_64/link.x index 1fedd92e..7a836aa0 100644 --- a/crates/eonix_hal/src/arch/x86_64/link.x +++ b/crates/eonix_hal/src/arch/x86_64/link.x @@ -37,10 +37,12 @@ SECTIONS { SECTIONS { .vdso ALIGN(0x1000) : ALIGN(0x1000) { + VDSO_START = .; + KEEP(*(.vdso .vdso.*)); . = ALIGN(0x1000); - } > VDSO AT> REGION_TEXT + } > REGION_DATA AT> REGION_DATA VDSO_PADDR = LOADADDR(.vdso); } From fde5a436a0d782304d2eeaaa27a871639e761552 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 01:08:19 +0800 Subject: [PATCH 042/153] x86, TLS: adapt for the new TLS design TLS should be a kernel feature instead of HAL work. Maybe we should provide a more native interface for x86? The changes should have been included in the commit. Pick them in now. Fixes: 9a0eeb00ae79 ("tls: rework of arch's UserTLS design") Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/cpu.rs | 39 ++----------------------- crates/eonix_hal/src/arch/x86_64/gdt.rs | 10 ++++++- 2 files changed, 12 insertions(+), 37 deletions(-) diff --git a/crates/eonix_hal/src/arch/x86_64/cpu.rs b/crates/eonix_hal/src/arch/x86_64/cpu.rs index 03ca78d2..528935a8 100644 --- a/crates/eonix_hal/src/arch/x86_64/cpu.rs +++ b/crates/eonix_hal/src/arch/x86_64/cpu.rs @@ -35,17 +35,6 @@ pub(crate) struct TSS { _pinned: PhantomPinned, } -#[derive(Debug, Clone)] -pub enum UserTLS { - /// TODO: This is not used yet. - #[allow(dead_code)] - TLS64(u64), - TLS32 { - base: u64, - desc: GDTEntry, - }, -} - /// Architecture-specific cpu status data. pub struct CPU { cpuid: usize, @@ -54,24 +43,6 @@ pub struct CPU { interrupt: InterruptControl, } -impl UserTLS { - /// # Return - /// Returns the TLS descriptor and the index of the TLS segment. - pub fn new32( - base: u32, limit: u32, is_limit_in_pages: bool, - ) -> (Self, u32) { - let flags = if is_limit_in_pages { 0xc } else { 0x4 }; - - ( - Self::TLS32 { - base: base as u64, - desc: GDTEntry::new(base, limit, 0xf2, flags), - }, - 7, - ) - } -} - impl CPU { pub fn new() -> Self { let (interrupt_control, cpuid) = InterruptControl::new(); @@ -126,18 +97,14 @@ impl CPU { } } - pub fn set_tls32(self: Pin<&mut Self>, user_tls: &UserTLS) { - let UserTLS::TLS32 { desc, base } = user_tls else { - unimplemented!("TLS64 is not supported yet") - }; - + pub fn set_tls32(self: Pin<&mut Self>, desc: GDTEntry, base: u64) { unsafe { // SAFETY: We don't move the GDT object. - self.get_unchecked_mut().gdt.set_tls32(*desc); + self.get_unchecked_mut().gdt.set_tls32(desc); } const IA32_KERNEL_GS_BASE: u32 = 0xc0000102; - wrmsr(IA32_KERNEL_GS_BASE, *base); + wrmsr(IA32_KERNEL_GS_BASE, base); } pub fn cpuid(&self) -> usize { diff --git a/crates/eonix_hal/src/arch/x86_64/gdt.rs b/crates/eonix_hal/src/arch/x86_64/gdt.rs index 4543e3e0..a455d5b0 100644 --- a/crates/eonix_hal/src/arch/x86_64/gdt.rs +++ b/crates/eonix_hal/src/arch/x86_64/gdt.rs @@ -33,6 +33,14 @@ impl GDTEntry { GDTEntry(entry) } + pub const fn new_tls(base: u32, limit_in_bytes: u32) -> Self { + Self::new(base, limit_in_bytes, 0xf2, 0x4) + } + + pub const fn new_tls_page_limit(base: u32, limit_in_pages: u32) -> Self { + Self::new(base, limit_in_pages, 0xf2, 0xc) + } + pub const fn new_ldt(base: u64, limit: u32) -> [Self; 2] { let first = Self::new(base as u32, limit, 0x82, 0x0); let second = Self(base >> 32); @@ -48,7 +56,7 @@ impl GDTEntry { impl GDT { const LEN: usize = 10; - const TLS32_INDEX: usize = 7; + pub const TLS32_INDEX: usize = 7; const TSS_INDEX: usize = 8; pub fn new() -> Self { From 60d18e510a047c3cb3e24482a81a42d04a865fde Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 01:12:00 +0800 Subject: [PATCH 043/153] hal, x86: add cpu count global variable Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/cpu.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/eonix_hal/src/arch/x86_64/cpu.rs b/crates/eonix_hal/src/arch/x86_64/cpu.rs index 528935a8..951c8f32 100644 --- a/crates/eonix_hal/src/arch/x86_64/cpu.rs +++ b/crates/eonix_hal/src/arch/x86_64/cpu.rs @@ -2,6 +2,7 @@ use core::arch::asm; use core::marker::PhantomPinned; use core::mem::size_of; use core::pin::Pin; +use core::sync::atomic::AtomicUsize; use eonix_preempt::PreemptGuard; use eonix_sync_base::LazyLock; @@ -10,6 +11,8 @@ use super::gdt::{GDTEntry, GDT}; use super::interrupt::InterruptControl; use super::trap::TrapContext; +pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(0); + #[eonix_percpu::define_percpu] static LOCAL_CPU: LazyLock = LazyLock::new(CPU::new); From 4ccaa6beeb43536c1f8d043e2ce933d9098d8c39 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 01:15:12 +0800 Subject: [PATCH 044/153] hal, x86: support the new folio abstraction Check f9b5b3a3dfe6 ("mem: introduce new `Folio` abstraction") for more details. Signed-off-by: greatbridf --- .../src/arch/x86_64/bootstrap/init.rs | 62 +++++++++---------- crates/eonix_hal/src/arch/x86_64/mm.rs | 42 +++++++++---- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs index f12a98f0..2f92ee8e 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs @@ -9,17 +9,20 @@ use acpi::{AcpiHandler, AcpiTables, PhysicalMapping, PlatformInfo}; use eonix_hal_traits::mm::Memory; use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VRange}; use eonix_mm::page_table::{PageAttribute, PagingMode, PTE as _}; -use eonix_mm::paging::{Page, PageAccess, PageAlloc, PAGE_SIZE}; +use eonix_mm::paging::{Folio, FrameAlloc, PageAccess, PAGE_SIZE}; use eonix_percpu::PercpuArea; use crate::arch::bootstrap::{EARLY_GDT_DESCRIPTOR, KERNEL_PML4}; use crate::arch::cpu::{wrmsr, CPU}; use crate::arch::io::Port8; -use crate::arch::mm::{ArchPhysAccess, GLOBAL_PAGE_TABLE, V_KERNEL_BSS_START}; +use crate::arch::mm::{ + with_global_page_table, ArchPhysAccess, PageAccessImpl, GLOBAL_PAGE_TABLE, + V_KERNEL_BSS_START, +}; use crate::bootstrap::BootStrapData; +use crate::extern_symbol_value; use crate::mm::{ - ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, - ScopedAllocator, + ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, }; static BSP_PAGE_ALLOC: AtomicPtr> = @@ -122,14 +125,17 @@ fn enable_sse() { } } -fn setup_cpu(alloc: impl PageAlloc) { +fn setup_cpu(alloc: impl FrameAlloc) { let mut percpu_area = PercpuArea::new(|layout| { // TODO: Use page size defined in `arch`. let page_count = layout.size().div_ceil(PAGE_SIZE); - let page = Page::alloc_at_least_in(page_count, alloc); + let folio = alloc.alloc_at_least(page_count).unwrap(); - let ptr = ArchPhysAccess::get_ptr_for_page(&page).cast(); - page.into_raw(); + let ptr = unsafe { + // TODO: safety + ArchPhysAccess::as_ptr(folio.start()) + }; + folio.into_raw(); ptr }); @@ -225,7 +231,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { let stack_range = { let page_alloc = BasicPageAllocRef::new(&page_alloc); - let ap_stack = Page::alloc_order_in(4, page_alloc); + let ap_stack = page_alloc.alloc_order(4).unwrap(); let stack_range = ap_stack.range(); ap_stack.into_raw(); @@ -269,44 +275,38 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell) { } pub extern "C" fn kernel_init() -> ! { - let global_page_table = &GLOBAL_PAGE_TABLE; - let paging_levels = ArchPagingMode::LEVELS; - enable_sse(); let real_allocator = RefCell::new(BasicPageAlloc::new()); let alloc = BasicPageAllocRef::new(&real_allocator); - unsafe extern "C" { - fn BSS_LENGTH(); - } + let bss_length = extern_symbol_value!(BSS_LENGTH); + let bss_range = VRange::from(V_KERNEL_BSS_START).grow(bss_length); for range in ArchMemory::free_ram() { real_allocator.borrow_mut().add_range(range); } // Map kernel BSS - for pte in global_page_table.iter_kernel_in( - VRange::from(V_KERNEL_BSS_START).grow(BSS_LENGTH as usize), - paging_levels, - &alloc, - ) { - let attr = PageAttribute::PRESENT - | PageAttribute::WRITE - | PageAttribute::READ - | PageAttribute::HUGE - | PageAttribute::GLOBAL; - - let page = Page::alloc_in(&alloc); - pte.set(page.into_raw(), attr.into()); - } + with_global_page_table(alloc.clone(), PageAccessImpl, |table| { + for pte in table.iter_kernel(bss_range) { + let attr = PageAttribute::PRESENT + | PageAttribute::WRITE + | PageAttribute::READ + | PageAttribute::HUGE + | PageAttribute::GLOBAL; + + let page = alloc.alloc().unwrap(); + pte.set(page.into_raw(), attr.into()); + } + }); unsafe { // SAFETY: We've just mapped the area with sufficient length. core::ptr::write_bytes( - V_KERNEL_BSS_START.addr() as *mut (), + bss_range.start().addr() as *mut u8, 0, - BSS_LENGTH as usize, + bss_length, ); } diff --git a/crates/eonix_hal/src/arch/x86_64/mm.rs b/crates/eonix_hal/src/arch/x86_64/mm.rs index a991cecc..11bc2996 100644 --- a/crates/eonix_hal/src/arch/x86_64/mm.rs +++ b/crates/eonix_hal/src/arch/x86_64/mm.rs @@ -6,10 +6,10 @@ use eonix_mm::address::{ Addr as _, AddrOps as _, PAddr, PRange, PhysAccess, VAddr, }; use eonix_mm::page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, - RawPageTable, TableAttribute, PTE, + PageAttribute, PageTable, PageTableAlloc, PageTableLevel, PagingMode, + RawAttribute, RawPageTable, TableAttribute, PTE, }; -use eonix_mm::paging::{NoAlloc, Page, PageBlock, PAGE_SIZE, PFN}; +use eonix_mm::paging::{BasicFolio, PageAccess, PageBlock, PAGE_SIZE, PFN}; use eonix_sync_base::LazyLock; use crate::traits::mm::Memory; @@ -35,14 +35,7 @@ pub const P_KIMAGE_START: PAddr = PAddr::from_val(0x200000); pub const V_KERNEL_BSS_START: VAddr = VAddr::from(0xffffffffc0200000); const KERNEL_PML4_PFN: PFN = PFN::from_val(0x1000 >> 12); - -pub static GLOBAL_PAGE_TABLE: LazyLock< - PageTable, -> = LazyLock::new(|| unsafe { - Page::with_raw(KERNEL_PML4_PFN, |root_table_page| { - PageTable::with_root_table(root_table_page.clone()) - }) -}); +pub const GLOBAL_PAGE_TABLE: BasicFolio = BasicFolio::new(KERNEL_PML4_PFN, 0); #[repr(transparent)] pub struct PTE64(u64); @@ -56,6 +49,9 @@ pub struct PagingMode4Levels; pub struct ArchPhysAccess; +#[derive(Clone)] +pub struct PageAccessImpl; + pub struct ArchMemory; #[repr(C)] @@ -124,6 +120,9 @@ impl<'a> RawPageTable<'a> for RawPageTable4Levels<'a> { } } +unsafe impl Send for RawPageTable4Levels<'_> {} +unsafe impl Sync for RawPageTable4Levels<'_> {} + impl RawAttribute for PageAttribute64 { fn null() -> Self { Self(0) @@ -278,6 +277,12 @@ impl PhysAccess for ArchPhysAccess { } } +impl PageAccess for PageAccessImpl { + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull { + unsafe { ArchPhysAccess::as_ptr(PAddr::from(pfn)) } + } +} + impl E820MemMapEntry { const ENTRY_FREE: u32 = 1; // const ENTRY_USED: u32 = 2; @@ -425,3 +430,18 @@ pub fn set_root_page_table_pfn(pfn: PFN) { ); } } + +pub fn with_global_page_table( + alloc: A, access: X, + func: impl FnOnce(&mut PageTable), +) where + A: PageTableAlloc, + X: PageAccess, +{ + let mut global_page_table = + PageTable::new(GLOBAL_PAGE_TABLE.clone(), alloc, access); + + func(&mut global_page_table); + + core::mem::forget(global_page_table); +} From 697295347a77f69e7fbc014fc39cdbaa9e4ad582 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 01:18:28 +0800 Subject: [PATCH 045/153] hal, x86: use a new trap handling method Similar to what we've done on riscv64 and loongarch64 platforms, we adapt captured traps to x86. x86 has a really limited number of registers. So we reserved 2 words in the percpu area as scratches and save some of the registers there if needed. When the trap_return is called, instead of using `TaskContext::switch`, we handwrite context switch codes so we can save a lot of boilerplate codes. We push the current flags register on our current stack, load all registers and save the current stack pointer to %rax's place for later trap_entry use. Nothing really different for this. For trap_entry, non-captured traps and captured userspace traps behave almost like the previous one in the new method except we make some optimizations on saved registers and so on. What really makes a difference is kernel space captured ones. x86 will push the interrupt frame onto the current stack unconditionally and this is cumbersome! We need to load the actual TrapContext from the percpu area, move the trap frame there, and proceed with our handler. In fact, this accounts for 66.7% of the percpu areas needed or 100% of the scratches needed. So might be great to discover some way we can get rid of it. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/interrupt.rs | 4 +- crates/eonix_hal/src/arch/x86_64/link.x | 4 + crates/eonix_hal/src/arch/x86_64/trap/mod.rs | 659 ++++++++++++------ .../src/arch/x86_64/trap/trap_context.rs | 34 +- 4 files changed, 484 insertions(+), 217 deletions(-) diff --git a/crates/eonix_hal/src/arch/x86_64/interrupt.rs b/crates/eonix_hal/src/arch/x86_64/interrupt.rs index 6397e9ec..bc2692fd 100644 --- a/crates/eonix_hal/src/arch/x86_64/interrupt.rs +++ b/crates/eonix_hal/src/arch/x86_64/interrupt.rs @@ -4,6 +4,8 @@ use core::pin::Pin; use core::ptr::NonNull; use crate::arch::cpu::rdmsr; +use crate::arch::trap::trap_stubs; +use crate::symbol_addr; #[repr(C)] #[derive(Clone, Copy)] @@ -113,7 +115,7 @@ impl InterruptControl { /// # Return /// Returns a tuple of InterruptControl and the cpu id of the current cpu. pub fn new() -> (Self, usize) { - let trap_stubs_base = super::trap::trap_stubs_start as usize; + let trap_stubs_base = symbol_addr!(trap_stubs); let idt = core::array::from_fn(|idx| match idx { 0..0x80 => IDTEntry::new(trap_stubs_base + 8 * idx, 0x08, 0x8e), diff --git a/crates/eonix_hal/src/arch/x86_64/link.x b/crates/eonix_hal/src/arch/x86_64/link.x index 7a836aa0..d4ac1bef 100644 --- a/crates/eonix_hal/src/arch/x86_64/link.x +++ b/crates/eonix_hal/src/arch/x86_64/link.x @@ -91,6 +91,10 @@ SECTIONS { __spercpu = .; QUAD(0); /* Reserved for x86 percpu pointer */ + QUAD(0); /* Reserved for x86 percpu capturer trap context */ + + QUAD(0); /* Reserved for x86 percpu capturer scratch */ + QUAD(0); /* Reserved for x86 percpu capturer scratch */ . = ALIGN(16); diff --git a/crates/eonix_hal/src/arch/x86_64/trap/mod.rs b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs index 20f1472f..1c32a965 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/mod.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs @@ -3,55 +3,26 @@ mod trap_context; use core::arch::{asm, global_asm, naked_asm}; use eonix_hal_traits::context::RawTaskContext; -use eonix_hal_traits::trap::{IrqState as IrqStateTrait, TrapReturn}; +use eonix_hal_traits::trap::{ + IrqState as IrqStateTrait, RawTrapContext, TrapReturn, +}; pub use trap_context::TrapContext; use super::context::TaskContext; +use super::cpu::CPU; unsafe extern "C" { + /// Default handler handles the trap on the current stack and returns + /// to the context before interrut. fn _default_trap_handler(trap_context: &mut TrapContext); - pub fn trap_stubs_start(); - fn _raw_trap_return(); } -#[eonix_percpu::define_percpu] -static TRAP_HANDLER: unsafe extern "C" fn() = default_trap_handler; - -#[eonix_percpu::define_percpu] -static CAPTURER_CONTEXT: TaskContext = TaskContext::new(); - -/// This value will never be used. -static mut DIRTY_TRAP_CONTEXT: TaskContext = TaskContext::new(); - /// State of the interrupt flag. pub struct IrqState(u64); -global_asm!( - r" - .set RAX, 0x00 - .set RBX, 0x08 - .set RCX, 0x10 - .set RDX, 0x18 - .set RDI, 0x20 - .set RSI, 0x28 - .set R8, 0x30 - .set R9, 0x38 - .set R10, 0x40 - .set R11, 0x48 - .set R12, 0x50 - .set R13, 0x58 - .set R14, 0x60 - .set R15, 0x68 - .set RBP, 0x70 - .set INT_NO, 0x78 - .set ERRCODE, 0x80 - .set RIP, 0x88 - .set CS, 0x90 - .set FLAGS, 0x98 - .set RSP, 0xa0 - .set SS, 0xa8 - - .macro cfi_all_same_value +macro_rules! cfi_all_same_value { + () => { + " .cfi_same_value %rax .cfi_same_value %rbx .cfi_same_value %rcx @@ -67,44 +38,50 @@ global_asm!( .cfi_same_value %r14 .cfi_same_value %r15 .cfi_same_value %rbp - .endm + " + }; +} - .globl {trap_stubs_start} - {trap_stubs_start}: +#[unsafe(naked)] +pub unsafe extern "C" fn trap_stubs() { + naked_asm!( + " .altmacro .macro build_isr_no_err name .align 8 - .globl ISR\name - .type ISR\name @function - ISR\name: + .globl ISR\\name + .type ISR\\name @function + ISR\\name: .cfi_startproc .cfi_signal_frame .cfi_def_cfa_offset 0x08 .cfi_offset %rsp, 0x10 - - cfi_all_same_value + ", + cfi_all_same_value!(), + " push %rbp # push placeholder for error code .cfi_def_cfa_offset 0x10 - call _raw_trap_entry + call {entry} .cfi_endproc .endm .altmacro .macro build_isr_err name .align 8 - .globl ISR\name - .type ISR\name @function - ISR\name: + .globl ISR\\name + .type ISR\\name @function + ISR\\name: .cfi_startproc .cfi_signal_frame .cfi_def_cfa_offset 0x10 .cfi_offset %rsp, 0x10 + ", + cfi_all_same_value!(), + " - cfi_all_same_value - - call _raw_trap_entry + call {entry} .cfi_endproc .endm @@ -146,185 +123,442 @@ global_asm!( build_isr_no_err %i .set i, i+1 .endr + ", + entry = sym raw_trap_entry, + options(att_syntax), + ) +} + +/// Offset of the capturer trap context in the percpu area. +const OFFSET_CAPTURER: usize = 8; + +#[unsafe(naked)] +unsafe extern "C" fn raw_trap_entry() { + naked_asm!( + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "cmpq $0x08, 0x18(%rsp)", + "je 2f", + "swapgs", + "", + "2:", + "subq ${trap_stubs}, (%rsp)", + "shrq $3, (%rsp)", + "", + "cmpq $0, %gs:{offset_capturer}", + "je {default_entry}", + "", + "cmpq $0x08, 0x18(%rsp)", + "je {captured_kernel_entry}", + "jmp {captured_user_entry}", + ".cfi_endproc", + trap_stubs = sym trap_stubs, + default_entry = sym default_trap_entry, + captured_kernel_entry = sym captured_trap_entry_kernel, + captured_user_entry = sym captured_trap_entry_user, + offset_capturer = const OFFSET_CAPTURER, + options(att_syntax), + ) +} - .globl _raw_trap_entry - .type _raw_trap_entry @function - _raw_trap_entry: - .cfi_startproc - .cfi_signal_frame - .cfi_def_cfa %rsp, 0x18 - .cfi_offset %rsp, 0x10 - - cfi_all_same_value - - sub $0x78, %rsp - .cfi_def_cfa_offset CS - - mov %rax, RAX(%rsp) - .cfi_rel_offset %rax, RAX - mov %rbx, RBX(%rsp) - .cfi_rel_offset %rbx, RBX - mov %rcx, RCX(%rsp) - .cfi_rel_offset %rcx, RCX - mov %rdx, RDX(%rsp) - .cfi_rel_offset %rdx, RDX - mov %rdi, RDI(%rsp) - .cfi_rel_offset %rdi, RDI - mov %rsi, RSI(%rsp) - .cfi_rel_offset %rsi, RSI - mov %r8, R8(%rsp) - .cfi_rel_offset %r8, R8 - mov %r9, R9(%rsp) - .cfi_rel_offset %r9, R9 - mov %r10, R10(%rsp) - .cfi_rel_offset %r10, R10 - mov %r11, R11(%rsp) - .cfi_rel_offset %r11, R11 - mov %r12, R12(%rsp) - .cfi_rel_offset %r12, R12 - mov %r13, R13(%rsp) - .cfi_rel_offset %r13, R13 - mov %r14, R14(%rsp) - .cfi_rel_offset %r14, R14 - mov %r15, R15(%rsp) - .cfi_rel_offset %r15, R15 - mov %rbp, RBP(%rsp) - .cfi_rel_offset %rbp, RBP - - mov INT_NO(%rsp), %rcx - sub ${trap_stubs_start}, %rcx - shr $3, %rcx - mov %rcx, INT_NO(%rsp) - - cmpq $0x08, CS(%rsp) - je 2f - swapgs - - 2: - mov %gs:0, %rcx - add ${handler}, %rcx - mov (%rcx), %rcx - - jmp *%rcx - .cfi_endproc - - _raw_trap_return: - .cfi_startproc - .cfi_def_cfa %rsp, CS - .cfi_rel_offset %rax, RAX - .cfi_rel_offset %rbx, RBX - .cfi_rel_offset %rcx, RCX - .cfi_rel_offset %rdx, RDX - .cfi_rel_offset %rdi, RDI - .cfi_rel_offset %rsi, RSI - .cfi_rel_offset %r8, R8 - .cfi_rel_offset %r9, R9 - .cfi_rel_offset %r10, R10 - .cfi_rel_offset %r11, R11 - .cfi_rel_offset %r12, R12 - .cfi_rel_offset %r13, R13 - .cfi_rel_offset %r14, R14 - .cfi_rel_offset %r15, R15 - .cfi_rel_offset %rbp, RBP - .cfi_rel_offset %rsp, RSP - - mov RAX(%rsp), %rax - .cfi_restore %rax - mov RBX(%rsp), %rbx - .cfi_restore %rbx - mov RCX(%rsp), %rcx - .cfi_restore %rcx - mov RDX(%rsp), %rdx - .cfi_restore %rdx - mov RDI(%rsp), %rdi - .cfi_restore %rdi - mov RSI(%rsp), %rsi - .cfi_restore %rsi - mov R8(%rsp), %r8 - .cfi_restore %r8 - mov R9(%rsp), %r9 - .cfi_restore %r9 - mov R10(%rsp), %r10 - .cfi_restore %r10 - mov R11(%rsp), %r11 - .cfi_restore %r11 - mov R12(%rsp), %r12 - .cfi_restore %r12 - mov R13(%rsp), %r13 - .cfi_restore %r13 - mov R14(%rsp), %r14 - .cfi_restore %r14 - mov R15(%rsp), %r15 - .cfi_restore %r15 - mov RBP(%rsp), %rbp - .cfi_restore %rbp - - cmpq $0x08, CS(%rsp) - je 2f - swapgs - - 2: - lea RIP(%rsp), %rsp - .cfi_def_cfa %rsp, 0x08 - .cfi_offset %rsp, 0x10 - - iretq - .cfi_endproc - ", - trap_stubs_start = sym trap_stubs_start, - handler = sym _percpu_inner_TRAP_HANDLER, - options(att_syntax), -); - -/// Default handler handles the trap on the current stack and returns -/// to the context before interrut. #[unsafe(naked)] -unsafe extern "C" fn default_trap_handler() { +unsafe extern "C" fn default_trap_entry() { naked_asm!( ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "sub ${INT_NO}, %rsp", + ".cfi_def_cfa_offset {CS}", + "", + "mov %rcx, {RCX}(%rsp)", + ".cfi_rel_offset %rcx, {RCX}", + "mov %rdx, {RDX}(%rsp)", + ".cfi_rel_offset %rdx, {RDX}", + "mov %rdi, {RDI}(%rsp)", + ".cfi_rel_offset %rdi, {RDI}", + "mov %rsi, {RSI}(%rsp)", + ".cfi_rel_offset %rsi, {RSI}", + "mov %r8, {R8}(%rsp)", + ".cfi_rel_offset %r8, {R8}", + "mov %r9, {R9}(%rsp)", + ".cfi_rel_offset %r9, {R9}", + "mov %r10, {R10}(%rsp)", + ".cfi_rel_offset %r10, {R10}", + "mov %r11, {R11}(%rsp)", + ".cfi_rel_offset %r11, {R11}", + "mov %rbx, {RBX}(%rsp)", + ".cfi_rel_offset %rbx, {RBX}", + "mov %r12, {R12}(%rsp)", + ".cfi_rel_offset %r12, {R12}", + "", + "mov %rax, %r12", + ".cfi_register %rax, %r12", "mov %rsp, %rbx", ".cfi_def_cfa_register %rbx", "", - "and $~0xf, %rsp", - "", + "and $-0x10, %rsp", "mov %rbx, %rdi", - "call {handle_trap}", + "", + "call {default_entry}", "", "mov %rbx, %rsp", ".cfi_def_cfa_register %rsp", + "mov %r12, %rax", + ".cfi_restore %rax", + "", + "mov {RCX}(%rsp), %rcx", + ".cfi_restore %rcx", + "mov {RDX}(%rsp), %rdx", + ".cfi_restore %rdx", + "mov {RDI}(%rsp), %rdi", + ".cfi_restore %rdi", + "mov {RSI}(%rsp), %rsi", + ".cfi_restore %rsi", + "mov {R8}(%rsp), %r8", + ".cfi_restore %r8", + "mov {R9}(%rsp), %r9", + ".cfi_restore %r9", + "mov {R10}(%rsp), %r10", + ".cfi_restore %r10", + "mov {R11}(%rsp), %r11", + ".cfi_restore %r11", + "mov {RBX}(%rsp), %rbx", + ".cfi_restore %rbx", + "mov {R12}(%rsp), %r12", + ".cfi_restore %r12", + "", + "cmpq $0x08, {CS}(%rsp)", + "je 2f", + "swapgs", + "", + "2:", + "lea {RIP}(%rsp), %rsp", + ".cfi_def_cfa %rsp, 0x08", + ".cfi_offset %rsp, 0x10", "", - "jmp {trap_return}", + "iretq", ".cfi_endproc", - handle_trap = sym _default_trap_handler, - trap_return = sym _raw_trap_return, + default_entry = sym _default_trap_handler, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + INT_NO = const TrapContext::OFFSET_INT_NO, + RIP = const TrapContext::OFFSET_RIP, + CS = const TrapContext::OFFSET_CS, options(att_syntax), - ); + ) +} + +#[unsafe(naked)] +unsafe extern "C" fn captured_trap_entry_kernel() { + naked_asm!( + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "mov %rsi, %gs:0x10", + ".cfi_undefined %rsi", + "mov %rsp, %rsi", + ".cfi_def_cfa_register %rsi", + ".cfi_register %rsp, %rsi", + + "mov %gs:0x08, %rsp", + // Save and load registers. + "mov %rcx, {RCX}(%rsp)", + ".cfi_rel_offset %rcx, {RCX}", + "", + "mov %gs:0x10, %rcx", + ".cfi_register %rsi, %rcx", + "", + "mov %rdx, {RDX}(%rsp)", + ".cfi_rel_offset %rdx, {RDX}", + "mov %rdi, {RDI}(%rsp)", + ".cfi_rel_offset %rdi, {RDI}", + "mov %rcx, {RSI}(%rsp)", + ".cfi_rel_offset %rsi, {RSI}", + "mov %r8, {R8}(%rsp)", + ".cfi_rel_offset %r8, {R8}", + "mov %r9, {R9}(%rsp)", + ".cfi_rel_offset %r9, {R9}", + "mov %r10, {R10}(%rsp)", + ".cfi_rel_offset %r10, {R10}", + "mov %r11, {R11}(%rsp)", + ".cfi_rel_offset %r11, {R11}", + "", + "xchg %rax, {RAX}(%rsp)", + ".cfi_rel_offset %rax, {RAX}", + "xchg %rbx, {RBX}(%rsp)", + ".cfi_rel_offset %rbx, {RBX}", + "xchg %r12, {R12}(%rsp)", + ".cfi_rel_offset %r12, {R12}", + "xchg %r13, {R13}(%rsp)", + ".cfi_rel_offset %r13, {R13}", + "xchg %r14, {R14}(%rsp)", + ".cfi_rel_offset %r14, {R14}", + "xchg %r15, {R15}(%rsp)", + ".cfi_rel_offset %r15, {R15}", + "xchg %rbp, {RBP}(%rsp)", + ".cfi_rel_offset %rbp, {RBP}", + "", + "lea {INT_NO}(%rsp), %rdi", + "mov $7, %rcx", + "cld", + "rep movsq", + "", + "mov %rax, %rsp", + ".cfi_def_cfa %rsp, 0x10", + ".cfi_undefined %rax", + ".cfi_restore %rbx", + ".cfi_undefined %rcx", + ".cfi_undefined %rdx", + ".cfi_undefined %rdi", + ".cfi_undefined %rsi", + ".cfi_undefined %r8", + ".cfi_undefined %r9", + ".cfi_undefined %r10", + ".cfi_undefined %r11", + ".cfi_restore %r12", + ".cfi_restore %r13", + ".cfi_restore %r14", + ".cfi_restore %r15", + ".cfi_restore %rbp", + "", + "popf", + ".cfi_def_cfa_offset 0x08", + "", + "ret", + ".cfi_endproc", + RAX = const TrapContext::OFFSET_RAX, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + R13 = const TrapContext::OFFSET_R13, + R14 = const TrapContext::OFFSET_R14, + R15 = const TrapContext::OFFSET_R15, + RBP = const TrapContext::OFFSET_RBP, + INT_NO = const TrapContext::OFFSET_INT_NO, + options(att_syntax), + ) } #[unsafe(naked)] -unsafe extern "C" fn captured_trap_handler() { +unsafe extern "C" fn captured_trap_entry_user() { naked_asm!( - "mov ${from_context}, %rdi", - "mov %gs:0, %rsi", - "add ${to_context}, %rsi", + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "sub ${INT_NO}, %rsp", + ".cfi_def_cfa_offset {CS}", + "", + // Save and load registers. + "mov %rcx, {RCX}(%rsp)", + ".cfi_rel_offset %rcx, {RCX}", + "mov %rdx, {RDX}(%rsp)", + ".cfi_rel_offset %rdx, {RDX}", + "mov %rdi, {RDI}(%rsp)", + ".cfi_rel_offset %rdi, {RDI}", + "mov %rsi, {RSI}(%rsp)", + ".cfi_rel_offset %rsi, {RSI}", + "mov %r8, {R8}(%rsp)", + ".cfi_rel_offset %r8, {R8}", + "mov %r9, {R9}(%rsp)", + ".cfi_rel_offset %r9, {R9}", + "mov %r10, {R10}(%rsp)", + ".cfi_rel_offset %r10, {R10}", + "mov %r11, {R11}(%rsp)", + ".cfi_rel_offset %r11, {R11}", + "", + "xchg %rax, {RAX}(%rsp)", + ".cfi_rel_offset %rax, {RAX}", + "xchg %rbx, {RBX}(%rsp)", + ".cfi_rel_offset %rbx, {RBX}", + "xchg %r12, {R12}(%rsp)", + ".cfi_rel_offset %r12, {R12}", + "xchg %r13, {R13}(%rsp)", + ".cfi_rel_offset %r13, {R13}", + "xchg %r14, {R14}(%rsp)", + ".cfi_rel_offset %r14, {R14}", + "xchg %r15, {R15}(%rsp)", + ".cfi_rel_offset %r15, {R15}", + "xchg %rbp, {RBP}(%rsp)", + ".cfi_rel_offset %rbp, {RBP}", + "", + "mov %rax, %rsp", + ".cfi_def_cfa %rsp, 0x10", + ".cfi_undefined %rax", + ".cfi_restore %rbx", + ".cfi_undefined %rcx", + ".cfi_undefined %rdx", + ".cfi_undefined %rdi", + ".cfi_undefined %rsi", + ".cfi_undefined %r8", + ".cfi_undefined %r9", + ".cfi_undefined %r10", + ".cfi_undefined %r11", + ".cfi_restore %r12", + ".cfi_restore %r13", + ".cfi_restore %r14", + ".cfi_restore %r15", + ".cfi_restore %rbp", "", - "mov %rdi, %rsp", // We need a temporary stack to use `switch()`. + "popf", + ".cfi_def_cfa_offset 0x08", "", - "jmp {switch}", - from_context = sym DIRTY_TRAP_CONTEXT, - to_context = sym _percpu_inner_CAPTURER_CONTEXT, - switch = sym TaskContext::switch, + "ret", + ".cfi_endproc", + RAX = const TrapContext::OFFSET_RAX, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + R13 = const TrapContext::OFFSET_R13, + R14 = const TrapContext::OFFSET_R14, + R15 = const TrapContext::OFFSET_R15, + RBP = const TrapContext::OFFSET_RBP, + INT_NO = const TrapContext::OFFSET_INT_NO, + CS = const TrapContext::OFFSET_CS, options(att_syntax), - ); + ) } #[unsafe(naked)] -unsafe extern "C" fn captured_trap_return(trap_context: usize) -> ! { +unsafe extern "C" fn captured_trap_return(trap_context: &mut TrapContext) { naked_asm!( - "jmp {trap_return}", - trap_return = sym _raw_trap_return, + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x08", + "", + "pushf", + ".cfi_def_cfa_offset 0x10", + "", + "mov %rdi, %rax", + ".cfi_def_cfa %rax, {CS}", + ".cfi_rel_offset %rcx, {RCX}", + ".cfi_rel_offset %rdx, {RDX}", + ".cfi_rel_offset %rdi, {RDI}", + ".cfi_rel_offset %rsi, {RSI}", + ".cfi_rel_offset %r8, {R8}", + ".cfi_rel_offset %r9, {R9}", + ".cfi_rel_offset %r10, {R10}", + ".cfi_rel_offset %r11, {R11}", + ".cfi_rel_offset %rax, {RAX}", + ".cfi_rel_offset %rbx, {RBX}", + ".cfi_rel_offset %r12, {R12}", + ".cfi_rel_offset %r13, {R13}", + ".cfi_rel_offset %r14, {R14}", + ".cfi_rel_offset %r15, {R15}", + ".cfi_rel_offset %rbp, {RBP}", + ".cfi_rel_offset %rflags, {FLAGS}", + ".cfi_rel_offset %rsp, {RSP}", + "", + "mov {RCX}(%rax), %rcx", + ".cfi_restore %rcx", + "mov {RDX}(%rax), %rdx", + ".cfi_restore %rdx", + "mov {RDI}(%rax), %rdi", + ".cfi_restore %rdi", + "mov {RSI}(%rax), %rsi", + ".cfi_restore %rsi", + "mov {R8}(%rax), %r8", + ".cfi_restore %r8", + "mov {R9}(%rax), %r9", + ".cfi_restore %r9", + "mov {R10}(%rax), %r10", + ".cfi_restore %r10", + "mov {R11}(%rax), %r11", + ".cfi_restore %r11", + "", + "xchg %rax, %rsp", + "xchg %rax, {RAX}(%rsp)", + ".cfi_restore %rax", + "xchg %rbx, {RBX}(%rsp)", + ".cfi_restore %rbx", + "xchg %r12, {R12}(%rsp)", + ".cfi_restore %r12", + "xchg %r13, {R13}(%rsp)", + ".cfi_restore %r13", + "xchg %r14, {R14}(%rsp)", + ".cfi_restore %r14", + "xchg %r15, {R15}(%rsp)", + ".cfi_restore %r15", + "xchg %rbp, {RBP}(%rsp)", + ".cfi_restore %rbp", + "", + "cmpq $0x08, {CS}(%rsp)", + "je 2f", + "swapgs", + "", + "2:", + "lea {RIP}(%rsp), %rsp", + ".cfi_def_cfa %rsp, 0x08", + "iretq", + ".cfi_endproc", + RAX = const TrapContext::OFFSET_RAX, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + R13 = const TrapContext::OFFSET_R13, + R14 = const TrapContext::OFFSET_R14, + R15 = const TrapContext::OFFSET_R15, + RBP = const TrapContext::OFFSET_RBP, + RIP = const TrapContext::OFFSET_RIP, + CS = const TrapContext::OFFSET_CS, + FLAGS = const TrapContext::OFFSET_FLAGS, + RSP = const TrapContext::OFFSET_RSP, + options(att_syntax), + ); +} + +unsafe fn swap_percpu_capturer(new_capturer: usize) -> usize { + let old_capturer: usize; + asm!( + "mov %gs:0x08, {old}", + "mov {new}, %gs:0x08", + new = in(reg) new_capturer, + old = out(reg) old_capturer, options(att_syntax), ); + + old_capturer } impl TrapReturn for TrapContext { @@ -332,18 +566,17 @@ impl TrapReturn for TrapContext { unsafe fn trap_return(&mut self) { let irq_states = disable_irqs_save(); - let old_handler = TRAP_HANDLER.swap(captured_trap_handler); - - let mut to_ctx = TaskContext::new(); - to_ctx.set_program_counter(captured_trap_return as _); - to_ctx.set_stack_pointer(&raw mut *self as usize); - to_ctx.set_interrupt_enabled(false); + let old_handler = swap_percpu_capturer(self as *mut _ as usize); unsafe { - TaskContext::switch(CAPTURER_CONTEXT.as_mut(), &mut to_ctx); + CPU::local() + .as_mut() + .load_interrupt_stack(self as *mut _ as usize as u64); } - TRAP_HANDLER.set(old_handler); + captured_trap_return(self); + + swap_percpu_capturer(old_handler); irq_states.restore(); } } diff --git a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs index 176f2de1..4d30c0dc 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs @@ -1,4 +1,5 @@ use core::arch::asm; +use core::mem::offset_of; use eonix_hal_traits::fault::{Fault, PageFaultErrorCode}; use eonix_hal_traits::trap::{RawTrapContext, TrapType}; @@ -6,11 +7,9 @@ use eonix_mm::address::VAddr; use crate::processor::CPU; -#[derive(Clone, Copy, Default)] #[repr(C, align(16))] +#[derive(Clone, Copy, Default)] pub struct TrapContext { - rax: u64, - rbx: u64, rcx: u64, rdx: u64, rdi: u64, @@ -19,11 +18,17 @@ pub struct TrapContext { r9: u64, r10: u64, r11: u64, + + /// on `trap_return`: save capturer %rsp, load %rax + /// on `trap_entry`: save %rax, load capturer %rsp + rax: u64, + rbx: u64, r12: u64, r13: u64, r14: u64, r15: u64, rbp: u64, + int_no: u64, errcode: u64, rip: u64, @@ -34,6 +39,29 @@ pub struct TrapContext { } impl TrapContext { + pub const OFFSET_RAX: usize = offset_of!(TrapContext, rax); + pub const OFFSET_RBX: usize = offset_of!(TrapContext, rbx); + pub const OFFSET_RCX: usize = offset_of!(TrapContext, rcx); + pub const OFFSET_RDX: usize = offset_of!(TrapContext, rdx); + pub const OFFSET_RDI: usize = offset_of!(TrapContext, rdi); + pub const OFFSET_RSI: usize = offset_of!(TrapContext, rsi); + pub const OFFSET_R8: usize = offset_of!(TrapContext, r8); + pub const OFFSET_R9: usize = offset_of!(TrapContext, r9); + pub const OFFSET_R10: usize = offset_of!(TrapContext, r10); + pub const OFFSET_R11: usize = offset_of!(TrapContext, r11); + pub const OFFSET_R12: usize = offset_of!(TrapContext, r12); + pub const OFFSET_R13: usize = offset_of!(TrapContext, r13); + pub const OFFSET_R14: usize = offset_of!(TrapContext, r14); + pub const OFFSET_R15: usize = offset_of!(TrapContext, r15); + pub const OFFSET_RBP: usize = offset_of!(TrapContext, rbp); + pub const OFFSET_INT_NO: usize = offset_of!(TrapContext, int_no); + pub const OFFSET_ERRCODE: usize = offset_of!(TrapContext, errcode); + pub const OFFSET_RIP: usize = offset_of!(TrapContext, rip); + pub const OFFSET_CS: usize = offset_of!(TrapContext, cs); + pub const OFFSET_FLAGS: usize = offset_of!(TrapContext, flags); + pub const OFFSET_RSP: usize = offset_of!(TrapContext, rsp); + pub const OFFSET_SS: usize = offset_of!(TrapContext, ss); + fn get_fault_type(&self) -> Fault { match self.int_no { 6 | 8 => Fault::InvalidOp, From 19821e5aaa629f03e7cc85b08b70939c1b8e8833 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 27 Jan 2026 23:32:14 +0800 Subject: [PATCH 046/153] runtime: remove unused ExecutionContext and Stack Simplify code. No functional changes. Signed-off-by: greatbridf --- crates/eonix_runtime/src/context.rs | 51 ----------- crates/eonix_runtime/src/executor.rs | 88 +++++++++++++++---- crates/eonix_runtime/src/executor/builder.rs | 63 ------------- .../src/executor/output_handle.rs | 64 -------------- crates/eonix_runtime/src/executor/stack.rs | 6 -- crates/eonix_runtime/src/lib.rs | 1 - src/kernel/task.rs | 1 - src/kernel/task/kernel_stack.rs | 10 +-- src/lib.rs | 1 - 9 files changed, 71 insertions(+), 214 deletions(-) delete mode 100644 crates/eonix_runtime/src/context.rs delete mode 100644 crates/eonix_runtime/src/executor/builder.rs delete mode 100644 crates/eonix_runtime/src/executor/output_handle.rs delete mode 100644 crates/eonix_runtime/src/executor/stack.rs diff --git a/crates/eonix_runtime/src/context.rs b/crates/eonix_runtime/src/context.rs deleted file mode 100644 index 402d032f..00000000 --- a/crates/eonix_runtime/src/context.rs +++ /dev/null @@ -1,51 +0,0 @@ -use core::{cell::UnsafeCell, mem::transmute}; -use eonix_hal::context::TaskContext; -use eonix_hal::traits::context::RawTaskContext; - -#[derive(Debug)] -pub struct ExecutionContext(UnsafeCell); - -unsafe impl Sync for ExecutionContext {} - -impl ExecutionContext { - pub const fn new() -> Self { - Self(UnsafeCell::new(TaskContext::new())) - } - - pub fn set_ip(&mut self, ip: usize) { - let Self(context) = self; - context.get_mut().set_program_counter(ip); - } - - pub fn set_sp(&mut self, sp: usize) { - let Self(context) = self; - context.get_mut().set_stack_pointer(sp); - } - - pub fn set_interrupt(&mut self, is_enabled: bool) { - let Self(context) = self; - context.get_mut().set_interrupt_enabled(is_enabled); - } - - pub fn call1(&mut self, func: unsafe extern "C" fn(T) -> !, arg: usize) { - let Self(context) = self; - context - .get_mut() - .call(unsafe { transmute(func as *mut ()) }, arg); - } - - pub fn switch_to(&self, to: &Self) { - let Self(from_ctx) = self; - let Self(to_ctx) = to; - unsafe { - TaskContext::switch(&mut *from_ctx.get(), &mut *to_ctx.get()); - } - } - - pub fn switch_noreturn(&self) -> ! { - let Self(to_ctx) = self; - unsafe { - TaskContext::switch_to_noreturn(&mut *to_ctx.get()); - } - } -} diff --git a/crates/eonix_runtime/src/executor.rs b/crates/eonix_runtime/src/executor.rs index 3b858a47..fcb6ef29 100644 --- a/crates/eonix_runtime/src/executor.rs +++ b/crates/eonix_runtime/src/executor.rs @@ -1,27 +1,34 @@ -// mod builder; -mod output_handle; -mod stack; - -use alloc::{ - boxed::Box, - sync::{Arc, Weak}, -}; -use core::{ - marker::PhantomData, - pin::Pin, - task::{Context, Poll}, -}; -use eonix_sync::Spin; +use alloc::boxed::Box; +use alloc::sync::{Arc, Weak}; +use core::marker::PhantomData; +use core::pin::Pin; +use core::task::{Context, Poll, Waker}; -pub use output_handle::OutputHandle; -pub use stack::Stack; +use eonix_sync::Spin; /// An `Executor` executes a Future object in a separate thread of execution. /// -/// When the Future is finished, the `Executor` will call the `OutputHandle` to commit the output. -/// Then the `Executor` will release the resources associated with the Future. +/// When the Future is finished, the `Executor` will call the `OutputHandle` to +/// commit the output. Then the `Executor` will release the resources +/// associated with the Future. pub struct Executor(Option>>); +enum OutputState +where + Output: Send, +{ + Waiting(Option), + Finished(Option), + TakenOut, +} + +pub struct OutputHandle +where + Output: Send, +{ + inner: OutputState, +} + trait TypeErasedExecutor: Send { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; } @@ -92,3 +99,48 @@ impl Executor { } } } + +impl OutputHandle +where + Output: Send, +{ + pub fn new() -> Arc> { + Arc::new(Spin::new(Self { + inner: OutputState::Waiting(None), + })) + } + + pub fn try_resolve(&mut self) -> Option { + let output = match &mut self.inner { + OutputState::Waiting(_) => return None, + OutputState::Finished(output) => output.take(), + OutputState::TakenOut => panic!("Output already taken out"), + }; + + self.inner = OutputState::TakenOut; + if let Some(output) = output { + Some(output) + } else { + unreachable!("Output should be present") + } + } + + pub fn register_waiter(&mut self, waker: Waker) { + if let OutputState::Waiting(inner_waker) = &mut self.inner { + inner_waker.replace(waker); + } else { + panic!("Output is not waiting"); + } + } + + pub fn commit_output(&mut self, output: Output) { + if let OutputState::Waiting(inner_waker) = &mut self.inner { + if let Some(waker) = inner_waker.take() { + waker.wake(); + } + self.inner = OutputState::Finished(Some(output)); + } else { + panic!("Output is not waiting"); + } + } +} diff --git a/crates/eonix_runtime/src/executor/builder.rs b/crates/eonix_runtime/src/executor/builder.rs deleted file mode 100644 index 2729270b..00000000 --- a/crates/eonix_runtime/src/executor/builder.rs +++ /dev/null @@ -1,63 +0,0 @@ -use super::{Executor, OutputHandle, RealExecutor, Stack}; -use crate::context::ExecutionContext; -use alloc::{boxed::Box, sync::Arc}; -use core::{pin::Pin, sync::atomic::AtomicBool}; -use eonix_sync::Spin; - -pub struct ExecutorBuilder { - stack: Option, - runnable: Option, -} - -impl ExecutorBuilder -where - S: Stack, - R::Output: Send, -{ - pub fn new() -> Self { - Self { - stack: None, - runnable: None, - } - } - - pub fn stack(mut self, stack: S) -> Self { - self.stack.replace(stack); - self - } - - pub fn runnable(mut self, runnable: R) -> Self { - self.runnable.replace(runnable); - self - } - - pub fn build( - mut self, - ) -> ( - Pin>, - ExecutionContext, - Arc>>, - ) { - let stack = self.stack.take().expect("Stack is required"); - let runnable = self.runnable.take().expect("Runnable is required"); - - let mut execution_context = ExecutionContext::new(); - let output_handle = OutputHandle::new(); - - execution_context.set_sp(stack.get_bottom().addr().get() as _); - - let executor = Box::pin(RealExecutor { - _stack: stack, - runnable, - output_handle: Arc::downgrade(&output_handle), - finished: AtomicBool::new(false), - }); - - execution_context.call1( - RealExecutor::::execute, - executor.as_ref().get_ref() as *const _ as usize, - ); - - (executor, execution_context, output_handle) - } -} diff --git a/crates/eonix_runtime/src/executor/output_handle.rs b/crates/eonix_runtime/src/executor/output_handle.rs deleted file mode 100644 index d0548fd2..00000000 --- a/crates/eonix_runtime/src/executor/output_handle.rs +++ /dev/null @@ -1,64 +0,0 @@ -use alloc::sync::Arc; -use core::task::Waker; -use eonix_sync::Spin; - -enum OutputState -where - Output: Send, -{ - Waiting(Option), - Finished(Option), - TakenOut, -} - -pub struct OutputHandle -where - Output: Send, -{ - inner: OutputState, -} - -impl OutputHandle -where - Output: Send, -{ - pub fn new() -> Arc> { - Arc::new(Spin::new(Self { - inner: OutputState::Waiting(None), - })) - } - - pub fn try_resolve(&mut self) -> Option { - let output = match &mut self.inner { - OutputState::Waiting(_) => return None, - OutputState::Finished(output) => output.take(), - OutputState::TakenOut => panic!("Output already taken out"), - }; - - self.inner = OutputState::TakenOut; - if let Some(output) = output { - Some(output) - } else { - unreachable!("Output should be present") - } - } - - pub fn register_waiter(&mut self, waker: Waker) { - if let OutputState::Waiting(inner_waker) = &mut self.inner { - inner_waker.replace(waker); - } else { - panic!("Output is not waiting"); - } - } - - pub fn commit_output(&mut self, output: Output) { - if let OutputState::Waiting(inner_waker) = &mut self.inner { - if let Some(waker) = inner_waker.take() { - waker.wake(); - } - self.inner = OutputState::Finished(Some(output)); - } else { - panic!("Output is not waiting"); - } - } -} diff --git a/crates/eonix_runtime/src/executor/stack.rs b/crates/eonix_runtime/src/executor/stack.rs deleted file mode 100644 index cfd0825a..00000000 --- a/crates/eonix_runtime/src/executor/stack.rs +++ /dev/null @@ -1,6 +0,0 @@ -use core::ptr::NonNull; - -pub trait Stack: Sized + Send { - fn new() -> Self; - fn get_bottom(&self) -> NonNull<()>; -} diff --git a/crates/eonix_runtime/src/lib.rs b/crates/eonix_runtime/src/lib.rs index 458e01d9..c5da3014 100644 --- a/crates/eonix_runtime/src/lib.rs +++ b/crates/eonix_runtime/src/lib.rs @@ -1,6 +1,5 @@ #![no_std] -pub mod context; pub mod executor; mod ready_queue; pub mod scheduler; diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 6bbccb28..7fffc008 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -99,7 +99,6 @@ where use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; use eonix_hal::trap::TrapContext; use eonix_preempt::assert_preempt_enabled; - use eonix_runtime::executor::Stack; use eonix_runtime::task::Task; use thread::wait_for_wakeups; diff --git a/src/kernel/task/kernel_stack.rs b/src/kernel/task/kernel_stack.rs index f00b91bd..d6325b50 100644 --- a/src/kernel/task/kernel_stack.rs +++ b/src/kernel/task/kernel_stack.rs @@ -1,7 +1,5 @@ use core::ptr::NonNull; -use eonix_runtime::executor::Stack; - use crate::kernel::mem::FolioOwned; #[derive(Debug)] @@ -19,14 +17,8 @@ impl KernelStack { folio: FolioOwned::alloc_order(Self::KERNEL_STACK_ORDER), } } -} - -impl Stack for KernelStack { - fn new() -> Self { - Self::new() - } - fn get_bottom(&self) -> NonNull<()> { + pub fn get_bottom(&self) -> NonNull<()> { let ptr = self.folio.get_bytes_ptr(); let len = ptr.len(); diff --git a/src/lib.rs b/src/lib.rs index bce90900..5792c3ee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,7 +37,6 @@ use eonix_hal::traits::context::RawTaskContext; use eonix_hal::traits::trap::IrqState; use eonix_hal::trap::disable_irqs_save; use eonix_mm::address::PRange; -use eonix_runtime::executor::Stack; use eonix_runtime::scheduler::RUNTIME; use kernel::mem::GlobalPageAlloc; use kernel::task::{KernelStack, ProcessList, ProgramLoader}; From b877fff904b65ae81e802e825200e7971ee51c71 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 28 Jan 2026 01:18:57 +0800 Subject: [PATCH 047/153] hal: remove dependency of TaskContext for TrapContext TaskContext is useless except for in the `trap_return()` implementation. But actually, this is unnecessary and makes the code ugly and weird. Rewrite the `trap_return()` with native code to remove dependency. Signed-off-by: greatbridf --- crates/eonix_hal/eonix_hal_traits/src/trap.rs | 14 +- .../src/arch/riscv64/trap/captured.rs | 284 ++++++++++++------ crates/eonix_hal/src/arch/riscv64/trap/mod.rs | 41 ++- .../src/arch/riscv64/trap/trap_context.rs | 33 +- src/lib.rs | 16 +- 5 files changed, 244 insertions(+), 144 deletions(-) diff --git a/crates/eonix_hal/eonix_hal_traits/src/trap.rs b/crates/eonix_hal/eonix_hal_traits/src/trap.rs index 74ed9a28..0619d3c0 100644 --- a/crates/eonix_hal/eonix_hal_traits/src/trap.rs +++ b/crates/eonix_hal/eonix_hal_traits/src/trap.rs @@ -2,7 +2,6 @@ use core::marker::PhantomData; use eonix_mm::address::VAddr; -use crate::context::RawTaskContext; use crate::fault::Fault; /// A raw trap context. @@ -40,8 +39,6 @@ pub trait RawTrapContext: Copy { #[doc(notable_trait)] pub trait TrapReturn { - type TaskContext: RawTaskContext; - /// Return to the context before the trap occurred. /// /// # Safety @@ -50,6 +47,17 @@ pub trait TrapReturn { /// points to a valid stack frame and the program counter points to some /// valid instruction. unsafe fn trap_return(&mut self); + + /// Switch to the context before the trap occurred. + /// This function will NOT capture traps and will never return. + /// + /// # Safety + /// This function is unsafe because the caller MUST ensure that the + /// context before the trap is valid, that is, that the stack pointer + /// points to a valid stack frame and the program counter points to some + /// valid instruction. Besides, the caller MUST ensure that all variables + /// in the current context are released. + unsafe fn trap_return_noreturn(&mut self) -> !; } pub trait IrqState { diff --git a/crates/eonix_hal/src/arch/riscv64/trap/captured.rs b/crates/eonix_hal/src/arch/riscv64/trap/captured.rs index d4c00e9f..00b3a35a 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/captured.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/captured.rs @@ -1,61 +1,110 @@ -use crate::{arch::trap::Registers, context::TaskContext, trap::TrapContext}; -use core::{arch::naked_asm, mem::MaybeUninit}; -use eonix_hal_traits::context::RawTaskContext; +use core::arch::naked_asm; +use core::mem::MaybeUninit; -static mut DIRTY_TASK_CONTEXT: MaybeUninit = MaybeUninit::uninit(); +use crate::arch::trap::Registers; +use crate::trap::TrapContext; // If captured trap context is present, we use it directly. -// We need to restore the kernel tp from that TrapContext but sp is -// fine since we will use TaskContext::switch. +// We need to restore the callee saved registers from that TrapContext. #[unsafe(naked)] pub(super) unsafe extern "C" fn _captured_trap_entry() -> ! { naked_asm!( - "csrrw t0, sscratch, t0", - "sd tp, {tp}(t0)", - "ld tp, {ra}(t0)", // Load kernel tp from trap_ctx.ra - "sd ra, {ra}(t0)", - "ld ra, {sp}(t0)", // Load capturer task context from trap_ctx.sp - "sd sp, {sp}(t0)", - "sd gp, {gp}(t0)", - "sd a0, {a0}(t0)", - "sd a1, {a1}(t0)", - "sd a2, {a2}(t0)", - "sd a3, {a3}(t0)", - "sd a4, {a4}(t0)", - "sd t1, {t1}(t0)", - "sd a5, {a5}(t0)", - "sd a6, {a6}(t0)", - "sd a7, {a7}(t0)", - "sd t3, {t3}(t0)", - "sd t4, {t4}(t0)", - "sd t5, {t5}(t0)", - "sd t2, {t2}(t0)", - "sd t6, {t6}(t0)", - "sd s0, {s0}(t0)", - "sd s1, {s1}(t0)", - "sd s2, {s2}(t0)", - "sd s3, {s3}(t0)", - "sd s4, {s4}(t0)", - "sd s5, {s5}(t0)", - "sd s6, {s6}(t0)", - "sd s7, {s7}(t0)", - "sd s8, {s8}(t0)", - "sd s9, {s9}(t0)", - "sd s10, {s10}(t0)", - "sd s11, {s11}(t0)", - "csrr t2, sstatus", - "csrr t3, sepc", - "csrr t4, scause", - "csrr t5, stval", - "csrrw t6, sscratch, t0", - "sd t6, {t0}(t0)", - "sd t2, {sstatus}(t0)", - "sd t3, {sepc}(t0)", - "sd t4, {scause}(t0)", - "sd t5, {stval}(t0)", - "la a0, {dirty_task_context}", - "mv a1, ra", - "j {task_context_switch}", + "csrrw gp, sscratch, gp", + + "sd a0, {a0}(gp)", + "mv a0, s0", + "ld s0, {s0}(gp)", + "sd a0, {s0}(gp)", + + "sd a1, {a1}(gp)", + "mv a1, s1", + "ld s1, {s1}(gp)", + "sd a1, {s1}(gp)", + + "sd a2, {a2}(gp)", + "mv a2, s2", + "ld s2, {s2}(gp)", + "sd a2, {s2}(gp)", + + "sd a3, {a3}(gp)", + "mv a3, s3", + "ld s3, {s3}(gp)", + "sd a3, {s3}(gp)", + + "sd a4, {a4}(gp)", + "mv a4, s4", + "ld s4, {s4}(gp)", + "sd a4, {s4}(gp)", + + "sd a5, {a5}(gp)", + "mv a5, s5", + "ld s5, {s5}(gp)", + "sd a5, {s5}(gp)", + + "sd a6, {a6}(gp)", + "mv a6, s6", + "ld s6, {s6}(gp)", + "sd a6, {s6}(gp)", + + "sd a7, {a7}(gp)", + "mv a7, s7", + "ld s7, {s7}(gp)", + "sd a7, {s7}(gp)", + + "sd t0, {t0}(gp)", + "mv t0, s8", + "ld s8, {s8}(gp)", + "sd t0, {s8}(gp)", + + "sd t1, {t1}(gp)", + "mv t1, s9", + "ld s9, {s9}(gp)", + "sd t1, {s9}(gp)", + + "sd t2, {t2}(gp)", + "mv t2, s10", + "ld s10, {s10}(gp)", + "sd t2, {s10}(gp)", + + "sd t3, {t3}(gp)", + "mv t3, s11", + "ld s11, {s11}(gp)", + "sd t3, {s11}(gp)", + + "sd t4, {t4}(gp)", + "mv t4, ra", + "ld ra, {ra}(gp)", + "sd t4, {ra}(gp)", + + "sd t5, {t5}(gp)", + "mv t5, tp", + "ld tp, {tp}(gp)", + "sd t5, {tp}(gp)", + + "sd t6, {t6}(gp)", + "mv t6, sp", + "ld sp, {sp}(gp)", + "sd t6, {sp}(gp)", + + "ld t5, {sstatus}(gp)", + + "csrrw t0, sscratch, gp", + "sd t0, {gp}(gp)", + + "csrr t1, sstatus", + "sd t1, {sstatus}(gp)", + + "csrr t2, sepc", + "sd t2, {sepc}(gp)", + + "csrr t3, scause", + "sd t3, {scause}(gp)", + + "csrr t4, stval", + "sd t4, {stval}(gp)", + + "csrw sstatus, t5", + "ret", ra = const Registers::OFFSET_RA, sp = const Registers::OFFSET_SP, gp = const Registers::OFFSET_GP, @@ -91,54 +140,99 @@ pub(super) unsafe extern "C" fn _captured_trap_entry() -> ! { sepc = const TrapContext::OFFSET_SEPC, scause = const TrapContext::OFFSET_SCAUSE, stval = const TrapContext::OFFSET_STVAL, - dirty_task_context = sym DIRTY_TASK_CONTEXT, - task_context_switch = sym TaskContext::switch, ); } #[unsafe(naked)] -pub(super) unsafe extern "C" fn _captured_trap_return(ctx: &mut TrapContext) -> ! { +pub(super) unsafe extern "C" fn _captured_trap_return(ctx: &mut TrapContext) { naked_asm!( - "csrr t0, sscratch", - "ld t1, {sstatus}(t0)", - "ld t2, {sepc}(t0)", - "csrw sstatus, t1", - "csrw sepc, t2", - "mv t4, tp", - "mv t5, sp", - "ld tp, {tp}(t0)", - "ld ra, {ra}(t0)", - "ld sp, {sp}(t0)", - "sd t4, {ra}(t0)", // Store kernel tp to trap_ctx.ra - "sd t5, {sp}(t0)", // Store capturer task context to trap_ctx.sp - "ld gp, {gp}(t0)", - "ld a0, {a0}(t0)", - "ld a1, {a1}(t0)", - "ld a2, {a2}(t0)", - "ld a3, {a3}(t0)", - "ld a4, {a4}(t0)", - "ld t1, {t1}(t0)", - "ld a5, {a5}(t0)", - "ld a6, {a6}(t0)", - "ld a7, {a7}(t0)", - "ld t3, {t3}(t0)", - "ld t4, {t4}(t0)", - "ld t5, {t5}(t0)", - "ld t2, {t2}(t0)", - "ld t6, {t6}(t0)", - "ld s0, {s0}(t0)", - "ld s1, {s1}(t0)", - "ld s2, {s2}(t0)", - "ld s3, {s3}(t0)", - "ld s4, {s4}(t0)", - "ld s5, {s5}(t0)", - "ld s6, {s6}(t0)", - "ld s7, {s7}(t0)", - "ld s8, {s8}(t0)", - "ld s9, {s9}(t0)", - "ld s10, {s10}(t0)", - "ld s11, {s11}(t0)", - "ld t0, {t0}(t0)", + "mv t6, a0", + + "mv a0, s0", + "ld s0, {s0}(t6)", + "sd a0, {s0}(t6)", + + "mv a1, s1", + "ld s1, {s1}(t6)", + "sd a1, {s1}(t6)", + + "mv a2, s2", + "ld s2, {s2}(t6)", + "sd a2, {s2}(t6)", + + "mv a3, s3", + "ld s3, {s3}(t6)", + "sd a3, {s3}(t6)", + + "mv a4, s4", + "ld s4, {s4}(t6)", + "sd a4, {s4}(t6)", + + "mv a5, s5", + "ld s5, {s5}(t6)", + "sd a5, {s5}(t6)", + + "mv a6, s6", + "ld s6, {s6}(t6)", + "sd a6, {s6}(t6)", + + "mv a7, s7", + "ld s7, {s7}(t6)", + "sd a7, {s7}(t6)", + + "mv t0, s8", + "ld s8, {s8}(t6)", + "sd t0, {s8}(t6)", + + "mv t1, s9", + "ld s9, {s9}(t6)", + "sd t1, {s9}(t6)", + + "mv t2, s10", + "ld s10, {s10}(t6)", + "sd t2, {s10}(t6)", + + "mv t3, s11", + "ld s11, {s11}(t6)", + "sd t3, {s11}(t6)", + + "mv t4, ra", + "ld ra, {ra}(t6)", + "sd t4, {ra}(t6)", + + "mv t5, tp", + "ld tp, {tp}(t6)", + "sd t5, {tp}(t6)", + + "mv a0, sp", + "ld sp, {sp}(t6)", + "sd a0, {sp}(t6)", + + "csrr t4, sstatus", + "ld t5, {sstatus}(t6)", + "sd t4, {sstatus}(t6)", + "ld t4, {sepc}(t6)", + + "ld gp, {gp}(t6)", + "ld a0, {a0}(t6)", + "ld a1, {a1}(t6)", + "ld a2, {a2}(t6)", + "ld a3, {a3}(t6)", + "ld a4, {a4}(t6)", + "ld a5, {a5}(t6)", + "ld a6, {a6}(t6)", + "ld a7, {a7}(t6)", + + "csrw sstatus, t5", + "csrw sepc, t4", + + "ld t0, {t0}(t6)", + "ld t1, {t1}(t6)", + "ld t2, {t2}(t6)", + "ld t3, {t3}(t6)", + "ld t4, {t4}(t6)", + "ld t5, {t5}(t6)", + "ld t6, {t6}(t6)", "sret", ra = const Registers::OFFSET_RA, sp = const Registers::OFFSET_SP, diff --git a/crates/eonix_hal/src/arch/riscv64/trap/mod.rs b/crates/eonix_hal/src/arch/riscv64/trap/mod.rs index 28689111..a52652a1 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/mod.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/mod.rs @@ -2,32 +2,25 @@ mod captured; mod default; mod trap_context; -use super::config::platform::virt::*; -use super::context::TaskContext; -use captured::{_captured_trap_entry, _captured_trap_return}; use core::arch::{global_asm, naked_asm}; use core::mem::{offset_of, size_of}; use core::num::NonZero; use core::ptr::NonNull; + +use captured::{_captured_trap_entry, _captured_trap_return}; use default::_default_trap_entry; -use eonix_hal_traits::{ - context::RawTaskContext, - trap::{IrqState as IrqStateTrait, TrapReturn}, -}; +use eonix_hal_traits::context::RawTaskContext; +use eonix_hal_traits::trap::{IrqState as IrqStateTrait, TrapReturn}; +use riscv::asm::sfence_vma_all; use riscv::register::sstatus::{self, Sstatus}; -use riscv::register::stvec::TrapMode; +use riscv::register::stvec::{self, Stvec, TrapMode}; use riscv::register::{scause, sepc, sscratch, stval}; -use riscv::{ - asm::sfence_vma_all, - register::stvec::{self, Stvec}, -}; use sbi::SbiError; - pub use trap_context::*; -impl TrapReturn for TrapContext { - type TaskContext = TaskContext; +use super::config::platform::virt::*; +impl TrapReturn for TrapContext { unsafe fn trap_return(&mut self) { let irq_states = disable_irqs_save(); @@ -42,14 +35,8 @@ impl TrapReturn for TrapContext { let old_trap_ctx = sscratch::read(); sscratch::write(&raw mut *self as usize); - let mut from_ctx = TaskContext::new(); - let mut to_ctx = TaskContext::new(); - to_ctx.set_program_counter(_captured_trap_return as usize); - to_ctx.set_stack_pointer(&raw mut from_ctx as usize); - to_ctx.set_interrupt_enabled(false); - unsafe { - TaskContext::switch(&mut from_ctx, &mut to_ctx); + _captured_trap_return(self); } sscratch::write(old_trap_ctx); @@ -57,6 +44,16 @@ impl TrapReturn for TrapContext { irq_states.restore(); } + + unsafe fn trap_return_noreturn(&mut self) -> ! { + disable_irqs(); + + unsafe { + _captured_trap_return(self); + } + + unreachable!("trap_return_noreturn should not return"); + } } fn setup_trap_handler(trap_entry_addr: usize) { diff --git a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs index 027db567..88d67c85 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs @@ -16,37 +16,38 @@ use crate::processor::CPU; #[repr(C)] #[derive(Default, Clone, Copy)] pub struct Registers { - tp: u64, + s0: u64, + s1: u64, + s2: u64, + s3: u64, + s4: u64, + s5: u64, + s6: u64, + s7: u64, + s8: u64, + s9: u64, + s10: u64, + s11: u64, ra: u64, + tp: u64, sp: u64, gp: u64, + a0: u64, a1: u64, a2: u64, a3: u64, a4: u64, - t1: u64, a5: u64, a6: u64, a7: u64, + t0: u64, + t1: u64, + t2: u64, t3: u64, t4: u64, t5: u64, - t2: u64, t6: u64, - s0: u64, - s1: u64, - s2: u64, - s3: u64, - s4: u64, - s5: u64, - s6: u64, - s7: u64, - s8: u64, - s9: u64, - s10: u64, - s11: u64, - t0: u64, } /// Saved CPU context when a trap (interrupt or exception) occurs on RISC-V 64. diff --git a/src/lib.rs b/src/lib.rs index 5792c3ee..2fa1000b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,12 +30,10 @@ use alloc::ffi::CString; use core::hint::spin_loop; use core::sync::atomic::{AtomicBool, Ordering}; -use eonix_hal::context::TaskContext; use eonix_hal::processor::CPU; use eonix_hal::symbol_addr; -use eonix_hal::traits::context::RawTaskContext; -use eonix_hal::traits::trap::IrqState; -use eonix_hal::trap::disable_irqs_save; +use eonix_hal::traits::trap::{IrqState, RawTrapContext, TrapReturn}; +use eonix_hal::trap::{disable_irqs_save, TrapContext}; use eonix_mm::address::PRange; use eonix_runtime::scheduler::RUNTIME; use kernel::mem::GlobalPageAlloc; @@ -68,7 +66,7 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { drop(data); - let mut ctx = TaskContext::new(); + let mut ctx = TrapContext::new(); let stack_bottom = { let stack = KernelStack::new(); let bottom = stack.get_bottom().addr().get(); @@ -77,11 +75,12 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { bottom }; ctx.set_interrupt_enabled(true); + ctx.set_user_mode(false); ctx.set_program_counter(symbol_addr!(standard_main)); ctx.set_stack_pointer(stack_bottom); unsafe { - TaskContext::switch_to_noreturn(&mut ctx); + ctx.trap_return_noreturn(); } } @@ -94,7 +93,7 @@ fn kernel_ap_main(_stack_range: PRange) -> ! { println_debug!("AP{} started", CPU::local().cpuid()); - let mut ctx = TaskContext::new(); + let mut ctx = TrapContext::new(); let stack_bottom = { let stack = KernelStack::new(); let bottom = stack.get_bottom().addr().get(); @@ -103,11 +102,12 @@ fn kernel_ap_main(_stack_range: PRange) -> ! { bottom }; ctx.set_interrupt_enabled(true); + ctx.set_user_mode(false); ctx.set_program_counter(symbol_addr!(standard_main)); ctx.set_stack_pointer(stack_bottom); unsafe { - TaskContext::switch_to_noreturn(&mut ctx); + ctx.trap_return_noreturn(); } } From 1f9d2468f6cf12f3f7b5dc94719866e3d8a666c7 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 28 Jan 2026 01:22:44 +0800 Subject: [PATCH 048/153] hal: remove struct TaskContext from riscv64 Commit cc4c9378f47b ("hal: remove dependency of TaskContext for TrapContext") has removed all usages of TaskContext. Remove this struct completely. We don't care about loongarch64 at all so keep it there... x86 will pick this soon. Signed-off-by: greatbridf --- .../eonix_hal/eonix_hal_traits/src/context.rs | 41 ------- crates/eonix_hal/eonix_hal_traits/src/lib.rs | 1 - crates/eonix_hal/src/arch/riscv64/context.rs | 111 ------------------ crates/eonix_hal/src/arch/riscv64/mod.rs | 3 +- crates/eonix_hal/src/arch/riscv64/trap/mod.rs | 1 - crates/eonix_hal/src/context.rs | 1 - crates/eonix_hal/src/lib.rs | 1 - 7 files changed, 1 insertion(+), 158 deletions(-) delete mode 100644 crates/eonix_hal/eonix_hal_traits/src/context.rs delete mode 100644 crates/eonix_hal/src/arch/riscv64/context.rs delete mode 100644 crates/eonix_hal/src/context.rs diff --git a/crates/eonix_hal/eonix_hal_traits/src/context.rs b/crates/eonix_hal/eonix_hal_traits/src/context.rs deleted file mode 100644 index 1d5e73ea..00000000 --- a/crates/eonix_hal/eonix_hal_traits/src/context.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[doc(notable_trait)] -pub trait RawTaskContext: Sized { - /// Creates a new instance of the task context with interrupt enabled and the program - /// counter and stack pointer set to zero (a.k.a. some invalid state). - /// - /// Using the created context without setting the program counter and stack pointer - /// will result in undefined behavior. - fn new() -> Self; - - fn set_program_counter(&mut self, pc: usize); - fn set_stack_pointer(&mut self, sp: usize); - - fn is_interrupt_enabled(&self) -> bool; - fn set_interrupt_enabled(&mut self, is_enabled: bool); - - /// Sets the instruction pointer to the given function and prepares the context - /// to call it with the given argument. - fn call(&mut self, func: unsafe extern "C" fn(usize) -> !, arg: usize); - - /// Switch the execution context from `from` to `to`. - /// - /// # Safety - /// This function is unsafe because it performs a context switch, which can lead to - /// undefined behavior if the contexts are not properly set up. - unsafe extern "C" fn switch(from: &mut Self, to: &mut Self); - - /// Switches the execution context to `to` where we will not return. - /// - /// # Safety - /// This function is unsafe because it performs a context switch that does not return. - /// The caller must ensure that the `to` context is properly set up and that it will not - /// return to the caller. - unsafe extern "C" fn switch_to_noreturn(to: &mut Self) -> ! { - let mut from_ctx = Self::new(); - unsafe { - Self::switch(&mut from_ctx, to); - } - - unreachable!("We should never return from `switch_to_noreturn()`"); - } -} diff --git a/crates/eonix_hal/eonix_hal_traits/src/lib.rs b/crates/eonix_hal/eonix_hal_traits/src/lib.rs index a8cede83..5a5b4a23 100644 --- a/crates/eonix_hal/eonix_hal_traits/src/lib.rs +++ b/crates/eonix_hal/eonix_hal_traits/src/lib.rs @@ -1,7 +1,6 @@ #![no_std] #![feature(doc_notable_trait)] -pub mod context; pub mod fault; pub mod fpu; pub mod mm; diff --git a/crates/eonix_hal/src/arch/riscv64/context.rs b/crates/eonix_hal/src/arch/riscv64/context.rs deleted file mode 100644 index e7a53ea3..00000000 --- a/crates/eonix_hal/src/arch/riscv64/context.rs +++ /dev/null @@ -1,111 +0,0 @@ -use core::arch::naked_asm; -use eonix_hal_traits::context::RawTaskContext; -use riscv::register::sstatus::Sstatus; - -#[repr(C)] -#[derive(Debug)] -pub struct TaskContext { - // s0-11 - s: [u64; 12], - sp: u64, - ra: u64, - sstatus: Sstatus, -} - -impl RawTaskContext for TaskContext { - fn new() -> Self { - Self::new() - } - - fn set_program_counter(&mut self, pc: usize) { - self.ra = pc as u64; - } - - fn set_stack_pointer(&mut self, sp: usize) { - self.sp = sp as u64; - } - - fn is_interrupt_enabled(&self) -> bool { - self.sstatus.sie() - } - - fn set_interrupt_enabled(&mut self, is_enabled: bool) { - self.sstatus.set_sie(is_enabled); - } - - fn call(&mut self, func: unsafe extern "C" fn(usize) -> !, arg: usize) { - self.s[0] = func as u64; - self.s[1] = arg as u64; - - self.set_program_counter(Self::do_call as usize); - } - - #[unsafe(naked)] - unsafe extern "C" fn switch(from: &mut Self, to: &mut Self) { - // Input arguments `from` and `to` will be in `a0` (x10) and `a1` (x11). - naked_asm!( - // Save current task's callee-saved registers to `from` context - "sd s0, 0(a0)", - "sd s1, 8(a0)", - "sd s2, 16(a0)", - "sd s3, 24(a0)", - "sd s4, 32(a0)", - "sd s5, 40(a0)", - "sd s6, 48(a0)", - "sd s7, 56(a0)", - "sd s8, 64(a0)", - "sd s9, 72(a0)", - "sd s10, 80(a0)", - "sd s11, 88(a0)", - "sd sp, 96(a0)", - "sd ra, 104(a0)", - "csrr t0, sstatus", - "sd t0, 112(a0)", - "", - "ld s0, 0(a1)", - "ld s1, 8(a1)", - "ld s2, 16(a1)", - "ld s3, 24(a1)", - "ld s4, 32(a1)", - "ld s5, 40(a1)", - "ld s6, 48(a1)", - "ld s7, 56(a1)", - "ld s8, 64(a1)", - "ld s9, 72(a1)", - "ld s10, 80(a1)", - "ld s11, 88(a1)", - "ld sp, 96(a1)", - "ld ra, 104(a1)", - "ld t0, 112(a1)", - "csrw sstatus, t0", - "ret", - ); - } -} - -impl TaskContext { - pub const fn new() -> Self { - Self { - s: [0; 12], - sp: 0, - ra: 0, - sstatus: Sstatus::from_bits((1 << 13) | (1 << 18)), // FS = Initial, SUM = 1. - } - } - - #[unsafe(naked)] - /// Maximum of 5 arguments supported. - unsafe extern "C" fn do_call() -> ! { - naked_asm!( - "mv t0, s0", // Function pointer in s0. - "mv a0, s1", // Args - "mv a1, s2", - "mv a2, s3", - "mv a3, s4", - "mv a4, s5", - "mv fp, zero", // Set frame pointer to 0. - "mv ra, zero", - "jr t0", - ); - } -} diff --git a/crates/eonix_hal/src/arch/riscv64/mod.rs b/crates/eonix_hal/src/arch/riscv64/mod.rs index a933c1a7..49574cce 100644 --- a/crates/eonix_hal/src/arch/riscv64/mod.rs +++ b/crates/eonix_hal/src/arch/riscv64/mod.rs @@ -1,12 +1,11 @@ pub mod bootstrap; mod config; pub mod console; -pub mod context; pub mod cpu; pub mod fdt; pub mod fence; pub mod fpu; pub mod interrupt; pub mod mm; -pub mod trap; pub mod time; +pub mod trap; diff --git a/crates/eonix_hal/src/arch/riscv64/trap/mod.rs b/crates/eonix_hal/src/arch/riscv64/trap/mod.rs index a52652a1..c62a2ff3 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/mod.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/mod.rs @@ -9,7 +9,6 @@ use core::ptr::NonNull; use captured::{_captured_trap_entry, _captured_trap_return}; use default::_default_trap_entry; -use eonix_hal_traits::context::RawTaskContext; use eonix_hal_traits::trap::{IrqState as IrqStateTrait, TrapReturn}; use riscv::asm::sfence_vma_all; use riscv::register::sstatus::{self, Sstatus}; diff --git a/crates/eonix_hal/src/context.rs b/crates/eonix_hal/src/context.rs deleted file mode 100644 index c7c1e14a..00000000 --- a/crates/eonix_hal/src/context.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::arch::context::TaskContext; diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index d8696994..3d34137e 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -6,7 +6,6 @@ pub(crate) mod arch; pub mod bootstrap; -pub mod context; pub mod mm; pub mod trap; From d2cb45cc7ad789bbe02af40c75931600173973ae Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 28 Jan 2026 01:32:44 +0800 Subject: [PATCH 049/153] hal: remove struct TaskContext for x86 Check commit e16090e191b4 ("hal: remove struct TaskContext from riscv64") for more details. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/context.rs | 114 ------------------- crates/eonix_hal/src/arch/x86_64/mod.rs | 1 - crates/eonix_hal/src/arch/x86_64/trap/mod.rs | 18 ++- 3 files changed, 14 insertions(+), 119 deletions(-) delete mode 100644 crates/eonix_hal/src/arch/x86_64/context.rs diff --git a/crates/eonix_hal/src/arch/x86_64/context.rs b/crates/eonix_hal/src/arch/x86_64/context.rs deleted file mode 100644 index f55ede84..00000000 --- a/crates/eonix_hal/src/arch/x86_64/context.rs +++ /dev/null @@ -1,114 +0,0 @@ -use core::arch::naked_asm; - -use eonix_hal_traits::context::RawTaskContext; - -/// Necessary hardware states of task for doing context switches. -#[repr(C)] -#[derive(Debug, Default)] -pub struct TaskContext { - r12: u64, - r13: u64, - r14: u64, - r15: u64, - rbx: u64, - rbp: u64, - rsp: u64, - rip: u64, // Should we save rip here? - rflags: u64, // Should we save rflags here? -} - -impl TaskContext { - /// Create a new task context with the given entry point and stack pointer. - /// The entry point is the function to be called when the task is scheduled. - /// The stack pointer is the address of the top of the stack. - /// The stack pointer should be aligned to 16 bytes. - pub const fn new() -> Self { - Self { - r12: 0, - r13: 0, - r14: 0, - r15: 0, - rbx: 0, - rbp: 0, - rsp: 0, - rip: 0, - rflags: 0x200, // IF = 1 by default. - } - } - - #[unsafe(naked)] - unsafe extern "C" fn do_call() { - naked_asm!( - "mov %r12, %rdi", - "push %rbp", // NULL return address. - "jmp *%rbx", - options(att_syntax), - ); - } -} - -impl RawTaskContext for TaskContext { - fn new() -> Self { - Self::new() - } - - fn set_program_counter(&mut self, pc: usize) { - self.rip = pc as u64; - } - - fn set_stack_pointer(&mut self, sp: usize) { - self.rsp = sp as u64; - } - - fn is_interrupt_enabled(&self) -> bool { - (self.rflags & 0x200) != 0 // IF = 1 - } - - fn set_interrupt_enabled(&mut self, is_enabled: bool) { - if is_enabled { - self.rflags |= 0x200; // IF = 1 - } else { - self.rflags &= !0x200; // IF = 0 - } - } - - fn call(&mut self, func: unsafe extern "C" fn(usize) -> !, arg: usize) { - self.set_program_counter(Self::do_call as _); - self.rbx = func as _; - self.r12 = arg as _; - self.rbp = 0; // NULL previous stack frame - } - - #[unsafe(naked)] - unsafe extern "C" fn switch(from: &mut Self, to: &mut Self) { - naked_asm!( - "pop %rax", - "pushf", - "pop %rcx", - "mov %r12, (%rdi)", - "mov %r13, 8(%rdi)", - "mov %r14, 16(%rdi)", - "mov %r15, 24(%rdi)", - "mov %rbx, 32(%rdi)", - "mov %rbp, 40(%rdi)", - "mov %rsp, 48(%rdi)", - "mov %rax, 56(%rdi)", - "mov %rcx, 64(%rdi)", - "", - "mov (%rsi), %r12", - "mov 8(%rsi), %r13", - "mov 16(%rsi), %r14", - "mov 24(%rsi), %r15", - "mov 32(%rsi), %rbx", - "mov 40(%rsi), %rbp", - "mov 48(%rsi), %rdi", // store next stack pointer - "mov 56(%rsi), %rax", - "mov 64(%rsi), %rcx", - "push %rcx", - "popf", - "xchg %rdi, %rsp", // switch to new stack - "jmp *%rax", - options(att_syntax), - ); - } -} diff --git a/crates/eonix_hal/src/arch/x86_64/mod.rs b/crates/eonix_hal/src/arch/x86_64/mod.rs index 177576fc..a7f760dc 100644 --- a/crates/eonix_hal/src/arch/x86_64/mod.rs +++ b/crates/eonix_hal/src/arch/x86_64/mod.rs @@ -1,5 +1,4 @@ pub mod bootstrap; -pub mod context; pub mod cpu; pub mod fence; pub mod fpu; diff --git a/crates/eonix_hal/src/arch/x86_64/trap/mod.rs b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs index 1c32a965..008d996e 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/mod.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs @@ -2,13 +2,11 @@ mod trap_context; use core::arch::{asm, global_asm, naked_asm}; -use eonix_hal_traits::context::RawTaskContext; use eonix_hal_traits::trap::{ IrqState as IrqStateTrait, RawTrapContext, TrapReturn, }; pub use trap_context::TrapContext; -use super::context::TaskContext; use super::cpu::CPU; unsafe extern "C" { @@ -562,8 +560,6 @@ unsafe fn swap_percpu_capturer(new_capturer: usize) -> usize { } impl TrapReturn for TrapContext { - type TaskContext = TaskContext; - unsafe fn trap_return(&mut self) { let irq_states = disable_irqs_save(); let old_handler = swap_percpu_capturer(self as *mut _ as usize); @@ -579,6 +575,20 @@ impl TrapReturn for TrapContext { swap_percpu_capturer(old_handler); irq_states.restore(); } + + unsafe fn trap_return_noreturn(&mut self) -> ! { + disable_irqs(); + + unsafe { + CPU::local() + .as_mut() + .load_interrupt_stack(self as *mut _ as usize as u64); + } + + captured_trap_return(self); + + unreachable!("trap_return_noreturn should not return"); + } } impl IrqStateTrait for IrqState { From 8a6acc4fe70400eec7d3e660bed5c7ee8d01591c Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 28 Jan 2026 22:12:44 +0800 Subject: [PATCH 050/153] hal: support breakpoint trap type Breakpoint is INT 3 on x86. Also insert the int 3 instruction in stackful(). Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs | 1 + src/kernel/task.rs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs index 4d30c0dc..b917a123 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs @@ -117,6 +117,7 @@ impl RawTrapContext for TrapContext { fn trap_type(&self) -> TrapType { match self.int_no { + 3 => TrapType::Breakpoint, 0..0x20 => TrapType::Fault(self.get_fault_type()), 0x40 => TrapType::Timer { callback: |handler| { diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 7fffc008..f7c66854 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -164,6 +164,9 @@ where #[cfg(target_arch = "loongarch64")] core::arch::asm!("break 1"); + + #[cfg(target_arch = "x86_64")] + core::arch::asm!("int 3"); } } } @@ -183,6 +186,9 @@ where #[cfg(target_arch = "loongarch64")] core::arch::asm!("break 1"); + + #[cfg(target_arch = "x86_64")] + core::arch::asm!("int 3"); } unreachable!() From 425291d85af3a7ca2166c0f976fd6610a471f679 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 29 Jan 2026 00:01:17 +0800 Subject: [PATCH 051/153] hal: add a function to setup kernel call frames Setting up call frames manually is highly architecture dependent, such as stack alignment at function entry and argument passing rules. Leave this work to the HAL crate to relief the end users from complexity. Signed-off-by: greatbridf --- crates/eonix_hal/eonix_hal_traits/src/trap.rs | 9 ++++++ .../src/arch/riscv64/trap/trap_context.rs | 27 +++++++++++++++++- .../src/arch/x86_64/trap/trap_context.rs | 28 ++++++++++++++++++- src/kernel/task.rs | 8 ++---- src/kernel/task/kernel_stack.rs | 8 ++++++ src/lib.rs | 24 ++++++---------- 6 files changed, 81 insertions(+), 23 deletions(-) diff --git a/crates/eonix_hal/eonix_hal_traits/src/trap.rs b/crates/eonix_hal/eonix_hal_traits/src/trap.rs index 0619d3c0..6788bd94 100644 --- a/crates/eonix_hal/eonix_hal_traits/src/trap.rs +++ b/crates/eonix_hal/eonix_hal_traits/src/trap.rs @@ -4,6 +4,10 @@ use eonix_mm::address::VAddr; use crate::fault::Fault; +pub trait Stack { + fn get_bottom(&self) -> *mut usize; +} + /// A raw trap context. /// /// This should be implemented by the architecture-specific trap context @@ -35,6 +39,11 @@ pub trait RawTrapContext: Copy { &mut self, pc: usize, sp: Option, ra: Option, args: &[usize], write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, ) -> Result<(), E>; + + fn set_kernel_call_frame( + &mut self, pc: usize, stack: &impl Stack, ra: Option, + args: &[usize], + ); } #[doc(notable_trait)] diff --git a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs index 88d67c85..ec67b4ca 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs @@ -2,7 +2,7 @@ use core::arch::asm; use core::mem::offset_of; use eonix_hal_traits::fault::{Fault, PageFaultErrorCode}; -use eonix_hal_traits::trap::{RawTrapContext, TrapType}; +use eonix_hal_traits::trap::{RawTrapContext, Stack, TrapType}; use eonix_mm::address::VAddr; use riscv::interrupt::{Exception, Interrupt, Trap}; use riscv::register::scause::{self, Scause}; @@ -276,6 +276,31 @@ impl RawTrapContext for TrapContext { Ok(()) } + + fn set_kernel_call_frame( + &mut self, pc: usize, sp: &impl Stack, ra: Option, + args: &[usize], + ) { + self.set_program_counter(pc); + self.set_stack_pointer(sp.get_bottom().addr()); + + if let Some(ra) = ra { + self.regs.ra = ra as u64; + } + + let arg_regs = [ + &mut self.regs.a0, + &mut self.regs.a1, + &mut self.regs.a2, + &mut self.regs.a3, + &mut self.regs.a4, + &mut self.regs.a5, + ]; + + for (&arg, reg) in args.iter().zip(arg_regs.into_iter()) { + *reg = arg as u64; + } + } } impl TrapContext { diff --git a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs index b917a123..113bdf70 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs @@ -2,7 +2,7 @@ use core::arch::asm; use core::mem::offset_of; use eonix_hal_traits::fault::{Fault, PageFaultErrorCode}; -use eonix_hal_traits::trap::{RawTrapContext, TrapType}; +use eonix_hal_traits::trap::{RawTrapContext, Stack, TrapType}; use eonix_mm::address::VAddr; use crate::processor::CPU; @@ -221,4 +221,30 @@ impl RawTrapContext for TrapContext { self.set_stack_pointer(sp); Ok(()) } + + fn set_kernel_call_frame( + &mut self, pc: usize, stack: &impl Stack, ra: Option, + args: &[usize], + ) { + let sp = stack.get_bottom().wrapping_sub(1); + + self.set_program_counter(pc); + self.set_stack_pointer(sp.addr()); + + unsafe { + sp.write(ra.map(|f| f as usize).unwrap_or(0)); + } + + if let Some(&arg) = args.get(0) { + self.rdi = arg as _; + } + + if let Some(&arg) = args.get(1) { + self.rsi = arg as _; + } + + if args.len() > 2 { + unimplemented!("More than 2 arguments are not supported for now"); + } + } } diff --git a/src/kernel/task.rs b/src/kernel/task.rs index f7c66854..6be91a54 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -109,7 +109,7 @@ where let stack = KernelStack::new(); - fn execute( + unsafe extern "C" fn execute( mut future: Pin<&mut F>, output_ptr: NonNull>, ) -> ! where @@ -194,19 +194,17 @@ where unreachable!() } - let sp = stack.get_bottom(); let mut output = UnsafeCell::new(None); let mut trap_ctx = TrapContext::new(); trap_ctx.set_user_mode(false); trap_ctx.set_interrupt_enabled(true); - let _ = trap_ctx.set_user_call_frame( + trap_ctx.set_kernel_call_frame( symbol_addr!(execute::), - Some(sp.addr().get()), + &stack, None, &[(&raw mut future) as usize, output.get() as usize], - |_, _| Ok::<(), u32>(()), ); loop { diff --git a/src/kernel/task/kernel_stack.rs b/src/kernel/task/kernel_stack.rs index d6325b50..30545552 100644 --- a/src/kernel/task/kernel_stack.rs +++ b/src/kernel/task/kernel_stack.rs @@ -1,5 +1,7 @@ use core::ptr::NonNull; +use eonix_hal::traits::trap::Stack; + use crate::kernel::mem::FolioOwned; #[derive(Debug)] @@ -26,3 +28,9 @@ impl KernelStack { unsafe { ptr.cast().byte_add(len) } } } + +impl Stack for KernelStack { + fn get_bottom(&self) -> *mut usize { + self.get_bottom().cast().as_ptr() + } +} diff --git a/src/lib.rs b/src/lib.rs index 2fa1000b..aec10738 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,17 +67,13 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { drop(data); let mut ctx = TrapContext::new(); - let stack_bottom = { - let stack = KernelStack::new(); - let bottom = stack.get_bottom().addr().get(); - core::mem::forget(stack); + let stack = KernelStack::new(); - bottom - }; ctx.set_interrupt_enabled(true); ctx.set_user_mode(false); - ctx.set_program_counter(symbol_addr!(standard_main)); - ctx.set_stack_pointer(stack_bottom); + ctx.set_kernel_call_frame(symbol_addr!(standard_main), &stack, None, &[]); + + core::mem::forget(stack); unsafe { ctx.trap_return_noreturn(); @@ -94,17 +90,13 @@ fn kernel_ap_main(_stack_range: PRange) -> ! { println_debug!("AP{} started", CPU::local().cpuid()); let mut ctx = TrapContext::new(); - let stack_bottom = { - let stack = KernelStack::new(); - let bottom = stack.get_bottom().addr().get(); - core::mem::forget(stack); + let stack = KernelStack::new(); - bottom - }; ctx.set_interrupt_enabled(true); ctx.set_user_mode(false); - ctx.set_program_counter(symbol_addr!(standard_main)); - ctx.set_stack_pointer(stack_bottom); + ctx.set_kernel_call_frame(symbol_addr!(standard_main), &stack, None, &[]); + + core::mem::forget(stack); unsafe { ctx.trap_return_noreturn(); From 42b5a096e828dd0ae76183106835751d4ca400e3 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 29 Jan 2026 00:10:49 +0800 Subject: [PATCH 052/153] style: reformat thread.rs No functional change. Signed-off-by: greatbridf --- src/kernel/task/thread.rs | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 76c56dcc..4b03466a 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -176,16 +176,14 @@ impl ThreadBuilder { } pub fn set_child_tid( - mut self, - set_child_tid: Option>, + mut self, set_child_tid: Option>, ) -> Self { self.set_child_tid = set_child_tid; self } pub fn clear_child_tid( - mut self, - clear_child_tid: Option>, + mut self, clear_child_tid: Option>, ) -> Self { self.clear_child_tid = clear_child_tid; self @@ -214,9 +212,7 @@ impl ThreadBuilder { /// Clone the thread from another thread. pub fn clone_from( - self, - thread: &Thread, - clone_args: &CloneArgs, + self, thread: &Thread, clone_args: &CloneArgs, ) -> KResult { let inner = thread.inner.lock(); @@ -331,8 +327,7 @@ impl Thread { } pub fn set_robust_list( - &self, - robust_list_address: Option>, + &self, robust_list_address: Option>, ) { self.inner.lock().robust_list_address = robust_list_address; } @@ -365,10 +360,7 @@ impl Thread { } pub async fn handle_syscall( - &self, - thd_alloc: ThreadAlloc<'_>, - no: usize, - args: [usize; 6], + &self, thd_alloc: ThreadAlloc<'_>, no: usize, args: [usize; 6], ) -> Option { match syscall_handlers().get(no) { Some(Some(SyscallHandler { @@ -549,8 +541,7 @@ impl Thread { unsafe impl Allocator for ThreadAlloc<'_> { fn allocate( - &self, - layout: core::alloc::Layout, + &self, layout: core::alloc::Layout, ) -> Result, alloc::alloc::AllocError> { self.0.allocate(layout) } @@ -569,8 +560,7 @@ pub async fn yield_now() { type Output = (); fn poll( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, + mut self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll { if self.as_mut().yielded { Poll::Ready(()) From 0fefeff6abeae1c57766c70a6c72c0e101f170e5 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 29 Jan 2026 00:12:06 +0800 Subject: [PATCH 053/153] hal: set int state and user mode in TrapContext::new We always need to specify interrupt enabled state and whether we're in user mode after we create a trap context otherwise the behavior may alter on different platforms. So make it default. Signed-off-by: greatbridf --- crates/eonix_hal/eonix_hal_traits/src/trap.rs | 21 ++++++++++++++++++- .../src/arch/riscv64/trap/trap_context.rs | 2 +- .../src/arch/x86_64/trap/trap_context.rs | 2 +- src/kernel/syscall/procops.rs | 4 +--- src/kernel/task.rs | 4 +--- src/kernel/task/thread.rs | 4 +--- src/lib.rs | 8 ++----- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/crates/eonix_hal/eonix_hal_traits/src/trap.rs b/crates/eonix_hal/eonix_hal_traits/src/trap.rs index 6788bd94..f7c6b149 100644 --- a/crates/eonix_hal/eonix_hal_traits/src/trap.rs +++ b/crates/eonix_hal/eonix_hal_traits/src/trap.rs @@ -17,7 +17,26 @@ pub trait RawTrapContext: Copy { type FIrq: FnOnce(fn(irqno: usize)); type FTimer: FnOnce(fn()); - fn new() -> Self; + /// **Don't use this function unless you know what you're doing** + /// + /// Create a blank trap context. + /// + /// The context should be in a state that is ready to be used but whether + /// the interrupt is enabled or the context is in user mode is unspecified. + fn blank() -> Self; + + /// Create a new trap context. + /// + /// The context will be in a state that is ready to be used. Whether the + /// interrupt is enabled or the context is in user mode is specified by + /// the arguments. + fn new(int_enabled: bool, user: bool) -> Self { + let mut me = Self::blank(); + me.set_interrupt_enabled(int_enabled); + me.set_user_mode(user); + + me + } fn trap_type(&self) -> TrapType; diff --git a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs index ec67b4ca..bb871be5 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs @@ -122,7 +122,7 @@ impl RawTrapContext for TrapContext { type FIrq = fn(handler: fn(irqno: usize)); type FTimer = fn(handler: fn()); - fn new() -> Self { + fn blank() -> Self { let mut sstatus = Sstatus::from_bits(0); sstatus.set_fs(FS::Initial); sstatus.set_sum(true); diff --git a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs index 113bdf70..0792bf67 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs @@ -109,7 +109,7 @@ impl RawTrapContext for TrapContext { type FIrq = impl FnOnce(fn(irqno: usize)); type FTimer = fn(handler: fn()); - fn new() -> Self { + fn blank() -> Self { Self { ..Default::default() } diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 83b14081..a207a3db 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -245,10 +245,8 @@ async fn execve( thread.set_name(dentry.get_name()); let mut trap_ctx = thread.trap_ctx.borrow(); - *trap_ctx = TrapContext::new(); + *trap_ctx = TrapContext::new(true, true); - trap_ctx.set_user_mode(true); - trap_ctx.set_interrupt_enabled(true); trap_ctx.set_program_counter(load_info.entry_ip.addr()); trap_ctx.set_stack_pointer(load_info.sp.addr()); diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 6be91a54..dd391405 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -196,10 +196,8 @@ where let mut output = UnsafeCell::new(None); - let mut trap_ctx = TrapContext::new(); + let mut trap_ctx = TrapContext::new(true, false); - trap_ctx.set_user_mode(false); - trap_ctx.set_interrupt_enabled(true); trap_ctx.set_kernel_call_frame( symbol_addr!(execute::), &stack, diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 4b03466a..fac9b8f1 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -200,11 +200,9 @@ impl ThreadBuilder { } pub fn entry(mut self, entry: VAddr, stack_pointer: VAddr) -> Self { - let mut trap_ctx = TrapContext::new(); - trap_ctx.set_user_mode(true); + let mut trap_ctx = TrapContext::new(true, true); trap_ctx.set_program_counter(entry.addr()); trap_ctx.set_stack_pointer(stack_pointer.addr()); - trap_ctx.set_interrupt_enabled(true); self.trap_ctx = Some(trap_ctx); self diff --git a/src/lib.rs b/src/lib.rs index aec10738..e4c56923 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,11 +66,9 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { drop(data); - let mut ctx = TrapContext::new(); + let mut ctx = TrapContext::new(true, false); let stack = KernelStack::new(); - ctx.set_interrupt_enabled(true); - ctx.set_user_mode(false); ctx.set_kernel_call_frame(symbol_addr!(standard_main), &stack, None, &[]); core::mem::forget(stack); @@ -89,11 +87,9 @@ fn kernel_ap_main(_stack_range: PRange) -> ! { println_debug!("AP{} started", CPU::local().cpuid()); - let mut ctx = TrapContext::new(); + let mut ctx = TrapContext::new(true, false); let stack = KernelStack::new(); - ctx.set_interrupt_enabled(true); - ctx.set_user_mode(false); ctx.set_kernel_call_frame(symbol_addr!(standard_main), &stack, None, &[]); core::mem::forget(stack); From 11c66036026f29f1926c2ae74a0b293729295eee Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 29 Jan 2026 00:17:13 +0800 Subject: [PATCH 054/153] pipeline: remove loongarch64 and add x86 build and test We don't really care about loongarch64... Signed-off-by: greatbridf --- .github/workflows/test-build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index 61289e57..f496dae4 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -20,9 +20,9 @@ jobs: - arch: riscv64 target: riscv64gc-unknown-none-elf platform: virt - - arch: loongarch64 - target: loongarch64-unknown-none-softfloat - platform: virt + - arch: x86_64 + target: x86_64-unknown-none + platform: acpi_compat steps: - name: Checkout code uses: actions/checkout@v4 From d2a4d7909a587baa1975473f122ddfcc402c7de1 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 12 Feb 2026 00:03:29 +0800 Subject: [PATCH 055/153] style: reformat mm_area.rs and mm_list.rs Signed-off-by: greatbridf --- src/kernel/mem/mm_area.rs | 24 +++++---- src/kernel/mem/mm_list.rs | 100 ++++++++++++++++++++++---------------- 2 files changed, 72 insertions(+), 52 deletions(-) diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs index 782c5ef7..d453078c 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_area.rs @@ -36,7 +36,10 @@ impl Clone for MMArea { } impl MMArea { - pub fn new(range: VRange, mapping: Mapping, permission: Permission, is_shared: bool) -> Self { + pub fn new( + range: VRange, mapping: Mapping, permission: Permission, + is_shared: bool, + ) -> Self { Self { range: range.into(), mapping, @@ -80,12 +83,15 @@ impl MMArea { permission: self.permission, mapping: match &self.mapping { Mapping::Anonymous => Mapping::Anonymous, - Mapping::File(mapping) => Mapping::File(mapping.offset(diff)), + Mapping::File(mapping) => { + Mapping::File(mapping.offset(diff)) + } }, is_shared: self.is_shared, }; - let new_range = self.range_borrow().shrink(self.range_borrow().end() - at); + let new_range = + self.range_borrow().shrink(self.range_borrow().end() - at); *self.range.get_mut() = new_range; (Some(self), Some(right)) @@ -136,10 +142,7 @@ impl MMArea { /// # Arguments /// * `offset`: The offset from the start of the mapping, aligned to 4KB boundary. pub async fn handle_mmap( - &self, - pfn: &mut PFN, - attr: &mut PageAttribute, - offset: usize, + &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: usize, write: bool, ) -> KResult<()> { let Mapping::File(file_mapping) = &self.mapping else { @@ -196,8 +199,11 @@ impl MMArea { Ok(()) } - pub async fn handle(&self, pte: &mut impl PTE, offset: usize, write: bool) -> KResult<()> { - let mut attr = pte.get_attr().as_page_attr().expect("Not a page attribute"); + pub async fn handle( + &self, pte: &mut impl PTE, offset: usize, write: bool, + ) -> KResult<()> { + let mut attr = + pte.get_attr().as_page_attr().expect("Not a page attribute"); let mut pfn = pte.get_pfn(); if attr.contains(PageAttribute::COPY_ON_WRITE) { diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index f073025b..a3b3635f 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -7,7 +7,8 @@ use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; use eonix_hal::mm::{ - flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, GLOBAL_PAGE_TABLE, + flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, + GLOBAL_PAGE_TABLE, }; use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; @@ -59,7 +60,9 @@ impl MMListInner { addr.is_user() && self.overlapping_addr(addr).is_none() } - fn overlapping_range(&self, range: VRange) -> impl DoubleEndedIterator + '_ { + fn overlapping_range( + &self, range: VRange, + ) -> impl DoubleEndedIterator + '_ { self.areas.range(range.into_bounds()) } @@ -112,7 +115,9 @@ impl MMListInner { // TODO: Write back dirty pages. self.areas.retain(|area| { - let Some((left, mid, right)) = area.range().mask_with_checked(&range_to_unmap) else { + let Some((left, mid, right)) = + area.range().mask_with_checked(&range_to_unmap) + else { return true; }; @@ -136,7 +141,8 @@ impl MMListInner { } (None, Some(right)) => { assert!(right_remaining.is_none()); - let (_, Some(right)) = area.clone().split(right.start()) else { + let (_, Some(right)) = area.clone().split(right.start()) + else { unreachable!("`right.start()` is within the area"); }; @@ -145,7 +151,9 @@ impl MMListInner { (Some(left), Some(right)) => { assert!(left_remaining.is_none()); assert!(right_remaining.is_none()); - let (Some(left), Some(mid)) = area.clone().split(left.end()) else { + let (Some(left), Some(mid)) = + area.clone().split(left.end()) + else { unreachable!("`left.end()` is within the area"); }; @@ -171,7 +179,9 @@ impl MMListInner { Ok(pages_to_free) } - fn protect(&mut self, start: VAddr, len: usize, permission: Permission) -> KResult<()> { + fn protect( + &mut self, start: VAddr, len: usize, permission: Permission, + ) -> KResult<()> { assert_eq!(start.floor(), start); assert!(len != 0); @@ -184,7 +194,9 @@ impl MMListInner { let mut found = false; let old_areas = core::mem::take(&mut self.areas); for mut area in old_areas { - let Some((left, mid, right)) = area.range().mask_with_checked(&range_to_protect) else { + let Some((left, mid, right)) = + area.range().mask_with_checked(&range_to_protect) + else { self.areas.insert(area); continue; }; @@ -201,7 +213,8 @@ impl MMListInner { } if let Some(right) = right { - let (Some(left), Some(right)) = area.split(right.start()) else { + let (Some(left), Some(right)) = area.split(right.start()) + else { unreachable!("`right.start()` is within the area"); }; @@ -210,9 +223,13 @@ impl MMListInner { } for pte in self.page_table.iter_user(mid) { - let mut page_attr = pte.get_attr().as_page_attr().expect("Not a page attribute"); + let mut page_attr = pte + .get_attr() + .as_page_attr() + .expect("Not a page attribute"); - if !permission.read && !permission.write && !permission.execute { + if !permission.read && !permission.write && !permission.execute + { // If no permissions are set, we just remove the page. page_attr.remove( PageAttribute::PRESENT @@ -248,24 +265,24 @@ impl MMListInner { } fn mmap( - &mut self, - at: VAddr, - len: usize, - mapping: Mapping, - permission: Permission, - is_shared: bool, + &mut self, at: VAddr, len: usize, mapping: Mapping, + permission: Permission, is_shared: bool, ) -> KResult<()> { assert_eq!(at.floor(), at); assert_eq!(len & (PAGE_SIZE - 1), 0); let range = VRange::new(at, at + len); // We are doing a area marker insertion. - if len == 0 && !self.check_overlapping_addr(at) || !self.check_overlapping_range(range) { + if len == 0 && !self.check_overlapping_addr(at) + || !self.check_overlapping_range(range) + { return Err(EEXIST); } match &mapping { - Mapping::Anonymous => self.page_table.set_anonymous(range, permission), + Mapping::Anonymous => { + self.page_table.set_anonymous(range, permission) + } Mapping::File(_) => self.page_table.set_mmapped(range, permission), } @@ -462,7 +479,8 @@ impl MMList { /// No need to do invalidation manually, `PageTable` already does it. pub async fn unmap(&self, start: VAddr, len: usize) -> KResult<()> { - let pages_to_free = self.inner.borrow().lock().await.unmap(start, len)?; + let pages_to_free = + self.inner.borrow().lock().await.unmap(start, len)?; // We need to assure that the pages are not accessed anymore. // The ones having these pages in their TLB could read from or write to them. @@ -475,7 +493,9 @@ impl MMList { Ok(()) } - pub async fn protect(&self, start: VAddr, len: usize, prot: Permission) -> KResult<()> { + pub async fn protect( + &self, start: VAddr, len: usize, prot: Permission, + ) -> KResult<()> { self.inner.borrow().lock().await.protect(start, len, prot)?; // flush the tlb due to the pte attribute changes @@ -526,12 +546,8 @@ impl MMList { } pub async fn mmap_hint( - &self, - hint: VAddr, - len: usize, - mapping: Mapping, - permission: Permission, - is_shared: bool, + &self, hint: VAddr, len: usize, mapping: Mapping, + permission: Permission, is_shared: bool, ) -> KResult { let inner = self.inner.borrow(); let mut inner = inner.lock().await; @@ -554,11 +570,7 @@ impl MMList { } pub async fn mmap_fixed( - &self, - at: VAddr, - len: usize, - mapping: Mapping, - permission: Permission, + &self, at: VAddr, len: usize, mapping: Mapping, permission: Permission, is_shared: bool, ) -> KResult { self.inner @@ -614,14 +626,11 @@ impl MMList { program_break.grow(len); - inner.page_table.set_anonymous( - range_to_grow, - Permission { - read: true, - write: true, - execute: false, - }, - ); + inner.page_table.set_anonymous(range_to_grow, Permission { + read: true, + write: true, + execute: false, + }); inner.break_pos = Some(pos); pos @@ -639,7 +648,9 @@ impl MMList { /// Access the memory area with the given function. /// The function will be called with the offset of the area and the slice of the area. - pub async fn access_mut(&self, start: VAddr, len: usize, func: F) -> KResult<()> + pub async fn access_mut( + &self, start: VAddr, len: usize, func: F, + ) -> KResult<()> where F: Fn(usize, &mut [u8]), { @@ -695,7 +706,8 @@ impl MMList { // SAFETY: We are sure that the page is valid and we have the right to access it. Folio::with_raw(pte.get_pfn(), |page| { let mut pg = page.lock(); - let page_data = &mut pg.as_bytes_mut()[start_offset..end_offset]; + let page_data = + &mut pg.as_bytes_mut()[start_offset..end_offset]; func(offset + idx * 0x1000, page_data); }); @@ -784,7 +796,8 @@ where fn set_mapped(&mut self, execute: bool) { // Writable flag is set during page fault handling while executable flag is // preserved across page faults, so we set executable flag now. - let mut attr = PageAttribute::READ | PageAttribute::USER | PageAttribute::MAPPED; + let mut attr = + PageAttribute::READ | PageAttribute::USER | PageAttribute::MAPPED; attr.set(PageAttribute::EXECUTE, execute); self.set(EMPTY_PAGE.clone().into_raw(), T::Attr::from(attr)); @@ -796,7 +809,8 @@ where .as_page_attr() .expect("Not a page attribute"); - if !from_attr.intersects(PageAttribute::PRESENT | PageAttribute::MAPPED) { + if !from_attr.intersects(PageAttribute::PRESENT | PageAttribute::MAPPED) + { return; } From 7747be0822bafe5fec1de1b479770443c2250f27 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 12 Feb 2026 00:08:41 +0800 Subject: [PATCH 056/153] style, mem: move mod mm_area into mm_list Signed-off-by: greatbridf --- src/kernel/mem.rs | 6 +++--- src/kernel/mem/mm_list.rs | 8 +++++--- src/kernel/mem/{ => mm_list}/mm_area.rs | 3 +-- 3 files changed, 9 insertions(+), 8 deletions(-) rename src/kernel/mem/{ => mm_list}/mm_area.rs (99%) diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index 47b864bb..41d5a17d 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -4,15 +4,15 @@ mod access; mod address; mod allocator; mod folio; -mod mm_area; mod mm_list; mod page_alloc; mod page_cache; pub use access::PhysAccess; pub use folio::{Folio, FolioOwned, LockedFolio}; -pub(self) use mm_area::MMArea; -pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; +pub use mm_list::{ + handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission, +}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache, PageOffset}; pub use paging::PageBuffer; diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index a3b3635f..c5e2fbf4 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -1,4 +1,5 @@ mod mapping; +mod mm_area; mod page_fault; mod page_table; @@ -14,12 +15,13 @@ use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; use eonix_sync::{LazyLock, Mutex}; -pub use mapping::{FileMapping, Mapping}; -pub use page_fault::handle_kernel_page_fault; use page_table::KernelPageTable; +pub use self::mapping::{FileMapping, Mapping}; +use self::mm_area::MMArea; +pub use self::page_fault::handle_kernel_page_fault; use super::address::{VAddrExt as _, VRangeExt as _}; -use super::{Folio, FolioOwned, MMArea}; +use super::{Folio, FolioOwned}; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; use crate::prelude::*; use crate::sync::ArcSwap; diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs similarity index 99% rename from src/kernel/mem/mm_area.rs rename to src/kernel/mem/mm_list/mm_area.rs index d453078c..86fc7ee3 100644 --- a/src/kernel/mem/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -7,8 +7,7 @@ use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{Folio as _, PFN}; -use super::mm_list::EMPTY_PAGE; -use super::{Mapping, Permission}; +use super::{Mapping, Permission, EMPTY_PAGE}; use crate::kernel::mem::folio::Folio; use crate::kernel::mem::{CachePage, FolioOwned, PageOffset}; use crate::prelude::KResult; From fe9ef4a4e1e77d8aad33e573dff01356181a8f0c Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 13 Feb 2026 19:36:14 +0800 Subject: [PATCH 057/153] style: reformat page_fault.rs Signed-off-by: greatbridf --- src/kernel/mem/mm_list/page_fault.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index 5a56efbc..755de32a 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -30,7 +30,8 @@ impl FixEntry { } fn entries() -> &'static [FixEntry] { - let fix_seg_len_bytes = extern_symbol_addr!(FIX_END) - extern_symbol_addr!(FIX_START); + let fix_seg_len_bytes = + extern_symbol_addr!(FIX_END) - extern_symbol_addr!(FIX_START); unsafe { // SAFETY: `FIX_START` and `FIX_END` are defined in the linker script @@ -46,9 +47,7 @@ impl FixEntry { impl MMList { /// Handle a user page fault. pub async fn handle_user_page_fault( - &self, - addr: VAddr, - error: PageFaultErrorCode, + &self, addr: VAddr, error: PageFaultErrorCode, ) -> Result<(), Signal> { debug_assert!( error.contains(PageFaultErrorCode::UserAccess), @@ -58,7 +57,8 @@ impl MMList { let inner = self.inner.borrow(); let inner = inner.lock().await; - let area = inner.areas.get(&VRange::from(addr)).ok_or(Signal::SIGBUS)?; + let area = + inner.areas.get(&VRange::from(addr)).ok_or(Signal::SIGBUS)?; // Check user access permission. if error.contains(PageFaultErrorCode::Read) && !area.permission.read { @@ -73,7 +73,9 @@ impl MMList { Err(Signal::SIGSEGV)? } - if error.contains(PageFaultErrorCode::InstructionFetch) && !area.permission.execute { + if error.contains(PageFaultErrorCode::InstructionFetch) + && !area.permission.execute + { Err(Signal::SIGSEGV)? } @@ -124,9 +126,7 @@ fn kernel_page_fault_die(vaddr: VAddr, pc: VAddr) -> ! { } pub async fn handle_kernel_page_fault( - fault_pc: VAddr, - addr: VAddr, - error: PageFaultErrorCode, + fault_pc: VAddr, addr: VAddr, error: PageFaultErrorCode, ) -> Option { debug_assert!( !error.contains(PageFaultErrorCode::UserAccess), @@ -156,7 +156,9 @@ pub async fn handle_kernel_page_fault( .page_table .iter_user(VRange::from(addr.floor()).grow(PAGE_SIZE)) .next() - .expect("If we can find the mapped area, we should be able to find the PTE"); + .expect( + "If we can find the mapped area, we should be able to find the PTE", + ); if let Err(_) = area .handle( From 78aaafa8f9c9db32103a4943fa4b24255bbdf5b0 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 13 Feb 2026 19:40:49 +0800 Subject: [PATCH 058/153] mem: use intrusive lists for `MemArea`s - Rename MMArea to MemArea - Use intrusive RBTree to store MemAreas - Introduce `RangeProtected` to get access to area ranges if we have MMListInner lock held. - Introduce `AreaFlags` in replace of `Permission` and `is_shared`. Old interfaces stay the same but we will migrate slowly to the new method. - Add functions for backwards compatibility. Those will be removed later. Using intrusive lists can save us time from moving the structs around and gives us potential to optimize the code's structure. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 133 ++++++------ src/kernel/mem/mm_list/mm_area.rs | 313 +++++++++++++++++++++------ src/kernel/mem/mm_list/page_fault.rs | 20 +- 3 files changed, 328 insertions(+), 138 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index c5e2fbf4..e9509622 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -3,7 +3,7 @@ mod mm_area; mod page_fault; mod page_table; -use alloc::collections::btree_set::BTreeSet; +use alloc::sync::Arc; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -15,10 +15,11 @@ use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; use eonix_sync::{LazyLock, Mutex}; +use mm_area::AreaList; use page_table::KernelPageTable; pub use self::mapping::{FileMapping, Mapping}; -use self::mm_area::MMArea; +pub use self::mm_area::{AreaFlags, MemArea}; pub use self::page_fault::handle_kernel_page_fault; use super::address::{VAddrExt as _, VRangeExt as _}; use super::{Folio, FolioOwned}; @@ -39,8 +40,8 @@ pub struct Permission { pub execute: bool, } -struct MMListInner { - areas: BTreeSet, +pub struct MMListInner { + areas: AreaList, page_table: KernelPageTable, break_start: Option, break_pos: Option, @@ -54,22 +55,16 @@ pub struct MMList { } impl MMListInner { - fn overlapping_addr(&self, addr: VAddr) -> Option<&MMArea> { - self.areas.get(&VRange::from(addr)) + fn overlapping_addr(&self, addr: VAddr) -> Option> { + self.areas.get(addr) } fn check_overlapping_addr(&self, addr: VAddr) -> bool { - addr.is_user() && self.overlapping_addr(addr).is_none() - } - - fn overlapping_range( - &self, range: VRange, - ) -> impl DoubleEndedIterator + '_ { - self.areas.range(range.into_bounds()) + addr.is_user() && !self.areas.contains(addr) } fn check_overlapping_range(&self, range: VRange) -> bool { - range.is_user() && self.overlapping_range(range).next().is_none() + range.is_user() && !self.areas.contains_range(&range) } fn random_start(&self) -> VAddr { @@ -85,19 +80,23 @@ impl MMListInner { hint = hint.floor(); } - let mut range = VRange::from(hint).grow(len); - loop { - if !range.is_user() { + let end = hint + len; + + if !VRange::new(hint, end).is_user() { return None; } - match self.overlapping_range(range).next_back() { - None => return Some(range.start()), - Some(area) => { - range = VRange::from(area.range().end().ceil()).grow(len); - } + let Some(ub) = self.areas.upper_bound(end) else { + return Some(hint); + }; + + let ub_end = ub.range.as_ref(self).end().ceil(); + if ub_end <= hint { + return Some(hint); } + + hint = ub_end; } } @@ -117,8 +116,10 @@ impl MMListInner { // TODO: Write back dirty pages. self.areas.retain(|area| { + let range = unsafe { area.range.as_ref_unchecked() }; + let Some((left, mid, right)) = - area.range().mask_with_checked(&range_to_unmap) + range.mask_with_checked(&range_to_unmap) else { return true; }; @@ -194,10 +195,12 @@ impl MMListInner { } let mut found = false; - let old_areas = core::mem::take(&mut self.areas); - for mut area in old_areas { + let old_areas = self.areas.take(); + for area in old_areas { + let mut area = area.as_ref().clone(); + let Some((left, mid, right)) = - area.range().mask_with_checked(&range_to_protect) + area.range.as_ref(self).mask_with_checked(&range_to_protect) else { self.areas.insert(area); continue; @@ -255,7 +258,7 @@ impl MMListInner { pte.set_attr(page_attr.into()); } - area.permission = permission; + area.set_permission(permission); self.areas.insert(area); } @@ -288,8 +291,12 @@ impl MMListInner { Mapping::File(_) => self.page_table.set_mmapped(range, permission), } - self.areas - .insert(MMArea::new(range, mapping, permission, is_shared)); + self.areas.insert(MemArea::new( + range, + AreaFlags::from_old(permission, is_shared), + mapping, + )); + Ok(()) } } @@ -297,9 +304,11 @@ impl MMListInner { impl Drop for MMListInner { fn drop(&mut self) { // May buggy - for area in &self.areas { - if area.is_shared { - for pte in self.page_table.iter_user(area.range()) { + for area in self.areas.iter() { + let range = area.range.as_ref(self).clone(); + + if area.is_shared() { + for pte in self.page_table.iter_user(range) { // XXX: Fix me let _ = pte.take(); // let raw_page = RawPagePtr::from(pfn); @@ -308,7 +317,7 @@ impl Drop for MMListInner { // } } } else { - for pte in self.page_table.iter_user(area.range()) { + for pte in self.page_table.iter_user(range) { let (pfn, _) = pte.take(); unsafe { Folio::from_raw(pfn) }; } @@ -351,7 +360,7 @@ impl MMList { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { - areas: BTreeSet::new(), + areas: AreaList::new(), page_table, break_start: None, break_pos: None, @@ -368,7 +377,7 @@ impl MMList { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { - areas: inner.areas.clone(), + areas: inner.areas.deep_clone(), page_table, break_start: inner.break_start, break_pos: inner.break_pos, @@ -379,15 +388,15 @@ impl MMList { let list_inner = list.inner.borrow(); let list_inner = list_inner.lock().await; + let pgtable = &list_inner.page_table; + for area in list_inner.areas.iter() { - if !area.is_shared { - list_inner - .page_table - .set_copy_on_write(&mut inner.page_table, area.range()); + let range = area.range.as_ref(&list_inner).clone(); + + if !area.is_shared() { + pgtable.set_copy_on_write(&mut inner.page_table, range); } else { - list_inner - .page_table - .set_copied(&mut inner.page_table, area.range()); + pgtable.set_copied(&mut inner.page_table, range); } } } @@ -605,26 +614,25 @@ impl MMList { return current_break; } - if !inner.areas.contains(&break_start) { - inner.areas.insert(MMArea::new( - break_start, - Mapping::Anonymous, - Permission { - read: true, - write: true, - execute: false, - }, - false, - )); - } - - let program_break = inner - .areas - .get(&break_start) - .expect("Program break area should be valid"); + let program_break = + inner.areas.get_or_insert(break_start.start(), || { + MemArea::new( + break_start, + AreaFlags::from_old( + Permission { + read: true, + write: true, + execute: false, + }, + false, + ), + Mapping::Anonymous, + ) + }); let len = pos - current_break; - let range_to_grow = VRange::from(program_break.range().end()).grow(len); + let range_to_grow = + VRange::from(program_break.range.as_ref(&inner).end()).grow(len); program_break.grow(len); @@ -672,8 +680,9 @@ impl MMList { while remaining > 0 { let area = inner.overlapping_addr(current).ok_or(EFAULT)?; - let area_start = area.range().start(); - let area_end = area.range().end(); + let range = area.range.as_ref(&inner); + let area_start = range.start(); + let area_end = range.end(); let area_remaining = area_end - current; let access_len = remaining.min(area_remaining); diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 86fc7ee3..3b563d27 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -1,4 +1,4 @@ -use core::borrow::Borrow; +use alloc::sync::Arc; use core::cell::UnsafeCell; use core::cmp; use core::sync::atomic::Ordering; @@ -6,91 +6,299 @@ use core::sync::atomic::Ordering; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{Folio as _, PFN}; +use intrusive_collections::rbtree::Entry; +use intrusive_collections::{ + intrusive_adapter, Bound, KeyAdapter, PointerOps, RBTree, RBTreeAtomicLink, +}; -use super::{Mapping, Permission, EMPTY_PAGE}; +use super::{Mapping, EMPTY_PAGE}; use crate::kernel::mem::folio::Folio; -use crate::kernel::mem::{CachePage, FolioOwned, PageOffset}; +use crate::kernel::mem::mm_list::MMListInner; +use crate::kernel::mem::{CachePage, FolioOwned, PageOffset, Permission}; use crate::prelude::KResult; -#[derive(Debug)] -pub struct MMArea { - range: UnsafeCell, - pub(super) mapping: Mapping, - pub(super) permission: Permission, - pub is_shared: bool, +bitflags::bitflags! { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub struct AreaFlags: u32 { + const READ = 1 << 0; + const WRITE = 1 << 1; + const EXECUTE = 1 << 2; + const SHARED = 1 << 3; + } } -unsafe impl Send for MMArea {} -unsafe impl Sync for MMArea {} +/// # Lock +/// Protected by [`MMListInner`] lock. +pub struct RangeProtected(UnsafeCell); + +unsafe impl Send for RangeProtected {} +unsafe impl Sync for RangeProtected {} -impl Clone for MMArea { +impl Clone for RangeProtected { fn clone(&self) -> Self { + // TODO: CHANGE THIS + let range = unsafe { self.as_ref_unchecked().clone() }; + + Self(UnsafeCell::new(range)) + } +} + +#[derive(Clone)] +pub struct MemArea { + /// # Lock + /// Protected by [`MMListInner`] lock. + pub range: RangeProtected, + pub flags: AreaFlags, + areas_link: RBTreeAtomicLink, + + mapping: Mapping, +} + +intrusive_adapter!(AreasAdapter = Arc: MemArea { areas_link: RBTreeAtomicLink }); + +impl<'a> KeyAdapter<'a> for AreasAdapter { + type Key = VRange; + + fn get_key( + &self, value: &'a ::Value, + ) -> Self::Key { + unsafe { + // SAFETY: The [`AreaList`] is within the [`MMListInner`]. + // References to the areas list implies holding the lock of + // [`MMListInner`], so it's safe to read the range. + *value.range.as_ref_unchecked() + } + } +} + +pub struct AreaList { + areas: RBTree, +} + +impl AreaFlags { + pub fn from_old(perm: Permission, is_shared: bool) -> Self { + let mut flags = AreaFlags::empty(); + + flags.set(AreaFlags::READ, perm.read); + flags.set(AreaFlags::WRITE, perm.write); + flags.set(AreaFlags::EXECUTE, perm.execute); + flags.set(AreaFlags::SHARED, is_shared); + + flags + } +} + +impl RangeProtected { + pub const fn new(range: VRange) -> Self { + Self(UnsafeCell::new(range)) + } + + pub unsafe fn as_ref_unchecked<'a>(&self) -> &'a VRange { + unsafe { &*self.0.get() } + } + + pub unsafe fn as_mut_unchecked<'a>(&self) -> &'a mut VRange { + unsafe { &mut *self.0.get() } + } + + pub fn as_ref(&self, _list_read_lock: &MMListInner) -> &VRange { + unsafe { + // SAFETY: If we are holding the list's read lock, we won't modify + // the areas and so won't the range, so it's safe to return a + // reference. + self.as_ref_unchecked() + } + } + + pub fn as_mut<'a>( + &self, _list_write_lock: &'a mut MMListInner, + ) -> &'a mut VRange { + unsafe { + // SAFETY: If we are holding the list's write lock, we can guarantee + // that no one else is reading or writing the areas, so it's safe to + // return a mutable reference. + self.as_mut_unchecked() + } + } + + pub fn get_mut(&mut self) -> &mut VRange { + self.0.get_mut() + } +} + +impl AreaList { + pub const fn new() -> Self { Self { - range: UnsafeCell::new(self.range()), - mapping: self.mapping.clone(), - permission: self.permission, - is_shared: self.is_shared, + areas: RBTree::new(AreasAdapter::NEW), + } + } + + pub fn get_or_insert( + &mut self, addr: VAddr, insert: impl FnOnce() -> MemArea, + ) -> Arc { + match self.areas.entry(&VRange::from(addr)) { + Entry::Vacant(entry) => { + let area = Arc::new(insert()); + entry.insert(area.clone()); + area + } + Entry::Occupied(cursor) => { + cursor.as_cursor().clone_pointer().unwrap() + } + } + } + + pub fn get(&self, addr: VAddr) -> Option> { + self.areas.find(&VRange::from(addr)).clone_pointer() + } + + pub fn contains(&self, addr: VAddr) -> bool { + !self.areas.find(&VRange::from(addr)).is_null() + } + + /// Return the mem area with the highest start address that is below or + /// overlaps with the given address. + pub fn upper_bound(&self, addr: VAddr) -> Option> { + self.areas + .upper_bound(Bound::Included(&VRange::from(addr))) + .clone_pointer() + } + + pub fn contains_range(&self, range: &VRange) -> bool { + let Some(ub) = self.upper_bound(range.end()) else { + return false; + }; + + let range = unsafe { + // SAFETY: Check `KeyAdapter` impl above. + ub.range.as_ref_unchecked() + }; + + range.end() <= range.start() + } + + // TODO: For backwards compatibility. Remove this. + pub fn retain(&mut self, mut pred: impl FnMut(&MemArea) -> bool) { + let mut cursor = self.areas.front_mut(); + + while let Some(area) = cursor.get() { + if !pred(area) { + cursor.remove(); + } + + cursor.move_next(); + } + } + + // TODO: For backwards compatibility. Remove this. + pub fn insert(&mut self, area: MemArea) { + let range = unsafe { area.range.as_ref_unchecked() }; + + match self.areas.entry(range) { + Entry::Occupied(_) => panic!("Overlapping mem area: {:?}.", range), + Entry::Vacant(insert_cursor) => { + insert_cursor.insert(Arc::new(area)); + } + } + } + + // TODO: For backwards compatibility. Remove this. + pub fn deep_clone(&self) -> Self { + let mut areas = RBTree::new(AreasAdapter::NEW); + + for area in self.areas.iter() { + areas.insert(Arc::new(area.clone())); } + + Self { areas } + } + + pub fn iter(&self) -> impl Iterator { + let mut cursor = self.areas.front(); + + core::iter::from_fn(move || { + let area = cursor.get()?; + cursor.move_next(); + + Some(area) + }) + } + + // TODO: For backwards compatibility. Remove this. + pub fn take(&mut self) -> impl IntoIterator> { + self.areas.take() } } -impl MMArea { - pub fn new( - range: VRange, mapping: Mapping, permission: Permission, - is_shared: bool, +impl MemArea { + pub const fn new( + range: VRange, flags: AreaFlags, mapping: Mapping, ) -> Self { Self { - range: range.into(), + range: RangeProtected::new(range), + flags, + areas_link: RBTreeAtomicLink::new(), mapping, - permission, - is_shared, } } - fn range_borrow(&self) -> &VRange { - // SAFETY: The only way we get a reference to `MMArea` object is through `MMListInner`. - // And `MMListInner` is locked with IRQ disabled. - unsafe { self.range.get().as_ref().unwrap() } + pub fn set_permission(&mut self, perm: Permission) { + self.flags.set(AreaFlags::READ, perm.read); + self.flags.set(AreaFlags::WRITE, perm.write); + self.flags.set(AreaFlags::EXECUTE, perm.execute); + } + + pub fn is_shared(&self) -> bool { + self.flags.contains(AreaFlags::SHARED) } - pub fn range(&self) -> VRange { - *self.range_borrow() + pub fn can_read(&self) -> bool { + self.flags.contains(AreaFlags::READ) + } + + pub fn can_write(&self) -> bool { + self.flags.contains(AreaFlags::WRITE) + } + + pub fn can_execute(&self) -> bool { + self.flags.contains(AreaFlags::EXECUTE) } /// # Safety /// This function should be called only when we can guarantee that the range /// won't overlap with any other range in some scope. pub fn grow(&self, count: usize) { - let range = unsafe { self.range.get().as_mut().unwrap() }; - range.clone_from(&self.range_borrow().grow(count)); + // TODO: Remove this. + let range = unsafe { self.range.as_mut_unchecked() }; + range.clone_from(&range.grow(count)); } pub fn split(mut self, at: VAddr) -> (Option, Option) { assert!(at.is_page_aligned()); + let range = self.range.get_mut(); - match self.range_borrow().cmp(&VRange::from(at)) { + match (*range).cmp(&VRange::from(at)) { cmp::Ordering::Less => (Some(self), None), cmp::Ordering::Greater => (None, Some(self)), cmp::Ordering::Equal => { - let diff = at - self.range_borrow().start(); + let diff = at - range.start(); if diff == 0 { return (None, Some(self)); } let right = Self { - range: VRange::new(at, self.range_borrow().end()).into(), - permission: self.permission, + range: RangeProtected::new(VRange::new(at, range.end())), + flags: self.flags.clone(), + areas_link: RBTreeAtomicLink::new(), mapping: match &self.mapping { Mapping::Anonymous => Mapping::Anonymous, Mapping::File(mapping) => { Mapping::File(mapping.offset(diff)) } }, - is_shared: self.is_shared, }; - let new_range = - self.range_borrow().shrink(self.range_borrow().end() - at); + let new_range = range.shrink(range.end() - at); *self.range.get_mut() = new_range; (Some(self), Some(right)) @@ -102,7 +310,7 @@ impl MMArea { assert!(attr.contains(PageAttribute::COPY_ON_WRITE)); attr.remove(PageAttribute::COPY_ON_WRITE); - attr.set(PageAttribute::WRITE, self.permission.write); + attr.set(PageAttribute::WRITE, self.can_write()); let page = unsafe { Folio::from_raw(*pfn) }; @@ -153,14 +361,14 @@ impl MMArea { let file_offset = file_mapping.offset + offset; let map_page = |cache_page: &CachePage| { - if !self.permission.write { + if !self.can_write() { assert!(!write, "Write fault on read-only mapping"); *pfn = cache_page.add_mapping(); return; } - if self.is_shared { + if self.is_shared() { // We don't process dirty flags in write faults. // Simply assume that page will eventually be dirtied. // So here we can set the dirty flag now. @@ -224,26 +432,3 @@ impl MMArea { Ok(()) } } - -impl Eq for MMArea {} -impl PartialEq for MMArea { - fn eq(&self, other: &Self) -> bool { - self.range_borrow().eq(other.range_borrow()) - } -} -impl PartialOrd for MMArea { - fn partial_cmp(&self, other: &Self) -> Option { - self.range_borrow().partial_cmp(other.range_borrow()) - } -} -impl Ord for MMArea { - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - self.range_borrow().cmp(other.range_borrow()) - } -} - -impl Borrow for MMArea { - fn borrow(&self) -> &VRange { - self.range_borrow() - } -} diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index 755de32a..db13bcc7 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -57,11 +57,10 @@ impl MMList { let inner = self.inner.borrow(); let inner = inner.lock().await; - let area = - inner.areas.get(&VRange::from(addr)).ok_or(Signal::SIGBUS)?; + let area = inner.areas.get(addr).ok_or(Signal::SIGBUS)?; // Check user access permission. - if error.contains(PageFaultErrorCode::Read) && !area.permission.read { + if error.contains(PageFaultErrorCode::Read) && !area.can_read() { // Under x86_64, we don't have a way to distinguish // between a read fault and a non-present fault. But it should be OK // since non-readable pages are not allowed under x86 and if we read @@ -69,12 +68,12 @@ impl MMList { Err(Signal::SIGSEGV)? } - if error.contains(PageFaultErrorCode::Write) && !area.permission.write { + if error.contains(PageFaultErrorCode::Write) && !area.can_write() { Err(Signal::SIGSEGV)? } if error.contains(PageFaultErrorCode::InstructionFetch) - && !area.permission.execute + && !area.can_execute() { Err(Signal::SIGSEGV)? } @@ -87,7 +86,7 @@ impl MMList { area.handle( pte, - addr.floor() - area.range().start(), + addr.floor() - area.range.as_ref(&inner).start(), error.contains(PageFaultErrorCode::Write), ) .await @@ -145,11 +144,8 @@ pub async fn handle_kernel_page_fault( let inner = mms.inner.borrow(); let inner = inner.lock().await; - let area = match inner.areas.get(&VRange::from(addr)) { - Some(area) => area, - None => { - return Some(try_page_fault_fix(fault_pc, addr)); - } + let Some(area) = inner.areas.get(addr) else { + return Some(try_page_fault_fix(fault_pc, addr)); }; let pte = inner @@ -163,7 +159,7 @@ pub async fn handle_kernel_page_fault( if let Err(_) = area .handle( pte, - addr.floor() - area.range().start(), + addr.floor() - area.range.as_ref(&inner).start(), error.contains(PageFaultErrorCode::Write), ) .await From 7ff4d544c52dd17fbf4d57d97f13d7322215a6c4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 20 Feb 2026 23:03:29 +0800 Subject: [PATCH 059/153] mem: add a function to insert Arcs into the list Eventually, we'd like to replace the `insert` function. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 3b563d27..59aabb4e 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -133,6 +133,16 @@ impl AreaList { } } + pub fn insert_new(&mut self, area: Arc) { + let range = unsafe { area.range.as_ref_unchecked() }; + match self.areas.entry(range) { + Entry::Occupied(_) => panic!("Overlapping mem area: {range:?}."), + Entry::Vacant(insert_cursor) => { + insert_cursor.insert(area); + } + } + } + pub fn get_or_insert( &mut self, addr: VAddr, insert: impl FnOnce() -> MemArea, ) -> Arc { From 727dcc0a5cbf1b6aaad43775eaa27dac2fc0ac77 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 20 Feb 2026 23:05:17 +0800 Subject: [PATCH 060/153] style: reformat interrupt.rs Signed-off-by: greatbridf --- src/kernel/interrupt.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/kernel/interrupt.rs b/src/kernel/interrupt.rs index 2092bfcb..a3237239 100644 --- a/src/kernel/interrupt.rs +++ b/src/kernel/interrupt.rs @@ -39,7 +39,9 @@ pub fn default_fault_handler(fault_type: Fault, trap_ctx: &mut TrapContext) { } => { let fault_pc = VAddr::from(trap_ctx.get_program_counter()); - if let Some(new_pc) = block_on(handle_kernel_page_fault(fault_pc, vaddr, error_code)) { + if let Some(new_pc) = + block_on(handle_kernel_page_fault(fault_pc, vaddr, error_code)) + { trap_ctx.set_program_counter(new_pc.addr()); } } @@ -50,7 +52,9 @@ pub fn default_fault_handler(fault_type: Fault, trap_ctx: &mut TrapContext) { #[eonix_hal::default_trap_handler] pub fn interrupt_handler(trap_ctx: &mut TrapContext) { match trap_ctx.trap_type() { - TrapType::Syscall { no, .. } => unreachable!("Syscall {} in kernel space.", no), + TrapType::Syscall { no, .. } => { + unreachable!("Syscall {} in kernel space.", no) + } TrapType::Breakpoint => unreachable!("Breakpoint in kernel space."), TrapType::Fault(fault) => default_fault_handler(fault, trap_ctx), TrapType::Irq { callback } => callback(default_irq_handler), From d632919e57c19c4e7a9847265b3b42fecc52bffd Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 20 Feb 2026 23:05:33 +0800 Subject: [PATCH 061/153] fix/interrupt: use lock_irq in default_irq_handler `default_irq_handler` may be called from anywhere including IRQ and interrupts enabled contexts. We need to use `lock_irq` to prevent nested deadlocks. Signed-off-by: greatbridf --- src/kernel/interrupt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel/interrupt.rs b/src/kernel/interrupt.rs index a3237239..09cfc3c2 100644 --- a/src/kernel/interrupt.rs +++ b/src/kernel/interrupt.rs @@ -19,7 +19,7 @@ pub fn default_irq_handler(irqno: usize) { assert!(irqno < 16); { - let handlers = IRQ_HANDLERS.lock(); + let handlers = IRQ_HANDLERS.lock_irq(); for handler in handlers[irqno].iter() { handler(); From 803bfb6bed4dde2de20258f52497330079adfa9e Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 20 Feb 2026 23:07:22 +0800 Subject: [PATCH 062/153] mem: move brk related codes to brk.rs Move codes to a dedicated module and split the logic into more functions for finer granularity. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 75 ++-------------- src/kernel/mem/mm_list/brk.rs | 155 ++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 70 deletions(-) create mode 100644 src/kernel/mem/mm_list/brk.rs diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index e9509622..42919510 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -1,3 +1,4 @@ +mod brk; mod mapping; mod mm_area; mod page_fault; @@ -24,6 +25,7 @@ pub use self::page_fault::handle_kernel_page_fault; use super::address::{VAddrExt as _, VRangeExt as _}; use super::{Folio, FolioOwned}; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; +use crate::kernel::mem::mm_list::brk::ProgramBreak; use crate::prelude::*; use crate::sync::ArcSwap; @@ -43,8 +45,7 @@ pub struct Permission { pub struct MMListInner { areas: AreaList, page_table: KernelPageTable, - break_start: Option, - break_pos: Option, + prog_break: ProgramBreak, } pub struct MMList { @@ -361,9 +362,8 @@ impl MMList { user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { areas: AreaList::new(), + prog_break: ProgramBreak::null(), page_table, - break_start: None, - break_pos: None, })), } } @@ -379,8 +379,7 @@ impl MMList { inner: ArcSwap::new(Mutex::new(MMListInner { areas: inner.areas.deep_clone(), page_table, - break_start: inner.break_start, - break_pos: inner.break_pos, + prog_break: inner.prog_break.clone(), })), }; @@ -592,70 +591,6 @@ impl MMList { .map(|_| at) } - pub async fn set_break(&self, pos: Option) -> VAddr { - let inner = self.inner.borrow(); - let mut inner = inner.lock().await; - - // SAFETY: `set_break` is only called in syscalls, where program break should be valid. - assert!(inner.break_start.is_some() && inner.break_pos.is_some()); - let break_start = inner.break_start.unwrap(); - let current_break = inner.break_pos.unwrap(); - let pos = match pos { - None => return current_break, - Some(pos) => pos.ceil(), - }; - - if current_break > pos { - return current_break; - } - - let range = VRange::new(current_break, pos); - if !inner.check_overlapping_range(range) { - return current_break; - } - - let program_break = - inner.areas.get_or_insert(break_start.start(), || { - MemArea::new( - break_start, - AreaFlags::from_old( - Permission { - read: true, - write: true, - execute: false, - }, - false, - ), - Mapping::Anonymous, - ) - }); - - let len = pos - current_break; - let range_to_grow = - VRange::from(program_break.range.as_ref(&inner).end()).grow(len); - - program_break.grow(len); - - inner.page_table.set_anonymous(range_to_grow, Permission { - read: true, - write: true, - execute: false, - }); - - inner.break_pos = Some(pos); - pos - } - - /// This should be called only **once** for every thread. - pub async fn register_break(&self, start: VAddr) { - let inner = self.inner.borrow(); - let mut inner = inner.lock().await; - assert!(inner.break_start.is_none() && inner.break_pos.is_none()); - - inner.break_start = Some(start.into()); - inner.break_pos = Some(start); - } - /// Access the memory area with the given function. /// The function will be called with the offset of the area and the slice of the area. pub async fn access_mut( diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs new file mode 100644 index 00000000..c3799782 --- /dev/null +++ b/src/kernel/mem/mm_list/brk.rs @@ -0,0 +1,155 @@ +use alloc::sync::Arc; + +use eonix_mm::address::{VAddr, VRange}; + +use crate::kernel::mem::address::VRangeExt; +use crate::kernel::mem::mm_list::mm_area::AreaList; +use crate::kernel::mem::mm_list::page_table::KernelPageTable; +use crate::kernel::mem::mm_list::{AreaFlags, MemArea, PageTableExt}; +use crate::kernel::mem::{MMList, Mapping, Permission}; + +#[derive(Clone)] +pub struct ProgramBreak { + start: VAddr, + current: VAddr, +} + +impl ProgramBreak { + pub const fn new(start: VAddr) -> Self { + Self { + start, + current: start, + } + } + + pub const fn null() -> Self { + Self::new(VAddr::NULL) + } + + pub fn get(&self) -> VAddr { + self.current + } + + pub fn is_null(&self) -> bool { + self.current == VAddr::NULL + } + + fn set(&mut self, pos: VAddr) { + self.current = pos; + } + + fn reset(&mut self, start: VAddr) { + self.start = start; + self.current = start; + } +} + +fn set_break( + areas: &mut AreaList, brk: &mut ProgramBreak, + page_table: &mut KernelPageTable, newbrk: Option, +) -> VAddr { + assert!( + !brk.is_null(), + "set_break called before program break is registered" + ); + + let curbrk = brk.get(); + let Some(newbrk) = newbrk else { + // Reject NULL brks + return curbrk; + }; + + if newbrk == curbrk { + // Nothing to do + return curbrk; + } + + if newbrk < curbrk { + // TODO: We should allow shrinking the break if there aren't non-brk + // mappings within the range. Reject now to keep things simple. + return curbrk; + } + + let new_range = VRange::new(curbrk, newbrk); + if !new_range.is_user() || areas.contains_range(&new_range) { + return curbrk; + } + + let Some(area) = areas.upper_bound(curbrk) else { + return expand_create_area(areas, brk, page_table, new_range); + }; + + let range = unsafe { + // SAFETY: We have a reference to `area`. + area.range.as_ref_unchecked().clone() + }; + + if range.end() != curbrk { + // Someone might have unmapped the brk area. + return expand_create_area(areas, brk, page_table, new_range); + } + + area.grow(new_range.len()); + map_break_area(page_table, new_range); + + brk.set(newbrk); + newbrk +} + +fn expand_create_area( + areas: &mut AreaList, brk: &mut ProgramBreak, + page_table: &mut KernelPageTable, new_range: VRange, +) -> VAddr { + let area = Arc::new(MemArea::new( + new_range, + AreaFlags::from_old( + Permission { + read: true, + write: true, + execute: false, + }, + false, + ), + Mapping::Anonymous, + )); + + areas.insert_new(area); + + map_break_area(page_table, new_range); + + brk.set(new_range.end()); + new_range.end() +} + +fn map_break_area(page_table: &mut KernelPageTable, range: VRange) { + page_table.set_anonymous(range, Permission { + read: true, + write: true, + execute: false, + }); +} + +impl MMList { + pub async fn set_break(&self, newbrk: Option) -> VAddr { + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + let inner = &mut *inner; + + set_break( + &mut inner.areas, + &mut inner.prog_break, + &mut inner.page_table, + newbrk, + ) + } + + /// This should be called only **once** for every thread. + pub async fn register_break(&self, start: VAddr) { + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + let brk = &mut inner.prog_break; + + assert!(brk.is_null(), "program break already registered"); + brk.reset(start); + } +} From cf7011cd63bc968c345ebeda72e7bf40ef525e10 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 23 Feb 2026 16:01:02 +0800 Subject: [PATCH 063/153] mem: remove unused `AreaList::get_or_insert()` The function is previously used by brk codes. Remove this since we don't handle that this way any more. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 59aabb4e..835cca98 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -143,21 +143,6 @@ impl AreaList { } } - pub fn get_or_insert( - &mut self, addr: VAddr, insert: impl FnOnce() -> MemArea, - ) -> Arc { - match self.areas.entry(&VRange::from(addr)) { - Entry::Vacant(entry) => { - let area = Arc::new(insert()); - entry.insert(area.clone()); - area - } - Entry::Occupied(cursor) => { - cursor.as_cursor().clone_pointer().unwrap() - } - } - } - pub fn get(&self, addr: VAddr) -> Option> { self.areas.find(&VRange::from(addr)).clone_pointer() } From 4d2699924e644ed985b393a7bde0183e3fc3d259 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 23 Feb 2026 15:23:51 +0800 Subject: [PATCH 064/153] mem: protect `MemArea::grow()` with AreaList lock Remove `RangeProtected::as_mut_unchecked()` and change the argument type passed to `RangeProtected::as_mut()` to `&mut AreaList` so that we can exclude all concurrent readers and writers. Note that there still exists an issue that we can't prove that MMList is locked in `MemArea::handle()`. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/brk.rs | 2 +- src/kernel/mem/mm_list/mm_area.rs | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index c3799782..0f2294cd 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -89,7 +89,7 @@ fn set_break( return expand_create_area(areas, brk, page_table, new_range); } - area.grow(new_range.len()); + area.grow(new_range.len(), areas); map_break_area(page_table, new_range); brk.set(newbrk); diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 835cca98..ee0410eb 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -97,10 +97,6 @@ impl RangeProtected { unsafe { &*self.0.get() } } - pub unsafe fn as_mut_unchecked<'a>(&self) -> &'a mut VRange { - unsafe { &mut *self.0.get() } - } - pub fn as_ref(&self, _list_read_lock: &MMListInner) -> &VRange { unsafe { // SAFETY: If we are holding the list's read lock, we won't modify @@ -111,13 +107,13 @@ impl RangeProtected { } pub fn as_mut<'a>( - &self, _list_write_lock: &'a mut MMListInner, + &self, _list_write_lock: &'a mut AreaList, ) -> &'a mut VRange { unsafe { // SAFETY: If we are holding the list's write lock, we can guarantee // that no one else is reading or writing the areas, so it's safe to // return a mutable reference. - self.as_mut_unchecked() + &mut *self.0.get() } } @@ -262,10 +258,10 @@ impl MemArea { /// # Safety /// This function should be called only when we can guarantee that the range /// won't overlap with any other range in some scope. - pub fn grow(&self, count: usize) { + pub fn grow(&self, count: usize, areas: &mut AreaList) { // TODO: Remove this. - let range = unsafe { self.range.as_mut_unchecked() }; - range.clone_from(&range.grow(count)); + let range = self.range.as_mut(areas); + *range = range.grow(count); } pub fn split(mut self, at: VAddr) -> (Option, Option) { From bd7a70c88adc0587569f9531f523c191bff6f67c Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 23 Feb 2026 15:51:36 +0800 Subject: [PATCH 065/153] mem: use a ZST MemListLock to avoid using &MMListInner Add a `MemListLock` so we can avoid taking the whole immutable reference to the list. This way we can replace many `as_ref_unchecked()` calls with safe calls. Make the unchecked version private inside the module. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 27 +++++++++++++++++++------ src/kernel/mem/mm_list/brk.rs | 5 +---- src/kernel/mem/mm_list/mm_area.rs | 30 +++++++++++++++++----------- src/kernel/mem/mm_list/page_fault.rs | 4 ++-- 4 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 42919510..ffb62961 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -42,7 +42,12 @@ pub struct Permission { pub execute: bool, } +pub struct MemListLock { + _phantom: (), +} + pub struct MMListInner { + pub lock: MemListLock, areas: AreaList, page_table: KernelPageTable, prog_break: ProgramBreak, @@ -55,6 +60,12 @@ pub struct MMList { root_page_table: AtomicUsize, } +impl MemListLock { + const fn _new() -> Self { + Self { _phantom: () } + } +} + impl MMListInner { fn overlapping_addr(&self, addr: VAddr) -> Option> { self.areas.get(addr) @@ -92,7 +103,8 @@ impl MMListInner { return Some(hint); }; - let ub_end = ub.range.as_ref(self).end().ceil(); + let ub_range = ub.range.as_ref(&self.lock); + let ub_end = ub_range.end().ceil(); if ub_end <= hint { return Some(hint); } @@ -117,7 +129,7 @@ impl MMListInner { // TODO: Write back dirty pages. self.areas.retain(|area| { - let range = unsafe { area.range.as_ref_unchecked() }; + let range = area.range.as_ref(&self.lock); let Some((left, mid, right)) = range.mask_with_checked(&range_to_unmap) @@ -199,9 +211,10 @@ impl MMListInner { let old_areas = self.areas.take(); for area in old_areas { let mut area = area.as_ref().clone(); + let range = area.range.as_ref(&self.lock); let Some((left, mid, right)) = - area.range.as_ref(self).mask_with_checked(&range_to_protect) + range.mask_with_checked(&range_to_protect) else { self.areas.insert(area); continue; @@ -306,7 +319,7 @@ impl Drop for MMListInner { fn drop(&mut self) { // May buggy for area in self.areas.iter() { - let range = area.range.as_ref(self).clone(); + let range = area.range.as_ref(&self.lock).clone(); if area.is_shared() { for pte in self.page_table.iter_user(range) { @@ -361,6 +374,7 @@ impl MMList { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { + lock: MemListLock::_new(), areas: AreaList::new(), prog_break: ProgramBreak::null(), page_table, @@ -377,6 +391,7 @@ impl MMList { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { + lock: MemListLock::_new(), areas: inner.areas.deep_clone(), page_table, prog_break: inner.prog_break.clone(), @@ -390,7 +405,7 @@ impl MMList { let pgtable = &list_inner.page_table; for area in list_inner.areas.iter() { - let range = area.range.as_ref(&list_inner).clone(); + let range = area.range.as_ref(&list_inner.lock).clone(); if !area.is_shared() { pgtable.set_copy_on_write(&mut inner.page_table, range); @@ -615,7 +630,7 @@ impl MMList { while remaining > 0 { let area = inner.overlapping_addr(current).ok_or(EFAULT)?; - let range = area.range.as_ref(&inner); + let range = area.range.as_ref(&inner.lock); let area_start = range.start(); let area_end = range.end(); let area_remaining = area_end - current; diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index 0f2294cd..de7bcc5c 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -79,10 +79,7 @@ fn set_break( return expand_create_area(areas, brk, page_table, new_range); }; - let range = unsafe { - // SAFETY: We have a reference to `area`. - area.range.as_ref_unchecked().clone() - }; + let range = area.range.as_ref_list(areas); if range.end() != curbrk { // Someone might have unmapped the brk area. diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index ee0410eb..398f4078 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -13,7 +13,7 @@ use intrusive_collections::{ use super::{Mapping, EMPTY_PAGE}; use crate::kernel::mem::folio::Folio; -use crate::kernel::mem::mm_list::MMListInner; +use crate::kernel::mem::mm_list::MemListLock; use crate::kernel::mem::{CachePage, FolioOwned, PageOffset, Permission}; use crate::prelude::KResult; @@ -93,11 +93,11 @@ impl RangeProtected { Self(UnsafeCell::new(range)) } - pub unsafe fn as_ref_unchecked<'a>(&self) -> &'a VRange { + unsafe fn as_ref_unchecked<'a>(&self) -> &'a VRange { unsafe { &*self.0.get() } } - pub fn as_ref(&self, _list_read_lock: &MMListInner) -> &VRange { + pub fn as_ref<'a>(&self, _list_read_lock: &'a MemListLock) -> &'a VRange { unsafe { // SAFETY: If we are holding the list's read lock, we won't modify // the areas and so won't the range, so it's safe to return a @@ -106,6 +106,15 @@ impl RangeProtected { } } + pub fn as_ref_list<'a>(&self, _list_read_lock: &'a AreaList) -> &'a VRange { + unsafe { + // SAFETY: `self.range` is write protected by both the list's lock + // and the area's lock, so it's safe to return an immutable + // reference to the range with any of them held. + self.as_ref_unchecked() + } + } + pub fn as_mut<'a>( &self, _list_write_lock: &'a mut AreaList, ) -> &'a mut VRange { @@ -130,8 +139,9 @@ impl AreaList { } pub fn insert_new(&mut self, area: Arc) { - let range = unsafe { area.range.as_ref_unchecked() }; - match self.areas.entry(range) { + let range = area.range.as_ref_list(self).clone(); + + match self.areas.entry(&range) { Entry::Occupied(_) => panic!("Overlapping mem area: {range:?}."), Entry::Vacant(insert_cursor) => { insert_cursor.insert(area); @@ -160,11 +170,7 @@ impl AreaList { return false; }; - let range = unsafe { - // SAFETY: Check `KeyAdapter` impl above. - ub.range.as_ref_unchecked() - }; - + let range = ub.range.as_ref_list(self); range.end() <= range.start() } @@ -183,9 +189,9 @@ impl AreaList { // TODO: For backwards compatibility. Remove this. pub fn insert(&mut self, area: MemArea) { - let range = unsafe { area.range.as_ref_unchecked() }; + let range = area.range.as_ref_list(self).clone(); - match self.areas.entry(range) { + match self.areas.entry(&range) { Entry::Occupied(_) => panic!("Overlapping mem area: {:?}.", range), Entry::Vacant(insert_cursor) => { insert_cursor.insert(Arc::new(area)); diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index db13bcc7..28c2c5ef 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -86,7 +86,7 @@ impl MMList { area.handle( pte, - addr.floor() - area.range.as_ref(&inner).start(), + addr.floor() - area.range.as_ref(&inner.lock).start(), error.contains(PageFaultErrorCode::Write), ) .await @@ -159,7 +159,7 @@ pub async fn handle_kernel_page_fault( if let Err(_) = area .handle( pte, - addr.floor() - area.range.as_ref(&inner).start(), + addr.floor() - area.range.as_ref(&inner.lock).start(), error.contains(PageFaultErrorCode::Write), ) .await From 0aa3dff728c4c37943392ce7668cf342c8cbe593 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 23 Feb 2026 16:13:18 +0800 Subject: [PATCH 066/153] mem: protect `MemArea::clone()` with MMList's lock Don't use Rust's Clone trait and add a function called clone to implement this on our own. This function will be soon removed when we eliminate retain calls in `unmap()` and `protect()`. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 13 ++++++++----- src/kernel/mem/mm_list/mm_area.rs | 23 +++++++++++------------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index ffb62961..43635440 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -149,7 +149,9 @@ impl MMListInner { (None, None) => {} (Some(left), None) => { assert!(left_remaining.is_none()); - let (Some(left), _) = area.clone().split(left.end()) else { + let (Some(left), _) = + area.clone(&self.lock).split(left.end()) + else { unreachable!("`left.end()` is within the area"); }; @@ -157,7 +159,8 @@ impl MMListInner { } (None, Some(right)) => { assert!(right_remaining.is_none()); - let (_, Some(right)) = area.clone().split(right.start()) + let (_, Some(right)) = + area.clone(&self.lock).split(right.start()) else { unreachable!("`right.start()` is within the area"); }; @@ -168,7 +171,7 @@ impl MMListInner { assert!(left_remaining.is_none()); assert!(right_remaining.is_none()); let (Some(left), Some(mid)) = - area.clone().split(left.end()) + area.clone(&self.lock).split(left.end()) else { unreachable!("`left.end()` is within the area"); }; @@ -210,7 +213,7 @@ impl MMListInner { let mut found = false; let old_areas = self.areas.take(); for area in old_areas { - let mut area = area.as_ref().clone(); + let mut area = area.as_ref().clone(&self.lock); let range = area.range.as_ref(&self.lock); let Some((left, mid, right)) = @@ -392,7 +395,7 @@ impl MMList { user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { lock: MemListLock::_new(), - areas: inner.areas.deep_clone(), + areas: inner.areas.deep_clone(&inner.lock), page_table, prog_break: inner.prog_break.clone(), })), diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 398f4078..3f5d130c 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -34,16 +34,6 @@ pub struct RangeProtected(UnsafeCell); unsafe impl Send for RangeProtected {} unsafe impl Sync for RangeProtected {} -impl Clone for RangeProtected { - fn clone(&self) -> Self { - // TODO: CHANGE THIS - let range = unsafe { self.as_ref_unchecked().clone() }; - - Self(UnsafeCell::new(range)) - } -} - -#[derive(Clone)] pub struct MemArea { /// # Lock /// Protected by [`MMListInner`] lock. @@ -200,11 +190,11 @@ impl AreaList { } // TODO: For backwards compatibility. Remove this. - pub fn deep_clone(&self) -> Self { + pub fn deep_clone(&self, lock: &MemListLock) -> Self { let mut areas = RBTree::new(AreasAdapter::NEW); for area in self.areas.iter() { - areas.insert(Arc::new(area.clone())); + areas.insert(Arc::new(area.clone(lock))); } Self { areas } @@ -261,6 +251,15 @@ impl MemArea { self.flags.contains(AreaFlags::EXECUTE) } + pub fn clone(&self, lock: &MemListLock) -> Self { + Self { + range: RangeProtected::new(self.range.as_ref(lock).clone()), + flags: self.flags, + areas_link: self.areas_link.clone(), + mapping: self.mapping.clone(), + } + } + /// # Safety /// This function should be called only when we can guarantee that the range /// won't overlap with any other range in some scope. From a1c264023089197d888635efa40a8991fb6771db Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 23 Feb 2026 16:31:02 +0800 Subject: [PATCH 067/153] mem: unify the two kind of range read locks Add `RangeReadLock` trait and impl the trait for &MemListLock so that we can use and extend it more conveniently. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/brk.rs | 2 +- src/kernel/mem/mm_list/mm_area.rs | 36 ++++++++++++++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index de7bcc5c..ccebabd9 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -79,7 +79,7 @@ fn set_break( return expand_create_area(areas, brk, page_table, new_range); }; - let range = area.range.as_ref_list(areas); + let range = area.range.as_ref(&*areas); if range.end() != curbrk { // Someone might have unmapped the brk area. diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 3f5d130c..6d32ed36 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -27,6 +27,19 @@ bitflags::bitflags! { } } +mod sealed { + pub trait RangeReadLock {} +} + +/// Some lock that can provide read protection for the area range, including a +/// reference to the MemList, its lock and the area list lock. +pub trait RangeReadLock: sealed::RangeReadLock {} + +impl RangeReadLock for &MemListLock {} +impl RangeReadLock for &AreaList {} +impl sealed::RangeReadLock for &MemListLock {} +impl sealed::RangeReadLock for &AreaList {} + /// # Lock /// Protected by [`MMListInner`] lock. pub struct RangeProtected(UnsafeCell); @@ -56,7 +69,7 @@ impl<'a> KeyAdapter<'a> for AreasAdapter { // SAFETY: The [`AreaList`] is within the [`MMListInner`]. // References to the areas list implies holding the lock of // [`MMListInner`], so it's safe to read the range. - *value.range.as_ref_unchecked() + value.range.as_ref_unchecked().clone() } } } @@ -87,20 +100,9 @@ impl RangeProtected { unsafe { &*self.0.get() } } - pub fn as_ref<'a>(&self, _list_read_lock: &'a MemListLock) -> &'a VRange { - unsafe { - // SAFETY: If we are holding the list's read lock, we won't modify - // the areas and so won't the range, so it's safe to return a - // reference. - self.as_ref_unchecked() - } - } - - pub fn as_ref_list<'a>(&self, _list_read_lock: &'a AreaList) -> &'a VRange { + pub fn as_ref<'a>(&self, _lock: impl RangeReadLock + 'a) -> &'a VRange { unsafe { - // SAFETY: `self.range` is write protected by both the list's lock - // and the area's lock, so it's safe to return an immutable - // reference to the range with any of them held. + // SAFETY: We are holding some kind of read safety lock. self.as_ref_unchecked() } } @@ -129,7 +131,7 @@ impl AreaList { } pub fn insert_new(&mut self, area: Arc) { - let range = area.range.as_ref_list(self).clone(); + let range = area.range.as_ref(&*self).clone(); match self.areas.entry(&range) { Entry::Occupied(_) => panic!("Overlapping mem area: {range:?}."), @@ -160,7 +162,7 @@ impl AreaList { return false; }; - let range = ub.range.as_ref_list(self); + let range = ub.range.as_ref(self); range.end() <= range.start() } @@ -179,7 +181,7 @@ impl AreaList { // TODO: For backwards compatibility. Remove this. pub fn insert(&mut self, area: MemArea) { - let range = area.range.as_ref_list(self).clone(); + let range = area.range.as_ref(&*self).clone(); match self.areas.entry(&range) { Entry::Occupied(_) => panic!("Overlapping mem area: {:?}.", range), From def1914473546afdbf394ee83afb1c7c454e86c3 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 25 Feb 2026 23:52:04 +0800 Subject: [PATCH 068/153] mem: introduce per MemArea lock Introduce `mem_area.lock` to protect faults from concurrent range changes. Allow getting immutable references to area's range from the area lock. Make `RangeProtected::as_mut()` rely on all the three locks to avoid Rust's borrow rule violation (&mut MemListLock and &AreaList). Signed-off-by: greatbridf --- src/kernel/mem/mm_list/brk.rs | 15 ++++++++---- src/kernel/mem/mm_list/mm_area.rs | 39 ++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index ccebabd9..2e463ef1 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -5,7 +5,9 @@ use eonix_mm::address::{VAddr, VRange}; use crate::kernel::mem::address::VRangeExt; use crate::kernel::mem::mm_list::mm_area::AreaList; use crate::kernel::mem::mm_list::page_table::KernelPageTable; -use crate::kernel::mem::mm_list::{AreaFlags, MemArea, PageTableExt}; +use crate::kernel::mem::mm_list::{ + AreaFlags, MemArea, MemListLock, PageTableExt, +}; use crate::kernel::mem::{MMList, Mapping, Permission}; #[derive(Clone)] @@ -44,8 +46,8 @@ impl ProgramBreak { } } -fn set_break( - areas: &mut AreaList, brk: &mut ProgramBreak, +async fn set_break( + areas: &mut AreaList, brk: &mut ProgramBreak, mm_lock: &mut MemListLock, page_table: &mut KernelPageTable, newbrk: Option, ) -> VAddr { assert!( @@ -86,7 +88,10 @@ fn set_break( return expand_create_area(areas, brk, page_table, new_range); } - area.grow(new_range.len(), areas); + let mut area_lock = area.lock.lock().await; + let range = area.range.as_mut(mm_lock, areas, &mut area_lock); + *range = range.grow(new_range.len()); + map_break_area(page_table, new_range); brk.set(newbrk); @@ -135,9 +140,11 @@ impl MMList { set_break( &mut inner.areas, &mut inner.prog_break, + &mut inner.lock, &mut inner.page_table, newbrk, ) + .await } /// This should be called only **once** for every thread. diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 6d32ed36..df20793f 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -6,6 +6,7 @@ use core::sync::atomic::Ordering; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{Folio as _, PFN}; +use eonix_sync::Mutex; use intrusive_collections::rbtree::Entry; use intrusive_collections::{ intrusive_adapter, Bound, KeyAdapter, PointerOps, RBTree, RBTreeAtomicLink, @@ -32,13 +33,15 @@ mod sealed { } /// Some lock that can provide read protection for the area range, including a -/// reference to the MemList, its lock and the area list lock. +/// reference to the MemList, its lock, the area list lock and the area lock. pub trait RangeReadLock: sealed::RangeReadLock {} impl RangeReadLock for &MemListLock {} impl RangeReadLock for &AreaList {} +impl RangeReadLock for &AreaLock {} impl sealed::RangeReadLock for &MemListLock {} impl sealed::RangeReadLock for &AreaList {} +impl sealed::RangeReadLock for &AreaLock {} /// # Lock /// Protected by [`MMListInner`] lock. @@ -47,11 +50,16 @@ pub struct RangeProtected(UnsafeCell); unsafe impl Send for RangeProtected {} unsafe impl Sync for RangeProtected {} +pub struct AreaLock { + _phantom: (), +} + pub struct MemArea { /// # Lock /// Protected by [`MMListInner`] lock. pub range: RangeProtected, pub flags: AreaFlags, + pub lock: Mutex, areas_link: RBTreeAtomicLink, mapping: Mapping, @@ -78,6 +86,12 @@ pub struct AreaList { areas: RBTree, } +impl AreaLock { + const fn _new() -> Self { + Self { _phantom: () } + } +} + impl AreaFlags { pub fn from_old(perm: Permission, is_shared: bool) -> Self { let mut flags = AreaFlags::empty(); @@ -108,7 +122,8 @@ impl RangeProtected { } pub fn as_mut<'a>( - &self, _list_write_lock: &'a mut AreaList, + &self, _mm_list_lock: &'a mut MemListLock, + _list_write_lock: &'a mut AreaList, _area_lock: &'a mut AreaLock, ) -> &'a mut VRange { unsafe { // SAFETY: If we are holding the list's write lock, we can guarantee @@ -226,6 +241,7 @@ impl MemArea { Self { range: RangeProtected::new(range), flags, + lock: Mutex::new(AreaLock::_new()), areas_link: RBTreeAtomicLink::new(), mapping, } @@ -257,20 +273,12 @@ impl MemArea { Self { range: RangeProtected::new(self.range.as_ref(lock).clone()), flags: self.flags, + lock: Mutex::new(AreaLock::_new()), areas_link: self.areas_link.clone(), mapping: self.mapping.clone(), } } - /// # Safety - /// This function should be called only when we can guarantee that the range - /// won't overlap with any other range in some scope. - pub fn grow(&self, count: usize, areas: &mut AreaList) { - // TODO: Remove this. - let range = self.range.as_mut(areas); - *range = range.grow(count); - } - pub fn split(mut self, at: VAddr) -> (Option, Option) { assert!(at.is_page_aligned()); let range = self.range.get_mut(); @@ -287,6 +295,7 @@ impl MemArea { let right = Self { range: RangeProtected::new(VRange::new(at, range.end())), flags: self.flags.clone(), + lock: Mutex::new(AreaLock::_new()), areas_link: RBTreeAtomicLink::new(), mapping: match &self.mapping { Mapping::Anonymous => Mapping::Anonymous, @@ -407,6 +416,14 @@ impl MemArea { pub async fn handle( &self, pte: &mut impl PTE, offset: usize, write: bool, ) -> KResult<()> { + // Exclude concurrent modifications and faults. + // TODO: concurrent faults should be acceptable... + let lock = self.lock.lock().await; + assert!( + offset < self.range.as_ref(&*lock).len(), + "Offset out of range" + ); + let mut attr = pte.get_attr().as_page_attr().expect("Not a page attribute"); let mut pfn = pte.get_pfn(); From a3fc06206a662434179987fd2b02362c09fc0a06 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 26 Feb 2026 22:07:42 +0800 Subject: [PATCH 069/153] mem: add a lock field in AreaList In order to prevent area_list.areas references conflict with `&mut area_list` used by `range.as_mut()`, we add a lock field in the list. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/brk.rs | 4 ++-- src/kernel/mem/mm_list/mm_area.rs | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index 2e463ef1..5c345b59 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -81,7 +81,7 @@ async fn set_break( return expand_create_area(areas, brk, page_table, new_range); }; - let range = area.range.as_ref(&*areas); + let range = area.range.as_ref(&areas.lock); if range.end() != curbrk { // Someone might have unmapped the brk area. @@ -89,7 +89,7 @@ async fn set_break( } let mut area_lock = area.lock.lock().await; - let range = area.range.as_mut(mm_lock, areas, &mut area_lock); + let range = area.range.as_mut(mm_lock, &mut areas.lock, &mut area_lock); *range = range.grow(new_range.len()); map_break_area(page_table, new_range); diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index df20793f..41c1c91d 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -37,10 +37,10 @@ mod sealed { pub trait RangeReadLock: sealed::RangeReadLock {} impl RangeReadLock for &MemListLock {} -impl RangeReadLock for &AreaList {} +impl RangeReadLock for &AreaListLock {} impl RangeReadLock for &AreaLock {} impl sealed::RangeReadLock for &MemListLock {} -impl sealed::RangeReadLock for &AreaList {} +impl sealed::RangeReadLock for &AreaListLock {} impl sealed::RangeReadLock for &AreaLock {} /// # Lock @@ -54,6 +54,10 @@ pub struct AreaLock { _phantom: (), } +pub struct AreaListLock { + _phantom: (), +} + pub struct MemArea { /// # Lock /// Protected by [`MMListInner`] lock. @@ -84,6 +88,7 @@ impl<'a> KeyAdapter<'a> for AreasAdapter { pub struct AreaList { areas: RBTree, + pub lock: AreaListLock, } impl AreaLock { @@ -123,7 +128,7 @@ impl RangeProtected { pub fn as_mut<'a>( &self, _mm_list_lock: &'a mut MemListLock, - _list_write_lock: &'a mut AreaList, _area_lock: &'a mut AreaLock, + _list_write_lock: &'a mut AreaListLock, _area_lock: &'a mut AreaLock, ) -> &'a mut VRange { unsafe { // SAFETY: If we are holding the list's write lock, we can guarantee @@ -142,11 +147,12 @@ impl AreaList { pub const fn new() -> Self { Self { areas: RBTree::new(AreasAdapter::NEW), + lock: AreaListLock { _phantom: () }, } } pub fn insert_new(&mut self, area: Arc) { - let range = area.range.as_ref(&*self).clone(); + let range = area.range.as_ref(&self.lock).clone(); match self.areas.entry(&range) { Entry::Occupied(_) => panic!("Overlapping mem area: {range:?}."), @@ -177,7 +183,7 @@ impl AreaList { return false; }; - let range = ub.range.as_ref(self); + let range = ub.range.as_ref(&self.lock); range.end() <= range.start() } @@ -196,7 +202,7 @@ impl AreaList { // TODO: For backwards compatibility. Remove this. pub fn insert(&mut self, area: MemArea) { - let range = area.range.as_ref(&*self).clone(); + let range = area.range.as_ref(&self.lock).clone(); match self.areas.entry(&range) { Entry::Occupied(_) => panic!("Overlapping mem area: {:?}.", range), @@ -214,7 +220,10 @@ impl AreaList { areas.insert(Arc::new(area.clone(lock))); } - Self { areas } + Self { + areas, + lock: AreaListLock { _phantom: () }, + } } pub fn iter(&self) -> impl Iterator { From eddb269ed123a1bdb920c12d27607b271728284d Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 27 Feb 2026 22:20:08 +0800 Subject: [PATCH 070/153] WIP: mem: add a helper to isolate mem areas - Iterate over the overlapping areas, for each area: - If the area is fully covered, remove and return it. - If the area exceeds the range in one side, resize the old area, create the overlapping area and return. - If the area exceeds the range in both sides, resize the area on the left, create the overlapping area and return, then create the area on the right and insert it back. - Add a helper to clone and then modify the area with a mut ref. There are some problems: - `isolate()` is not Send because we hold `Cursor` across awaits and `Cursor`s are not Send. - The RBTree link can't be used by LinkedLists without reinitializing. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 87 ++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 41c1c91d..c5b81dcb 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -9,7 +9,8 @@ use eonix_mm::paging::{Folio as _, PFN}; use eonix_sync::Mutex; use intrusive_collections::rbtree::Entry; use intrusive_collections::{ - intrusive_adapter, Bound, KeyAdapter, PointerOps, RBTree, RBTreeAtomicLink, + intrusive_adapter, Bound, KeyAdapter, LinkedList, LinkedListAtomicLink, + PointerOps, RBTree, RBTreeAtomicLink, }; use super::{Mapping, EMPTY_PAGE}; @@ -69,6 +70,8 @@ pub struct MemArea { mapping: Mapping, } +// SAFETY: `areas_link` is larger in size than `LinkedListAtomicLink`. +intrusive_adapter!(ListAdapter = Arc: MemArea { areas_link: LinkedListAtomicLink }); intrusive_adapter!(AreasAdapter = Arc: MemArea { areas_link: RBTreeAtomicLink }); impl<'a> KeyAdapter<'a> for AreasAdapter { @@ -187,6 +190,76 @@ impl AreaList { range.end() <= range.start() } + /// Isolate the given range by splitting the areas that overlap with it, and + /// return the areas that are fully covered by the range and removed. + pub async fn isolate( + &mut self, isolate_range: &VRange, lock: &mut MemListLock, + ) -> impl Iterator> { + let mut ret_areas = LinkedList::new(ListAdapter::NEW); + let begin = VRange::from(isolate_range.start()); + + let mut cursor = self.areas.upper_bound_mut(Bound::Included(&begin)); + + while !cursor.is_null() { + let range = { + let area = cursor.get().unwrap(); + let range = area.range.as_ref(&*lock); + + if range.start() >= isolate_range.end() { + break; + } + + if range.end() <= isolate_range.start() { + cursor.move_next(); + continue; + } + + range.clone() + }; + + let area = cursor.as_cursor().clone_pointer().unwrap(); + let list_lock = &mut self.lock; + let mut area_lock = area.lock.lock().await; + let area_lock = &mut area_lock; + + let (l, m, r) = range.mask_with_checked(isolate_range).unwrap(); + + match (l, r) { + (None, None) => { + // Fully covered, just remove it. + let area = cursor.remove().unwrap(); + ret_areas.push_back(area); + continue; + } + (None, Some(rem)) | (Some(rem), None) => { + // Overflow on one side, change the old area's range and + // return the newly created area. + let range = area.range.as_mut(lock, list_lock, area_lock); + *range = rem; + } + (Some(left), Some(right)) => { + // Overflow on both sides, change the old area's range to + // the left part, create a new area for the right part and + // return the middle part. + let range = area.range.as_mut(lock, list_lock, area_lock); + *range = left; + + cursor.insert_after(area.clone_and_modify(lock, |area| { + area.range = RangeProtected::new(right); + })); + } + } + + let ret_range = area.clone_and_modify(lock, |area| { + area.range = RangeProtected::new(m); + }); + + ret_areas.push_back(ret_range); + } + + ret_areas.into_iter() + } + // TODO: For backwards compatibility. Remove this. pub fn retain(&mut self, mut pred: impl FnMut(&MemArea) -> bool) { let mut cursor = self.areas.front_mut(); @@ -288,6 +361,18 @@ impl MemArea { } } + fn clone_and_modify( + &self, lock: &MemListLock, modify: impl FnOnce(&mut MemArea), + ) -> Arc { + let mut arc = Arc::new(self.clone(lock)); + let arc_mut = unsafe { + // SAFETY: We are the only owner. + Arc::get_mut(&mut arc).unwrap_unchecked() + }; + modify(arc_mut); + arc + } + pub fn split(mut self, at: VAddr) -> (Option, Option) { assert!(at.is_page_aligned()); let range = self.range.get_mut(); From c8a7a59cca2c4b95ecd867bb590cc107fcbeeea4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 27 Feb 2026 22:25:53 +0800 Subject: [PATCH 071/153] WIP: mem: wrap and force the Future returned to be Send Brutal but effective. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index c5b81dcb..4d445fd9 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -194,6 +194,29 @@ impl AreaList { /// return the areas that are fully covered by the range and removed. pub async fn isolate( &mut self, isolate_range: &VRange, lock: &mut MemListLock, + ) -> impl Iterator> { + struct SendFuture(F); + unsafe impl Send for SendFuture {} + unsafe impl Sync for SendFuture {} + impl core::future::Future for SendFuture + where + F: core::future::Future, + { + type Output = F::Output; + + fn poll( + self: core::pin::Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll { + unsafe { self.map_unchecked_mut(|s| &mut s.0).poll(cx) } + } + } + + SendFuture(self.isolate_(isolate_range, lock)).await + } + + async fn isolate_( + &mut self, isolate_range: &VRange, lock: &mut MemListLock, ) -> impl Iterator> { let mut ret_areas = LinkedList::new(ListAdapter::NEW); let begin = VRange::from(isolate_range.start()); From 2a43d31a9055b95997cb0a8410da303cb352bb83 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 27 Feb 2026 22:29:52 +0800 Subject: [PATCH 072/153] WIP: mem: provide multi-usage link struct Use union to provide both RBTree and LinkedList link. Users can call `to_list()` or `to_rbtree()` on unlinked links to convert between the two, no runtime safety checks done. Address the link type problem by converting the link to LinkedList link when removed and to be inserted to the linked list, and converted back when we are returning the list. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 66 +++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 4d445fd9..909c00f0 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -1,6 +1,7 @@ use alloc::sync::Arc; use core::cell::UnsafeCell; use core::cmp; +use core::mem::ManuallyDrop; use core::sync::atomic::Ordering; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; @@ -59,20 +60,25 @@ pub struct AreaListLock { _phantom: (), } +union Link { + rbtree: ManuallyDrop, + list: ManuallyDrop, +} + pub struct MemArea { /// # Lock /// Protected by [`MMListInner`] lock. pub range: RangeProtected, pub flags: AreaFlags, pub lock: Mutex, - areas_link: RBTreeAtomicLink, + link: Link, mapping: Mapping, } // SAFETY: `areas_link` is larger in size than `LinkedListAtomicLink`. -intrusive_adapter!(ListAdapter = Arc: MemArea { areas_link: LinkedListAtomicLink }); -intrusive_adapter!(AreasAdapter = Arc: MemArea { areas_link: RBTreeAtomicLink }); +intrusive_adapter!(ListAdapter = Arc: MemArea { link: LinkedListAtomicLink }); +intrusive_adapter!(AreasAdapter = Arc: MemArea { link: RBTreeAtomicLink }); impl<'a> KeyAdapter<'a> for AreasAdapter { type Key = VRange; @@ -251,6 +257,9 @@ impl AreaList { (None, None) => { // Fully covered, just remove it. let area = cursor.remove().unwrap(); + unsafe { + area.link.to_list(); + } ret_areas.push_back(area); continue; } @@ -275,12 +284,19 @@ impl AreaList { let ret_range = area.clone_and_modify(lock, |area| { area.range = RangeProtected::new(m); + area.link = Link::list(); }); ret_areas.push_back(ret_range); } - ret_areas.into_iter() + // We are returning the area, and the area should be by default + // in rbtrees, so we need to convert the link back to rbtree + // link so the caller won't be surprised. + ret_areas.into_iter().inspect(|area| unsafe { + // SAFETY: `into_iter()` removes the area from the list. + area.link.to_rbtree() + }) } // TODO: For backwards compatibility. Remove this. @@ -347,7 +363,7 @@ impl MemArea { range: RangeProtected::new(range), flags, lock: Mutex::new(AreaLock::_new()), - areas_link: RBTreeAtomicLink::new(), + link: Link::rbtree(), mapping, } } @@ -379,7 +395,7 @@ impl MemArea { range: RangeProtected::new(self.range.as_ref(lock).clone()), flags: self.flags, lock: Mutex::new(AreaLock::_new()), - areas_link: self.areas_link.clone(), + link: Link::rbtree(), mapping: self.mapping.clone(), } } @@ -413,7 +429,7 @@ impl MemArea { range: RangeProtected::new(VRange::new(at, range.end())), flags: self.flags.clone(), lock: Mutex::new(AreaLock::_new()), - areas_link: RBTreeAtomicLink::new(), + link: Link::rbtree(), mapping: match &self.mapping { Mapping::Anonymous => Mapping::Anonymous, Mapping::File(mapping) => { @@ -564,3 +580,39 @@ impl MemArea { Ok(()) } } + +impl Link { + const fn rbtree() -> Self { + Self { + rbtree: ManuallyDrop::new(RBTreeAtomicLink::new()), + } + } + + const fn list() -> Self { + Self { + list: ManuallyDrop::new(LinkedListAtomicLink::new()), + } + } + + /// Convert from RBTree link to LinkedList link. + /// + /// # Safety + /// The caller must guarantee that the link is currently not linked. + /// Otherwise, this is an undefined behavior. + unsafe fn to_list(&self) { + unsafe { + self.list.force_unlink(); + } + } + + /// Convert from LinkedList link to RBTree link. + /// + /// # Safety + /// The caller must guarantee that the link is currently not linked. + /// Otherwise, this is an undefined behavior. + unsafe fn to_rbtree(&self) { + unsafe { + self.rbtree.force_unlink(); + } + } +} From 1f82101be4ed3d799ae492b4ade103575364d52e Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 27 Feb 2026 22:39:37 +0800 Subject: [PATCH 073/153] WIP: mem: reimpl `MMList::unmap` using `isolate()` Reimplement the unmapping (and `protect()` is doable also) using `isolate()`. `AreaList::retain()` can be removed now. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 76 +++++-------------------------- src/kernel/mem/mm_list/mm_area.rs | 13 ------ 2 files changed, 12 insertions(+), 77 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 43635440..eb4c9c2c 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -113,7 +113,7 @@ impl MMListInner { } } - fn unmap(&mut self, start: VAddr, len: usize) -> KResult> { + async fn unmap(&mut self, start: VAddr, len: usize) -> KResult> { assert_eq!(start.floor(), start); let end = (start + len).ceil(); let range_to_unmap = VRange::new(start, end); @@ -121,78 +121,22 @@ impl MMListInner { return Err(EINVAL); } - let mut left_remaining = None; - let mut right_remaining = None; - + // TODO: Write back dirty pages. let mut pages_to_free = Vec::new(); - // TODO: Write back dirty pages. + let isolated_areas = + self.areas.isolate(&range_to_unmap, &mut self.lock).await; - self.areas.retain(|area| { + for area in isolated_areas { let range = area.range.as_ref(&self.lock); - let Some((left, mid, right)) = - range.mask_with_checked(&range_to_unmap) - else { - return true; - }; - - for pte in self.page_table.iter_user(mid) { + for pte in self.page_table.iter_user(*range) { let (pfn, _) = pte.take(); pages_to_free.push(unsafe { // SAFETY: We got the pfn from a valid page table entry, so it should be valid. Folio::from_raw(pfn) }); } - - match (left, right) { - (None, None) => {} - (Some(left), None) => { - assert!(left_remaining.is_none()); - let (Some(left), _) = - area.clone(&self.lock).split(left.end()) - else { - unreachable!("`left.end()` is within the area"); - }; - - left_remaining = Some(left); - } - (None, Some(right)) => { - assert!(right_remaining.is_none()); - let (_, Some(right)) = - area.clone(&self.lock).split(right.start()) - else { - unreachable!("`right.start()` is within the area"); - }; - - right_remaining = Some(right); - } - (Some(left), Some(right)) => { - assert!(left_remaining.is_none()); - assert!(right_remaining.is_none()); - let (Some(left), Some(mid)) = - area.clone(&self.lock).split(left.end()) - else { - unreachable!("`left.end()` is within the area"); - }; - - let (_, Some(right)) = mid.split(right.start()) else { - unreachable!("`right.start()` is within the area"); - }; - - left_remaining = Some(left); - right_remaining = Some(right); - } - } - - false - }); - - if let Some(front) = left_remaining { - self.areas.insert(front); - } - if let Some(back) = right_remaining { - self.areas.insert(back); } Ok(pages_to_free) @@ -507,8 +451,12 @@ impl MMList { /// No need to do invalidation manually, `PageTable` already does it. pub async fn unmap(&self, start: VAddr, len: usize) -> KResult<()> { - let pages_to_free = - self.inner.borrow().lock().await.unmap(start, len)?; + let pages_to_free = { + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + + inner.unmap(start, len).await? + }; // We need to assure that the pages are not accessed anymore. // The ones having these pages in their TLB could read from or write to them. diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 909c00f0..7c3ec2b8 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -299,19 +299,6 @@ impl AreaList { }) } - // TODO: For backwards compatibility. Remove this. - pub fn retain(&mut self, mut pred: impl FnMut(&MemArea) -> bool) { - let mut cursor = self.areas.front_mut(); - - while let Some(area) = cursor.get() { - if !pred(area) { - cursor.remove(); - } - - cursor.move_next(); - } - } - // TODO: For backwards compatibility. Remove this. pub fn insert(&mut self, area: MemArea) { let range = area.range.as_ref(&self.lock).clone(); From 96d25cfc2102c5e2622b49fb014b68e79e872ab6 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 28 Feb 2026 23:35:06 +0800 Subject: [PATCH 074/153] mem: make MemArea flags atomic We use atomics for MemArea flags so we can remove mut restriction of `MemArea::set_permission()`. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 79 ++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 7c3ec2b8..daf25dfc 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -2,7 +2,7 @@ use alloc::sync::Arc; use core::cell::UnsafeCell; use core::cmp; use core::mem::ManuallyDrop; -use core::sync::atomic::Ordering; +use core::sync::atomic::{AtomicU32, Ordering}; use eonix_mm::address::{AddrOps as _, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; @@ -30,6 +30,8 @@ bitflags::bitflags! { } } +struct AtomicFlags(AtomicU32); + mod sealed { pub trait RangeReadLock {} } @@ -69,7 +71,7 @@ pub struct MemArea { /// # Lock /// Protected by [`MMListInner`] lock. pub range: RangeProtected, - pub flags: AreaFlags, + flags: AtomicFlags, pub lock: Mutex, link: Link, @@ -107,6 +109,8 @@ impl AreaLock { } impl AreaFlags { + const RWX: Self = Self::READ.union(Self::WRITE).union(Self::EXECUTE); + pub fn from_old(perm: Permission, is_shared: bool) -> Self { let mut flags = AreaFlags::empty(); @@ -119,6 +123,44 @@ impl AreaFlags { } } +impl AtomicFlags { + const fn new(flags: AreaFlags) -> Self { + Self(AtomicU32::new(flags.bits())) + } + + /// Load with Acquire semantics. + fn load(&self) -> AreaFlags { + AreaFlags::from_bits_retain(self.0.load(Ordering::Acquire)) + } + + /// Do atomic read-modify-write with AcqRel semantics on success and + /// Relaxed semantics on failure. + fn update( + &self, mut try_modify: impl FnMut(AreaFlags) -> AreaFlags, + ) -> AreaFlags { + loop { + let Ok(old_raw) = self.0.fetch_update( + Ordering::AcqRel, + Ordering::Relaxed, + |raw| { + let flags = AreaFlags::from_bits_retain(raw); + Some(try_modify(flags).bits()) + }, + ) else { + continue; + }; + + return AreaFlags::from_bits_retain(old_raw); + } + } +} + +impl Clone for AtomicFlags { + fn clone(&self) -> Self { + Self::new(self.load()) + } +} + impl RangeProtected { pub const fn new(range: VRange) -> Self { Self(UnsafeCell::new(range)) @@ -348,39 +390,52 @@ impl MemArea { ) -> Self { Self { range: RangeProtected::new(range), - flags, + flags: AtomicFlags::new(flags), lock: Mutex::new(AreaLock::_new()), link: Link::rbtree(), mapping, } } - pub fn set_permission(&mut self, perm: Permission) { - self.flags.set(AreaFlags::READ, perm.read); - self.flags.set(AreaFlags::WRITE, perm.write); - self.flags.set(AreaFlags::EXECUTE, perm.execute); + pub fn set_permission(&self, perm: Permission) { + let mut new_flags = AreaFlags::empty(); + + if perm.read { + new_flags.insert(AreaFlags::READ); + } + + if perm.write { + new_flags.insert(AreaFlags::WRITE); + } + + if perm.execute { + new_flags.insert(AreaFlags::EXECUTE); + } + + self.flags + .update(|flags| (flags & !AreaFlags::RWX) | new_flags); } pub fn is_shared(&self) -> bool { - self.flags.contains(AreaFlags::SHARED) + self.flags.load().contains(AreaFlags::SHARED) } pub fn can_read(&self) -> bool { - self.flags.contains(AreaFlags::READ) + self.flags.load().contains(AreaFlags::READ) } pub fn can_write(&self) -> bool { - self.flags.contains(AreaFlags::WRITE) + self.flags.load().contains(AreaFlags::WRITE) } pub fn can_execute(&self) -> bool { - self.flags.contains(AreaFlags::EXECUTE) + self.flags.load().contains(AreaFlags::EXECUTE) } pub fn clone(&self, lock: &MemListLock) -> Self { Self { range: RangeProtected::new(self.range.as_ref(lock).clone()), - flags: self.flags, + flags: self.flags.clone(), lock: Mutex::new(AreaLock::_new()), link: Link::rbtree(), mapping: self.mapping.clone(), From f37b288240bafb802bf3ea58c3bc014eb56e846b Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 28 Feb 2026 23:37:22 +0800 Subject: [PATCH 075/153] WIP: mem: reimpl `MMList::protect` using `isolate()` Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 46 ++++++++++----------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index eb4c9c2c..c10036ba 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -142,7 +142,7 @@ impl MMListInner { Ok(pages_to_free) } - fn protect( + async fn protect( &mut self, start: VAddr, len: usize, permission: Permission, ) -> KResult<()> { assert_eq!(start.floor(), start); @@ -154,41 +154,15 @@ impl MMListInner { return Err(EINVAL); } + let isolated_areas = + self.areas.isolate(&range_to_protect, &mut self.lock).await; + let mut found = false; - let old_areas = self.areas.take(); - for area in old_areas { - let mut area = area.as_ref().clone(&self.lock); + for area in isolated_areas { let range = area.range.as_ref(&self.lock); - - let Some((left, mid, right)) = - range.mask_with_checked(&range_to_protect) - else { - self.areas.insert(area); - continue; - }; - found = true; - if let Some(left) = left { - let (Some(left), Some(right)) = area.split(left.end()) else { - unreachable!("`left.end()` is within the area"); - }; - - self.areas.insert(left); - area = right; - } - - if let Some(right) = right { - let (Some(left), Some(right)) = area.split(right.start()) - else { - unreachable!("`right.start()` is within the area"); - }; - - self.areas.insert(right); - area = left; - } - - for pte in self.page_table.iter_user(mid) { + for pte in self.page_table.iter_user(*range) { let mut page_attr = pte .get_attr() .as_page_attr() @@ -220,7 +194,9 @@ impl MMListInner { } area.set_permission(permission); - self.areas.insert(area); + + // Insert it back. + self.areas.insert_new(area); } if !found { @@ -472,7 +448,9 @@ impl MMList { pub async fn protect( &self, start: VAddr, len: usize, prot: Permission, ) -> KResult<()> { - self.inner.borrow().lock().await.protect(start, len, prot)?; + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + inner.protect(start, len, prot).await?; // flush the tlb due to the pte attribute changes self.flush_user_tlbs().await; From 9b18fc867f2ec78e2d2ef1705154d92e8a5c9ad4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 28 Feb 2026 23:39:09 +0800 Subject: [PATCH 076/153] WIP: mem: remove unused MemArea member functions Remove - `MemArea::split()` - `AreaList::take()` - `RangeProtected::get_mut()` Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 46 +------------------------------ 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index daf25dfc..8892b7d4 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -1,10 +1,9 @@ use alloc::sync::Arc; use core::cell::UnsafeCell; -use core::cmp; use core::mem::ManuallyDrop; use core::sync::atomic::{AtomicU32, Ordering}; -use eonix_mm::address::{AddrOps as _, VAddr, VRange}; +use eonix_mm::address::{VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{Folio as _, PFN}; use eonix_sync::Mutex; @@ -188,10 +187,6 @@ impl RangeProtected { &mut *self.0.get() } } - - pub fn get_mut(&mut self) -> &mut VRange { - self.0.get_mut() - } } impl AreaList { @@ -377,11 +372,6 @@ impl AreaList { Some(area) }) } - - // TODO: For backwards compatibility. Remove this. - pub fn take(&mut self) -> impl IntoIterator> { - self.areas.take() - } } impl MemArea { @@ -454,40 +444,6 @@ impl MemArea { arc } - pub fn split(mut self, at: VAddr) -> (Option, Option) { - assert!(at.is_page_aligned()); - let range = self.range.get_mut(); - - match (*range).cmp(&VRange::from(at)) { - cmp::Ordering::Less => (Some(self), None), - cmp::Ordering::Greater => (None, Some(self)), - cmp::Ordering::Equal => { - let diff = at - range.start(); - if diff == 0 { - return (None, Some(self)); - } - - let right = Self { - range: RangeProtected::new(VRange::new(at, range.end())), - flags: self.flags.clone(), - lock: Mutex::new(AreaLock::_new()), - link: Link::rbtree(), - mapping: match &self.mapping { - Mapping::Anonymous => Mapping::Anonymous, - Mapping::File(mapping) => { - Mapping::File(mapping.offset(diff)) - } - }, - }; - - let new_range = range.shrink(range.end() - at); - - *self.range.get_mut() = new_range; - (Some(self), Some(right)) - } - } - } - pub fn handle_cow(&self, pfn: &mut PFN, attr: &mut PageAttribute) { assert!(attr.contains(PageAttribute::COPY_ON_WRITE)); From 2a5bcaecb221aa6dd03501a2d8d65dbd4eefe9da Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 1 Mar 2026 15:42:12 +0800 Subject: [PATCH 077/153] style: reformat chardev.rs dentry.rs vfs/file/mod.rs No functional change. Signed-off-by: greatbridf --- src/kernel/chardev.rs | 8 ++--- src/kernel/vfs/dentry.rs | 61 ++++++++++++++++++++++++-------------- src/kernel/vfs/file/mod.rs | 16 +++++++--- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/kernel/chardev.rs b/src/kernel/chardev.rs index e4a6e1b3..bf993c7a 100644 --- a/src/kernel/chardev.rs +++ b/src/kernel/chardev.rs @@ -59,9 +59,7 @@ impl CharDevice { } pub fn register( - devid: DeviceId, - name: Arc, - device: CharDeviceType, + devid: DeviceId, name: Arc, device: CharDeviceType, ) -> KResult<()> { match CHAR_DEVICES.lock().entry(devid) { Entry::Vacant(entry) => { @@ -73,9 +71,7 @@ impl CharDevice { } pub async fn open( - self: &Arc, - thread: &Thread, - flags: OpenFlags, + self: &Arc, thread: &Thread, flags: OpenFlags, ) -> KResult { let file = match &self.device { CharDeviceType::Virtual(_) => { diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index a401f4f7..d661b3ba 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -21,7 +21,9 @@ use super::FsContext; use crate::hash::KernelHasher; use crate::io::{Buffer, Stream}; use crate::kernel::block::BlockDevice; -use crate::kernel::constants::{EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE}; +use crate::kernel::constants::{ + EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE, +}; use crate::kernel::CharDevice; use crate::path::Path; use crate::prelude::*; @@ -201,7 +203,9 @@ impl Dentry { self.inode.load().is_some() } - pub async fn open_check(self: &Arc, flags: OpenFlags, perm: Permission) -> KResult<()> { + pub async fn open_check( + self: &Arc, flags: OpenFlags, perm: Permission, + ) -> KResult<()> { match self.inode.load() { Some(_) => { if flags.contains(OpenFlags::O_CREAT | OpenFlags::O_EXCL) { @@ -224,19 +228,14 @@ impl Dentry { impl Dentry { pub async fn open( - context: &FsContext, - path: &Path, - follow_symlinks: bool, + context: &FsContext, path: &Path, follow_symlinks: bool, ) -> KResult> { let cwd = context.cwd.lock().clone(); Self::open_at(context, &cwd, path, follow_symlinks).await } pub async fn open_at( - context: &FsContext, - at: &Arc, - path: &Path, - follow_symlinks: bool, + context: &FsContext, at: &Arc, path: &Path, follow_symlinks: bool, ) -> KResult> { let mut found = context.start_recursive_walk(at, path).await?; @@ -245,16 +244,17 @@ impl Dentry { } loop { - match found.inode.load() { - Some((DentryKind::Symlink, inode)) => { - found = context.follow_symlink(found.aref(), inode, 0).await?; - } - _ => return Ok(found), - } + let Some((DentryKind::Symlink, inode)) = found.inode.load() else { + return Ok(found); + }; + + found = context.follow_symlink(found.aref(), inode, 0).await?; } } - pub fn get_path(self: &Arc, context: &FsContext, buffer: &mut dyn Buffer) -> KResult<()> { + pub fn get_path( + self: &Arc, context: &FsContext, buffer: &mut dyn Buffer, + ) -> KResult<()> { let rcu_read = rcu_read_lock(); let mut path = vec![]; @@ -285,7 +285,9 @@ impl Dentry { } impl Dentry { - pub async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { + pub async fn read( + &self, buffer: &mut dyn Buffer, offset: usize, + ) -> KResult { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents @@ -304,19 +306,25 @@ impl Dentry { } } - pub async fn write(&self, stream: &mut dyn Stream, offset: WriteOffset<'_>) -> KResult { + pub async fn write( + &self, stream: &mut dyn Stream, offset: WriteOffset<'_>, + ) -> KResult { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents match inode.format { Format::DIR => Err(EISDIR), Format::REG => inode.write(stream, offset).await, Format::BLK => Err(EINVAL), // TODO - Format::CHR => CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream), + Format::CHR => { + CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream) + } _ => Err(EINVAL), } } - pub async fn readdir(&self, offset: usize, mut for_each_entry: F) -> KResult> + pub async fn readdir( + &self, offset: usize, mut for_each_entry: F, + ) -> KResult> where F: FnMut(&[u8], Ino) -> KResult + Send, { @@ -380,7 +388,9 @@ impl Dentry { self.get_inode()?.chown(uid, gid).await } - pub async fn rename(self: &Arc, new: &Arc, flags: RenameFlags) -> KResult<()> { + pub async fn rename( + self: &Arc, new: &Arc, flags: RenameFlags, + ) -> KResult<()> { if Arc::ptr_eq(self, new) { return Ok(()); } @@ -423,8 +433,13 @@ impl DentryKind { Self::from_raw(Self::as_atomic(me).load(Ordering::Acquire)) } - fn atomic_swap_acqrel(me: &UnsafeCell>, kind: Option) -> Option { - Self::from_raw(Self::as_atomic(me).swap(kind.map_or(0, Self::into_raw), Ordering::AcqRel)) + fn atomic_swap_acqrel( + me: &UnsafeCell>, kind: Option, + ) -> Option { + Self::from_raw( + Self::as_atomic(me) + .swap(kind.map_or(0, Self::into_raw), Ordering::AcqRel), + ) } } diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index 799b9848..912b52c2 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -51,7 +51,9 @@ bitflags! { } impl FileType { - pub async fn read(&self, buffer: &mut dyn Buffer, offset: Option) -> KResult { + pub async fn read( + &self, buffer: &mut dyn Buffer, offset: Option, + ) -> KResult { match self { FileType::Inode(inode) => inode.read(buffer, offset).await, FileType::PipeRead(pipe) => pipe.read(buffer).await, @@ -76,7 +78,9 @@ impl FileType { // } // } - pub async fn write(&self, stream: &mut dyn Stream, offset: Option) -> KResult { + pub async fn write( + &self, stream: &mut dyn Stream, offset: Option, + ) -> KResult { match self { FileType::Inode(inode) => inode.write(stream, offset).await, FileType::PipeWrite(pipe) => pipe.write(stream).await, @@ -93,7 +97,9 @@ impl FileType { } } - pub async fn sendfile(&self, dest_file: &Self, count: usize) -> KResult { + pub async fn sendfile( + &self, dest_file: &Self, count: usize, + ) -> KResult { let mut buffer_page = FolioOwned::alloc(); let buffer = buffer_page.as_bytes_mut(); @@ -126,7 +132,9 @@ impl FileType { pub async fn ioctl(&self, request: usize, arg3: usize) -> KResult { match self { - FileType::Terminal(tty) => tty.ioctl(request, arg3).await.map(|_| 0), + FileType::Terminal(tty) => { + tty.ioctl(request, arg3).await.map(|_| 0) + } _ => Err(ENOTTY), } } From 5a0d054042219e84a1a710635774e77eef98714f Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 1 Mar 2026 15:43:08 +0800 Subject: [PATCH 078/153] chardev: make `CharDevice::read()` async In order to remove `block_on()` calls, we need to make the function async and replace `block_on()`s with `.await`s. `CharDevice::read()` is used in dentry IO operations and file IO operations, where sleeps are acceptable. So this won't cause problems. Signed-off-by: greatbridf --- src/kernel/chardev.rs | 6 ++---- src/kernel/vfs/dentry.rs | 2 +- src/kernel/vfs/file/mod.rs | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/kernel/chardev.rs b/src/kernel/chardev.rs index bf993c7a..08a21774 100644 --- a/src/kernel/chardev.rs +++ b/src/kernel/chardev.rs @@ -33,11 +33,9 @@ static CHAR_DEVICES: Spin>> = Spin::new(BTreeMap::new()); impl CharDevice { - pub fn read(&self, buffer: &mut dyn Buffer) -> KResult { + pub async fn read(&self, buffer: &mut dyn Buffer) -> KResult { match &self.device { - CharDeviceType::Terminal(terminal) => { - block_on(terminal.read(buffer)) - } + CharDeviceType::Terminal(terminal) => terminal.read(buffer).await, CharDeviceType::Virtual(device) => device.read(buffer), } } diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index d661b3ba..7741c55a 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -300,7 +300,7 @@ impl Dentry { } Format::CHR => { let device = CharDevice::get(inode.devid()?).ok_or(EPERM)?; - device.read(buffer) + device.read(buffer).await } _ => Err(EINVAL), } diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index 912b52c2..0af79dc8 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -58,7 +58,7 @@ impl FileType { FileType::Inode(inode) => inode.read(buffer, offset).await, FileType::PipeRead(pipe) => pipe.read(buffer).await, FileType::Terminal(tty) => tty.read(buffer).await, - FileType::CharDev(device) => device.read(buffer), + FileType::CharDev(device) => device.read(buffer).await, _ => Err(EBADF), } } From e0a0f3ec8852d25f8eb154627b79d6ee1446dc41 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 3 Mar 2026 00:07:35 +0800 Subject: [PATCH 079/153] hal, x86_64: increase global CPU_COUNT in setup_cpu In the previous version, we forgot to inc CPU_COUNT by one. As a result, the CPU_COUNT is always 0 on x86_64 platforms, which will cause problems when we use the count. Fixes: dd32b9396356 ("feat(hal): smp initialization") Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs index 2f92ee8e..1945d2b6 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs @@ -24,6 +24,7 @@ use crate::extern_symbol_value; use crate::mm::{ ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, }; +use crate::processor::CPU_COUNT; static BSP_PAGE_ALLOC: AtomicPtr> = AtomicPtr::new(core::ptr::null_mut()); @@ -126,6 +127,8 @@ fn enable_sse() { } fn setup_cpu(alloc: impl FrameAlloc) { + CPU_COUNT.fetch_add(1, Ordering::AcqRel); + let mut percpu_area = PercpuArea::new(|layout| { // TODO: Use page size defined in `arch`. let page_count = layout.size().div_ceil(PAGE_SIZE); From d5ec1b54d86f95ffd5f05380bf31ff1ab4af6024 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 3 Mar 2026 00:10:37 +0800 Subject: [PATCH 080/153] style: reformat rcu.rs No functional change. Signed-off-by: greatbridf --- src/rcu.rs | 54 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/src/rcu.rs b/src/rcu.rs index b06db9e2..ee797d9e 100644 --- a/src/rcu.rs +++ b/src/rcu.rs @@ -1,16 +1,17 @@ -use crate::{kernel::task::block_on, prelude::*}; use alloc::sync::Arc; +use core::ops::Deref; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicPtr, Ordering}; + use arcref::ArcRef; -use core::{ - ops::Deref, - ptr::NonNull, - sync::atomic::{AtomicPtr, Ordering}, -}; use eonix_preempt::PreemptGuard; use eonix_runtime::scheduler::RUNTIME; use eonix_sync::{RwLock, RwLockReadGuard}; use pointers::BorrowedArc; +use crate::kernel::task::block_on; +use crate::prelude::*; + /// The RCU Read Lock. Holding a reference to an instance of the struct assures /// you that any RCU protected data would not be dropped. /// @@ -90,9 +91,10 @@ impl> RCUList { new_node.rcu_next().store(old_head, Ordering::Release); if let Some(old_head) = unsafe { old_head.as_ref() } { - old_head - .rcu_prev() - .store(Arc::into_raw(new_node.clone()) as *mut _, Ordering::Release); + old_head.rcu_prev().store( + Arc::into_raw(new_node.clone()) as *mut _, + Ordering::Release, + ); } self.head @@ -112,8 +114,8 @@ impl> RCUList { } { - let prev_next = - unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) }.unwrap_or(&self.head); + let prev_next = unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) } + .unwrap_or(&self.head); let me = prev_next.swap(next, Ordering::AcqRel); debug_assert!(me == Arc::as_ptr(&node) as *mut _); @@ -136,19 +138,23 @@ impl> RCUList { new_node.rcu_next().store(next, Ordering::Release); { - let prev_next = - unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) }.unwrap_or(&self.head); + let prev_next = unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) } + .unwrap_or(&self.head); - let old = prev_next.swap(Arc::into_raw(new_node.clone()) as *mut _, Ordering::AcqRel); + let old = prev_next.swap( + Arc::into_raw(new_node.clone()) as *mut _, + Ordering::AcqRel, + ); debug_assert!(old == Arc::as_ptr(&old_node) as *mut _); unsafe { Arc::from_raw(old) }; } if let Some(next) = unsafe { next.as_ref() } { - let old = next - .rcu_prev() - .swap(Arc::into_raw(new_node.clone()) as *mut _, Ordering::AcqRel); + let old = next.rcu_prev().swap( + Arc::into_raw(new_node.clone()) as *mut _, + Ordering::AcqRel, + ); debug_assert!(old == Arc::as_ptr(&old_node) as *mut _); unsafe { Arc::from_raw(old) }; @@ -162,7 +168,9 @@ impl> RCUList { .store(core::ptr::null_mut(), Ordering::Release); } - pub fn iter<'a, 'r>(&'a self, _lock: &'r RCUReadLock) -> RCUIterator<'a, 'r, T> { + pub fn iter<'a, 'r>( + &'a self, _lock: &'r RCUReadLock, + ) -> RCUIterator<'a, 'r, T> { RCUIterator { cur: NonNull::new(self.head.load(Ordering::Acquire)), _phantom: PhantomData, @@ -185,7 +193,8 @@ impl<'rcu, T: RCUNode> Iterator for RCUIterator<'_, 'rcu, T> { pointer.as_ref() }; - self.cur = NonNull::new(reference.rcu_next().load(Ordering::Acquire)); + self.cur = + NonNull::new(reference.rcu_next().load(Ordering::Acquire)); unsafe { // SAFETY: We have the read lock so the node is still alive. @@ -234,7 +243,9 @@ where .map(|p| RCUReadGuard::lock(unsafe { BorrowedArc::from_raw(p) })) } - pub fn dereference<'r, 'a: 'r>(&self, _lock: &'a RCUReadLock) -> Option> { + pub fn dereference<'r, 'a: 'r>( + &self, _lock: &'a RCUReadLock, + ) -> Option> { NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { // SAFETY: We have a read lock, so the node is still alive. ArcRef::new_unchecked(p.as_ptr()) @@ -244,7 +255,8 @@ where /// # Safety /// Caller must ensure no writers are updating the pointer. pub unsafe fn load_locked<'lt>(&self) -> Option> { - NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { BorrowedArc::from_raw(p) }) + NonNull::new(self.0.load(Ordering::Acquire)) + .map(|p| unsafe { BorrowedArc::from_raw(p) }) } /// # Safety From ca216c8757afa99c1251b1190ff8cd54b797e086 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 3 Mar 2026 00:12:21 +0800 Subject: [PATCH 081/153] sync, rcu: reuse `RCUReadGuardNew` for `RCUReadGuard` In ea2122331ee2 ("vfs, rcu: rework path walking with new rcu syntax"), we introduced the `RCUReadGuardNew` lock complex, which encapsulates RCU guard and the preempt guard together. We should reuse the existing interface to do RCU read locks. Signed-off-by: greatbridf --- src/rcu.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rcu.rs b/src/rcu.rs index ee797d9e..7fa5ee70 100644 --- a/src/rcu.rs +++ b/src/rcu.rs @@ -26,7 +26,7 @@ pub struct RCUReadGuardNew { pub struct RCUReadGuard<'data, T: 'data> { value: T, - _guard: RwLockReadGuard<'static, RCUReadLock>, + _guard: RCUReadGuardNew, _phantom: PhantomData<&'data T>, } @@ -36,7 +36,7 @@ impl<'data, T> RCUReadGuard<'data, BorrowedArc<'data, T>> { fn lock(value: BorrowedArc<'data, T>) -> Self { Self { value, - _guard: block_on(GLOBAL_RCU_SEM.read()), + _guard: rcu_read_lock(), _phantom: PhantomData, } } From 682c4c9c5aed76bb5b5ecd78ab8f760f01cb43a9 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 3 Mar 2026 00:41:49 +0800 Subject: [PATCH 082/153] style: reformat clone.rs No functional change. Signed-off-by: greatbridf --- src/kernel/task/clone.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/kernel/task/clone.rs b/src/kernel/task/clone.rs index dd6f538d..28edbc4d 100644 --- a/src/kernel/task/clone.rs +++ b/src/kernel/task/clone.rs @@ -8,7 +8,9 @@ use posix_types::signal::Signal; use super::{UserTLS, UserTLSDescriptor}; use crate::kernel::syscall::UserMut; -use crate::kernel::task::{alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder}; +use crate::kernel::task::{ + alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder, +}; use crate::kernel::user::UserPointerMut; use crate::KResult; @@ -63,11 +65,8 @@ impl CloneArgs { const MASK: usize = 0xff; pub fn for_clone( - flags: usize, - sp: usize, - child_tid_ptr: UserMut, - parent_tid_ptr: UserMut, - tls: PtrT, + flags: usize, sp: usize, child_tid_ptr: UserMut, + parent_tid_ptr: UserMut, tls: PtrT, ) -> KResult { let clone_flags = CloneFlags::from_bits_truncate(flags & !Self::MASK); let exit_signal = flags & Self::MASK; @@ -139,7 +138,8 @@ impl CloneArgs { pub async fn do_clone(thread: &Thread, clone_args: CloneArgs) -> KResult { let mut procs = ProcessList::get().write().await; - let thread_builder = ThreadBuilder::new().clone_from(&thread, &clone_args)?; + let thread_builder = + ThreadBuilder::new().clone_from(&thread, &clone_args)?; let current_process = thread.process.clone(); let new_pid = alloc_pid(); From 38671c141918e5329eaeb0d8088915a19c1eb5b3 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 3 Mar 2026 01:03:17 +0800 Subject: [PATCH 083/153] task: don't hold thread.inner spinlock for too long Instead of acquiring the spinlock at the begin and hold it throughout the `ThreadBuilder::clone_from()`, we lock and use it only when we need to access `inner.name`. This can reduce contention. Signed-off-by: greatbridf --- src/kernel/task/thread.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index fac9b8f1..12189833 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -212,8 +212,6 @@ impl ThreadBuilder { pub fn clone_from( self, thread: &Thread, clone_args: &CloneArgs, ) -> KResult { - let inner = thread.inner.lock(); - let mut trap_ctx = thread.trap_ctx.borrow().clone(); trap_ctx.set_user_return_value(0); @@ -250,7 +248,7 @@ impl ThreadBuilder { .files(files) .fs_context(fs_context) .signal_list(signal_list) - .name(inner.name.clone()) + .name(thread.inner.lock().name.clone()) .tls(clone_args.tls.clone()) .set_child_tid(clone_args.set_tid_ptr) .clear_child_tid(clone_args.clear_tid_ptr) From 5b2a3afb5471fb2172b50c6fd824f290db4efe18 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 3 Mar 2026 01:05:18 +0800 Subject: [PATCH 084/153] task: clone the parent thread's TLS if not given If TLS is not given in clone syscall arguments, the userspace programs may read undefined value on some platforms (for example, x86, because the TLS is not stored in TrapContexts). And this causes NULL pointer derefs when the userspace program uses the TLS. When we are building a thread, use the parent's TLS if not given from `CloneArgs` to address this problem. Fixes: 5bcdb8040bee ("feat: implement clone") Signed-off-by: greatbridf --- src/kernel/task/clone.rs | 1 - src/kernel/task/thread.rs | 9 ++++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/kernel/task/clone.rs b/src/kernel/task/clone.rs index 28edbc4d..392787de 100644 --- a/src/kernel/task/clone.rs +++ b/src/kernel/task/clone.rs @@ -148,7 +148,6 @@ pub async fn do_clone(thread: &Thread, clone_args: CloneArgs) -> KResult { let new_thread = thread_builder .process(current_process) .tid(new_pid) - .tls(clone_args.tls) .build(&mut procs); new_thread } else { diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 12189833..d9a21fc1 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -244,12 +244,19 @@ impl ThreadBuilder { SignalList::new_cloned(&thread.signal_list) }; + let mut tls = clone_args.tls.clone(); + + // Respect `clone_args`'s TLS if given. Otherwise, clone from the thread. + if tls.is_none() { + tls = thread.inner.lock().tls.clone(); + } + Ok(self .files(files) .fs_context(fs_context) .signal_list(signal_list) .name(thread.inner.lock().name.clone()) - .tls(clone_args.tls.clone()) + .tls(tls) .set_child_tid(clone_args.set_tid_ptr) .clear_child_tid(clone_args.clear_tid_ptr) .trap_ctx(trap_ctx) From 1316f9c730b18b5a5d98e7f9c6ac3155c020eaba Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 00:31:13 +0800 Subject: [PATCH 085/153] kernel: introduce a late init mechanism Add a macro to define late init functions. Function addresses are collected into an array and called after kernel initialization. Async functions can also be done in the future: just return a boxed Future in the helper function. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/loongarch64/link.x | 14 +++ crates/eonix_hal/src/arch/riscv64/link.x | 14 +++ crates/eonix_hal/src/arch/x86_64/link.x | 14 +++ macros/src/lib.rs | 90 +++++++++++++++----- src/kernel.rs | 2 + src/kernel/late_init.rs | 12 +++ src/lib.rs | 4 + 7 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 src/kernel/late_init.rs diff --git a/crates/eonix_hal/src/arch/loongarch64/link.x b/crates/eonix_hal/src/arch/loongarch64/link.x index db95248e..b4b3e441 100644 --- a/crates/eonix_hal/src/arch/loongarch64/link.x +++ b/crates/eonix_hal/src/arch/loongarch64/link.x @@ -67,6 +67,20 @@ SECTIONS { RAW_SYSCALL_HANDLERS_SIZE = ABSOLUTE(__raw_syscall_handlers_end - __raw_syscall_handlers_start); } > REGION_RODATA AT> RAM + + .rodata.late_init : + { + . = ALIGN(16); + __raw_late_init_start = .; + + LATE_INIT_FUNCTIONS = .; + KEEP(*(.late_init)); + + __raw_late_init_end = .; + + LATE_INIT_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + } > REGION_RODATA AT> RAM } INSERT AFTER .rodata; diff --git a/crates/eonix_hal/src/arch/riscv64/link.x b/crates/eonix_hal/src/arch/riscv64/link.x index a74f0d0d..71368659 100644 --- a/crates/eonix_hal/src/arch/riscv64/link.x +++ b/crates/eonix_hal/src/arch/riscv64/link.x @@ -62,6 +62,20 @@ SECTIONS { RAW_SYSCALL_HANDLERS_SIZE = ABSOLUTE(__raw_syscall_handlers_end - __raw_syscall_handlers_start); } > REGION_RODATA AT> RAM + + .rodata.late_init : + { + . = ALIGN(16); + __raw_late_init_start = .; + + LATE_INIT_FUNCTIONS = .; + KEEP(*(.late_init)); + + __raw_late_init_end = .; + + LATE_INIT_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + } > REGION_RODATA AT> RAM } INSERT AFTER .rodata; diff --git a/crates/eonix_hal/src/arch/x86_64/link.x b/crates/eonix_hal/src/arch/x86_64/link.x index d4ac1bef..38a11e3a 100644 --- a/crates/eonix_hal/src/arch/x86_64/link.x +++ b/crates/eonix_hal/src/arch/x86_64/link.x @@ -82,6 +82,20 @@ SECTIONS { RAW_SYSCALL_HANDLERS_SIZE = ABSOLUTE(__raw_syscall_handlers_end - __raw_syscall_handlers_start); } > REGION_RODATA + + .rodata.late_init : + { + . = ALIGN(16); + __raw_late_init_start = .; + + LATE_INIT_FUNCTIONS = .; + KEEP(*(.late_init)); + + __raw_late_init_end = .; + + LATE_INIT_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + } > REGION_RODATA } INSERT AFTER .rodata; diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 09e12f99..2592acd5 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -6,7 +6,9 @@ use syn::{parse2, FnArg, Ident, ItemFn, LitStr}; fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { if attrs.is_empty() { - panic!("`define_syscall` attribute should take one argument: `syscall_no`"); + panic!( + "`define_syscall` attribute should take one argument: `syscall_no`" + ); } let syscall_no = parse2::(attrs).expect("Invalid syscall number"); @@ -37,37 +39,43 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { } }); - let args_call = item - .sig - .inputs - .iter() - .enumerate() - .map(|(idx, arg)| match arg { - FnArg::Receiver(_) => panic!("&self is not permitted."), - FnArg::Typed(_) => { - let arg_ident = Ident::new(&format!("arg_{}", idx), Span::call_site()); - quote! { #arg_ident } - } - }); + let args_call = + item.sig + .inputs + .iter() + .enumerate() + .map(|(idx, arg)| match arg { + FnArg::Receiver(_) => panic!("&self is not permitted."), + FnArg::Typed(_) => { + let arg_ident = + Ident::new(&format!("arg_{}", idx), Span::call_site()); + quote! { #arg_ident } + } + }); let syscall_name = item.sig.ident; - let syscall_name_str = LitStr::new(&syscall_name.to_string(), Span::call_site()); + let syscall_name_str = + LitStr::new(&syscall_name.to_string(), Span::call_site()); let body = item.block; - let helper_fn = Ident::new(&format!("_do_syscall_{}", syscall_name), Span::call_site()); + let helper_fn = + Ident::new(&format!("_do_syscall_{}", syscall_name), Span::call_site()); let helper_fn_pointer = Ident::new( &format!("_SYSCALL_ENTRY_{}", syscall_name.to_string().to_uppercase()), Span::call_site(), ); - let real_fn = Ident::new(&format!("sys_{}", syscall_name), Span::call_site()); + let real_fn = + Ident::new(&format!("sys_{}", syscall_name), Span::call_site()); let raw_syscall_section = LitStr::new( &format!(".raw_syscalls.{}", syscall_name), Span::call_site(), ); - let syscall_fn_section = - LitStr::new(&format!(".syscall_fns.{}", syscall_name), Span::call_site()); + let syscall_fn_section = LitStr::new( + &format!(".syscall_fns.{}", syscall_name), + Span::call_site(), + ); let trace_format_string = { let arg_count = item.sig.inputs.len(); @@ -84,7 +92,8 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { let args = args.enumerate().map(|(idx, arg)| match arg { FnArg::Receiver(_) => panic!("&self is not permitted."), FnArg::Typed(_) => { - let arg_ident = Ident::new(&format!("arg_{}", idx), Span::call_site()); + let arg_ident = + Ident::new(&format!("arg_{}", idx), Span::call_site()); quote! { #arg_ident } } }); @@ -175,8 +184,47 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { /// ``` #[proc_macro_attribute] pub fn define_syscall( - attrs: proc_macro::TokenStream, - item: proc_macro::TokenStream, + attrs: proc_macro::TokenStream, item: proc_macro::TokenStream, ) -> proc_macro::TokenStream { define_syscall_impl(attrs.into(), item.into()).into() } + +fn define_late_init_impl(attrs: TokenStream, func: TokenStream) -> TokenStream { + if !attrs.is_empty() { + panic!("attributes not allowed here"); + } + + let func_parsed = + parse2::(func.clone()).expect("expected function definition"); + + let func_ident = &func_parsed.sig.ident; + let func_name = func_parsed.sig.ident.to_string(); + + let static_ident = Ident::new( + &format!("__LATE_INIT_{}", func_name.to_uppercase()), + Span::call_site(), + ); + + quote! { + #func + + #[used] + #[doc(hidden)] + #[link_section = ".late_init"] + static #static_ident : fn() = #func_ident; + } +} + +/// Define a function to run after kernel initialization. +/// +/// # Note +/// Keep in mind that all the functions defined with this macro may be run in +/// **ANY** order. +/// +/// So if order matters, try another way other than this! +#[proc_macro_attribute] +pub fn define_late_init( + attrs: proc_macro::TokenStream, func: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + define_late_init_impl(attrs.into(), func.into()).into() +} diff --git a/src/kernel.rs b/src/kernel.rs index 39e3f9d3..a973af67 100644 --- a/src/kernel.rs +++ b/src/kernel.rs @@ -12,10 +12,12 @@ pub mod user; pub mod vfs; mod chardev; +mod late_init; mod power; mod terminal; #[allow(unused_imports)] pub use chardev::{CharDevice, CharDeviceType, VirtualCharDevice}; +pub use late_init::run_late_init; pub use power::shutdown_system; pub use terminal::{Terminal, TerminalDevice}; diff --git a/src/kernel/late_init.rs b/src/kernel/late_init.rs new file mode 100644 index 00000000..3b990d73 --- /dev/null +++ b/src/kernel/late_init.rs @@ -0,0 +1,12 @@ +use eonix_hal::{extern_symbol_addr, extern_symbol_value}; + +pub fn run_late_init() { + let count = extern_symbol_value!(LATE_INIT_FUNCTION_COUNT); + let addr = extern_symbol_addr!(LATE_INIT_FUNCTIONS, fn()); + + let functions = unsafe { core::slice::from_raw_parts(addr, count) }; + + for func in functions { + func(); + } +} diff --git a/src/lib.rs b/src/lib.rs index e4c56923..f8121cab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,8 @@ use kernel_init::setup_memory; use path::Path; use prelude::*; +use crate::kernel::run_late_init; + static BSP_OK: AtomicBool = AtomicBool::new(false); #[eonix_hal::main] @@ -148,6 +150,8 @@ async fn init_process(early_kstack: PRange) { fs::fat32::init(); // fs::ext4::init(); + run_late_init(); + let load_info = { // mount fat32 /mnt directory let fs_context = FsContext::global(); From 7d0adb2cbcc9ca31e519969c335b335229b36d1f Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 22:17:50 +0800 Subject: [PATCH 086/153] kernel: support async functions for late init Add another array storing async function stubs that wrap the async function with `Box::pin()`. Call these functions after system initialization. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/loongarch64/link.x | 10 +++ crates/eonix_hal/src/arch/riscv64/link.x | 10 +++ crates/eonix_hal/src/arch/x86_64/link.x | 10 +++ macros/src/lib.rs | 81 ++++++++++++++++++-- src/kernel.rs | 2 +- src/kernel/late_init.rs | 18 +++++ src/lib.rs | 3 +- 7 files changed, 126 insertions(+), 8 deletions(-) diff --git a/crates/eonix_hal/src/arch/loongarch64/link.x b/crates/eonix_hal/src/arch/loongarch64/link.x index b4b3e441..e6bccdda 100644 --- a/crates/eonix_hal/src/arch/loongarch64/link.x +++ b/crates/eonix_hal/src/arch/loongarch64/link.x @@ -78,8 +78,18 @@ SECTIONS { __raw_late_init_end = .; + __raw_late_init_async_start = .; + + LATE_INIT_ASYNC_FUNCTIONS = .; + KEEP(*(.late_init_async)); + + __raw_late_init_async_end = .; + LATE_INIT_FUNCTION_COUNT = ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + + LATE_INIT_ASYNC_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_async_end - __raw_late_init_async_start) / 8); } > REGION_RODATA AT> RAM } INSERT AFTER .rodata; diff --git a/crates/eonix_hal/src/arch/riscv64/link.x b/crates/eonix_hal/src/arch/riscv64/link.x index 71368659..f7cb8f7f 100644 --- a/crates/eonix_hal/src/arch/riscv64/link.x +++ b/crates/eonix_hal/src/arch/riscv64/link.x @@ -73,8 +73,18 @@ SECTIONS { __raw_late_init_end = .; + __raw_late_init_async_start = .; + + LATE_INIT_ASYNC_FUNCTIONS = .; + KEEP(*(.late_init_async)); + + __raw_late_init_async_end = .; + LATE_INIT_FUNCTION_COUNT = ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + + LATE_INIT_ASYNC_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_async_end - __raw_late_init_async_start) / 8); } > REGION_RODATA AT> RAM } INSERT AFTER .rodata; diff --git a/crates/eonix_hal/src/arch/x86_64/link.x b/crates/eonix_hal/src/arch/x86_64/link.x index 38a11e3a..d706a1b7 100644 --- a/crates/eonix_hal/src/arch/x86_64/link.x +++ b/crates/eonix_hal/src/arch/x86_64/link.x @@ -93,8 +93,18 @@ SECTIONS { __raw_late_init_end = .; + __raw_late_init_async_start = .; + + LATE_INIT_ASYNC_FUNCTIONS = .; + KEEP(*(.late_init_async)); + + __raw_late_init_async_end = .; + LATE_INIT_FUNCTION_COUNT = ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + + LATE_INIT_ASYNC_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_async_end - __raw_late_init_async_start) / 8); } > REGION_RODATA } INSERT AFTER .rodata; diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 2592acd5..42f72213 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,7 +2,7 @@ extern crate proc_macro; use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{parse2, FnArg, Ident, ItemFn, LitStr}; +use syn::{parse2, FnArg, Ident, ItemFn, LitStr, ReturnType, Signature}; fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { if attrs.is_empty() { @@ -189,14 +189,30 @@ pub fn define_syscall( define_syscall_impl(attrs.into(), item.into()).into() } -fn define_late_init_impl(attrs: TokenStream, func: TokenStream) -> TokenStream { - if !attrs.is_empty() { - panic!("attributes not allowed here"); +fn check_late_init_sig(sig: &Signature) { + assert!(sig.constness.is_none(), "no const function allowed"); + assert!(sig.inputs.is_empty(), "no arguments allowed"); + + if let Some(abi) = &sig.abi { + if let Some(name) = &abi.name { + assert_eq!(name.value(), "Rust", "expected Rust ABI"); + } } - let func_parsed = - parse2::(func.clone()).expect("expected function definition"); + assert!( + matches!(&sig.output, ReturnType::Default), + "no return types allowed" + ); + + let generics = &sig.generics; + + assert_eq!(generics.const_params().count(), 0, "no generics allowed"); + assert_eq!(generics.type_params().count(), 0, "no generics allowed") +} +fn define_late_init_normal( + func: &TokenStream, func_parsed: &ItemFn, +) -> TokenStream { let func_ident = &func_parsed.sig.ident; let func_name = func_parsed.sig.ident.to_string(); @@ -215,6 +231,59 @@ fn define_late_init_impl(attrs: TokenStream, func: TokenStream) -> TokenStream { } } +fn define_late_init_async( + func: &TokenStream, func_parsed: &ItemFn, +) -> TokenStream { + let func_ident = &func_parsed.sig.ident; + let func_name = func_parsed.sig.ident.to_string(); + + let helper_ident = Ident::new( + &format!("__late_init_async_wrapper_{}", func_name.to_uppercase()), + Span::call_site(), + ); + + let static_ident = Ident::new( + &format!("__LATE_INIT_ASYNC_{}", func_name.to_uppercase()), + Span::call_site(), + ); + + let pin_t = quote!(core::pin::Pin); + let box_t = quote!(alloc::boxed::Box); + let future_t = quote!(core::future::Future); + + quote! { + #func + + fn #helper_ident() -> + #pin_t<#box_t + Send>> { + #box_t::pin(#func_ident()) + } + + #[used] + #[doc(hidden)] + #[link_section = ".late_init_async"] + static #static_ident: fn() -> + #pin_t<#box_t + Send>> = #helper_ident; + } +} + +fn define_late_init_impl(attrs: TokenStream, func: TokenStream) -> TokenStream { + if !attrs.is_empty() { + panic!("attributes not allowed here"); + } + + let func_parsed = + parse2::(func.clone()).expect("expected function definition"); + + check_late_init_sig(&func_parsed.sig); + + if func_parsed.sig.asyncness.is_some() { + define_late_init_async(&func, &func_parsed) + } else { + define_late_init_normal(&func, &func_parsed) + } +} + /// Define a function to run after kernel initialization. /// /// # Note diff --git a/src/kernel.rs b/src/kernel.rs index a973af67..e5712856 100644 --- a/src/kernel.rs +++ b/src/kernel.rs @@ -18,6 +18,6 @@ mod terminal; #[allow(unused_imports)] pub use chardev::{CharDevice, CharDeviceType, VirtualCharDevice}; -pub use late_init::run_late_init; +pub use late_init::{run_late_init, run_late_init_async}; pub use power::shutdown_system; pub use terminal::{Terminal, TerminalDevice}; diff --git a/src/kernel/late_init.rs b/src/kernel/late_init.rs index 3b990d73..9c20be22 100644 --- a/src/kernel/late_init.rs +++ b/src/kernel/late_init.rs @@ -1,3 +1,7 @@ +use alloc::boxed::Box; +use core::future::Future; +use core::pin::Pin; + use eonix_hal::{extern_symbol_addr, extern_symbol_value}; pub fn run_late_init() { @@ -10,3 +14,17 @@ pub fn run_late_init() { func(); } } + +pub async fn run_late_init_async() { + let count = extern_symbol_value!(LATE_INIT_ASYNC_FUNCTION_COUNT); + let addr = extern_symbol_addr!( + LATE_INIT_ASYNC_FUNCTIONS, + fn() -> Pin + Send>> + ); + + let functions = unsafe { core::slice::from_raw_parts(addr, count) }; + + for func in functions { + func().await; + } +} diff --git a/src/lib.rs b/src/lib.rs index f8121cab..c6f0f92e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,7 +49,7 @@ use kernel_init::setup_memory; use path::Path; use prelude::*; -use crate::kernel::run_late_init; +use crate::kernel::{run_late_init, run_late_init_async}; static BSP_OK: AtomicBool = AtomicBool::new(false); @@ -151,6 +151,7 @@ async fn init_process(early_kstack: PRange) { // fs::ext4::init(); run_late_init(); + run_late_init_async().await; let load_info = { // mount fat32 /mnt directory From c46cba46ade15f1a18bed2d49c520a075b07ec4c Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 22:25:34 +0800 Subject: [PATCH 087/153] style: reformat mount.rs No functional change. Signed-off-by: greatbridf --- src/kernel/vfs/mount.rs | 42 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 6b171f81..03dbf917 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -30,7 +30,8 @@ const MOUNT_FLAGS: [(u64, &str); 6] = [ (MS_LAZYTIME, ",lazytime"), ]; -static MOUNT_CREATORS: Spin>> = Spin::new(BTreeMap::new()); +static MOUNT_CREATORS: Spin>> = + Spin::new(BTreeMap::new()); static MOUNTS: Spin, MountPointData)>> = Spin::new(vec![]); pub struct Mount { @@ -39,7 +40,9 @@ pub struct Mount { } impl Mount { - pub fn new(mp: &Dentry, sb: SbUse, root_inode: InodeUse) -> KResult { + pub fn new( + mp: &Dentry, sb: SbUse, root_inode: InodeUse, + ) -> KResult { let root_dentry = Dentry::create(mp.parent().clone(), &mp.get_name()); root_dentry.fill(root_inode); @@ -60,10 +63,14 @@ unsafe impl Sync for Mount {} #[async_trait] pub trait MountCreator: Send + Sync { fn check_signature(&self, first_block: &[u8]) -> KResult; - async fn create_mount(&self, source: &str, flags: u64, mp: &Arc) -> KResult; + async fn create_mount( + &self, source: &str, flags: u64, mp: &Arc, + ) -> KResult; } -pub fn register_filesystem(fstype: &str, creator: Arc) -> KResult<()> { +pub fn register_filesystem( + fstype: &str, creator: Arc, +) -> KResult<()> { let mut creators = MOUNT_CREATORS.lock(); if !creators.contains_key(fstype) { creators.insert(fstype.to_string(), creator); @@ -83,10 +90,7 @@ struct MountPointData { } pub async fn do_mount( - mountpoint: &Arc, - source: &str, - mountpoint_str: &str, - fstype: &str, + mountpoint: &Arc, source: &str, mountpoint_str: &str, fstype: &str, flags: u64, ) -> KResult<()> { let mut flags = flags; @@ -170,23 +174,21 @@ impl Dentry { .cloned() .expect("tmpfs not registered."); - let mount = block_on(creator.create_mount(&source, mount_flags, &DROOT)) - .expect("Failed to create root mount."); + let mount = + block_on(creator.create_mount(&source, mount_flags, &DROOT)) + .expect("Failed to create root mount."); let root_dentry = mount.root().clone(); dcache::d_add(root_dentry.clone()); - MOUNTS.lock().push(( - DROOT.clone(), - MountPointData { - mount, - source, - mountpoint: String::from("/"), - fstype, - flags: mount_flags, - }, - )); + MOUNTS.lock().push((DROOT.clone(), MountPointData { + mount, + source, + mountpoint: String::from("/"), + fstype, + flags: mount_flags, + })); root_dentry }); From dec29cfa928bf440dd50c581a4deec8027942161 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 22:26:12 +0800 Subject: [PATCH 088/153] style: reformat procfs.rs No functional change. Signed-off-by: greatbridf --- src/fs/procfs.rs | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 9a3933bb..e6acd2a9 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -10,7 +10,9 @@ use crate::kernel::mem::paging::PageBuffer; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; -use crate::kernel::vfs::mount::{dump_mounts, register_filesystem, Mount, MountCreator}; +use crate::kernel::vfs::mount::{ + dump_mounts, register_filesystem, Mount, MountCreator, +}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; use crate::prelude::*; @@ -39,11 +41,8 @@ impl InodeOps for Node { type SuperBlock = ProcFs; async fn read( - &self, - _: SbUse, - _: &InodeUse, - buffer: &mut dyn Buffer, - offset: usize, + &self, _: SbUse, _: &InodeUse, + buffer: &mut dyn Buffer, offset: usize, ) -> KResult { let NodeKind::File(file_inode) = &self.kind else { return Err(EISDIR); @@ -56,7 +55,8 @@ impl InodeOps for Node { let mut page_buffer = PageBuffer::new(); read_fn(&mut page_buffer)?; - let Some((_, data)) = page_buffer.data().split_at_checked(offset) else { + let Some((_, data)) = page_buffer.data().split_at_checked(offset) + else { return Ok(0); }; @@ -64,10 +64,7 @@ impl InodeOps for Node { } async fn lookup( - &self, - _: SbUse, - _: &InodeUse, - dentry: &Arc, + &self, _: SbUse, _: &InodeUse, dentry: &Arc, ) -> KResult> { let NodeKind::Dir(dir) = &self.kind else { return Err(ENOTDIR); @@ -86,10 +83,7 @@ impl InodeOps for Node { } async fn readdir( - &self, - _: SbUse, - _: &InodeUse, - offset: usize, + &self, _: SbUse, _: &InodeUse, offset: usize, callback: &mut (dyn FnMut(&[u8], Ino) -> KResult + Send), ) -> KResult> { let NodeKind::Dir(dir) = &self.kind else { @@ -113,8 +107,7 @@ impl InodeOps for Node { impl Node { pub fn new_file( - ino: Ino, - sb: SbRef, + ino: Ino, sb: SbRef, read: impl Fn(&mut PageBuffer) -> KResult<()> + Send + Sync + 'static, ) -> InodeUse { InodeUse::new( @@ -160,7 +153,9 @@ impl Node { } impl FileInode { - fn new(read: Box KResult<()> + Send + Sync>) -> Self { + fn new( + read: Box KResult<()> + Send + Sync>, + ) -> Self { Self { read: Some(read), write: None, @@ -206,7 +201,9 @@ struct ProcFsMountCreator; #[async_trait] impl MountCreator for ProcFsMountCreator { - async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc) -> KResult { + async fn create_mount( + &self, _source: &str, _flags: u64, mp: &Arc, + ) -> KResult { let fs = GLOBAL_PROCFS.clone(); let root_inode = fs.backend.root.clone(); @@ -232,7 +229,11 @@ where let mut entries = root.entries.write().await; entries.push(( name.clone(), - Node::new_file(procfs.assign_ino(), SbRef::from(&GLOBAL_PROCFS), read_fn), + Node::new_file( + procfs.assign_ino(), + SbRef::from(&GLOBAL_PROCFS), + read_fn, + ), )); } From 181c1896667d7646446ff2548dfb50b19283a564 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 22:33:49 +0800 Subject: [PATCH 089/153] vfs: move /proc/mounts population to mount.rs Now that we have late init, we can use this instead of contaminating the global namespace. Signed-off-by: greatbridf --- src/fs/procfs.rs | 12 ++---------- src/kernel/vfs/mount.rs | 37 +++++++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index e6acd2a9..5c0f5212 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -10,9 +10,7 @@ use crate::kernel::mem::paging::PageBuffer; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; -use crate::kernel::vfs::mount::{ - dump_mounts, register_filesystem, Mount, MountCreator, -}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; use crate::kernel::vfs::types::{DeviceId, Format, Permission}; use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; use crate::prelude::*; @@ -228,7 +226,7 @@ where let mut entries = root.entries.write().await; entries.push(( - name.clone(), + name, Node::new_file( procfs.assign_ino(), SbRef::from(&GLOBAL_PROCFS), @@ -239,10 +237,4 @@ where pub async fn init() { register_filesystem("procfs", Arc::new(ProcFsMountCreator)).unwrap(); - - populate_root(Arc::from(b"mounts".as_slice()), |buffer| { - dump_mounts(&mut buffer.get_writer()); - Ok(()) - }) - .await; } diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 03dbf917..d3d9fe69 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -3,12 +3,15 @@ use alloc::string::ToString as _; use alloc::sync::Arc; use async_trait::async_trait; +use eonix_macros::define_late_init; use eonix_sync::LazyLock; use super::dentry::{dcache, Dentry, DROOT}; use super::inode::InodeUse; use super::{SbUse, SuperBlock}; -use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; +use crate::fs::procfs::populate_root; +use crate::io::Buffer; +use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR, ERANGE}; use crate::kernel::task::block_on; use crate::prelude::*; @@ -148,17 +151,27 @@ fn mount_opts(flags: u64) -> String { out } -pub fn dump_mounts(buffer: &mut dyn core::fmt::Write) { - for (_, mpdata) in MOUNTS.lock().iter() { - dont_check!(writeln!( - buffer, - "{} {} {} {} 0 0", - mpdata.source, - mpdata.mountpoint, - mpdata.fstype, - mount_opts(mpdata.flags) - )) - } +#[define_late_init] +async fn populate_mounts_file() { + populate_root(Arc::from(b"mounts".as_slice()), |buffer| { + let mut writer = buffer.get_writer(); + + for (_, mpdata) in MOUNTS.lock().iter() { + let result = writeln!( + writer, + "{} {} {} {} 0 0", + mpdata.source, + mpdata.mountpoint, + mpdata.fstype, + mount_opts(mpdata.flags) + ); + + result.map_err(|_| ERANGE)?; + } + + Ok(()) + }) + .await; } impl Dentry { From 9c2568bafa9c544eb8c0d5c7c6682a92ca1ff58e Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:19:44 +0800 Subject: [PATCH 090/153] style: reformat e1000e.rs Signed-off-by: greatbridf --- src/driver/e1000e.rs | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index 923a4594..e81cd72f 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -152,7 +152,9 @@ impl netdev::Netdev for E1000eDev { match status & defs::STAT_SPEED_MASK { defs::STAT_SPEED_10M => self.speed = netdev::LinkSpeed::Speed10M, defs::STAT_SPEED_100M => self.speed = netdev::LinkSpeed::Speed100M, - defs::STAT_SPEED_1000M => self.speed = netdev::LinkSpeed::Speed1000M, + defs::STAT_SPEED_1000M => { + self.speed = netdev::LinkSpeed::Speed1000M + } _ => return Err(EINVAL), } @@ -209,7 +211,10 @@ impl netdev::Netdev for E1000eDev { let buffer = &self.rx_buffers[next_tail as usize]; let data = &buffer.as_bytes()[..len]; - println_debug!("e1000e: received {len} bytes, {:?}", PrintableBytes(data)); + println_debug!( + "e1000e: received {len} bytes, {:?}", + PrintableBytes(data) + ); self.rx_tail = Some(next_tail); } @@ -245,7 +250,9 @@ impl netdev::Netdev for E1000eDev { self.tx_tail = Some(next_tail); self.regs.write(defs::REG_TDT, next_tail); - unimplemented!("Check if the packets are sent and update self.tx_head state"); + unimplemented!( + "Check if the packets are sent and update self.tx_head state" + ); // Ok(()) } } @@ -374,12 +381,16 @@ impl E1000eDev { rx_head: None, rx_tail: None, tx_tail: None, - rx_buffers: Box::new(core::array::from_fn(|_| FolioOwned::alloc_order(2))), + rx_buffers: Box::new(core::array::from_fn(|_| { + FolioOwned::alloc_order(2) + })), tx_buffers: Box::new([const { None }; 32]), }; unsafe { - for (desc, page) in dev.rx_desc_table().into_iter().zip(dev.rx_buffers.iter()) { + for (desc, page) in + dev.rx_desc_table().into_iter().zip(dev.rx_buffers.iter()) + { desc.buffer = page.start().addr() as u64; desc.status = 0; } @@ -424,7 +435,9 @@ impl PCIDriver for Driver { self.dev_id } - async fn handle_device(&self, device: Arc>) -> Result<(), PciError> { + async fn handle_device( + &self, device: Arc>, + ) -> Result<(), PciError> { let Header::Endpoint(header) = device.header else { Err(EINVAL)? }; From 5732b46e9a58e2a9954457e9b7784061a79097e4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:20:26 +0800 Subject: [PATCH 091/153] util: remove dont_check macro This was used in `dump_mount()` and is now unused after rewrite of `dump_mount()` in commit 181c1896667d ("vfs: move /proc/mounts population to mount.rs")"). Remove it now. Signed-off-by: greatbridf --- src/prelude.rs | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/prelude.rs b/src/prelude.rs index 880489da..8588ede0 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,23 +1,14 @@ #[allow(dead_code)] pub type KResult = Result; -macro_rules! dont_check { - ($arg:expr) => { - match $arg { - Ok(_) => (), - Err(_) => (), - } - }; -} - -pub(crate) use dont_check; - -pub(crate) use crate::kernel::console::{ - print, println, println_debug, println_fatal, println_info, println_trace, println_warn, -}; - -pub(crate) use alloc::{boxed::Box, string::String, vec, vec::Vec}; - -pub(crate) use core::{fmt::Write, marker::PhantomData, str}; +pub(crate) use alloc::boxed::Box; +pub(crate) use alloc::string::String; +pub(crate) use alloc::vec; +pub(crate) use alloc::vec::Vec; +pub(crate) use core::fmt::Write; +pub(crate) use core::marker::PhantomData; +pub(crate) use core::str; + +pub(crate) use eonix_log::{println_debug, println_warn}; pub use crate::sync::Spin; From a4fc4ddabe0d806a78c399b23a3b47ca8b89eed1 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:24:43 +0800 Subject: [PATCH 092/153] style: clean old kernel console log macros These macros were kept for compatibility and is unused now. Remove them. Signed-off-by: greatbridf --- src/driver/e1000e.rs | 1 + src/kernel/console.rs | 78 +++---------------------------------------- src/lib.rs | 1 + src/prelude.rs | 2 +- 4 files changed, 7 insertions(+), 75 deletions(-) diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index e81cd72f..079b29ae 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -4,6 +4,7 @@ use core::ptr::NonNull; use async_trait::async_trait; use eonix_hal::fence::memory_barrier; +use eonix_log::println_debug; use eonix_mm::address::{Addr, PAddr}; use eonix_mm::paging::Folio as _; use eonix_sync::SpinIrq; diff --git a/src/kernel/console.rs b/src/kernel/console.rs index 85fb8b39..3ddef551 100644 --- a/src/kernel/console.rs +++ b/src/kernel/console.rs @@ -1,6 +1,9 @@ -use crate::prelude::*; use alloc::sync::Arc; +use super::constants::EEXIST; +use super::terminal::Terminal; +use crate::prelude::*; + static CONSOLE: Spin>> = Spin::new(None); pub fn set_console(terminal: Arc) -> KResult<()> { @@ -17,76 +20,3 @@ pub fn get_console() -> Option> { let console = CONSOLE.lock(); console.clone() } - -#[doc(hidden)] -pub fn _print(args: core::fmt::Arguments) { - // TODO!!!!!!!!!!!!!: REMOVE THIS AND USE `eonix_log`. - eonix_log::do_print(args); -} - -macro_rules! print { - ($($arg:tt)*) => { - $crate::kernel::console::_print(format_args!($($arg)*)) - }; -} - -macro_rules! println { - () => { - $crate::print!("\n") - }; - ($($arg:tt)*) => { - $crate::print!("{}\n", format_args!($($arg)*)) - }; -} - -macro_rules! println_warn { - ($($arg:tt)*) => { - $crate::println!("[kernel: warn] {}", format_args!($($arg)*)) - }; -} - -macro_rules! println_debug { - ($($arg:tt)*) => { - $crate::println!("[kernel:debug] {}", format_args!($($arg)*)) - }; -} - -#[allow(unused_macros)] -macro_rules! println_info { - ($($arg:tt)*) => { - $crate::println!("[kernel: info] {}", format_args!($($arg)*)) - }; -} - -macro_rules! println_fatal { - () => { - $crate::println!("[kernel:fatal] ") - }; - ($($arg:tt)*) => { - $crate::println!("[kernel:fatal] {}", format_args!($($arg)*)) - }; -} - -#[allow(unused_macros)] -macro_rules! println_trace { - ($feat:literal) => { - #[deny(unexpected_cfgs)] - { - #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] ") - } - }; - ($feat:literal, $($arg:tt)*) => {{ - #[deny(unexpected_cfgs)] - { - #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] {}", format_args!($($arg)*)) - } - }}; -} - -use super::{constants::EEXIST, terminal::Terminal}; - -pub(crate) use { - print, println, println_debug, println_fatal, println_info, println_trace, println_warn, -}; diff --git a/src/lib.rs b/src/lib.rs index c6f0f92e..a9087beb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,7 @@ use eonix_hal::processor::CPU; use eonix_hal::symbol_addr; use eonix_hal::traits::trap::{IrqState, RawTrapContext, TrapReturn}; use eonix_hal::trap::{disable_irqs_save, TrapContext}; +use eonix_log::println_debug; use eonix_mm::address::PRange; use eonix_runtime::scheduler::RUNTIME; use kernel::mem::GlobalPageAlloc; diff --git a/src/prelude.rs b/src/prelude.rs index 8588ede0..f7bf5868 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -9,6 +9,6 @@ pub(crate) use core::fmt::Write; pub(crate) use core::marker::PhantomData; pub(crate) use core::str; -pub(crate) use eonix_log::{println_debug, println_warn}; +pub(crate) use eonix_log::println_warn; pub use crate::sync::Spin; From b0040f4e3babbeb08681f4115cad3f11eb4490c9 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:27:22 +0800 Subject: [PATCH 093/153] style: reformat ahci/mod.rs Signed-off-by: greatbridf --- src/driver/ahci/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index 3ea44ed3..98373012 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -166,7 +166,9 @@ impl PCIDriver for AHCIDriver { DEVICE_AHCI } - async fn handle_device(&self, pcidev: Arc>) -> Result<(), PciError> { + async fn handle_device( + &self, pcidev: Arc>, + ) -> Result<(), PciError> { let Header::Endpoint(header) = pcidev.header else { Err(EINVAL)? }; @@ -201,7 +203,9 @@ impl PCIDriver for AHCIDriver { device.control.enable_interrupts(); let device_irq = device.clone(); - register_irq_handler(irqno as i32, move || device_irq.handle_interrupt())?; + register_irq_handler(irqno as i32, move || { + device_irq.handle_interrupt() + })?; device.probe_ports().await?; From 2e1774b2a4b1b6f220af879d9835c7d082eccef0 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:28:12 +0800 Subject: [PATCH 094/153] style: remove println_warn from prelude We hope to completely remove prelude module in the future. Do some of the cleanup work now. Signed-off-by: greatbridf --- src/driver/ahci/mod.rs | 1 + src/kernel/syscall/procops.rs | 1 + src/kernel/task/thread.rs | 1 + src/prelude.rs | 2 -- 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index 98373012..5fc83e3b 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -4,6 +4,7 @@ use alloc::sync::Arc; use async_trait::async_trait; use control::AdapterControl; use defs::*; +use eonix_log::println_warn; use eonix_mm::address::{AddrOps as _, PAddr}; use eonix_sync::SpinIrq as _; use port::AdapterPort; diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index a207a3db..72d09f98 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -5,6 +5,7 @@ use core::time::Duration; use bitflags::bitflags; use eonix_hal::traits::trap::RawTrapContext; use eonix_hal::trap::TrapContext; +use eonix_log::println_warn; use eonix_mm::address::Addr as _; use eonix_sync::AsProof as _; use posix_types::ctypes::PtrT; diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index d9a21fc1..c30f4b4b 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -12,6 +12,7 @@ use eonix_hal::traits::fault::Fault; use eonix_hal::traits::fpu::RawFpuState as _; use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; use eonix_hal::trap::TrapContext; +use eonix_log::println_warn; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::AsProofMut as _; use intrusive_collections::{intrusive_adapter, KeyAdapter, RBTreeAtomicLink}; diff --git a/src/prelude.rs b/src/prelude.rs index f7bf5868..097dac6b 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -9,6 +9,4 @@ pub(crate) use core::fmt::Write; pub(crate) use core::marker::PhantomData; pub(crate) use core::str; -pub(crate) use eonix_log::println_warn; - pub use crate::sync::Spin; From 2cfc80d4a6ba53718704bbe544a24903589ac9b7 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:29:41 +0800 Subject: [PATCH 095/153] style: reformat io.rs Signed-off-by: greatbridf --- src/io.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/io.rs b/src/io.rs index d7094f6d..b0156171 100644 --- a/src/io.rs +++ b/src/io.rs @@ -52,7 +52,9 @@ pub trait Buffer: Send { } pub trait Stream: Send { - fn poll_data<'a>(&mut self, buf: &'a mut [u8]) -> KResult>; + fn poll_data<'a>( + &mut self, buf: &'a mut [u8], + ) -> KResult>; fn ignore(&mut self, len: usize) -> KResult>; } @@ -64,9 +66,7 @@ pub trait IntoStream { pub trait StreamRead { fn read_till_end( - &mut self, - buffer: &mut [u8], - func: impl Fn(&mut [u8]) -> KResult<()>, + &mut self, buffer: &mut [u8], func: impl Fn(&mut [u8]) -> KResult<()>, ) -> KResult; fn ignore_all(&mut self) -> KResult; @@ -77,9 +77,7 @@ where T: Stream + ?Sized, { fn read_till_end( - &mut self, - buffer: &mut [u8], - func: impl Fn(&mut [u8]) -> KResult<()>, + &mut self, buffer: &mut [u8], func: impl Fn(&mut [u8]) -> KResult<()>, ) -> KResult { let mut total = 0; while let Some(data) = self.poll_data(buffer)? { @@ -143,7 +141,10 @@ impl<'lt, T: Copy + Sized> UninitBuffer<'lt, T> { Self { data, buffer: ByteBuffer::from(unsafe { - core::slice::from_raw_parts_mut(ptr as *mut u8, core::mem::size_of::()) + core::slice::from_raw_parts_mut( + ptr as *mut u8, + core::mem::size_of::(), + ) }), } } @@ -294,7 +295,9 @@ impl<'a> ByteStream<'a> { } impl<'a> Stream for ByteStream<'a> { - fn poll_data<'b>(&mut self, buf: &'b mut [u8]) -> KResult> { + fn poll_data<'b>( + &mut self, buf: &'b mut [u8], + ) -> KResult> { if self.cur >= self.data.len() { return Ok(None); } From 2e26c4a72b91707f22ff94f418b227493a347a0b Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:31:35 +0800 Subject: [PATCH 096/153] util: add buf_writeln helper macro `Buffer`s return `core::fmt::Result` when it overflows. The bothersome error handling code can be encapsulated in a macro. Besides, we don't have to import `core::fmt::Write` trait manually now. Signed-off-by: greatbridf --- src/io.rs | 10 ++++++++++ src/kernel/vfs/mount.rs | 14 +++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/io.rs b/src/io.rs index b0156171..74960154 100644 --- a/src/io.rs +++ b/src/io.rs @@ -333,3 +333,13 @@ impl<'a> IntoStream for &'a [u8] { ByteStream::new(self) } } + +macro_rules! buf_writeln { + ($buf:ident, $($args:expr),*) => {{ + use core::fmt::Write; + writeln!($buf.get_writer(), $($args),*) + .map_err(|_| $crate::kernel::constants::ERANGE) + }}; +} + +pub(crate) use buf_writeln; diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index d3d9fe69..0a7983d9 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -10,8 +10,8 @@ use super::dentry::{dcache, Dentry, DROOT}; use super::inode::InodeUse; use super::{SbUse, SuperBlock}; use crate::fs::procfs::populate_root; -use crate::io::Buffer; -use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR, ERANGE}; +use crate::io::{buf_writeln, Buffer}; +use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; use crate::kernel::task::block_on; use crate::prelude::*; @@ -154,19 +154,15 @@ fn mount_opts(flags: u64) -> String { #[define_late_init] async fn populate_mounts_file() { populate_root(Arc::from(b"mounts".as_slice()), |buffer| { - let mut writer = buffer.get_writer(); - for (_, mpdata) in MOUNTS.lock().iter() { - let result = writeln!( - writer, + buf_writeln!( + buffer, "{} {} {} {} 0 0", mpdata.source, mpdata.mountpoint, mpdata.fstype, mount_opts(mpdata.flags) - ); - - result.map_err(|_| ERANGE)?; + )?; } Ok(()) From f0f751898a4d0d5dfa5485c8fb26ed45eb0fcdfe Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 5 Mar 2026 23:47:04 +0800 Subject: [PATCH 097/153] style: reformat slab_allocator/lib.rs Make `make_slab_page()` private since it's not used outside. Signed-off-by: greatbridf --- crates/slab_allocator/src/lib.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index c3e7f392..733e4de5 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -114,8 +114,14 @@ where alloc: P, } -unsafe impl Send for SlabAlloc where P: SlabPageAlloc {} -unsafe impl Sync for SlabAlloc where P: SlabPageAlloc {} +unsafe impl Send for SlabAlloc where + P: SlabPageAlloc +{ +} +unsafe impl Sync for SlabAlloc where + P: SlabPageAlloc +{ +} impl SlabAlloc where @@ -124,7 +130,9 @@ where { pub fn new_in(alloc: L) -> Self { Self { - slabs: core::array::from_fn(|i| Spin::new(SlabList::new(1 << (i + 3)))), + slabs: core::array::from_fn(|i| { + Spin::new(SlabList::new(1 << (i + 3))) + }), alloc, } } @@ -199,7 +207,9 @@ where self.empty_list.push_tail(slab); } - fn alloc(&mut self, alloc: &impl SlabPageAlloc) -> NonNull { + fn alloc( + &mut self, alloc: &impl SlabPageAlloc, + ) -> NonNull { if !self.partial_list.is_empty() { return self.alloc_from_partial(); } @@ -211,7 +221,9 @@ where self.alloc_from_empty() } - unsafe fn dealloc(&mut self, ptr: NonNull, _alloc: &impl SlabPageAlloc) { + unsafe fn dealloc( + &mut self, ptr: NonNull, _alloc: &impl SlabPageAlloc, + ) { let slab_page = unsafe { // SAFETY: ::from_allocated(ptr) @@ -249,7 +261,9 @@ where } } -pub fn make_slab_page(page_ptr: NonNull<[u8]>, slot_size: usize) -> NonNull { +fn make_slab_page( + page_ptr: NonNull<[u8]>, slot_size: usize, +) -> NonNull { assert!( slot_size >= core::mem::size_of::(), "The minimum slot size is of a pointer's width" From b1099dc3f7a5b2afb95b10cc048827ce63269f50 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 6 Mar 2026 00:23:46 +0800 Subject: [PATCH 098/153] slab: count alloced and total {slab, object} SlabList have exclusiveness guarantee and we don't shrink slab page list for now. So we just: - Increase total folio and object count when new slabs are allocated. - Increase active folio count when we grab a new empty slab. - Increase active object count when allocation succeed. - Decrease active folio count when we push slabs back to empty list. - Decrease active object count on deallocation. and all should be fine. Signed-off-by: greatbridf --- crates/slab_allocator/src/lib.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index 733e4de5..15d8a50d 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -104,6 +104,12 @@ where partial_list: T, full_list: T, object_size: usize, + + total_objects: usize, + active_objects: usize, + + total_folios: usize, + active_folios: usize, } pub struct SlabAlloc @@ -164,6 +170,10 @@ where partial_list: T::NEW, full_list: T::NEW, object_size, + total_objects: 0, + active_objects: 0, + total_folios: 0, + active_folios: 0, } } } @@ -195,6 +205,8 @@ where self.partial_list.push_tail(head); } + self.active_folios += 1; + slot } @@ -204,12 +216,18 @@ where slab.set_free_slot(Some(free_slot)); + self.total_folios += 1; + self.total_objects += slab.get_data_ptr().len() / self.object_size; + self.empty_list.push_tail(slab); } fn alloc( &mut self, alloc: &impl SlabPageAlloc, ) -> NonNull { + // Increase active count here, since we won't fail for now... + self.active_objects += 1; + if !self.partial_list.is_empty() { return self.alloc_from_partial(); } @@ -256,6 +274,12 @@ where } } + if is_empty { + self.active_folios -= 1; + } + + self.active_objects -= 1; + // TODO: Check whether we should place some pages back with `alloc` if // the global free page count is below the watermark. } From 2564ca52bfa968dce9db9b793c27b03f7c10d057 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 6 Mar 2026 00:28:44 +0800 Subject: [PATCH 099/153] slab: support printing slab debug info - Add `SlabListStat` for debug use. - Add a function and copy the info from the list in the function. - Add /proc/slabinfo as a userspace interface, the format of which taken from Linux. Signed-off-by: greatbridf --- crates/slab_allocator/src/debug.rs | 52 ++++++++++++++++++++++++++++++ crates/slab_allocator/src/lib.rs | 2 ++ src/kernel/mem/allocator.rs | 36 +++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 crates/slab_allocator/src/debug.rs diff --git a/crates/slab_allocator/src/debug.rs b/crates/slab_allocator/src/debug.rs new file mode 100644 index 00000000..e4169482 --- /dev/null +++ b/crates/slab_allocator/src/debug.rs @@ -0,0 +1,52 @@ +use core::mem::MaybeUninit; + +use eonix_mm::paging::FolioList; + +use crate::{SlabAlloc, SlabList, SlabPageAlloc}; + +pub struct SlabListStat { + pub object_size: usize, + + pub total_slabs: usize, + pub active_slabs: usize, + pub total_objects: usize, + pub active_objects: usize, +} + +impl SlabList +where + T: FolioList, +{ + pub fn dump_stats(&self) -> SlabListStat { + SlabListStat { + object_size: self.object_size, + total_slabs: self.total_folios, + active_slabs: self.active_folios, + total_objects: self.total_objects, + active_objects: self.active_objects, + } + } +} + +impl SlabAlloc +where + P: SlabPageAlloc, +{ + pub fn dump_stats(&self) -> [SlabListStat; COUNT] { + let mut stats: MaybeUninit<[SlabListStat; COUNT]> = + MaybeUninit::uninit(); + + for (i, slab_list) in self.slabs.iter().enumerate() { + let ptr: *mut SlabListStat = stats.as_mut_ptr().cast(); + + unsafe { + ptr.wrapping_add(i).write(slab_list.lock().dump_stats()); + } + } + + unsafe { + // SAFETY: All `COUNT` items are initialized. + stats.assume_init() + } + } +} diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index 15d8a50d..1961f125 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -1,5 +1,7 @@ #![no_std] +pub mod debug; + use core::ptr::NonNull; use eonix_mm::paging::{FolioList, FolioListSized}; diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 3a70a8c2..63bd7715 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -1,7 +1,10 @@ +use alloc::format; +use alloc::sync::Arc; use core::alloc::{GlobalAlloc, Layout}; use core::ptr::NonNull; use eonix_hal::mm::ArchPhysAccess; +use eonix_macros::define_late_init; use eonix_mm::address::PhysAccess; use eonix_mm::paging::{Folio as _, PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; @@ -9,6 +12,8 @@ use slab_allocator::SlabAlloc; use super::folio::Folio; use super::GlobalPageAlloc; +use crate::fs::procfs::populate_root; +use crate::io::{buf_writeln, Buffer}; static SLAB_ALLOCATOR: LazyLock> = LazyLock::new(|| SlabAlloc::new_in(GlobalPageAlloc)); @@ -50,3 +55,34 @@ unsafe impl GlobalAlloc for Allocator { #[global_allocator] static ALLOCATOR: Allocator = Allocator; + +#[define_late_init] +async fn slabinfo_file() { + populate_root(Arc::from(&b"slabinfo"[..]), |buf| { + let stats = SLAB_ALLOCATOR.dump_stats(); + + buf_writeln!(buf, "slabinfo - version: 2.1")?; + buf_writeln!( + buf, + "# name\t\t\t\t\t" + )?; + + for (i, stat) in stats.iter().enumerate() { + let name = format!("kmalloc[{i}]"); + + buf_writeln!( + buf, + "{:15}\t{}\t{}\t{}\t{}\t{}", + name, + stat.active_objects, + stat.total_objects, + stat.object_size, + stat.active_slabs, + stat.total_slabs + )?; + } + + Ok(()) + }) + .await; +} From d30c88b46bdba6a894743e7bfd8497cb32788d48 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 6 Mar 2026 00:35:27 +0800 Subject: [PATCH 100/153] style: reformat buddy_allocator/lib.rs With slight style change, no functional change. Signed-off-by: greatbridf --- crates/buddy_allocator/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index 82a7d6c5..926daaa2 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -132,7 +132,9 @@ where pub fn alloc_order(&mut self, order: u32) -> Option<&'static mut Z::Page> { for current_order in order..AREAS as u32 { - let Some(folio) = self.free_areas[current_order as usize].get_free_folio() else { + let area = &mut self.free_areas[current_order as usize]; + + let Some(folio) = area.get_free_folio() else { continue; }; @@ -176,7 +178,9 @@ where /// - the buddy is within the same [`Zone`] as us. /// - the buddy is a free buddy (in some [`FreeArea`]) /// - the buddy has order [`order`] - fn try_get_buddy<'a>(&mut self, buddy_pfn: PFN, order: u32) -> Option<&'a mut F> { + fn try_get_buddy<'a>( + &mut self, buddy_pfn: PFN, order: u32, + ) -> Option<&'a mut F> { let mut buddy = self.zone.get_page(buddy_pfn)?; unsafe { From 7e9da0df67ab60f2d8e166c46e92db9164acf6be Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 6 Mar 2026 00:43:44 +0800 Subject: [PATCH 101/153] mem, buddy: count alloced pages Add a counter in each of `FreeArea`s. Increase the counter on page allocation and decrease on deallocation. Signed-off-by: greatbridf --- crates/buddy_allocator/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index 926daaa2..dc0f7d9c 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -23,6 +23,8 @@ where L: FolioList, { free_list: L, + + alloced: usize, count: usize, } @@ -142,6 +144,7 @@ where self.break_folio(folio, current_order, order); } + self.free_areas[order as usize].alloced += 1; return Some(folio); } @@ -157,6 +160,8 @@ where "Trying to free a folio that is already in the buddy system: {pfn:?}", ); + self.free_areas[order as usize].alloced -= 1; + while order < MAX_ORDER { let buddy_pfn = pfn.buddy_pfn(order); let Some(buddy) = self.try_get_buddy(buddy_pfn, order) else { @@ -210,6 +215,7 @@ where const fn new() -> Self { Self { free_list: L::NEW, + alloced: 0, count: 0, } } From ea1ff47983a6285cc6ce22b74b95b022dcfefe36 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 7 Mar 2026 14:30:23 +0800 Subject: [PATCH 102/153] runtime: reformat scheduler.rs No functional change. Signed-off-by: greatbridf --- crates/eonix_runtime/src/scheduler.rs | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/crates/eonix_runtime/src/scheduler.rs b/crates/eonix_runtime/src/scheduler.rs index b4b7960d..758a68b8 100644 --- a/crates/eonix_runtime/src/scheduler.rs +++ b/crates/eonix_runtime/src/scheduler.rs @@ -31,11 +31,13 @@ where impl Task { pub fn current<'a>() -> BorrowedArc<'a, Task> { unsafe { - // SAFETY: - // We should never "inspect" a change in `current`. - // The change of `CURRENT` will only happen in the scheduler. And if we are preempted, - // when we DO return, the `CURRENT` will be the same and remain valid. - BorrowedArc::from_raw(CURRENT_TASK.get().expect("Current task should be present")) + // SAFETY: We should never "inspect" a change in `current`: + // The change of `CURRENT` will only happen in the scheduler. + // And if we are preempted outside of the scheduler's context, when + // we return, `CURRENT` will be the same and remain valid. + BorrowedArc::from_raw( + CURRENT_TASK.get().expect("Current task should be present"), + ) } } } @@ -92,7 +94,9 @@ impl Runtime { } } - fn remove_and_enqueue_current(&self, rq: &mut impl DerefMut) { + fn remove_and_enqueue_current( + &self, rq: &mut impl DerefMut, + ) { let Some(current) = CURRENT_TASK .swap(None) .map(|cur| unsafe { Arc::from_raw(cur.as_ptr()) }) @@ -129,7 +133,9 @@ impl Runtime { } } - pub fn block_till_woken(set_waker: impl FnOnce(&Waker)) -> impl Future { + pub fn block_till_woken( + set_waker: impl FnOnce(&Waker), + ) -> impl Future { struct BlockTillWoken { set_waker: Option, slept: bool, @@ -138,7 +144,9 @@ impl Runtime { impl Future for BlockTillWoken { type Output = (); - fn poll(self: core::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + fn poll( + self: core::pin::Pin<&mut Self>, cx: &mut Context<'_>, + ) -> Poll<()> { if self.slept { Poll::Ready(()) } else { @@ -191,7 +199,9 @@ impl Runtime { ); unsafe { - CURRENT_TASK.set(Some(NonNull::new_unchecked(Arc::into_raw(next) as *mut _))); + CURRENT_TASK.set(Some(NonNull::new_unchecked( + Arc::into_raw(next) as *mut _, + ))); } drop(rq); From ded411a2f9ba3ff1db388c87afc8f234d1980cd5 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 7 Mar 2026 15:08:49 +0800 Subject: [PATCH 103/153] runtime: release finished Task to avoid memory leaks I notice a memory leak when the task is finished and we store a None to CURRENT_TASK without retrieving and release the arc refcount. Fix this by introducing `Task::_swap_current()` to swap and return the previous Option> to actually drop it. Replace other operations to CURRENT_TASK with this function as well. Fixes: e23c9eb1f24c ("runtime: new task sleep-wakeup method and some adaption")") Signed-off-by: greatbridf --- crates/eonix_runtime/src/scheduler.rs | 29 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/eonix_runtime/src/scheduler.rs b/crates/eonix_runtime/src/scheduler.rs index 758a68b8..d658d579 100644 --- a/crates/eonix_runtime/src/scheduler.rs +++ b/crates/eonix_runtime/src/scheduler.rs @@ -1,5 +1,6 @@ use alloc::sync::Arc; use alloc::task::Wake; +use core::hint::assert_unchecked; use core::ops::{Deref, DerefMut}; use core::ptr::NonNull; use core::task::{Context, Poll, Waker}; @@ -40,6 +41,21 @@ impl Task { ) } } + + #[inline(always)] + fn _swap_current(current: Option>) -> Option> { + let current_ptr = current.map(|arc| unsafe { + // SAFETY: `into_raw` always returns a valid pointer. + NonNull::new_unchecked(Arc::into_raw(arc) as *mut _) + }); + + let old = CURRENT_TASK.swap(current_ptr); + + old.map(|ptr| unsafe { + // SAFETY: The pointer always comes from Arc::into_raw + Arc::from_raw(ptr.as_ptr()) + }) + } } impl JoinHandle @@ -97,10 +113,7 @@ impl Runtime { fn remove_and_enqueue_current( &self, rq: &mut impl DerefMut, ) { - let Some(current) = CURRENT_TASK - .swap(None) - .map(|cur| unsafe { Arc::from_raw(cur.as_ptr()) }) - else { + let Some(current) = Task::_swap_current(None) else { return; }; @@ -198,10 +211,10 @@ impl Runtime { "Next task should be in READY state" ); + let _old_current = Task::_swap_current(Some(next)); + debug_assert!(_old_current.is_none()); unsafe { - CURRENT_TASK.set(Some(NonNull::new_unchecked( - Arc::into_raw(next) as *mut _, - ))); + assert_unchecked(_old_current.is_none()); } drop(rq); @@ -222,7 +235,7 @@ impl Runtime { self.remove_task(&Task::current()); - CURRENT_TASK.set(None); + Task::_swap_current(None); } } } From 1eb3109fb9b4294484147be86afc6019cad2fc4f Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 7 Mar 2026 16:04:10 +0800 Subject: [PATCH 104/153] style: reformat process.rs No functional change. Signed-off-by: greatbridf --- src/kernel/task/process.rs | 45 ++++++++++---------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/src/kernel/task/process.rs b/src/kernel/task/process.rs index 1385235d..6e99404a 100644 --- a/src/kernel/task/process.rs +++ b/src/kernel/task/process.rs @@ -241,9 +241,7 @@ impl ProcessBuilder { } pub async fn clone_from( - mut self, - process: Arc, - clone_args: &CloneArgs, + mut self, process: Arc, clone_args: &CloneArgs, ) -> Self { let mm_list = if clone_args.flags.contains(CloneFlags::CLONE_VM) { process.mm_list.new_shared().await @@ -294,8 +292,7 @@ impl ProcessBuilder { } pub fn build( - self, - process_list: &mut ProcessList, + self, process_list: &mut ProcessList, ) -> (Arc, Arc) { let mm_list = self.mm_list.unwrap_or_else(|| MMList::new()); @@ -368,28 +365,21 @@ impl Process { } pub fn add_child( - &self, - child: &Arc, - procs: ProofMut<'_, ProcessList>, + &self, child: &Arc, procs: ProofMut<'_, ProcessList>, ) { assert!(self.all_procs_link.is_linked(), "Dead process"); self.children.access_mut(procs).insert(child.clone()); } pub fn add_thread( - &self, - thread: &Arc, - procs: ProofMut<'_, ProcessList>, + &self, thread: &Arc, procs: ProofMut<'_, ProcessList>, ) { assert!(self.all_procs_link.is_linked(), "Dead process"); self.threads.access_mut(procs).insert(thread.clone()); } pub async fn wait( - &self, - wait_id: WaitId, - no_block: bool, - trace_stop: bool, + &self, wait_id: WaitId, no_block: bool, trace_stop: bool, trace_continue: bool, ) -> KResult> { let wait_object = { @@ -470,9 +460,7 @@ impl Process { /// /// This function does the actual work. fn do_setpgid( - self: &Arc, - pgid: u32, - procs: &mut ProcessList, + self: &Arc, pgid: u32, procs: &mut ProcessList, ) -> KResult<()> { // SAFETY: We are holding the process list lock. let session = unsafe { self.session.load_locked().unwrap() }; @@ -558,8 +546,7 @@ impl Process { /// Provide locked (consistent) access to the session. pub fn session<'r>( - &'r self, - _procs: Proof<'r, ProcessList>, + &'r self, _procs: Proof<'r, ProcessList>, ) -> BorrowedArc<'r, Session> { // SAFETY: We are holding the process list lock. unsafe { self.session.load_locked() }.unwrap() @@ -567,8 +554,7 @@ impl Process { /// Provide locked (consistent) access to the process group. pub fn pgroup<'r>( - &'r self, - _procs: Proof<'r, ProcessList>, + &'r self, _procs: Proof<'r, ProcessList>, ) -> BorrowedArc<'r, ProcessGroup> { // SAFETY: We are holding the process list lock. unsafe { self.pgroup.load_locked() }.unwrap() @@ -576,8 +562,7 @@ impl Process { /// Provide locked (consistent) access to the parent process. pub fn parent<'r>( - &'r self, - _procs: Proof<'r, ProcessList>, + &'r self, _procs: Proof<'r, ProcessList>, ) -> BorrowedArc<'r, Process> { // SAFETY: We are holding the process list lock. unsafe { self.parent.load_locked() }.unwrap() @@ -603,9 +588,7 @@ impl Process { } pub fn notify( - &self, - signal: Option, - wait: WaitObject, + &self, signal: Option, wait: WaitObject, procs: Proof<'_, ProcessList>, ) { self.wait_list.notify(wait); @@ -650,10 +633,7 @@ impl WaitList { /// Locks `ProcessList` and `WaitList` at the same time. When `wait` is called, /// releases the lock on `ProcessList` and `WaitList` and waits on `cv_wait_procs`. pub async fn entry( - &self, - wait_id: WaitId, - want_stop: bool, - want_continue: bool, + &self, wait_id: WaitId, want_stop: bool, want_continue: bool, ) -> Entry<'_, '_, '_> { Entry { process_list: ProcessList::get().read().await, @@ -704,8 +684,7 @@ impl Entry<'_, '_, '_> { } pub fn wait( - self, - no_block: bool, + self, no_block: bool, ) -> impl core::future::Future> + Send { let wait_procs = self.wait_procs.unlock(); From f6d98ec06c2e12b1c36eb6bd4f4934eaf721f92d Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 7 Mar 2026 16:04:38 +0800 Subject: [PATCH 105/153] task, process: remove thread from its list when removing process When we remove processes in wait syscalls, we forget to remove the thread from the process's threads list, which will introduce a memory leak since the Arc will never get released. Fix the issue by adding a `remove_thread` call to release it. Fixes: 66958893e55c ("rewrite: move Process, ProcessGroup, Session out of thread.rs")") Signed-off-by: greatbridf --- src/kernel/task/process.rs | 11 +++++++++++ src/kernel/task/process_list.rs | 10 ++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/kernel/task/process.rs b/src/kernel/task/process.rs index 6e99404a..6299f87f 100644 --- a/src/kernel/task/process.rs +++ b/src/kernel/task/process.rs @@ -378,6 +378,17 @@ impl Process { self.threads.access_mut(procs).insert(thread.clone()); } + pub fn remove_thread( + &self, thread: &Arc, procs: ProofMut<'_, ProcessList>, + ) { + let threads = self.threads.access_mut(procs); + + assert!( + threads.find_mut(&thread.tid).remove().is_some(), + "Not a member" + ); + } + pub async fn wait( &self, wait_id: WaitId, no_block: bool, trace_stop: bool, trace_continue: bool, diff --git a/src/kernel/task/process_list.rs b/src/kernel/task/process_list.rs index f3371f25..ccce8e2c 100644 --- a/src/kernel/task/process_list.rs +++ b/src/kernel/task/process_list.rs @@ -60,14 +60,16 @@ impl ProcessList { } pub fn remove_process(&mut self, pid: u32) { + let Some(proc) = self.procs.find_mut(&pid).remove() else { + panic!("Process {} not found", pid); + }; + // Thread group leader has the same tid as the pid. - let Some(_) = self.threads.find_mut(&pid).remove() else { + let Some(thd) = self.threads.find_mut(&pid).remove() else { panic!("Thread {} not found", pid); }; - let Some(proc) = self.procs.find_mut(&pid).remove() else { - panic!("Process {} not found", pid); - }; + proc.remove_thread(&thd, self.prove_mut()); // SAFETY: `call_rcu` below. let session = unsafe { proc.session.swap(None) }.unwrap(); From 0443358c984662590a41898eb136f0574cd6129a Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 7 Mar 2026 16:21:38 +0800 Subject: [PATCH 106/153] util: embed the import of trait Buffer into buf_writeln To use `buffer.get_writer()`, we need to import Buffer trait. But for users using buf_writeln macros, this can be confusing. add the use statement inside the macro can solve this. Signed-off-by: greatbridf --- src/io.rs | 1 + src/kernel/mem/allocator.rs | 2 +- src/kernel/vfs/mount.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/io.rs b/src/io.rs index 74960154..83cc7435 100644 --- a/src/io.rs +++ b/src/io.rs @@ -337,6 +337,7 @@ impl<'a> IntoStream for &'a [u8] { macro_rules! buf_writeln { ($buf:ident, $($args:expr),*) => {{ use core::fmt::Write; + use $crate::io::Buffer; writeln!($buf.get_writer(), $($args),*) .map_err(|_| $crate::kernel::constants::ERANGE) }}; diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 63bd7715..cdcda81e 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -13,7 +13,7 @@ use slab_allocator::SlabAlloc; use super::folio::Folio; use super::GlobalPageAlloc; use crate::fs::procfs::populate_root; -use crate::io::{buf_writeln, Buffer}; +use crate::io::buf_writeln; static SLAB_ALLOCATOR: LazyLock> = LazyLock::new(|| SlabAlloc::new_in(GlobalPageAlloc)); diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 0a7983d9..fcb5b622 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -10,7 +10,7 @@ use super::dentry::{dcache, Dentry, DROOT}; use super::inode::InodeUse; use super::{SbUse, SuperBlock}; use crate::fs::procfs::populate_root; -use crate::io::{buf_writeln, Buffer}; +use crate::io::buf_writeln; use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; use crate::kernel::task::block_on; use crate::prelude::*; From 610136b343e62404edfe18beba06cad546bc94b6 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 7 Mar 2026 16:23:23 +0800 Subject: [PATCH 107/153] util: add buf_write macro similar to buf_writeln In future patch of supporting buddyinfo procfs file, we need to print one line in many write calls. Support this by copying buf_writeln definition and replace `writeln!` with `write!`. Signed-off-by: greatbridf --- src/io.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/io.rs b/src/io.rs index 83cc7435..1fa79817 100644 --- a/src/io.rs +++ b/src/io.rs @@ -334,6 +334,15 @@ impl<'a> IntoStream for &'a [u8] { } } +macro_rules! buf_write { + ($buf:ident, $($args:expr),*) => {{ + use core::fmt::Write; + use $crate::io::Buffer; + write!($buf.get_writer(), $($args),*) + .map_err(|_| $crate::kernel::constants::ERANGE) + }}; +} + macro_rules! buf_writeln { ($buf:ident, $($args:expr),*) => {{ use core::fmt::Write; @@ -343,4 +352,4 @@ macro_rules! buf_writeln { }}; } -pub(crate) use buf_writeln; +pub(crate) use {buf_write, buf_writeln}; From 5e91a72b8388be442cc23a48e85cac0a501a1182 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 7 Mar 2026 16:49:01 +0800 Subject: [PATCH 108/153] buddy: count and print alloc and total info Similar to what we've done to the slab system, we count allocated and total folio count and add a procfs file to show stat. Signed-off-by: greatbridf --- crates/buddy_allocator/src/debug.rs | 36 +++++++++++++++++++++++++++++ crates/buddy_allocator/src/lib.rs | 2 ++ src/kernel/mem/page_alloc.rs | 35 ++++++++++++++++++++++++++-- 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 crates/buddy_allocator/src/debug.rs diff --git a/crates/buddy_allocator/src/debug.rs b/crates/buddy_allocator/src/debug.rs new file mode 100644 index 00000000..3942c67d --- /dev/null +++ b/crates/buddy_allocator/src/debug.rs @@ -0,0 +1,36 @@ +use eonix_mm::paging::{FolioList, Zone}; + +use crate::{BuddyAllocator, BuddyFolio, AREAS}; + +#[derive(Clone, Copy)] +pub struct BuddyStat { + pub alloced_count: usize, + pub free_count: usize, + pub order: u32, +} + +impl BuddyAllocator +where + Z: Zone, + L: FolioList, + F: BuddyFolio + 'static, +{ + pub fn dump_stat(&self) -> [BuddyStat; AREAS] { + let mut stats = [BuddyStat { + alloced_count: 0, + free_count: 0, + order: 0, + }; AREAS]; + + for i in 0..AREAS { + let area = &self.free_areas[i]; + let stat = &mut stats[i]; + + stat.alloced_count = area.alloced; + stat.free_count = area.count; + stat.order = i as u32; + } + + stats + } +} diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index dc0f7d9c..c4dfa7c9 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -1,5 +1,7 @@ #![no_std] +pub mod debug; + use core::hint::unreachable_unchecked; use eonix_mm::address::{AddrOps as _, PAddr, PRange}; diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index ac2485da..f60332d7 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -1,18 +1,24 @@ mod raw_page; mod zones; +use alloc::sync::Arc; use core::sync::atomic::Ordering; use buddy_allocator::BuddyAllocator; +use eonix_macros::define_late_init; use eonix_mm::address::PRange; use eonix_mm::page_table::PageTableAlloc; -use eonix_mm::paging::{FolioList, FolioListSized as _, FrameAlloc, GlobalFrameAlloc, PFN}; +use eonix_mm::paging::{ + FolioList, FolioListSized as _, FrameAlloc, GlobalFrameAlloc, PFN, +}; use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; pub use raw_page::{PageFlags, RawPage, RawPageList}; pub use zones::{GlobalZone, ZONE}; use super::folio::Folio; +use crate::fs::procfs::populate_root; +use crate::io::buf_write; const COSTLY_ORDER: u32 = 3; const AREAS: usize = COSTLY_ORDER as usize + 1; @@ -110,7 +116,8 @@ impl GlobalPageAlloc { let order = raw_page.order; unsafe { - PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()).free_pages(raw_page, order); + PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()) + .free_pages(raw_page, order); } } } @@ -145,3 +152,27 @@ impl PageTableAlloc for GlobalPageAlloc { unsafe { Folio::from_raw(pfn) } } } + +#[define_late_init] +async fn buddyinfo_file() { + populate_root(Arc::from(&b"buddyinfo"[..]), |buffer| { + let stats = BUDDY_ALLOC.lock().dump_stat(); + + buf_write!(buffer, "Node 0, zone {:>10}", "Normal")?; + + for stat in &stats { + buf_write!(buffer, " {:>6}", stat.free_count)?; + } + + buf_write!(buffer, "\nNode 0, zone {:>10}", "Alloced")?; + + for stat in &stats { + buf_write!(buffer, " {:>6}", stat.alloced_count)?; + } + + buf_write!(buffer, "\n")?; + + Ok(()) + }) + .await; +} From aa81a17bd06b9ec31b13d91559d8cfdf276471e0 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 16:40:41 +0800 Subject: [PATCH 109/153] style: reformat mm.rs and pte.rs No functional change. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/loongarch64/mm.rs | 50 +++++++++++---------- crates/eonix_mm/src/page_table/pte.rs | 4 +- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/crates/eonix_hal/src/arch/loongarch64/mm.rs b/crates/eonix_hal/src/arch/loongarch64/mm.rs index d5b00a6b..dc0a0df4 100644 --- a/crates/eonix_hal/src/arch/loongarch64/mm.rs +++ b/crates/eonix_hal/src/arch/loongarch64/mm.rs @@ -1,30 +1,29 @@ -use crate::traits::mm::Memory; -use core::{ - arch::asm, - marker::PhantomData, - ptr::NonNull, - sync::atomic::{compiler_fence, Ordering}, -}; -use eonix_mm::{ - address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}, - page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, - }, - paging::{NoAlloc, Page, PageBlock, PAGE_SIZE, PFN}, +use core::arch::asm; +use core::marker::PhantomData; +use core::ptr::NonNull; +use core::sync::atomic::{compiler_fence, Ordering}; + +use eonix_mm::address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, + RawPageTable, TableAttribute, PTE, }; +use eonix_mm::paging::{NoAlloc, Page, PageBlock, PAGE_SIZE, PFN}; use eonix_sync_base::LazyLock; use loongArch64::register::pgdl; +use crate::traits::mm::Memory; + pub const KIMAGE_OFFSET: usize = 0xffff_ffff_0000_0000; pub const ROOT_PAGE_TABLE_PFN: usize = 0x8000_1000 >> 12; pub const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); -pub static GLOBAL_PAGE_TABLE: LazyLock> = - LazyLock::new(|| unsafe { - Page::with_raw(PAGE_TABLE_BASE, |root_table_page| { - PageTable::with_root_table(root_table_page.clone()) - }) - }); +pub static GLOBAL_PAGE_TABLE: LazyLock< + PageTable, +> = LazyLock::new(|| unsafe { + Page::with_raw(PAGE_TABLE_BASE, |root_table_page| { + PageTable::with_root_table(root_table_page.clone()) + }) +}); pub const PA_VP: u64 = ((1 << 0) | (1 << 7)); pub const PA_D: u64 = 1 << 1; @@ -243,8 +242,12 @@ impl PhysAccess for ArchPhysAccess { impl Memory for ArchMemory { fn present_ram() -> impl Iterator { - let range1 = core::iter::once(PRange::from(PAddr::from_val(0)).grow(0x1000_0000)); - let range2 = core::iter::once(PRange::from(PAddr::from_val(0x8000_0000)).grow(0x3000_0000)); + let range1 = core::iter::once( + PRange::from(PAddr::from_val(0)).grow(0x1000_0000), + ); + let range2 = core::iter::once( + PRange::from(PAddr::from_val(0x8000_0000)).grow(0x3000_0000), + ); range2.chain(range1) } @@ -261,7 +264,8 @@ impl Memory for ArchMemory { Self::present_ram() .filter(move |range| { - range.end() <= kernel_start || range.end() > paddr_after_kimage_aligned + range.end() <= kernel_start + || range.end() > paddr_after_kimage_aligned }) .map(move |range| { if range.end() > paddr_after_kimage_aligned diff --git a/crates/eonix_mm/src/page_table/pte.rs b/crates/eonix_mm/src/page_table/pte.rs index c14d5738..e533b95b 100644 --- a/crates/eonix_mm/src/page_table/pte.rs +++ b/crates/eonix_mm/src/page_table/pte.rs @@ -29,7 +29,9 @@ bitflags! { } #[doc(notable_trait)] -pub trait RawAttribute: Copy + From + From { +pub trait RawAttribute: + Copy + From + From +{ /// Create a new attribute representing a non-present page. fn null() -> Self; From 5d86ac9b0ca575a9074f941fcb6b3cbdc02b8ca5 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 16:45:33 +0800 Subject: [PATCH 110/153] arch, mm: remove ANONYMOUS PTE flag The bit is in fact unused right now, and its meaning can be covered by the MemArea's type. So remove this. Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/loongarch64/mm.rs | 2 +- crates/eonix_hal/src/arch/riscv64/mm.rs | 5 ----- crates/eonix_hal/src/arch/x86_64/mm.rs | 6 ------ 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/crates/eonix_hal/src/arch/loongarch64/mm.rs b/crates/eonix_hal/src/arch/loongarch64/mm.rs index dc0a0df4..20ed28c8 100644 --- a/crates/eonix_hal/src/arch/loongarch64/mm.rs +++ b/crates/eonix_hal/src/arch/loongarch64/mm.rs @@ -185,7 +185,7 @@ impl From for PageAttribute64 { PageAttribute::GLOBAL => raw_attr |= PA_G, PageAttribute::COPY_ON_WRITE => raw_attr |= PA_COW, PageAttribute::MAPPED => raw_attr |= PA_MMAP, - PageAttribute::ACCESSED | PageAttribute::ANONYMOUS => {} + PageAttribute::ACCESSED => {} _ => unreachable!("Invalid page attribute"), } } diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 7891f094..731cfc28 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -184,10 +184,6 @@ impl RawAttribute for PageAttribute64 { page_attr |= PageAttribute::MAPPED; } - /*if self.0 & PA_ANON != 0 { - page_attr |= PageAttribute::ANONYMOUS; - }*/ - Some(page_attr) } } @@ -208,7 +204,6 @@ impl From for PageAttribute64 { PageAttribute::GLOBAL => raw_attr |= PA_G, PageAttribute::COPY_ON_WRITE => raw_attr |= PA_COW, PageAttribute::MAPPED => raw_attr |= PA_MMAP, - PageAttribute::ANONYMOUS => {} _ => unreachable!("Invalid page attribute"), } } diff --git a/crates/eonix_hal/src/arch/x86_64/mm.rs b/crates/eonix_hal/src/arch/x86_64/mm.rs index 11bc2996..9d7ddea4 100644 --- a/crates/eonix_hal/src/arch/x86_64/mm.rs +++ b/crates/eonix_hal/src/arch/x86_64/mm.rs @@ -27,7 +27,6 @@ pub const PA_PS: u64 = 0x080; pub const PA_G: u64 = 0x100; pub const PA_COW: u64 = 0x200; pub const PA_MMAP: u64 = 0x400; -pub const PA_ANON: u64 = 0x800; pub const PA_NXE: u64 = 0x8000_0000_0000_0000; pub const PA_MASK: u64 = 0xfff0_0000_0000_0fff; @@ -190,10 +189,6 @@ impl RawAttribute for PageAttribute64 { page_attr |= PageAttribute::MAPPED; } - if self.0 & PA_ANON != 0 { - page_attr |= PageAttribute::ANONYMOUS; - } - if self.0 & PA_PS != 0 { page_attr |= PageAttribute::HUGE; } @@ -218,7 +213,6 @@ impl From for PageAttribute64 { PageAttribute::GLOBAL => raw_attr |= PA_G, PageAttribute::COPY_ON_WRITE => raw_attr |= PA_COW, PageAttribute::MAPPED => raw_attr |= PA_MMAP, - PageAttribute::ANONYMOUS => raw_attr |= PA_ANON, PageAttribute::HUGE => raw_attr |= PA_PS, _ => unreachable!("Invalid page attribute"), } From 05a498a95c5e2075c7fad802939e02855ce353e8 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 17:31:29 +0800 Subject: [PATCH 111/153] mm: copy mmapped PTEs as a whole when doing CoWs The previous implementation in commit 576c43db2d2b ("fix(CoW): check both PRESENT and MAPPED bits when doing cow") decided to treat mmapped PTEs and normal anonymous PTEs the same, which leads to unnecessary gets to empty pages (since we don't treat mapped PTEs as CoW now so the extra reference will never be used or put in handle_mmap()) during the process we make the PTE CoW. This issue combines with yet another two bugs that: 1. We get a reference to the empty page when we setup mapped PTEs, so there will always be "enough" refcounts to the empty page. 2. We forget to check whether the PTE is present when we drop the MMList. Our current implementation happens to be working "just fine" and produces no errors and crashes. But the whole system is ill-formed and needs repairs. The solution is to add checks to skip non-present PTEs when we drop the MMList, and to copy the entire PTE to the new PTE when we are setting up CoW, if the PTE is a mapped one. So now, when the PTE needs special treatments in the fault handler, the entry is either a CoW one with its PFN valid, or a mapped one with its PFN unset. Fixes: 576c43db2d2b ("fix(CoW): check both PRESENT and MAPPED bits when doing cow") Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 40 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index c10036ba..caa0396c 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -245,19 +245,23 @@ impl Drop for MMListInner { let range = area.range.as_ref(&self.lock).clone(); if area.is_shared() { - for pte in self.page_table.iter_user(range) { - // XXX: Fix me - let _ = pte.take(); - // let raw_page = RawPagePtr::from(pfn); - // if raw_page.refcount().fetch_sub(1, Ordering::Relaxed) == 1 { - // unsafe { Page::from_raw(pfn) }; - // } - } - } else { - for pte in self.page_table.iter_user(range) { - let (pfn, _) = pte.take(); - unsafe { Folio::from_raw(pfn) }; + unimplemented!("Shared mapping is not yet implemented"); + } + + for pte in self.page_table.iter_user(range) { + let (pfn, raw_attr) = pte.take(); + let attr = raw_attr.as_page_attr().expect("Not a page"); + + if !attr.contains(PageAttribute::PRESENT) { + continue; } + + unsafe { + // SAFETY: Present PTEs always corresponding to a valid + // Folio that was previously installed through + // `Folio::into_raw()`. + Folio::from_raw(pfn) + }; } } @@ -699,11 +703,19 @@ where .as_page_attr() .expect("Not a page attribute"); - if !from_attr.intersects(PageAttribute::PRESENT | PageAttribute::MAPPED) - { + if from_attr.intersects(PageAttribute::MAPPED) { + // Copy non-installed mapped PTEs directly to the new PTE and delay + // its handling till the page fault. + let (pfn, attr) = from.get(); + self.set(pfn, attr); return; } + if !from_attr.intersects(PageAttribute::PRESENT) { + // Shouldn't really see this happen... + unreachable!("Non present PTE"); + } + from_attr.remove(PageAttribute::WRITE | PageAttribute::DIRTY); from_attr.insert(PageAttribute::COPY_ON_WRITE); From 18d53608ae2560789ee95f8cfa92f0394b514502 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 17:46:51 +0800 Subject: [PATCH 112/153] mm: set an invalid value for mapped PTEs to help debugging The PFN should never be used. We put a 0 here to avoid problems and help debugging by future attempts to check PFN validity in the Zone's get_page function. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index caa0396c..2222742c 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -694,7 +694,8 @@ where PageAttribute::READ | PageAttribute::USER | PageAttribute::MAPPED; attr.set(PageAttribute::EXECUTE, execute); - self.set(EMPTY_PAGE.clone().into_raw(), T::Attr::from(attr)); + // Set an obviously invalid PFN to help debugging... + self.set(PFN::from_val(0), T::Attr::from(attr)); } fn set_copy_on_write(&mut self, from: &mut Self) { From 8e6b08ba1bbe99dc176fc9a95a3dcd3f39173d41 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 17:58:23 +0800 Subject: [PATCH 113/153] mm: don't use MAPPED bit for checking mapped PTE I notice that the MAPPED bit is kind of useless here, since if we've found that the fault address corresponding to a valid MemArea, we must have set a PTE before. And the mapping's type can tell us whether it's an anon or a file PTE. Remove its usage in `set_mapped()` and `MemArea::handle()`. Replace the mapped entry check with present check: all non-present PTEs are copied as a whole when setting CoW. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 10 ++-------- src/kernel/mem/mm_list/mm_area.rs | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 2222742c..2c20cc50 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -690,8 +690,7 @@ where fn set_mapped(&mut self, execute: bool) { // Writable flag is set during page fault handling while executable flag is // preserved across page faults, so we set executable flag now. - let mut attr = - PageAttribute::READ | PageAttribute::USER | PageAttribute::MAPPED; + let mut attr = PageAttribute::READ | PageAttribute::USER; attr.set(PageAttribute::EXECUTE, execute); // Set an obviously invalid PFN to help debugging... @@ -704,7 +703,7 @@ where .as_page_attr() .expect("Not a page attribute"); - if from_attr.intersects(PageAttribute::MAPPED) { + if !from_attr.intersects(PageAttribute::PRESENT) { // Copy non-installed mapped PTEs directly to the new PTE and delay // its handling till the page fault. let (pfn, attr) = from.get(); @@ -712,11 +711,6 @@ where return; } - if !from_attr.intersects(PageAttribute::PRESENT) { - // Shouldn't really see this happen... - unreachable!("Non present PTE"); - } - from_attr.remove(PageAttribute::WRITE | PageAttribute::DIRTY); from_attr.insert(PageAttribute::COPY_ON_WRITE); diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 8892b7d4..f28db415 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -16,7 +16,9 @@ use intrusive_collections::{ use super::{Mapping, EMPTY_PAGE}; use crate::kernel::mem::folio::Folio; use crate::kernel::mem::mm_list::MemListLock; -use crate::kernel::mem::{CachePage, FolioOwned, PageOffset, Permission}; +use crate::kernel::mem::{ + CachePage, FileMapping, FolioOwned, PageOffset, Permission, +}; use crate::prelude::KResult; bitflags::bitflags! { @@ -488,14 +490,16 @@ impl MemArea { /// * `offset`: The offset from the start of the mapping, aligned to 4KB boundary. pub async fn handle_mmap( &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: usize, - write: bool, + write: bool, file_mapping: &FileMapping, ) -> KResult<()> { - let Mapping::File(file_mapping) = &self.mapping else { - panic!("Anonymous mapping should not be PA_MMAP"); - }; - assert!(offset < file_mapping.length, "Offset out of range"); + if attr.contains(PageAttribute::PRESENT) { + // Nothing we can do... Possibly race with another threads to try + // installing the PTE, or on some arch, fault to set accessed bit. + return Ok(()); + } + let file_offset = file_mapping.offset + offset; let map_page = |cache_page: &CachePage| { @@ -540,7 +544,6 @@ impl MemArea { .await?; attr.insert(PageAttribute::PRESENT); - attr.remove(PageAttribute::MAPPED); Ok(()) } @@ -563,8 +566,9 @@ impl MemArea { self.handle_cow(&mut pfn, &mut attr); } - if attr.contains(PageAttribute::MAPPED) { - self.handle_mmap(&mut pfn, &mut attr, offset, write).await?; + if let Mapping::File(mapping) = &self.mapping { + self.handle_mmap(&mut pfn, &mut attr, offset, write, mapping) + .await?; } attr.insert(PageAttribute::ACCESSED); From 7ddae5076d179b1cbf635d555e52f16649598485 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 21:31:11 +0800 Subject: [PATCH 114/153] mm: delay populating PTEs to fault handlers Instead of populating anon and file PTEs when we do mmaps, we keep them empty and delay it till the fault happens. Add handle_missing function to handle this situation, and old logic of handling non-missing PTEs to another function: handle_non_missing. Besides, add helper functions to check non-present PTEs when we remove mappings from page tables. And use these functions in fragile positions such as `unmap` or `protect`. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 95 +++++++++++++++---------------- src/kernel/mem/mm_list/brk.rs | 27 ++------- src/kernel/mem/mm_list/mm_area.rs | 91 +++++++++++++++++++++++------ 3 files changed, 125 insertions(+), 88 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 2c20cc50..9db61ebc 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -131,11 +131,11 @@ impl MMListInner { let range = area.range.as_ref(&self.lock); for pte in self.page_table.iter_user(*range) { - let (pfn, _) = pte.take(); - pages_to_free.push(unsafe { - // SAFETY: We got the pfn from a valid page table entry, so it should be valid. - Folio::from_raw(pfn) - }); + let Some(folio) = pte.take_if_present() else { + continue; + }; + + pages_to_free.push(folio); } } @@ -168,17 +168,15 @@ impl MMListInner { .as_page_attr() .expect("Not a page attribute"); + if !page_attr.contains(PageAttribute::PRESENT) { + // Skip PTEs that are not installed yet. + continue; + } + if !permission.read && !permission.write && !permission.execute { // If no permissions are set, we just remove the page. - page_attr.remove( - PageAttribute::PRESENT - | PageAttribute::READ - | PageAttribute::WRITE - | PageAttribute::EXECUTE, - ); - - pte.set_attr(page_attr.into()); + pte.release(); continue; } @@ -221,13 +219,6 @@ impl MMListInner { return Err(EEXIST); } - match &mapping { - Mapping::Anonymous => { - self.page_table.set_anonymous(range, permission) - } - Mapping::File(_) => self.page_table.set_mmapped(range, permission), - } - self.areas.insert(MemArea::new( range, AreaFlags::from_old(permission, is_shared), @@ -249,19 +240,7 @@ impl Drop for MMListInner { } for pte in self.page_table.iter_user(range) { - let (pfn, raw_attr) = pte.take(); - let attr = raw_attr.as_page_attr().expect("Not a page"); - - if !attr.contains(PageAttribute::PRESENT) { - continue; - } - - unsafe { - // SAFETY: Present PTEs always corresponding to a valid - // Folio that was previously installed through - // `Folio::into_raw()`. - Folio::from_raw(pfn) - }; + pte.release(); } } @@ -332,13 +311,15 @@ impl MMList { let pgtable = &list_inner.page_table; for area in list_inner.areas.iter() { + // Skip shared areas because they can be filled just as new + // entries in page faults. + if area.is_shared() { + continue; + } + let range = area.range.as_ref(&list_inner.lock).clone(); - if !area.is_shared() { - pgtable.set_copy_on_write(&mut inner.page_table, range); - } else { - pgtable.set_copied(&mut inner.page_table, range); - } + pgtable.set_copy_on_write(&mut inner.page_table, range); } } @@ -669,6 +650,8 @@ trait PTEExt { // file mapped or shared anonymous fn set_mapped(&mut self, execute: bool); fn set_copy_on_write(&mut self, from: &mut Self); + fn take_if_present(&mut self) -> Option; + fn release(&mut self); } impl PTEExt for T @@ -698,12 +681,10 @@ where } fn set_copy_on_write(&mut self, from: &mut Self) { - let mut from_attr = from - .get_attr() - .as_page_attr() - .expect("Not a page attribute"); + let (pfn, raw_attr) = from.get(); + let mut attr = raw_attr.as_page_attr().expect("Not a page attribute"); - if !from_attr.intersects(PageAttribute::PRESENT) { + if !attr.contains(PageAttribute::PRESENT) { // Copy non-installed mapped PTEs directly to the new PTE and delay // its handling till the page fault. let (pfn, attr) = from.get(); @@ -711,16 +692,32 @@ where return; } - from_attr.remove(PageAttribute::WRITE | PageAttribute::DIRTY); - from_attr.insert(PageAttribute::COPY_ON_WRITE); + attr.remove(PageAttribute::WRITE | PageAttribute::DIRTY); + attr.insert(PageAttribute::COPY_ON_WRITE); let pfn = unsafe { - // SAFETY: We get the pfn from a valid page table entry, so it should be valid as well. - Folio::with_raw(from.get_pfn(), |page| page.clone().into_raw()) + // SAFETY: We get the pfn from a valid page table entry, so it + // should be valid as well. + Folio::with_raw(pfn, |page| page.clone().into_raw()) }; - self.set(pfn, T::Attr::from(from_attr & !PageAttribute::ACCESSED)); + self.set(pfn, T::Attr::from(attr & !PageAttribute::ACCESSED)); + from.set_attr(T::Attr::from(attr)); + } + + fn take_if_present(&mut self) -> Option { + let (pfn, raw_attr) = self.take(); + let attr = raw_attr.as_page_attr().expect("Not a page"); + + attr.contains(PageAttribute::PRESENT).then(|| unsafe { + // SAFETY: Present PTEs always corresponding to a valid Folio that + // was previously installed through `Folio::into_raw()`. + Folio::from_raw(pfn) + }) + } - from.set_attr(T::Attr::from(from_attr)); + fn release(&mut self) { + // Drop the returned folio. + let _ = self.take_if_present(); } } diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index 5c345b59..ede89b19 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -4,10 +4,7 @@ use eonix_mm::address::{VAddr, VRange}; use crate::kernel::mem::address::VRangeExt; use crate::kernel::mem::mm_list::mm_area::AreaList; -use crate::kernel::mem::mm_list::page_table::KernelPageTable; -use crate::kernel::mem::mm_list::{ - AreaFlags, MemArea, MemListLock, PageTableExt, -}; +use crate::kernel::mem::mm_list::{AreaFlags, MemArea, MemListLock}; use crate::kernel::mem::{MMList, Mapping, Permission}; #[derive(Clone)] @@ -48,7 +45,7 @@ impl ProgramBreak { async fn set_break( areas: &mut AreaList, brk: &mut ProgramBreak, mm_lock: &mut MemListLock, - page_table: &mut KernelPageTable, newbrk: Option, + newbrk: Option, ) -> VAddr { assert!( !brk.is_null(), @@ -78,29 +75,26 @@ async fn set_break( } let Some(area) = areas.upper_bound(curbrk) else { - return expand_create_area(areas, brk, page_table, new_range); + return expand_create_area(areas, brk, new_range); }; let range = area.range.as_ref(&areas.lock); if range.end() != curbrk { // Someone might have unmapped the brk area. - return expand_create_area(areas, brk, page_table, new_range); + return expand_create_area(areas, brk, new_range); } let mut area_lock = area.lock.lock().await; let range = area.range.as_mut(mm_lock, &mut areas.lock, &mut area_lock); *range = range.grow(new_range.len()); - map_break_area(page_table, new_range); - brk.set(newbrk); newbrk } fn expand_create_area( - areas: &mut AreaList, brk: &mut ProgramBreak, - page_table: &mut KernelPageTable, new_range: VRange, + areas: &mut AreaList, brk: &mut ProgramBreak, new_range: VRange, ) -> VAddr { let area = Arc::new(MemArea::new( new_range, @@ -117,20 +111,10 @@ fn expand_create_area( areas.insert_new(area); - map_break_area(page_table, new_range); - brk.set(new_range.end()); new_range.end() } -fn map_break_area(page_table: &mut KernelPageTable, range: VRange) { - page_table.set_anonymous(range, Permission { - read: true, - write: true, - execute: false, - }); -} - impl MMList { pub async fn set_break(&self, newbrk: Option) -> VAddr { let inner = self.inner.borrow(); @@ -141,7 +125,6 @@ impl MMList { &mut inner.areas, &mut inner.prog_break, &mut inner.lock, - &mut inner.page_table, newbrk, ) .await diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index f28db415..6a1eb97c 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -452,7 +452,10 @@ impl MemArea { attr.remove(PageAttribute::COPY_ON_WRITE); attr.set(PageAttribute::WRITE, self.can_write()); - let page = unsafe { Folio::from_raw(*pfn) }; + let page = unsafe { + // SAFETY: CoW PTEs always have a valid PFN. + Folio::from_raw(*pfn) + }; // XXX: Change me!!! if page.refcount.load(Ordering::Relaxed) == 1 { @@ -488,17 +491,12 @@ impl MemArea { /// # Arguments /// * `offset`: The offset from the start of the mapping, aligned to 4KB boundary. - pub async fn handle_mmap( + async fn missing_file( &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: usize, write: bool, file_mapping: &FileMapping, ) -> KResult<()> { assert!(offset < file_mapping.length, "Offset out of range"); - - if attr.contains(PageAttribute::PRESENT) { - // Nothing we can do... Possibly race with another threads to try - // installing the PTE, or on some arch, fault to set accessed bit. - return Ok(()); - } + assert!(!attr.contains(PageAttribute::PRESENT)); let file_offset = file_mapping.offset + offset; @@ -544,6 +542,68 @@ impl MemArea { .await?; attr.insert(PageAttribute::PRESENT); + + if self.can_read() { + attr.insert(PageAttribute::READ); + } + + if self.can_execute() { + attr.insert(PageAttribute::EXECUTE); + } + + Ok(()) + } + + fn missing_anon(&self, pfn: &mut PFN, attr: &mut PageAttribute) { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + + *pfn = folio.share().into_raw(); + + attr.insert(PageAttribute::PRESENT); + + if self.can_read() { + attr.insert(PageAttribute::READ); + } + + if self.can_execute() { + attr.insert(PageAttribute::EXECUTE); + } + + if self.can_write() { + attr.insert(PageAttribute::WRITE); + } + } + + async fn handle_missing( + &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: usize, + write: bool, + ) -> KResult<()> { + assert!( + !attr.contains(PageAttribute::COPY_ON_WRITE), + "Missing PTEs should not have CoW set" + ); + + attr.insert(PageAttribute::USER); + + if let Mapping::File(mapping) = &self.mapping { + self.missing_file(pfn, attr, offset, write, mapping).await?; + } else { + self.missing_anon(pfn, attr); + } + + assert!(attr.contains(PageAttribute::PRESENT)); + + Ok(()) + } + + fn handle_non_missing( + &self, pfn: &mut PFN, attr: &mut PageAttribute, + ) -> KResult<()> { + if attr.contains(PageAttribute::COPY_ON_WRITE) { + self.handle_cow(pfn, attr); + } + Ok(()) } @@ -558,17 +618,14 @@ impl MemArea { "Offset out of range" ); - let mut attr = - pte.get_attr().as_page_attr().expect("Not a page attribute"); - let mut pfn = pte.get_pfn(); + let (mut pfn, raw_attr) = pte.get(); + let mut attr = raw_attr.as_page_attr().expect("Not a page"); - if attr.contains(PageAttribute::COPY_ON_WRITE) { - self.handle_cow(&mut pfn, &mut attr); - } - - if let Mapping::File(mapping) = &self.mapping { - self.handle_mmap(&mut pfn, &mut attr, offset, write, mapping) + if !attr.contains(PageAttribute::PRESENT) { + self.handle_missing(&mut pfn, &mut attr, offset, write) .await?; + } else { + self.handle_non_missing(&mut pfn, &mut attr)?; } attr.insert(PageAttribute::ACCESSED); From ea661c70b3d86df816870d0244ae86691fb9b606 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 22:03:11 +0800 Subject: [PATCH 115/153] mm: remove unused page table helpers Since we move the population of anon and file PTEs to missing_anon and missing_file, set_{anonymous, mapped} is no longer used. Remove those. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 51 --------------------------------------- 1 file changed, 51 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 9db61ebc..01a6fe6a 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -605,25 +605,10 @@ impl fmt::Debug for MMList { } trait PageTableExt { - fn set_anonymous(&self, range: VRange, permission: Permission); - fn set_mmapped(&self, range: VRange, permission: Permission); fn set_copy_on_write(&self, from: &Self, range: VRange); - fn set_copied(&self, from: &Self, range: VRange); } impl PageTableExt for KernelPageTable { - fn set_anonymous(&self, range: VRange, permission: Permission) { - for pte in self.iter_user(range) { - pte.set_anonymous(permission.execute); - } - } - - fn set_mmapped(&self, range: VRange, permission: Permission) { - for pte in self.iter_user(range) { - pte.set_mapped(permission.execute); - } - } - fn set_copy_on_write(&self, from: &Self, range: VRange) { let to_iter = self.iter_user(range); let from_iter = from.iter_user(range); @@ -632,23 +617,9 @@ impl PageTableExt for KernelPageTable { to.set_copy_on_write(from); } } - - fn set_copied(&self, from: &Self, range: VRange) { - let to_iter = self.iter_user(range); - let from_iter = from.iter_user(range); - - for (to, from) in to_iter.zip(from_iter) { - let (pfn, attr) = from.get(); - to.set(pfn, attr); - } - } } trait PTEExt { - // private anonymous - fn set_anonymous(&mut self, execute: bool); - // file mapped or shared anonymous - fn set_mapped(&mut self, execute: bool); fn set_copy_on_write(&mut self, from: &mut Self); fn take_if_present(&mut self) -> Option; fn release(&mut self); @@ -658,28 +629,6 @@ impl PTEExt for T where T: PTE, { - fn set_anonymous(&mut self, execute: bool) { - // Writable flag is set during page fault handling while executable flag is - // preserved across page faults, so we set executable flag now. - let mut attr = PageAttribute::PRESENT - | PageAttribute::READ - | PageAttribute::USER - | PageAttribute::COPY_ON_WRITE; - attr.set(PageAttribute::EXECUTE, execute); - - self.set(EMPTY_PAGE.clone().into_raw(), T::Attr::from(attr)); - } - - fn set_mapped(&mut self, execute: bool) { - // Writable flag is set during page fault handling while executable flag is - // preserved across page faults, so we set executable flag now. - let mut attr = PageAttribute::READ | PageAttribute::USER; - attr.set(PageAttribute::EXECUTE, execute); - - // Set an obviously invalid PFN to help debugging... - self.set(PFN::from_val(0), T::Attr::from(attr)); - } - fn set_copy_on_write(&mut self, from: &mut Self) { let (pfn, raw_attr) = from.get(); let mut attr = raw_attr.as_page_attr().expect("Not a page attribute"); From 6fd9c7a1a1afb81bdb45e3d4bccf3a3df147a950 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 8 Mar 2026 23:06:17 +0800 Subject: [PATCH 116/153] mm: remove shared EMPTY_PAGE We don't use it for initial anon mappings, so remove it. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 10 ++-------- src/kernel/mem/mm_list/mm_area.rs | 25 ++++++++----------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 01a6fe6a..2027cb64 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -15,7 +15,7 @@ use eonix_hal::mm::{ use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; -use eonix_sync::{LazyLock, Mutex}; +use eonix_sync::Mutex; use mm_area::AreaList; use page_table::KernelPageTable; @@ -23,18 +23,12 @@ pub use self::mapping::{FileMapping, Mapping}; pub use self::mm_area::{AreaFlags, MemArea}; pub use self::page_fault::handle_kernel_page_fault; use super::address::{VAddrExt as _, VRangeExt as _}; -use super::{Folio, FolioOwned}; +use super::Folio; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; use crate::kernel::mem::mm_list::brk::ProgramBreak; use crate::prelude::*; use crate::sync::ArcSwap; -pub static EMPTY_PAGE: LazyLock = LazyLock::new(|| { - let mut folio = FolioOwned::alloc(); - folio.as_bytes_mut().fill(0); - folio.share() -}); - #[derive(Debug, Clone, Copy)] pub struct Permission { pub read: bool, diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 6a1eb97c..61139ef0 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -13,7 +13,7 @@ use intrusive_collections::{ PointerOps, RBTree, RBTreeAtomicLink, }; -use super::{Mapping, EMPTY_PAGE}; +use super::Mapping; use crate::kernel::mem::folio::Folio; use crate::kernel::mem::mm_list::MemListLock; use crate::kernel::mem::{ @@ -466,24 +466,15 @@ impl MemArea { return; } - let mut new_page; - if *pfn == EMPTY_PAGE.pfn() { - new_page = { - let mut folio = FolioOwned::alloc(); - folio.as_bytes_mut().fill(0); - folio - }; - } else { - new_page = FolioOwned::alloc(); + let mut new_page = FolioOwned::alloc(); - unsafe { - // SAFETY: `page` is CoW, which means that others won't write to it. - let old_page_data = page.get_bytes_ptr().as_ref(); - let new_page_data = new_page.as_bytes_mut(); + unsafe { + // SAFETY: `page` is CoW, which means that others won't write to it. + let old_page_data = page.get_bytes_ptr().as_ref(); + let new_page_data = new_page.as_bytes_mut(); - new_page_data.copy_from_slice(old_page_data); - }; - } + new_page_data.copy_from_slice(old_page_data); + }; attr.remove(PageAttribute::ACCESSED); *pfn = new_page.share().into_raw(); From 34faa798d6a1ac6fd54f71c771143bc5f99f56c9 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 10 Mar 2026 02:33:59 +0800 Subject: [PATCH 117/153] style: reformat page_cache.rs No functional change. Signed-off-by: greatbridf --- src/kernel/mem/page_cache.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 3fe33d5b..4328a044 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -32,7 +32,9 @@ impl PageOffset { Self((offset + PAGE_SIZE - 1) >> PAGE_SIZE_BITS) } - pub fn iter_till(self, end: PageOffset) -> impl Iterator { + pub fn iter_till( + self, end: PageOffset, + ) -> impl Iterator { (self.0..end.0).map(PageOffset) } @@ -100,10 +102,10 @@ impl PageCache { } pub fn get_page_locked<'a>( - &self, - pages: &'a mut BTreeMap, + &self, pages: &'a mut BTreeMap, pgoff: PageOffset, - ) -> impl Future> + Send + use<'_, 'a> { + ) -> impl Future> + Send + use<'_, 'a> + { async move { match pages.entry(pgoff) { Entry::Occupied(ent) => Ok(ent.into_mut()), @@ -122,7 +124,9 @@ impl PageCache { } // TODO: Remove this. - pub async fn with_page(&self, pgoff: PageOffset, func: impl FnOnce(&CachePage)) -> KResult<()> { + pub async fn with_page( + &self, pgoff: PageOffset, func: impl FnOnce(&CachePage), + ) -> KResult<()> { let mut pages = self.pages.lock().await; if pgoff > PageOffset::from_byte_ceil(self.len()) { return Err(EINVAL); @@ -135,7 +139,9 @@ impl PageCache { Ok(()) } - pub async fn read(&self, buffer: &mut dyn Buffer, mut offset: usize) -> KResult { + pub async fn read( + &self, buffer: &mut dyn Buffer, mut offset: usize, + ) -> KResult { let mut pages = self.pages.lock().await; let total_len = self.len(); @@ -156,7 +162,10 @@ impl PageCache { let data_len = real_end - offset; if buffer - .fill(&page.lock().as_bytes()[inner_offset..inner_offset + data_len])? + .fill( + &page.lock().as_bytes() + [inner_offset..inner_offset + data_len], + )? .should_stop() || buffer.available() == 0 { @@ -169,7 +178,9 @@ impl PageCache { Ok(buffer.wrote()) } - pub async fn write(&self, stream: &mut dyn Stream, mut offset: usize) -> KResult { + pub async fn write( + &self, stream: &mut dyn Stream, mut offset: usize, + ) -> KResult { let mut pages = self.pages.lock().await; let mut total_written = 0; From 240a48b0a875dd87fe412dcab126072d1f90e448 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 10 Mar 2026 02:23:20 +0800 Subject: [PATCH 118/153] sync: add atomic! macro to help performing atomic operations If we want to assign different ordering to every atomic operation, it's bothersome to import Ordering. And the current version of Rust doesn't support optional argument. So we can't assign a default SeqCst ordering as well. Implement this helper macro in which Ordering is implicitly imported. We make the function called a dedicated argument so that IDEs can find their reference. Signed-off-by: greatbridf --- .../eonix_sync/eonix_sync_base/src/atomic.rs | 22 +++++++++++++++++++ crates/eonix_sync/eonix_sync_base/src/lib.rs | 1 + 2 files changed, 23 insertions(+) create mode 100644 crates/eonix_sync/eonix_sync_base/src/atomic.rs diff --git a/crates/eonix_sync/eonix_sync_base/src/atomic.rs b/crates/eonix_sync/eonix_sync_base/src/atomic.rs new file mode 100644 index 00000000..b32a88ae --- /dev/null +++ b/crates/eonix_sync/eonix_sync_base/src/atomic.rs @@ -0,0 +1,22 @@ +#[macro_export] +macro_rules! atomic { + (@$order:ident, $e:expr, $func:ident) => {{ + use core::sync::atomic::Ordering; + + $e.$func(Ordering::$order) + }}; + + (@$order:ident, $e:expr, $func:ident, $($arg:expr),*) => {{ + use core::sync::atomic::Ordering; + + $e.$func($($arg),*, Ordering::$order) + }}; + + ($e:expr, $func:ident) => { + $crate::atomic!(@SeqCst, $e, $func) + }; + + ($e:expr, $func:ident, $($arg:expr),*) => { + $crate::atomic!(@SeqCst, $e, $func, $($arg),*) + }; +} diff --git a/crates/eonix_sync/eonix_sync_base/src/lib.rs b/crates/eonix_sync/eonix_sync_base/src/lib.rs index 0574b0d2..c3a48729 100644 --- a/crates/eonix_sync/eonix_sync_base/src/lib.rs +++ b/crates/eonix_sync/eonix_sync_base/src/lib.rs @@ -1,5 +1,6 @@ #![no_std] +mod atomic; mod guard; mod lazy_lock; mod locked; From 9cdba49ec1ece541d09ed320131232ccd929acc4 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 10 Mar 2026 02:34:40 +0800 Subject: [PATCH 119/153] mm, sync: apply atomic! macro to PageFlags users With minor change that setting and clearing dirty flag of page cache folios are not sequential consistent. Signed-off-by: greatbridf --- src/kernel/mem/page_alloc/raw_page.rs | 29 ++++++++++++++++----------- src/kernel/mem/page_cache.rs | 8 ++++---- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 16d57714..9283199c 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -5,6 +5,7 @@ use buddy_allocator::BuddyFolio; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _}; use eonix_mm::paging::{FolioList, FolioListSized, Zone, PFN}; +use eonix_sync::atomic; use intrusive_list::{container_of, Link, List}; use slab_allocator::{SlabPage, SlabPageAlloc, SlabSlot}; @@ -55,16 +56,16 @@ impl PageFlags { pub const DIRTY: u32 = 1 << 4; pub const LOCAL: u32 = 1 << 6; - pub fn has(&self, flag: u32) -> bool { - (self.0.load(Ordering::Relaxed) & flag) == flag + pub fn has(&self, flag: u32, order: Ordering) -> bool { + (self.0.load(order) & flag) == flag } - pub fn set(&self, flag: u32) { - self.0.fetch_or(flag, Ordering::Relaxed); + pub fn set(&self, flag: u32, order: Ordering) { + self.0.fetch_or(flag, order); } - pub fn clear(&self, flag: u32) { - self.0.fetch_and(!flag, Ordering::Relaxed); + pub fn clear(&self, flag: u32, order: Ordering) { + self.0.fetch_and(!flag, order); } /// Set the flag and return whether it was already set. @@ -85,7 +86,8 @@ impl BuddyFolio for RawPage { } fn is_buddy(&self) -> bool { - self.flags.has(PageFlags::BUDDY) + // Check below `set_buddy`. + atomic!(@Relaxed, self.flags, has, PageFlags::BUDDY) } fn set_order(&mut self, order: u32) { @@ -93,10 +95,12 @@ impl BuddyFolio for RawPage { } fn set_buddy(&mut self, val: bool) { + // Ordering can be Relaxed since all these operations are performed with + // the allocator's mutable reference. So they are sequenced. if val { - self.flags.set(PageFlags::BUDDY); + atomic!(@Relaxed, self.flags, set, PageFlags::BUDDY); } else { - self.flags.clear(PageFlags::BUDDY) + atomic!(@Relaxed, self.flags, clear, PageFlags::BUDDY); } } } @@ -160,10 +164,11 @@ impl SlabPage for RawPage { impl PerCpuPage for RawPage { fn set_local(&mut self, val: bool) { + // Per cpu variables need no synchronization. if val { - self.flags.set(PageFlags::LOCAL) + atomic!(@Relaxed, self.flags, set, PageFlags::LOCAL); } else { - self.flags.clear(PageFlags::LOCAL) + atomic!(@Relaxed, self.flags, clear, PageFlags::LOCAL); } } } @@ -216,7 +221,7 @@ unsafe impl SlabPageAlloc for GlobalPageAlloc { fn alloc_slab_page(&self) -> &'static mut RawPage { let raw_page = self.alloc_raw_order(0).expect("Out of memory"); - raw_page.flags.set(PageFlags::SLAB); + atomic!(@Relaxed, raw_page.flags, set, PageFlags::SLAB); raw_page.shared_data.slab = SlabPageData::new(); raw_page diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 4328a044..b561946d 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -3,7 +3,7 @@ use core::future::Future; use core::ops::{Deref, DerefMut}; use eonix_mm::paging::{Folio as _, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; -use eonix_sync::Mutex; +use eonix_sync::{atomic, Mutex}; use super::page_alloc::PageFlags; use super::{Folio, FolioOwned}; @@ -62,14 +62,14 @@ impl CachePage { } pub fn is_dirty(&self) -> bool { - self.flags.has(PageFlags::DIRTY) + atomic!(self.flags, has, PageFlags::DIRTY) } pub fn set_dirty(&self, dirty: bool) { if dirty { - self.flags.set(PageFlags::DIRTY); + atomic!(self.flags, set, PageFlags::DIRTY); } else { - self.flags.clear(PageFlags::DIRTY); + atomic!(self.flags, clear, PageFlags::DIRTY); } } From 65340930ffb1a4d0b4ff84997fa0ea5299b0a8f6 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 9 Mar 2026 21:29:45 +0800 Subject: [PATCH 120/153] mm, folio: implement spinlock based folio lock Current blank implementation of folio locks is fragile and provides zero protection over concurrent accesses in smp environment. Let's add a basic implementation spinning the LOCKED bit in the folio's flag field. Signed-off-by: greatbridf --- src/kernel/mem/folio.rs | 46 ++++++++++++++++++++++++--- src/kernel/mem/page_alloc/raw_page.rs | 4 +-- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs index 8ab4d6be..9ab80c63 100644 --- a/src/kernel/mem/folio.rs +++ b/src/kernel/mem/folio.rs @@ -4,10 +4,14 @@ use core::ops::Deref; use core::ptr::NonNull; use core::sync::atomic::Ordering; -use eonix_mm::paging::{Folio as FolioTrait, FrameAlloc, GlobalFrameAlloc, Zone, PFN}; +use eonix_mm::paging::{ + Folio as FolioTrait, FrameAlloc, GlobalFrameAlloc, Zone, PFN, +}; +use eonix_sync::atomic; use super::page_alloc::ZONE; use super::{GlobalPageAlloc, PhysAccess as _, RawPage}; +use crate::kernel::mem::page_alloc::PageFlags; #[repr(transparent)] pub struct Folio(NonNull); @@ -79,8 +83,26 @@ impl Folio { } } + pub fn try_lock(&self) -> Option> { + if !atomic!(@AcqRel, self.flags, test_and_set, PageFlags::LOCKED) { + return Some(LockedFolio(self)); + } + + None + } + pub fn lock(&self) -> LockedFolio<'_> { - // TODO: actually perform the lock... + if let Some(locked) = self.try_lock() { + return locked; + } + + // XXX: Spinlock based folio locks may suffer severe contention. + // Find a way to optimize this. + + while atomic!(@AcqRel, self.flags, test_and_set, PageFlags::LOCKED) { + core::hint::spin_loop(); + } + LockedFolio(self) } @@ -123,7 +145,9 @@ impl Drop for Folio { fn drop(&mut self) { match self.refcount.fetch_sub(1, Ordering::AcqRel) { 0 => unreachable!("Refcount for an in-use page is 0"), - 1 => unsafe { GlobalPageAlloc::GLOBAL.dealloc_raw(self.0.as_mut()) }, + 1 => unsafe { + GlobalPageAlloc::GLOBAL.dealloc_raw(self.0.as_mut()) + }, _ => {} } } @@ -149,14 +173,20 @@ impl LockedFolio<'_> { pub fn as_bytes(&self) -> &[u8] { unsafe { // SAFETY: `self.start()` points to valid memory of length `self.len()`. - core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) + core::slice::from_raw_parts( + self.start().as_ptr().as_ptr(), + self.len(), + ) } } pub fn as_bytes_mut(&mut self) -> &mut [u8] { unsafe { // SAFETY: `self.start()` points to valid memory of length `self.len()`. - core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) + core::slice::from_raw_parts_mut( + self.start().as_ptr().as_ptr(), + self.len(), + ) } } } @@ -169,6 +199,12 @@ impl Deref for LockedFolio<'_> { } } +impl Drop for LockedFolio<'_> { + fn drop(&mut self) { + atomic!(@AcqRel, self.0.flags, clear, PageFlags::LOCKED); + } +} + impl FolioOwned { pub fn alloc() -> Self { Self(Folio::alloc()) diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 9283199c..24f7d561 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -71,8 +71,8 @@ impl PageFlags { /// Set the flag and return whether it was already set. /// /// If multiple flags are given, returns true if any of them were already set. - pub fn test_and_set(&self, flag: u32) -> bool { - (self.0.fetch_or(flag, Ordering::Relaxed) & flag) != 0 + pub fn test_and_set(&self, flag: u32, order: Ordering) -> bool { + (self.0.fetch_or(flag, order) & flag) != 0 } } From 00484b63148d1f33fb5560a8b0b536f2c89df2f0 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 9 Mar 2026 22:19:36 +0800 Subject: [PATCH 121/153] mm: extract logic for creating/removing mappings into functions To make changing its behavior easier, extract these logics into functions and replace its usages with these functions. Notice that in `handle_cow()`, originally, we take the folio and check its refcount, if it's 1, we simply forget the folio and return. This behavior is incorrect because `Folio`s may have different implementations, and it's actually unspecified whether using the raw pfn after doing from_raw is well-formed. So now, we remove the mapping, and if we are the only one, add the mapping back then. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 15 ++++++------ src/kernel/mem/mm_list/mapping.rs | 40 ++++++++++++++++++++++++++++--- src/kernel/mem/mm_list/mm_area.rs | 25 +++++++++---------- src/kernel/mem/page_cache.rs | 6 ++--- 4 files changed, 61 insertions(+), 25 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 2027cb64..7b3c2de0 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -19,7 +19,9 @@ use eonix_sync::Mutex; use mm_area::AreaList; use page_table::KernelPageTable; -pub use self::mapping::{FileMapping, Mapping}; +pub use self::mapping::{ + add_mapping, duplicate_mapping, remove_mapping, FileMapping, Mapping, +}; pub use self::mm_area::{AreaFlags, MemArea}; pub use self::page_fault::handle_kernel_page_fault; use super::address::{VAddrExt as _, VRangeExt as _}; @@ -639,9 +641,9 @@ where attr.insert(PageAttribute::COPY_ON_WRITE); let pfn = unsafe { - // SAFETY: We get the pfn from a valid page table entry, so it - // should be valid as well. - Folio::with_raw(pfn, |page| page.clone().into_raw()) + // SAFETY: We get the pfn from a valid page table entry, which + // should have been created via `add_mapping`. + duplicate_mapping(pfn) }; self.set(pfn, T::Attr::from(attr & !PageAttribute::ACCESSED)); @@ -653,9 +655,8 @@ where let attr = raw_attr.as_page_attr().expect("Not a page"); attr.contains(PageAttribute::PRESENT).then(|| unsafe { - // SAFETY: Present PTEs always corresponding to a valid Folio that - // was previously installed through `Folio::into_raw()`. - Folio::from_raw(pfn) + // SAFETY: Present PTEs are always created via `add_mapping`. + remove_mapping(pfn) }) } diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 2b837ae7..b9a9ad29 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,8 +1,8 @@ use alloc::sync::Arc; -use eonix_mm::paging::PAGE_SIZE; +use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; -use crate::kernel::mem::PageCache; +use crate::kernel::mem::{Folio, PageCache}; #[derive(Debug, Clone)] pub struct FileMapping { @@ -22,7 +22,9 @@ pub enum Mapping { } impl FileMapping { - pub fn new(page_cache: Arc, offset: usize, length: usize) -> Self { + pub fn new( + page_cache: Arc, offset: usize, length: usize, + ) -> Self { assert_eq!(offset & (PAGE_SIZE - 1), 0); Self { page_cache, @@ -43,3 +45,35 @@ impl FileMapping { } } } + +/// Turn the folio together with its refcount into a raw [`PFN`] that can be +/// mapped into some page table. +pub fn add_mapping(folio: Folio) -> PFN { + folio.into_raw() +} + +/// Duplicate the mapping in page tables. +/// +/// # Safety +/// `pfn` must be previously created via [`add_mapping`]. +/// +/// Otherwise this is undefined behavior. +pub unsafe fn duplicate_mapping(pfn: PFN) -> PFN { + unsafe { + // SAFETY: `pfn` is created via `add_mapping`, which uses `into_raw`. + Folio::with_raw(pfn, |folio| add_mapping(folio.clone())) + } +} + +/// Remove mappings in page tables. +/// +/// # Safety +/// `pfn` must be previously created via [`add_mapping`]. +/// +/// Otherwise this is undefined behavior. +pub unsafe fn remove_mapping(pfn: PFN) -> Folio { + unsafe { + // SAFETY: `pfn` is created via `add_mapping`, which uses `into_raw`. + Folio::from_raw(pfn) + } +} diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 61139ef0..213bf396 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -5,7 +5,7 @@ use core::sync::atomic::{AtomicU32, Ordering}; use eonix_mm::address::{VAddr, VRange}; use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; -use eonix_mm::paging::{Folio as _, PFN}; +use eonix_mm::paging::PFN; use eonix_sync::Mutex; use intrusive_collections::rbtree::Entry; use intrusive_collections::{ @@ -15,7 +15,8 @@ use intrusive_collections::{ use super::Mapping; use crate::kernel::mem::folio::Folio; -use crate::kernel::mem::mm_list::MemListLock; +use crate::kernel::mem::mm_list::mapping::add_mapping; +use crate::kernel::mem::mm_list::{remove_mapping, MemListLock}; use crate::kernel::mem::{ CachePage, FileMapping, FolioOwned, PageOffset, Permission, }; @@ -452,32 +453,32 @@ impl MemArea { attr.remove(PageAttribute::COPY_ON_WRITE); attr.set(PageAttribute::WRITE, self.can_write()); - let page = unsafe { - // SAFETY: CoW PTEs always have a valid PFN. - Folio::from_raw(*pfn) + let folio = unsafe { + // SAFETY: `pfn` is a mapping taken from some page table. + remove_mapping(*pfn) }; // XXX: Change me!!! - if page.refcount.load(Ordering::Relaxed) == 1 { + if folio.refcount.load(Ordering::Relaxed) == 1 { // SAFETY: This is actually safe. If we read `1` here and we have `MMList` lock // held, there couldn't be neither other processes sharing the page, nor other // threads making the page COW at the same time. - core::mem::forget(page); + *pfn = add_mapping(folio); return; } let mut new_page = FolioOwned::alloc(); unsafe { - // SAFETY: `page` is CoW, which means that others won't write to it. - let old_page_data = page.get_bytes_ptr().as_ref(); + // SAFETY: `folio` is CoW, which means that others won't write to it. + let old_page_data = folio.get_bytes_ptr().as_ref(); let new_page_data = new_page.as_bytes_mut(); new_page_data.copy_from_slice(old_page_data); }; attr.remove(PageAttribute::ACCESSED); - *pfn = new_page.share().into_raw(); + *pfn = add_mapping(new_page.share()); } /// # Arguments @@ -524,7 +525,7 @@ impl MemArea { .copy_from_slice(cache_page.lock().as_bytes()); attr.insert(PageAttribute::WRITE); - *pfn = new_page.share().into_raw(); + *pfn = add_mapping(new_page.share()); }; file_mapping @@ -549,7 +550,7 @@ impl MemArea { let mut folio = FolioOwned::alloc(); folio.as_bytes_mut().fill(0); - *pfn = folio.share().into_raw(); + *pfn = add_mapping(folio.share()); attr.insert(PageAttribute::PRESENT); diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index b561946d..e97e8f60 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -2,13 +2,14 @@ use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; use core::ops::{Deref, DerefMut}; -use eonix_mm::paging::{Folio as _, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; +use eonix_mm::paging::{PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::{atomic, Mutex}; use super::page_alloc::PageFlags; use super::{Folio, FolioOwned}; use crate::io::{Buffer, Stream}; use crate::kernel::constants::EINVAL; +use crate::kernel::mem::mm_list::add_mapping; use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; @@ -74,8 +75,7 @@ impl CachePage { } pub fn add_mapping(&self) -> PFN { - // TODO: Increase map_count - self.0.clone().into_raw() + add_mapping(self.0.clone()) } } From d0a22c0c2b96a83e9f17a29c5f20292442a44081 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 13 Mar 2026 00:19:00 +0800 Subject: [PATCH 122/153] mm: remove DerefMut impl for CachePage No one is ever going to use Folio's mutable references. Remove this for simplicity. Signed-off-by: greatbridf --- src/kernel/mem/page_cache.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index e97e8f60..6e1c7a00 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,6 +1,6 @@ use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; -use core::ops::{Deref, DerefMut}; +use core::ops::Deref; use eonix_mm::paging::{PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::{atomic, Mutex}; @@ -87,12 +87,6 @@ impl Deref for CachePage { } } -impl DerefMut for CachePage { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - impl PageCache { pub fn new(inode: InodeUse) -> Self { Self { From 888c0004e04aa4720ca663299b02ca5ba8dbeba6 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 13 Mar 2026 18:44:35 +0800 Subject: [PATCH 123/153] style: reformat syscall/mm.rs No functional change. Signed-off-by: greatbridf --- src/kernel/syscall/mm.rs | 47 +++++++++++++++------------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index 825440ef..bd756048 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -4,7 +4,9 @@ use eonix_mm::paging::PAGE_SIZE; use posix_types::syscall_no::*; use super::FromSyscallArg; -use crate::kernel::constants::{UserMmapFlags, UserMmapProtocol, EBADF, EINVAL}; +use crate::kernel::constants::{ + UserMmapFlags, UserMmapProtocol, EBADF, EINVAL, +}; use crate::kernel::mem::{FileMapping, Mapping, Permission}; use crate::kernel::task::Thread; use crate::kernel::vfs::filearray::FD; @@ -34,13 +36,8 @@ fn check_impl(condition: bool, err: u32) -> KResult<()> { } async fn do_mmap2( - thread: &Thread, - addr: usize, - len: usize, - prot: UserMmapProtocol, - flags: UserMmapFlags, - fd: FD, - pgoffset: usize, + thread: &Thread, addr: usize, len: usize, prot: UserMmapProtocol, + flags: UserMmapFlags, fd: FD, pgoffset: usize, ) -> KResult { let addr = VAddr::from(addr); if !addr.is_page_aligned() || pgoffset % PAGE_SIZE != 0 || len == 0 { @@ -97,12 +94,8 @@ async fn do_mmap2( #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] #[eonix_macros::define_syscall(SYS_MMAP)] async fn mmap( - addr: usize, - len: usize, - prot: UserMmapProtocol, - flags: UserMmapFlags, - fd: FD, - offset: usize, + addr: usize, len: usize, prot: UserMmapProtocol, flags: UserMmapFlags, + fd: FD, offset: usize, ) -> KResult { do_mmap2(thread, addr, len, prot, flags, fd, offset).await } @@ -110,12 +103,8 @@ async fn mmap( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MMAP2)] async fn mmap2( - addr: usize, - len: usize, - prot: UserMmapProtocol, - flags: UserMmapFlags, - fd: FD, - pgoffset: usize, + addr: usize, len: usize, prot: UserMmapProtocol, flags: UserMmapFlags, + fd: FD, pgoffset: usize, ) -> KResult { do_mmap2(thread, addr, len, prot, flags, fd, pgoffset).await } @@ -143,7 +132,9 @@ async fn madvise(_addr: usize, _len: usize, _advice: u32) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_MPROTECT)] -async fn mprotect(addr: usize, len: usize, prot: UserMmapProtocol) -> KResult<()> { +async fn mprotect( + addr: usize, len: usize, prot: UserMmapProtocol, +) -> KResult<()> { let addr = VAddr::from(addr); if !addr.is_page_aligned() || len == 0 { return Err(EINVAL); @@ -154,15 +145,11 @@ async fn mprotect(addr: usize, len: usize, prot: UserMmapProtocol) -> KResult<() thread .process .mm_list - .protect( - addr, - len, - Permission { - read: prot.contains(UserMmapProtocol::PROT_READ), - write: prot.contains(UserMmapProtocol::PROT_WRITE), - execute: prot.contains(UserMmapProtocol::PROT_EXEC), - }, - ) + .protect(addr, len, Permission { + read: prot.contains(UserMmapProtocol::PROT_READ), + write: prot.contains(UserMmapProtocol::PROT_WRITE), + execute: prot.contains(UserMmapProtocol::PROT_EXEC), + }) .await } From 878e9d056c2915ece6e728b7a8c886a2eaf65289 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 13 Mar 2026 18:46:16 +0800 Subject: [PATCH 124/153] style: reformat elf.rs No functional change. Signed-off-by: greatbridf --- src/kernel/task/loader/elf.rs | 94 +++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index 9f8aa166..9489940a 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -180,9 +180,11 @@ impl Elf { let ph_offset = self.ph_offset(); for program_header in &self.program_headers { if program_header.offset() <= ph_offset - && ph_offset < program_header.offset() + program_header.file_size() + && ph_offset + < program_header.offset() + program_header.file_size() { - return Ok(ph_offset - program_header.offset() + program_header.virtual_addr()); + return Ok(ph_offset - program_header.offset() + + program_header.virtual_addr()); } } Err(ENOEXEC) @@ -212,7 +214,9 @@ impl Elf { }) } - async fn load(&self, args: Vec, envs: Vec) -> KResult { + async fn load( + &self, args: Vec, envs: Vec, + ) -> KResult { let mm_list = MMList::new(); // Load Segments @@ -258,10 +262,7 @@ impl Elf { } async fn create_and_init_stack( - &self, - mm_list: &MMList, - args: Vec, - envs: Vec, + &self, mm_list: &MMList, args: Vec, envs: Vec, aux_vec: AuxVec, ) -> KResult { mm_list @@ -283,7 +284,9 @@ impl Elf { .await } - fn init_aux_vec(&self, elf_base: VAddr, ldso_base: Option) -> KResult> { + fn init_aux_vec( + &self, elf_base: VAddr, ldso_base: Option, + ) -> KResult> { let mut aux_vec: AuxVec = AuxVec::new(); let ph_addr = if self.is_shared_object() { elf_base.addr() + self.ph_addr()? @@ -294,7 +297,8 @@ impl Elf { aux_vec.set(AuxKey::AT_PAGESZ, E::Ea::from_usize(PAGE_SIZE))?; aux_vec.set(AuxKey::AT_PHDR, E::Ea::from_usize(ph_addr))?; aux_vec.set(AuxKey::AT_PHNUM, E::Ea::from_usize(self.ph_count()))?; - aux_vec.set(AuxKey::AT_PHENT, E::Ea::from_usize(self.ph_entry_size()))?; + aux_vec + .set(AuxKey::AT_PHENT, E::Ea::from_usize(self.ph_entry_size()))?; let elf_entry = if self.is_shared_object() { elf_base.addr() + self.entry_point() } else { @@ -307,13 +311,15 @@ impl Elf { )?; if let Some(ldso_base) = ldso_base { - aux_vec.set(AuxKey::AT_BASE, E::Ea::from_usize(ldso_base.addr()))?; + aux_vec + .set(AuxKey::AT_BASE, E::Ea::from_usize(ldso_base.addr()))?; } Ok(aux_vec) } async fn load_segments(&self, mm_list: &MMList) -> KResult<(VAddr, VAddr)> { - let base: VAddr = if self.is_shared_object() { E::DYN_BASE_ADDR } else { 0 }.into(); + let base: VAddr = + if self.is_shared_object() { E::DYN_BASE_ADDR } else { 0 }.into(); let mut segments_end = VAddr::NULL; @@ -321,7 +327,8 @@ impl Elf { let type_ = program_header.type_().map_err(|_| ENOEXEC)?; if type_ == program::Type::Load { - let segment_end = self.load_segment(program_header, mm_list, base).await?; + let segment_end = + self.load_segment(program_header, mm_list, base).await?; if segment_end > segments_end { segments_end = segment_end; @@ -333,10 +340,7 @@ impl Elf { } async fn load_segment( - &self, - program_header: &E::Ph, - mm_list: &MMList, - base_addr: VAddr, + &self, program_header: &E::Ph, mm_list: &MMList, base_addr: VAddr, ) -> KResult { let virtual_addr = base_addr + program_header.virtual_addr(); let vmem_vaddr_end = virtual_addr + program_header.mem_size(); @@ -377,7 +381,9 @@ impl Elf { zero_len = zero_len.min(vmem_vaddr_end - load_vaddr_end); mm_list - .access_mut(load_vaddr_end, zero_len, |_, data| data.fill(0)) + .access_mut(load_vaddr_end, zero_len, |_, data| { + data.fill(0) + }) .await?; } @@ -397,13 +403,19 @@ impl Elf { Ok(vmap_start + vmem_len) } - async fn load_ldso(&self, mm_list: &MMList) -> KResult> { + async fn load_ldso( + &self, mm_list: &MMList, + ) -> KResult> { let ldso_path = self.ldso_path().await?; if let Some(ldso_path) = ldso_path { let fs_context = FsContext::global(); - let ldso_file = - Dentry::open(fs_context, Path::new(ldso_path.as_bytes())?, true).await?; + let ldso_file = Dentry::open( + fs_context, + Path::new(ldso_path.as_bytes())?, + true, + ) + .await?; let ldso_elf = Elf::::parse(ldso_file).await?; let base = VAddr::from(E::LDSO_BASE_ADDR); @@ -412,7 +424,9 @@ impl Elf { let type_ = program_header.type_().map_err(|_| ENOEXEC)?; if type_ == program::Type::Load { - ldso_elf.load_segment(program_header, mm_list, base).await?; + ldso_elf + .load_segment(program_header, mm_list, base) + .await?; } } @@ -439,9 +453,13 @@ impl Elf { let mut ldso_vec = vec![0u8; file_size - 1]; // -1 due to '\0' self.file - .read(&mut ByteBuffer::from(ldso_vec.as_mut_slice()), file_offset) + .read( + &mut ByteBuffer::from(ldso_vec.as_mut_slice()), + file_offset, + ) .await?; - let ldso_path = String::from_utf8(ldso_vec).map_err(|_| ENOEXEC)?; + let ldso_path = + String::from_utf8(ldso_vec).map_err(|_| ENOEXEC)?; return Ok(Some(ldso_path)); } } @@ -469,7 +487,9 @@ impl ELF { } } - pub async fn load(&self, args: Vec, envs: Vec) -> KResult { + pub async fn load( + &self, args: Vec, envs: Vec, + ) -> KResult { match &self { ELF::Elf32(elf32) => elf32.load(args, envs).await, ELF::Elf64(elf64) => elf64.load(args, envs).await, @@ -487,10 +507,7 @@ struct StackInitializer<'a, T> { impl<'a, T: ElfAddr + Clone + Copy> StackInitializer<'a, T> { fn new( - mm_list: &'a MMList, - sp: usize, - args: Vec, - envs: Vec, + mm_list: &'a MMList, sp: usize, args: Vec, envs: Vec, aux_vec: AuxVec, ) -> Self { Self { @@ -524,7 +541,10 @@ impl<'a, T: ElfAddr + Clone + Copy> StackInitializer<'a, T> { self.sp -= len; self.mm_list .access_mut(VAddr::from(self.sp), len, |offset, data| { - data.copy_from_slice(&string.as_bytes_with_nul()[offset..offset + data.len()]) + data.copy_from_slice( + &string.as_bytes_with_nul() + [offset..offset + data.len()], + ) }) .await?; addrs.push(T::from_usize(self.sp)); @@ -540,7 +560,10 @@ impl<'a, T: ElfAddr + Clone + Copy> StackInitializer<'a, T> { self.sp -= len; self.mm_list .access_mut(VAddr::from(self.sp), len, |offset, data| { - data.copy_from_slice(&string.as_bytes_with_nul()[offset..offset + data.len()]) + data.copy_from_slice( + &string.as_bytes_with_nul() + [offset..offset + data.len()], + ) }) .await?; addrs.push(T::from_usize(self.sp)); @@ -550,11 +573,13 @@ impl<'a, T: ElfAddr + Clone + Copy> StackInitializer<'a, T> { } fn stack_alignment(&mut self) { - let aux_vec_size = (self.aux_vec.table().len() + 1) * (size_of::() * 2); + let aux_vec_size = + (self.aux_vec.table().len() + 1) * (size_of::() * 2); let envp_pointers_size = (self.envs.len() + 1) * size_of::(); let argv_pointers_size = (self.args.len() + 1) * size_of::(); let argc_size = size_of::(); - let all_size = aux_vec_size + envp_pointers_size + argv_pointers_size + argc_size; + let all_size = + aux_vec_size + envp_pointers_size + argv_pointers_size + argc_size; let align_sp = (self.sp - all_size).align_down(16); self.sp = align_sp + all_size; @@ -588,7 +613,10 @@ impl<'a, T: ElfAddr + Clone + Copy> StackInitializer<'a, T> { self.mm_list .access_mut(VAddr::from(self.sp), size_of::(), |_, data| { data.copy_from_slice(unsafe { - core::slice::from_raw_parts(&val as *const _ as *const u8, data.len()) + core::slice::from_raw_parts( + &val as *const _ as *const u8, + data.len(), + ) }) }) .await?; From 9f74ee04eeb5b9d2288a47103d665a79667d6c76 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 13 Mar 2026 18:47:29 +0800 Subject: [PATCH 125/153] mm: add blanket AnonMapping impl To provide more features to anonymous mappings, we introduce an AnonMapping struct in replace of the Mapping::Anonymous enum. The new struct contains still nothing, but we can add stuff to it later. Signed-off-by: greatbridf --- src/kernel/mem.rs | 3 ++- src/kernel/mem/mm_list.rs | 3 ++- src/kernel/mem/mm_list/brk.rs | 4 ++-- src/kernel/mem/mm_list/mapping.rs | 11 ++++++++++- src/kernel/syscall/mm.rs | 4 ++-- src/kernel/task/loader/elf.rs | 8 +++++--- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index 41d5a17d..ab0e150b 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -11,7 +11,8 @@ mod page_cache; pub use access::PhysAccess; pub use folio::{Folio, FolioOwned, LockedFolio}; pub use mm_list::{ - handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission, + handle_kernel_page_fault, AnonMapping, FileMapping, MMList, Mapping, + Permission, }; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache, PageOffset}; diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 7b3c2de0..56a5f069 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -20,7 +20,8 @@ use mm_area::AreaList; use page_table::KernelPageTable; pub use self::mapping::{ - add_mapping, duplicate_mapping, remove_mapping, FileMapping, Mapping, + add_mapping, duplicate_mapping, remove_mapping, AnonMapping, FileMapping, + Mapping, }; pub use self::mm_area::{AreaFlags, MemArea}; pub use self::page_fault::handle_kernel_page_fault; diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index ede89b19..1212046e 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -5,7 +5,7 @@ use eonix_mm::address::{VAddr, VRange}; use crate::kernel::mem::address::VRangeExt; use crate::kernel::mem::mm_list::mm_area::AreaList; use crate::kernel::mem::mm_list::{AreaFlags, MemArea, MemListLock}; -use crate::kernel::mem::{MMList, Mapping, Permission}; +use crate::kernel::mem::{AnonMapping, MMList, Permission}; #[derive(Clone)] pub struct ProgramBreak { @@ -106,7 +106,7 @@ fn expand_create_area( }, false, ), - Mapping::Anonymous, + AnonMapping::new(), )); areas.insert_new(area); diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index b9a9ad29..33b3c8a8 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -13,14 +13,23 @@ pub struct FileMapping { pub length: usize, } +#[derive(Debug, Clone)] +pub struct AnonMapping(); + #[derive(Debug, Clone)] pub enum Mapping { // private anonymous memory - Anonymous, + Anonymous(AnonMapping), // file-backed memory or shared anonymous memory(tmp file) File(FileMapping), } +impl AnonMapping { + pub const fn new() -> Mapping { + Mapping::Anonymous(Self()) + } +} + impl FileMapping { pub fn new( page_cache: Arc, offset: usize, length: usize, diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index bd756048..e98bb061 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -7,7 +7,7 @@ use super::FromSyscallArg; use crate::kernel::constants::{ UserMmapFlags, UserMmapProtocol, EBADF, EINVAL, }; -use crate::kernel::mem::{FileMapping, Mapping, Permission}; +use crate::kernel::mem::{AnonMapping, FileMapping, Mapping, Permission}; use crate::kernel::task::Thread; use crate::kernel::vfs::filearray::FD; use crate::prelude::*; @@ -54,7 +54,7 @@ async fn do_mmap2( } if !is_shared { - Mapping::Anonymous + AnonMapping::new() } else { unimplemented!("mmap MAP_ANONYMOUS | MAP_SHARED"); } diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index 9489940a..0da7675d 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -11,7 +11,9 @@ use xmas_elf::program::{self, ProgramHeader32, ProgramHeader64}; use super::{LoadInfo, ELF_MAGIC}; use crate::io::{ByteBuffer, UninitBuffer}; use crate::kernel::constants::ENOEXEC; -use crate::kernel::mem::{FileMapping, MMList, Mapping, Permission}; +use crate::kernel::mem::{ + AnonMapping, FileMapping, MMList, Mapping, Permission, +}; use crate::kernel::task::loader::aux_vec::{AuxKey, AuxVec}; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::FsContext; @@ -269,7 +271,7 @@ impl Elf { .mmap_fixed( VAddr::from(E::STACK_BASE_ADDR - INIT_STACK_SIZE), INIT_STACK_SIZE, - Mapping::Anonymous, + AnonMapping::new(), Permission { read: true, write: true, @@ -392,7 +394,7 @@ impl Elf { .mmap_fixed( vmap_start + file_len, vmem_len - file_len, - Mapping::Anonymous, + AnonMapping::new(), permission, false, ) From cae61f4c23e5f147c7c92386cfa9e0a0ae4e9148 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 13 Mar 2026 18:49:04 +0800 Subject: [PATCH 126/153] style: remove unused imports in mm_area.rs Remove Folio imports. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 213bf396..168a8836 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -14,7 +14,6 @@ use intrusive_collections::{ }; use super::Mapping; -use crate::kernel::mem::folio::Folio; use crate::kernel::mem::mm_list::mapping::add_mapping; use crate::kernel::mem::mm_list::{remove_mapping, MemListLock}; use crate::kernel::mem::{ From a654701783d0c676b03ed450f21a144694cf3e1a Mon Sep 17 00:00:00 2001 From: greatbridf Date: Tue, 17 Mar 2026 23:34:36 +0800 Subject: [PATCH 127/153] mm: rename insert_new() to insert() and remove the old one There used to be two `insert()` impls due to historical reasons that we need to operate on raw `MemArea`s. We hope to remove raw MemArea handles, so do the cleanups. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 2 +- src/kernel/mem/mm_list/brk.rs | 8 +++----- src/kernel/mem/mm_list/mm_area.rs | 31 ++++++++++--------------------- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 56a5f069..18eaa5be 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -191,7 +191,7 @@ impl MMListInner { area.set_permission(permission); // Insert it back. - self.areas.insert_new(area); + self.areas.insert(area); } if !found { diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index 1212046e..1fad4dcb 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -1,5 +1,3 @@ -use alloc::sync::Arc; - use eonix_mm::address::{VAddr, VRange}; use crate::kernel::mem::address::VRangeExt; @@ -96,7 +94,7 @@ async fn set_break( fn expand_create_area( areas: &mut AreaList, brk: &mut ProgramBreak, new_range: VRange, ) -> VAddr { - let area = Arc::new(MemArea::new( + let area = MemArea::new( new_range, AreaFlags::from_old( Permission { @@ -107,9 +105,9 @@ fn expand_create_area( false, ), AnonMapping::new(), - )); + ); - areas.insert_new(area); + areas.insert(area); brk.set(new_range.end()); new_range.end() diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 168a8836..30df0b92 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -199,12 +199,15 @@ impl AreaList { } } - pub fn insert_new(&mut self, area: Arc) { - let range = area.range.as_ref(&self.lock).clone(); + pub fn insert(&mut self, area: Arc) { + let lock = area.lock.try_lock().expect("Insert called on busy area"); - match self.areas.entry(&range) { - Entry::Occupied(_) => panic!("Overlapping mem area: {range:?}."), + match self.areas.entry(area.range.as_ref(&*lock)) { + Entry::Occupied(_) => { + panic!("Overlapping mem area: {:?}.", area.range.as_ref(&*lock)) + } Entry::Vacant(insert_cursor) => { + drop(lock); insert_cursor.insert(area); } } @@ -338,18 +341,6 @@ impl AreaList { }) } - // TODO: For backwards compatibility. Remove this. - pub fn insert(&mut self, area: MemArea) { - let range = area.range.as_ref(&self.lock).clone(); - - match self.areas.entry(&range) { - Entry::Occupied(_) => panic!("Overlapping mem area: {:?}.", range), - Entry::Vacant(insert_cursor) => { - insert_cursor.insert(Arc::new(area)); - } - } - } - // TODO: For backwards compatibility. Remove this. pub fn deep_clone(&self, lock: &MemListLock) -> Self { let mut areas = RBTree::new(AreasAdapter::NEW); @@ -377,16 +368,14 @@ impl AreaList { } impl MemArea { - pub const fn new( - range: VRange, flags: AreaFlags, mapping: Mapping, - ) -> Self { - Self { + pub fn new(range: VRange, flags: AreaFlags, mapping: Mapping) -> Arc { + Arc::new(Self { range: RangeProtected::new(range), flags: AtomicFlags::new(flags), lock: Mutex::new(AreaLock::_new()), link: Link::rbtree(), mapping, - } + }) } pub fn set_permission(&self, perm: Permission) { From b8af8192175c45ca5df81fefc4d6f9f02c708540 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 20 Mar 2026 12:01:52 +0800 Subject: [PATCH 128/153] mm: add helper function to create anon mapping Add a helper to create `AnonMapping` struct. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mapping.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 33b3c8a8..54b682b3 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -25,8 +25,12 @@ pub enum Mapping { } impl AnonMapping { + const fn _new() -> Self { + Self() + } + pub const fn new() -> Mapping { - Mapping::Anonymous(Self()) + Mapping::Anonymous(Self::_new()) } } From 74d6122acffef7539ecf127d5ed714d740fc3daa Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 20 Mar 2026 12:04:02 +0800 Subject: [PATCH 129/153] mm: update mappings' offsets when splitting areas I notice that when we isolate the areas for protecting or unmapping, we forgot to update the underlying mappings' offsets. This may cause wrong offset passed to read() in fault handlers, despite we haven't observed the problem yet. Fix this by refactoring the isolate() logic: we recreate every area within the isolate range, call newly defined split functions and handle the offset correctly. Fixes: 9b18fc867f2e ("WIP: mem: remove unused MemArea member functions") Fixes: f37b288240ba ("WIP: mem: reimpl `MMList::protect` using `isolate()`") Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mapping.rs | 40 ++++++++-- src/kernel/mem/mm_list/mm_area.rs | 119 +++++++++++++++++------------- 2 files changed, 100 insertions(+), 59 deletions(-) diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 54b682b3..298baa89 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -32,6 +32,10 @@ impl AnonMapping { pub const fn new() -> Mapping { Mapping::Anonymous(Self::_new()) } + + fn split(&self, _offset: usize) -> (Self, Self) { + (Self::_new(), Self::_new()) + } } impl FileMapping { @@ -46,15 +50,35 @@ impl FileMapping { } } - pub fn offset(&self, offset: usize) -> Self { - if self.length <= offset { - Self::new(self.page_cache.clone(), self.offset + self.length, 0) + fn split(&self, offset: usize) -> (Self, Self) { + let (left_len, right_len); + + if offset >= self.length { + left_len = self.length; + right_len = 0; } else { - Self::new( - self.page_cache.clone(), - self.offset + offset, - self.length - offset, - ) + left_len = offset; + right_len = self.length - offset; + } + + ( + Self::new(self.page_cache.clone(), self.offset, left_len), + Self::new(self.page_cache.clone(), self.offset + offset, right_len), + ) + } +} + +impl Mapping { + pub(super) fn split(&self, offset: usize) -> (Self, Self) { + match self { + Mapping::Anonymous(anon_mapping) => { + let (l, r) = anon_mapping.split(offset); + (Self::Anonymous(l), Self::Anonymous(r)) + } + Mapping::File(file_mapping) => { + let (l, r) = file_mapping.split(offset); + (Self::File(l), Self::File(r)) + } } } } diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 30df0b92..50fb841c 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -267,9 +267,9 @@ impl AreaList { &mut self, isolate_range: &VRange, lock: &mut MemListLock, ) -> impl Iterator> { let mut ret_areas = LinkedList::new(ListAdapter::NEW); - let begin = VRange::from(isolate_range.start()); - - let mut cursor = self.areas.upper_bound_mut(Bound::Included(&begin)); + let mut cursor = self.areas.upper_bound_mut(Bound::Included( + &VRange::from(isolate_range.start()), + )); while !cursor.is_null() { let range = { @@ -288,48 +288,44 @@ impl AreaList { range.clone() }; - let area = cursor.as_cursor().clone_pointer().unwrap(); - let list_lock = &mut self.lock; - let mut area_lock = area.lock.lock().await; - let area_lock = &mut area_lock; - - let (l, m, r) = range.mask_with_checked(isolate_range).unwrap(); - - match (l, r) { - (None, None) => { - // Fully covered, just remove it. - let area = cursor.remove().unwrap(); - unsafe { - area.link.to_list(); - } - ret_areas.push_back(area); - continue; - } - (None, Some(rem)) | (Some(rem), None) => { - // Overflow on one side, change the old area's range and - // return the newly created area. - let range = area.range.as_mut(lock, list_lock, area_lock); - *range = rem; - } - (Some(left), Some(right)) => { - // Overflow on both sides, change the old area's range to - // the left part, create a new area for the right part and - // return the middle part. - let range = area.range.as_mut(lock, list_lock, area_lock); - *range = left; - - cursor.insert_after(area.clone_and_modify(lock, |area| { - area.range = RangeProtected::new(right); - })); - } + let mut area = cursor.remove().unwrap(); + + { + // Exclude all concurrent faulters. + area.lock.lock().await; + + // XXX: This is slightly racy and the assertion may not hold if + // one hasn't dropped the arc after releasing the lock. + // Let's observe if we can actually trigger this... + assert_eq!(Arc::strong_count(&area), 1, "Shared isolated area"); } - let ret_range = area.clone_and_modify(lock, |area| { - area.range = RangeProtected::new(m); - area.link = Link::list(); - }); + let left_overflow = range.start() < isolate_range.start(); + let right_overflow = range.end() > isolate_range.end(); - ret_areas.push_back(ret_range); + if left_overflow { + let left; + let offset = isolate_range.start() - range.start(); + + (left, area) = area.split(offset, lock); + + cursor.insert_before(left); + } + + if right_overflow { + let right; + let offset = isolate_range.end() - range.start(); + + (area, right) = area.split(offset, lock); + + cursor.insert_before(right); + } + + unsafe { + // SAFETY: The area is unlinked and exclusive to us. + area.link.to_list(); + } + ret_areas.push_back(area); } // We are returning the area, and the area should be by default @@ -423,16 +419,37 @@ impl MemArea { } } - fn clone_and_modify( - &self, lock: &MemListLock, modify: impl FnOnce(&mut MemArea), - ) -> Arc { - let mut arc = Arc::new(self.clone(lock)); - let arc_mut = unsafe { - // SAFETY: We are the only owner. - Arc::get_mut(&mut arc).unwrap_unchecked() + fn split( + &self, offset: usize, lock: &MemListLock, + ) -> (Arc, Arc) { + let (begin, mid, end) = { + let range = self.range.as_ref(lock); + let begin = range.start(); + let mid = begin + offset; + let end = range.end(); + + (begin, mid, end) }; - modify(arc_mut); - arc + + let (left_mapping, right_mapping) = self.mapping.split(offset); + + let left = Arc::new(Self { + range: RangeProtected::new(VRange::new(begin, mid)), + flags: self.flags.clone(), + lock: Mutex::new(AreaLock::_new()), + link: Link::rbtree(), + mapping: left_mapping, + }); + + let right = Arc::new(Self { + range: RangeProtected::new(VRange::new(mid, end)), + flags: self.flags.clone(), + lock: Mutex::new(AreaLock::_new()), + link: Link::rbtree(), + mapping: right_mapping, + }); + + (left, right) } pub fn handle_cow(&self, pfn: &mut PFN, attr: &mut PageAttribute) { From f53d61c228812116ebdb67c031d10ec06396756e Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 21 Mar 2026 10:02:40 +0800 Subject: [PATCH 130/153] mm: create `MemArea`s using new() func Replace all MemArea construction with `Self::new()` calls so we can modify it more easily. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 34 +++++++++---------------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 50fb841c..2fd7471f 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -342,7 +342,7 @@ impl AreaList { let mut areas = RBTree::new(AreasAdapter::NEW); for area in self.areas.iter() { - areas.insert(Arc::new(area.clone(lock))); + areas.insert(area.clone(lock)); } Self { @@ -409,14 +409,12 @@ impl MemArea { self.flags.load().contains(AreaFlags::EXECUTE) } - pub fn clone(&self, lock: &MemListLock) -> Self { - Self { - range: RangeProtected::new(self.range.as_ref(lock).clone()), - flags: self.flags.clone(), - lock: Mutex::new(AreaLock::_new()), - link: Link::rbtree(), - mapping: self.mapping.clone(), - } + pub fn clone(&self, lock: &MemListLock) -> Arc { + Self::new( + self.range.as_ref(lock).clone(), + self.flags.load(), + self.mapping.clone(), + ) } fn split( @@ -431,23 +429,11 @@ impl MemArea { (begin, mid, end) }; + let flags = self.flags.load(); let (left_mapping, right_mapping) = self.mapping.split(offset); - let left = Arc::new(Self { - range: RangeProtected::new(VRange::new(begin, mid)), - flags: self.flags.clone(), - lock: Mutex::new(AreaLock::_new()), - link: Link::rbtree(), - mapping: left_mapping, - }); - - let right = Arc::new(Self { - range: RangeProtected::new(VRange::new(mid, end)), - flags: self.flags.clone(), - lock: Mutex::new(AreaLock::_new()), - link: Link::rbtree(), - mapping: right_mapping, - }); + let left = Self::new(VRange::new(begin, mid), flags, left_mapping); + let right = Self::new(VRange::new(mid, end), flags, right_mapping); (left, right) } From 0303cf3c29ba1407cfbf13dfd0ddd6527795d341 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 21 Mar 2026 10:29:20 +0800 Subject: [PATCH 131/153] mm: make handle_cow() private Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mm_area.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 2fd7471f..c155d0a6 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -438,7 +438,7 @@ impl MemArea { (left, right) } - pub fn handle_cow(&self, pfn: &mut PFN, attr: &mut PageAttribute) { + fn handle_cow(&self, pfn: &mut PFN, attr: &mut PageAttribute) { assert!(attr.contains(PageAttribute::COPY_ON_WRITE)); attr.remove(PageAttribute::COPY_ON_WRITE); From 5ff19c9e21c5114de2dc0d5ffd8175566cceecbf Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 21 Mar 2026 17:48:36 +0800 Subject: [PATCH 132/153] mm: add a helper to create new MMLists Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 18eaa5be..74d01a90 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -271,35 +271,39 @@ impl MMList { } } - pub fn new() -> Self { - let page_table = KernelPageTable::new(); + #[inline(always)] + fn _new( + areas: AreaList, page_table: KernelPageTable, prog_break: ProgramBreak, + ) -> Self { Self { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { lock: MemListLock::_new(), - areas: AreaList::new(), - prog_break: ProgramBreak::null(), + areas, page_table, + prog_break, })), } } + pub fn new() -> Self { + Self::_new( + AreaList::new(), + KernelPageTable::new(), + ProgramBreak::null(), + ) + } + pub async fn new_cloned(&self) -> Self { let inner = self.inner.borrow(); let mut inner = inner.lock().await; - let page_table = KernelPageTable::new(); - let list = Self { - root_page_table: AtomicUsize::from(page_table.addr().addr()), - user_count: AtomicUsize::new(0), - inner: ArcSwap::new(Mutex::new(MMListInner { - lock: MemListLock::_new(), - areas: inner.areas.deep_clone(&inner.lock), - page_table, - prog_break: inner.prog_break.clone(), - })), - }; + let list = Self::_new( + inner.areas.deep_clone(&inner.lock), + KernelPageTable::new(), + inner.prog_break.clone(), + ); { let list_inner = list.inner.borrow(); From 543d91298f8b90d8a8cf83289891293d12089eb5 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 21 Mar 2026 17:59:28 +0800 Subject: [PATCH 133/153] mm: rework MMList creation while doing clones I extract the part cloning the area list and processing cloned areas ( particularly creating new PTEs in the new pgtable and modifying the old ones in the old table) into a separate function where shared areas and private areas are handled differently. Then, creation of new MMLists can be put off till the area list and pgtable are ready. `MemArea::deep_clone()` exists only for creating cloned MMLists, thus can be removed. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 39 ++++++++++---------------- src/kernel/mem/mm_list/mm_area.rs | 46 +++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 74d01a90..9c2f765e 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -29,6 +29,7 @@ use super::address::{VAddrExt as _, VRangeExt as _}; use super::Folio; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; use crate::kernel::mem::mm_list::brk::ProgramBreak; +use crate::kernel::mem::mm_list::mm_area::dup_area_to_list; use crate::prelude::*; use crate::sync::ArcSwap; @@ -297,37 +298,25 @@ impl MMList { pub async fn new_cloned(&self) -> Self { let inner = self.inner.borrow(); - let mut inner = inner.lock().await; - - let list = Self::_new( - inner.areas.deep_clone(&inner.lock), - KernelPageTable::new(), - inner.prog_break.clone(), - ); - - { - let list_inner = list.inner.borrow(); - let list_inner = list_inner.lock().await; - - let pgtable = &list_inner.page_table; - - for area in list_inner.areas.iter() { - // Skip shared areas because they can be filled just as new - // entries in page faults. - if area.is_shared() { - continue; - } - - let range = area.range.as_ref(&list_inner.lock).clone(); + let inner = inner.lock().await; - pgtable.set_copy_on_write(&mut inner.page_table, range); - } + let mut new_areas = AreaList::new(); + let new_pgtable = KernelPageTable::new(); + + for area in inner.areas.iter() { + dup_area_to_list( + area, + &mut new_areas, + &inner.lock, + &inner.page_table, + &new_pgtable, + ); } // We've set some pages as CoW, so we need to invalidate all our users' TLB. self.flush_user_tlbs().await; - list + Self::_new(new_areas, new_pgtable, inner.prog_break.clone()) } pub async fn new_shared(&self) -> Self { diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index c155d0a6..00013871 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -15,7 +15,8 @@ use intrusive_collections::{ use super::Mapping; use crate::kernel::mem::mm_list::mapping::add_mapping; -use crate::kernel::mem::mm_list::{remove_mapping, MemListLock}; +use crate::kernel::mem::mm_list::page_table::KernelPageTable; +use crate::kernel::mem::mm_list::{remove_mapping, MemListLock, PageTableExt}; use crate::kernel::mem::{ CachePage, FileMapping, FolioOwned, PageOffset, Permission, }; @@ -337,20 +338,6 @@ impl AreaList { }) } - // TODO: For backwards compatibility. Remove this. - pub fn deep_clone(&self, lock: &MemListLock) -> Self { - let mut areas = RBTree::new(AreasAdapter::NEW); - - for area in self.areas.iter() { - areas.insert(area.clone(lock)); - } - - Self { - areas, - lock: AreaListLock { _phantom: () }, - } - } - pub fn iter(&self) -> impl Iterator { let mut cursor = self.areas.front(); @@ -658,3 +645,32 @@ impl Link { } } } + +fn dup_area_shared<'a, 'b: 'a>( + area: &'a MemArea, list: &mut AreaList, from_lock: &'b MemListLock, +) { + // Shared areas can be filled in faults anyway. + // Just cloning the area is enough. + list.insert(area.clone(from_lock)); +} + +fn dup_area_private<'a, 'b: 'a>( + area: &'a MemArea, list: &mut AreaList, from_lock: &'b MemListLock, + from_pgtable: &KernelPageTable, to_pgtable: &KernelPageTable, +) { + list.insert(area.clone(from_lock)); + + to_pgtable.set_copy_on_write(from_pgtable, *area.range.as_ref(from_lock)); +} + +pub fn dup_area_to_list<'a, 'b: 'a>( + area: &'a MemArea, list: &mut AreaList, from_lock: &'b MemListLock, + from_pgtable: &KernelPageTable, to_pgtable: &KernelPageTable, +) { + if area.is_shared() { + dup_area_shared(area, list, from_lock); + return; + } + + dup_area_private(area, list, from_lock, from_pgtable, to_pgtable); +} From 6f8e43c696312c4c88275748a7a437eef7d95cac Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 22 Mar 2026 14:24:06 +0800 Subject: [PATCH 134/153] mm: add a macro to impl Deref for transparent wrappers For repr(transparent) types that need to impl Deref trait and returns &self.0, we introduce a macro called TransparentDeref to derive Deref for them and use the macro. Signed-off-by: greatbridf --- macros/src/lib.rs | 35 +++++++++++++++++++++++++++- src/kernel/mem/folio.rs | 11 ++------- src/kernel/mem/mm_list/page_table.rs | 16 ++++--------- src/kernel/mem/page_cache.rs | 12 +++------- 4 files changed, 44 insertions(+), 30 deletions(-) diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 42f72213..78fa481e 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,7 +2,9 @@ extern crate proc_macro; use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{parse2, FnArg, Ident, ItemFn, LitStr, ReturnType, Signature}; +use syn::{ + parse2, FnArg, Ident, ItemFn, ItemStruct, LitStr, ReturnType, Signature, +}; fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { if attrs.is_empty() { @@ -297,3 +299,34 @@ pub fn define_late_init( ) -> proc_macro::TokenStream { define_late_init_impl(attrs.into(), func.into()).into() } + +fn define_transparent_deref_impl(items: TokenStream) -> TokenStream { + let def = parse2::(items).expect("expected struct definition"); + + let ident = &def.ident; + + assert!(def.fields.len() == 1, "expected only 1 field"); + + let field = def.fields.iter().next().unwrap(); + + assert!(field.ident.is_none(), "expected tuple structs"); + + let inner = &field.ty; + + quote! { + impl core::ops::Deref for #ident { + type Target = #inner; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + } +} + +#[proc_macro_derive(TransparentDeref)] +pub fn define_transparent_deref( + items: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + define_transparent_deref_impl(items.into()).into() +} diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs index 9ab80c63..e6f893be 100644 --- a/src/kernel/mem/folio.rs +++ b/src/kernel/mem/folio.rs @@ -4,6 +4,7 @@ use core::ops::Deref; use core::ptr::NonNull; use core::sync::atomic::Ordering; +use eonix_macros::TransparentDeref; use eonix_mm::paging::{ Folio as FolioTrait, FrameAlloc, GlobalFrameAlloc, Zone, PFN, }; @@ -16,7 +17,7 @@ use crate::kernel::mem::page_alloc::PageFlags; #[repr(transparent)] pub struct Folio(NonNull); -#[derive(Debug)] +#[derive(Debug, TransparentDeref)] #[repr(transparent)] pub struct FolioOwned(Folio); @@ -236,11 +237,3 @@ impl FolioOwned { self.0 } } - -impl Deref for FolioOwned { - type Target = Folio; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/src/kernel/mem/mm_list/page_table.rs b/src/kernel/mem/mm_list/page_table.rs index 8a2acc13..3f8a6c5f 100644 --- a/src/kernel/mem/mm_list/page_table.rs +++ b/src/kernel/mem/mm_list/page_table.rs @@ -1,14 +1,16 @@ -use core::ops::Deref; - use eonix_hal::arch_exported::mm::{ArchPagingMode, PageAccessImpl}; use eonix_hal::mm::GLOBAL_PAGE_TABLE; +use eonix_macros::TransparentDeref; use eonix_mm::page_table::PageTable; use eonix_mm::paging::{Folio, GlobalFrameAlloc}; use crate::kernel::mem::{FolioOwned, GlobalPageAlloc, PhysAccess}; #[repr(transparent)] -pub struct KernelPageTable(PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>); +#[derive(TransparentDeref)] +pub struct KernelPageTable( + PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>, +); impl KernelPageTable { pub fn new() -> Self { @@ -30,11 +32,3 @@ impl KernelPageTable { )) } } - -impl Deref for KernelPageTable { - type Target = PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 6e1c7a00..2ff4c11a 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,7 +1,7 @@ use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; -use core::ops::Deref; +use eonix_macros::TransparentDeref; use eonix_mm::paging::{PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::{atomic, Mutex}; @@ -22,6 +22,8 @@ pub struct PageCache { inode: InodeUse, } +#[repr(transparent)] +#[derive(TransparentDeref)] pub struct CachePage(Folio); impl PageOffset { @@ -79,14 +81,6 @@ impl CachePage { } } -impl Deref for CachePage { - type Target = Folio; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - impl PageCache { pub fn new(inode: InodeUse) -> Self { Self { From c4dc88480a69611af9208ac7d30f710b18b208e3 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 22 Mar 2026 14:37:14 +0800 Subject: [PATCH 135/153] mm: don't expose AnonMapping outside of mem module Signed-off-by: greatbridf --- src/kernel/mem.rs | 3 +-- src/kernel/mem/mm_list.rs | 3 +-- src/kernel/mem/mm_list/brk.rs | 4 ++-- src/kernel/mem/mm_list/mapping.rs | 12 ++++++------ src/kernel/syscall/mm.rs | 4 ++-- src/kernel/task/loader/elf.rs | 8 +++----- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index ab0e150b..41d5a17d 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -11,8 +11,7 @@ mod page_cache; pub use access::PhysAccess; pub use folio::{Folio, FolioOwned, LockedFolio}; pub use mm_list::{ - handle_kernel_page_fault, AnonMapping, FileMapping, MMList, Mapping, - Permission, + handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission, }; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache, PageOffset}; diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 9c2f765e..1d5c58f0 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -20,8 +20,7 @@ use mm_area::AreaList; use page_table::KernelPageTable; pub use self::mapping::{ - add_mapping, duplicate_mapping, remove_mapping, AnonMapping, FileMapping, - Mapping, + add_mapping, duplicate_mapping, remove_mapping, FileMapping, Mapping, }; pub use self::mm_area::{AreaFlags, MemArea}; pub use self::page_fault::handle_kernel_page_fault; diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs index 1fad4dcb..9640995e 100644 --- a/src/kernel/mem/mm_list/brk.rs +++ b/src/kernel/mem/mm_list/brk.rs @@ -3,7 +3,7 @@ use eonix_mm::address::{VAddr, VRange}; use crate::kernel::mem::address::VRangeExt; use crate::kernel::mem::mm_list::mm_area::AreaList; use crate::kernel::mem::mm_list::{AreaFlags, MemArea, MemListLock}; -use crate::kernel::mem::{AnonMapping, MMList, Permission}; +use crate::kernel::mem::{MMList, Mapping, Permission}; #[derive(Clone)] pub struct ProgramBreak { @@ -104,7 +104,7 @@ fn expand_create_area( }, false, ), - AnonMapping::new(), + Mapping::new_anon(), ); areas.insert(area); diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 298baa89..89104db2 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -25,16 +25,12 @@ pub enum Mapping { } impl AnonMapping { - const fn _new() -> Self { + const fn new() -> Self { Self() } - pub const fn new() -> Mapping { - Mapping::Anonymous(Self::_new()) - } - fn split(&self, _offset: usize) -> (Self, Self) { - (Self::_new(), Self::_new()) + (Self::new(), Self::new()) } } @@ -69,6 +65,10 @@ impl FileMapping { } impl Mapping { + pub fn new_anon() -> Self { + Self::Anonymous(AnonMapping::new()) + } + pub(super) fn split(&self, offset: usize) -> (Self, Self) { match self { Mapping::Anonymous(anon_mapping) => { diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index e98bb061..9f27cd8b 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -7,7 +7,7 @@ use super::FromSyscallArg; use crate::kernel::constants::{ UserMmapFlags, UserMmapProtocol, EBADF, EINVAL, }; -use crate::kernel::mem::{AnonMapping, FileMapping, Mapping, Permission}; +use crate::kernel::mem::{FileMapping, Mapping, Permission}; use crate::kernel::task::Thread; use crate::kernel::vfs::filearray::FD; use crate::prelude::*; @@ -54,7 +54,7 @@ async fn do_mmap2( } if !is_shared { - AnonMapping::new() + Mapping::new_anon() } else { unimplemented!("mmap MAP_ANONYMOUS | MAP_SHARED"); } diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index 0da7675d..e9efbfcd 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -11,9 +11,7 @@ use xmas_elf::program::{self, ProgramHeader32, ProgramHeader64}; use super::{LoadInfo, ELF_MAGIC}; use crate::io::{ByteBuffer, UninitBuffer}; use crate::kernel::constants::ENOEXEC; -use crate::kernel::mem::{ - AnonMapping, FileMapping, MMList, Mapping, Permission, -}; +use crate::kernel::mem::{FileMapping, MMList, Mapping, Permission}; use crate::kernel::task::loader::aux_vec::{AuxKey, AuxVec}; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::FsContext; @@ -271,7 +269,7 @@ impl Elf { .mmap_fixed( VAddr::from(E::STACK_BASE_ADDR - INIT_STACK_SIZE), INIT_STACK_SIZE, - AnonMapping::new(), + Mapping::new_anon(), Permission { read: true, write: true, @@ -394,7 +392,7 @@ impl Elf { .mmap_fixed( vmap_start + file_len, vmem_len - file_len, - AnonMapping::new(), + Mapping::new_anon(), permission, false, ) From 9c6626e3efaf5571353a7be3a1d1d64bb01c1d47 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 22 Mar 2026 14:47:54 +0800 Subject: [PATCH 136/153] mm: move PageOffset definition to mod types Signed-off-by: greatbridf --- src/kernel/mem.rs | 4 +++- src/kernel/mem/page_cache.rs | 31 ++---------------------------- src/kernel/mem/types.rs | 37 ++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 30 deletions(-) create mode 100644 src/kernel/mem/types.rs diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index 41d5a17d..033060ee 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -7,6 +7,7 @@ mod folio; mod mm_list; mod page_alloc; mod page_cache; +mod types; pub use access::PhysAccess; pub use folio::{Folio, FolioOwned, LockedFolio}; @@ -14,5 +15,6 @@ pub use mm_list::{ handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission, }; pub use page_alloc::{GlobalPageAlloc, RawPage}; -pub use page_cache::{CachePage, PageCache, PageOffset}; +pub use page_cache::{CachePage, PageCache}; pub use paging::PageBuffer; +pub use types::PageOffset; diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 2ff4c11a..432581fd 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -2,7 +2,7 @@ use alloc::collections::btree_map::{BTreeMap, Entry}; use core::future::Future; use eonix_macros::TransparentDeref; -use eonix_mm::paging::{PAGE_SIZE, PAGE_SIZE_BITS, PFN}; +use eonix_mm::paging::{PAGE_SIZE, PFN}; use eonix_sync::{atomic, Mutex}; use super::page_alloc::PageFlags; @@ -10,13 +10,10 @@ use super::{Folio, FolioOwned}; use crate::io::{Buffer, Stream}; use crate::kernel::constants::EINVAL; use crate::kernel::mem::mm_list::add_mapping; +use crate::kernel::mem::PageOffset; use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; -#[repr(transparent)] -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct PageOffset(usize); - pub struct PageCache { pages: Mutex>, inode: InodeUse, @@ -26,30 +23,6 @@ pub struct PageCache { #[derive(TransparentDeref)] pub struct CachePage(Folio); -impl PageOffset { - pub const fn from_byte_floor(offset: usize) -> Self { - Self(offset >> PAGE_SIZE_BITS) - } - - pub const fn from_byte_ceil(offset: usize) -> Self { - Self((offset + PAGE_SIZE - 1) >> PAGE_SIZE_BITS) - } - - pub fn iter_till( - self, end: PageOffset, - ) -> impl Iterator { - (self.0..end.0).map(PageOffset) - } - - pub fn page_count(self) -> usize { - self.0 - } - - pub fn byte_count(self) -> usize { - self.page_count() * PAGE_SIZE - } -} - impl CachePage { pub fn new() -> Self { CachePage(Folio::alloc()) diff --git a/src/kernel/mem/types.rs b/src/kernel/mem/types.rs new file mode 100644 index 00000000..88163335 --- /dev/null +++ b/src/kernel/mem/types.rs @@ -0,0 +1,37 @@ +use core::fmt::Debug; + +use eonix_mm::paging::{PAGE_SIZE, PAGE_SIZE_BITS}; + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct PageOffset(usize); + +impl PageOffset { + pub const fn from_byte_floor(offset: usize) -> Self { + Self(offset >> PAGE_SIZE_BITS) + } + + pub const fn from_byte_ceil(offset: usize) -> Self { + Self((offset + PAGE_SIZE - 1) >> PAGE_SIZE_BITS) + } + + pub fn iter_till( + self, end: PageOffset, + ) -> impl Iterator { + (self.0..end.0).map(PageOffset) + } + + pub fn page_count(self) -> usize { + self.0 + } + + pub fn byte_count(self) -> usize { + self.page_count() * PAGE_SIZE + } +} + +impl Debug for PageOffset { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "PageOffset({:#x})", self.0) + } +} From 68a0200150691f124ddcfe4357681fa4719de25b Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 22 Mar 2026 17:20:22 +0800 Subject: [PATCH 137/153] mm: don't expose FileMapping outside of mem module Signed-off-by: greatbridf --- src/kernel/mem.rs | 4 +--- src/kernel/mem/mm_list/mapping.rs | 10 +++++++--- src/kernel/mem/mm_list/mm_area.rs | 6 +++--- src/kernel/syscall/mm.rs | 4 ++-- src/kernel/task/loader/elf.rs | 6 +++--- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index 033060ee..e9b48390 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -11,9 +11,7 @@ mod types; pub use access::PhysAccess; pub use folio::{Folio, FolioOwned, LockedFolio}; -pub use mm_list::{ - handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission, -}; +pub use mm_list::{handle_kernel_page_fault, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache}; pub use paging::PageBuffer; diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 89104db2..506d8607 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -35,9 +35,7 @@ impl AnonMapping { } impl FileMapping { - pub fn new( - page_cache: Arc, offset: usize, length: usize, - ) -> Self { + fn new(page_cache: Arc, offset: usize, length: usize) -> Self { assert_eq!(offset & (PAGE_SIZE - 1), 0); Self { page_cache, @@ -69,6 +67,12 @@ impl Mapping { Self::Anonymous(AnonMapping::new()) } + pub fn new_file( + page_cache: Arc, offset: usize, length: usize, + ) -> Self { + Self::File(FileMapping::new(page_cache, offset, length)) + } + pub(super) fn split(&self, offset: usize) -> (Self, Self) { match self { Mapping::Anonymous(anon_mapping) => { diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 00013871..22ad2d54 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -16,10 +16,10 @@ use intrusive_collections::{ use super::Mapping; use crate::kernel::mem::mm_list::mapping::add_mapping; use crate::kernel::mem::mm_list::page_table::KernelPageTable; -use crate::kernel::mem::mm_list::{remove_mapping, MemListLock, PageTableExt}; -use crate::kernel::mem::{ - CachePage, FileMapping, FolioOwned, PageOffset, Permission, +use crate::kernel::mem::mm_list::{ + remove_mapping, FileMapping, MemListLock, PageTableExt, }; +use crate::kernel::mem::{CachePage, FolioOwned, PageOffset, Permission}; use crate::prelude::KResult; bitflags::bitflags! { diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index 9f27cd8b..2c699442 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -7,7 +7,7 @@ use super::FromSyscallArg; use crate::kernel::constants::{ UserMmapFlags, UserMmapProtocol, EBADF, EINVAL, }; -use crate::kernel::mem::{FileMapping, Mapping, Permission}; +use crate::kernel::mem::{Mapping, Permission}; use crate::kernel::task::Thread; use crate::kernel::vfs::filearray::FD; use crate::prelude::*; @@ -66,7 +66,7 @@ async fn do_mmap2( .get_inode()? .ok_or(EBADF)?; - Mapping::File(FileMapping::new(file.get_page_cache(), pgoffset, len)) + Mapping::new_file(file.get_page_cache(), pgoffset, len) }; let permission = Permission { diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index e9efbfcd..b42eb465 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -11,7 +11,7 @@ use xmas_elf::program::{self, ProgramHeader32, ProgramHeader64}; use super::{LoadInfo, ELF_MAGIC}; use crate::io::{ByteBuffer, UninitBuffer}; use crate::kernel::constants::ENOEXEC; -use crate::kernel::mem::{FileMapping, MMList, Mapping, Permission}; +use crate::kernel::mem::{MMList, Mapping, Permission}; use crate::kernel::task::loader::aux_vec::{AuxKey, AuxVec}; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::FsContext; @@ -364,11 +364,11 @@ impl Elf { .mmap_fixed( vmap_start, file_len, - Mapping::File(FileMapping::new( + Mapping::new_file( self.file.get_inode()?.get_page_cache(), file_offset, real_file_length, - )), + ), permission, false, ) From b9c4706c0958c4c47ff8a5346de3ca92d7259975 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 22 Mar 2026 17:26:30 +0800 Subject: [PATCH 138/153] mm: use PageOffset when creating file mappings Add a helper function to ensure alignment and create PageOffset. Then use PageOffset and the helper internally while creating file mappings. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mapping.rs | 39 +++++++++++++++++++++---------- src/kernel/mem/types.rs | 10 ++++++++ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 506d8607..22c1caa7 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -2,7 +2,7 @@ use alloc::sync::Arc; use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; -use crate::kernel::mem::{Folio, PageCache}; +use crate::kernel::mem::{Folio, PageCache, PageOffset}; #[derive(Debug, Clone)] pub struct FileMapping { @@ -29,35 +29,44 @@ impl AnonMapping { Self() } - fn split(&self, _offset: usize) -> (Self, Self) { + fn split(&self, _offset: PageOffset) -> (Self, Self) { (Self::new(), Self::new()) } } impl FileMapping { - fn new(page_cache: Arc, offset: usize, length: usize) -> Self { - assert_eq!(offset & (PAGE_SIZE - 1), 0); + fn new( + page_cache: Arc, offset: PageOffset, length: usize, + ) -> Self { Self { page_cache, - offset, + offset: offset.byte_count(), length, } } - fn split(&self, offset: usize) -> (Self, Self) { + fn split(&self, offset: PageOffset) -> (Self, Self) { let (left_len, right_len); - if offset >= self.length { + if offset.byte_count() >= self.length { left_len = self.length; right_len = 0; } else { - left_len = offset; - right_len = self.length - offset; + left_len = offset.byte_count(); + right_len = self.length - offset.byte_count(); } ( - Self::new(self.page_cache.clone(), self.offset, left_len), - Self::new(self.page_cache.clone(), self.offset + offset, right_len), + Self::new( + self.page_cache.clone(), + PageOffset::from_byte_floor(self.offset), + left_len, + ), + Self::new( + self.page_cache.clone(), + PageOffset::from_byte_floor(self.offset + offset.byte_count()), + right_len, + ), ) } } @@ -70,10 +79,16 @@ impl Mapping { pub fn new_file( page_cache: Arc, offset: usize, length: usize, ) -> Self { - Self::File(FileMapping::new(page_cache, offset, length)) + Self::File(FileMapping::new( + page_cache, + PageOffset::from_byte_aligned(offset), + length, + )) } pub(super) fn split(&self, offset: usize) -> (Self, Self) { + let offset = PageOffset::from_byte_aligned(offset); + match self { Mapping::Anonymous(anon_mapping) => { let (l, r) = anon_mapping.split(offset); diff --git a/src/kernel/mem/types.rs b/src/kernel/mem/types.rs index 88163335..c8301482 100644 --- a/src/kernel/mem/types.rs +++ b/src/kernel/mem/types.rs @@ -7,6 +7,16 @@ use eonix_mm::paging::{PAGE_SIZE, PAGE_SIZE_BITS}; pub struct PageOffset(usize); impl PageOffset { + /// Convert the byte offset into page offset. + /// + /// # Panics + /// The function call panics if `offset` is not aligned to page size. + pub fn from_byte_aligned(offset: usize) -> Self { + assert!((offset & (PAGE_SIZE - 1)) == 0, "Unaligned offset"); + + Self::from_byte_floor(offset) + } + pub const fn from_byte_floor(offset: usize) -> Self { Self(offset >> PAGE_SIZE_BITS) } From 750b909aa3477d68cc147d152237576390a508a6 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sun, 22 Mar 2026 17:35:40 +0800 Subject: [PATCH 139/153] mm: use PageOffset inside FileMapping and fault handlers Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mapping.rs | 18 +++++------------- src/kernel/mem/mm_list/mm_area.rs | 17 ++++++++++------- src/kernel/mem/types.rs | 9 +++++++++ 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 22c1caa7..2da3ec78 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,6 +1,6 @@ use alloc::sync::Arc; -use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; +use eonix_mm::paging::{Folio as _, PFN}; use crate::kernel::mem::{Folio, PageCache, PageOffset}; @@ -8,7 +8,7 @@ use crate::kernel::mem::{Folio, PageCache, PageOffset}; pub struct FileMapping { pub page_cache: Arc, /// Offset in the file, aligned to 4KB boundary. - pub offset: usize, + pub offset: PageOffset, /// Length of the mapping. Exceeding part will be zeroed. pub length: usize, } @@ -40,7 +40,7 @@ impl FileMapping { ) -> Self { Self { page_cache, - offset: offset.byte_count(), + offset, length, } } @@ -57,16 +57,8 @@ impl FileMapping { } ( - Self::new( - self.page_cache.clone(), - PageOffset::from_byte_floor(self.offset), - left_len, - ), - Self::new( - self.page_cache.clone(), - PageOffset::from_byte_floor(self.offset + offset.byte_count()), - right_len, - ), + Self::new(self.page_cache.clone(), self.offset, left_len), + Self::new(self.page_cache.clone(), self.offset + offset, right_len), ) } } diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 22ad2d54..d381e86f 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -459,13 +459,14 @@ impl MemArea { *pfn = add_mapping(new_page.share()); } - /// # Arguments - /// * `offset`: The offset from the start of the mapping, aligned to 4KB boundary. async fn missing_file( - &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: usize, + &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: PageOffset, write: bool, file_mapping: &FileMapping, ) -> KResult<()> { - assert!(offset < file_mapping.length, "Offset out of range"); + assert!( + offset.byte_count() < file_mapping.length, + "Offset out of range" + ); assert!(!attr.contains(PageAttribute::PRESENT)); let file_offset = file_mapping.offset + offset; @@ -508,7 +509,7 @@ impl MemArea { file_mapping .page_cache - .with_page(PageOffset::from_byte_floor(file_offset), map_page) + .with_page(file_offset, map_page) .await?; attr.insert(PageAttribute::PRESENT); @@ -546,7 +547,7 @@ impl MemArea { } async fn handle_missing( - &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: usize, + &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: PageOffset, write: bool, ) -> KResult<()> { assert!( @@ -580,11 +581,13 @@ impl MemArea { pub async fn handle( &self, pte: &mut impl PTE, offset: usize, write: bool, ) -> KResult<()> { + let offset = PageOffset::from_byte_aligned(offset); + // Exclude concurrent modifications and faults. // TODO: concurrent faults should be acceptable... let lock = self.lock.lock().await; assert!( - offset < self.range.as_ref(&*lock).len(), + offset.byte_count() < self.range.as_ref(&*lock).len(), "Offset out of range" ); diff --git a/src/kernel/mem/types.rs b/src/kernel/mem/types.rs index c8301482..b0b7d070 100644 --- a/src/kernel/mem/types.rs +++ b/src/kernel/mem/types.rs @@ -1,4 +1,5 @@ use core::fmt::Debug; +use core::ops::Add; use eonix_mm::paging::{PAGE_SIZE, PAGE_SIZE_BITS}; @@ -40,6 +41,14 @@ impl PageOffset { } } +impl Add for PageOffset { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + impl Debug for PageOffset { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "PageOffset({:#x})", self.0) From 855266310a54ff92cc3bd3e58f2760bf43cef53b Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 25 Mar 2026 22:46:46 +0800 Subject: [PATCH 140/153] mm: rework Mapping to divide it into 3 classes There are in fact 3 types of mapping: - File private: fault in from underlying files and modifications are not carried back to the file. After faulting in, the folios are associated to some anonymous mapping and CoW on forks. - File shared: fault in just as the previous one but all folios from the same mapping are shared from the page cache. - Anonymous: Always private. Shared anonymous mappings should use tmpfs + shared file mappings. Now add a variant to the enum Mapping, and unify the fault handling logic related to the mapping. Introduce new helpers to create the mappings. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mapping.rs | 52 ++++++++++++++++++++++++------- src/kernel/mem/mm_list/mm_area.rs | 12 ++++--- src/kernel/syscall/mm.rs | 29 +++++++++-------- src/kernel/task/loader/elf.rs | 2 +- 4 files changed, 64 insertions(+), 31 deletions(-) diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 2da3ec78..ecf8c98c 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -18,10 +18,21 @@ pub struct AnonMapping(); #[derive(Debug, Clone)] pub enum Mapping { - // private anonymous memory + /// Anonymous mappings that reside in memory only. + /// + /// All anonymous mappings are private. If shared mappings are needed, use + /// tmpfs together with shared file mappings. Anonymous(AnonMapping), - // file-backed memory or shared anonymous memory(tmp file) - File(FileMapping), + /// File backed mappings that is copy on write across duplications and not + /// written back to the underlying [Inode]. + /// + /// [Inode]: crate::kernel::vfs::inode::Inode + PrivateFile { + anon: AnonMapping, + file: FileMapping, + }, + /// Shared file mappings. + SharedFile(FileMapping), } impl AnonMapping { @@ -68,14 +79,17 @@ impl Mapping { Self::Anonymous(AnonMapping::new()) } - pub fn new_file( + pub fn new_file_priv( page_cache: Arc, offset: usize, length: usize, ) -> Self { - Self::File(FileMapping::new( - page_cache, - PageOffset::from_byte_aligned(offset), - length, - )) + Self::PrivateFile { + file: FileMapping::new( + page_cache, + PageOffset::from_byte_aligned(offset), + length, + ), + anon: AnonMapping::new(), + } } pub(super) fn split(&self, offset: usize) -> (Self, Self) { @@ -86,9 +100,25 @@ impl Mapping { let (l, r) = anon_mapping.split(offset); (Self::Anonymous(l), Self::Anonymous(r)) } - Mapping::File(file_mapping) => { + Mapping::PrivateFile { file, anon } => { + let (l_file, r_file) = file.split(offset); + let (l_anon, r_anon) = anon.split(offset); + + ( + Self::PrivateFile { + file: l_file, + anon: l_anon, + }, + Self::PrivateFile { + file: r_file, + anon: r_anon, + }, + ) + } + Mapping::SharedFile(file_mapping) => { let (l, r) = file_mapping.split(offset); - (Self::File(l), Self::File(r)) + + (Self::SharedFile(l), Self::SharedFile(r)) } } } diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index d381e86f..2727c013 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -557,10 +557,14 @@ impl MemArea { attr.insert(PageAttribute::USER); - if let Mapping::File(mapping) = &self.mapping { - self.missing_file(pfn, attr, offset, write, mapping).await?; - } else { - self.missing_anon(pfn, attr); + match &self.mapping { + Mapping::Anonymous(_) => { + self.missing_anon(pfn, attr); + } + Mapping::PrivateFile { file: mapping, .. } + | Mapping::SharedFile(mapping) => { + self.missing_file(pfn, attr, offset, write, mapping).await?; + } } assert!(attr.contains(PageAttribute::PRESENT)); diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index 2c699442..f0c19cde 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -46,28 +46,27 @@ async fn do_mmap2( let len = len.align_up(PAGE_SIZE); let mm_list = &thread.process.mm_list; + let mapping; + let is_shared = flags.contains(UserMmapFlags::MAP_SHARED); + let is_anonymous = flags.contains(UserMmapFlags::MAP_ANONYMOUS); + + if is_shared { + unimplemented!("Shared mappings"); + } - let mapping = if flags.contains(UserMmapFlags::MAP_ANONYMOUS) { + if is_anonymous { if pgoffset != 0 { return Err(EINVAL); } - if !is_shared { - Mapping::new_anon() - } else { - unimplemented!("mmap MAP_ANONYMOUS | MAP_SHARED"); - } + mapping = Mapping::new_anon(); } else { - let file = thread - .files - .get(fd) - .ok_or(EBADF)? - .get_inode()? - .ok_or(EBADF)?; - - Mapping::new_file(file.get_page_cache(), pgoffset, len) - }; + let fd = thread.files.get(fd).ok_or(EBADF)?; + let file = fd.get_inode()?.ok_or(EBADF)?; + + mapping = Mapping::new_file_priv(file.get_page_cache(), pgoffset, len); + } let permission = Permission { read: prot.contains(UserMmapProtocol::PROT_READ), diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index b42eb465..2f730a99 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -364,7 +364,7 @@ impl Elf { .mmap_fixed( vmap_start, file_len, - Mapping::new_file( + Mapping::new_file_priv( self.file.get_inode()?.get_page_cache(), file_offset, real_file_length, From 0c992f5ba91c5a25553f2b505848354db84e659b Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 26 Mar 2026 00:44:43 +0800 Subject: [PATCH 141/153] mm: don't read the PTE twice setting CoW The PTE is read to get the pfn and attr. We can reuse it if the PTE is not present and we need to copy the PTE as is. Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 1d5c58f0..5cc3c6f4 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -625,8 +625,7 @@ where if !attr.contains(PageAttribute::PRESENT) { // Copy non-installed mapped PTEs directly to the new PTE and delay // its handling till the page fault. - let (pfn, attr) = from.get(); - self.set(pfn, attr); + self.set(pfn, attr.into()); return; } From c7b260053ae578e37041bb32c9253c61478a7c7e Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 26 Mar 2026 00:59:14 +0800 Subject: [PATCH 142/153] mm: associate newly alloced anon folios with the anon mapping When handling missing anon or writings to private file PTEs, we need to alloc a new anon folio and add a mapping. Pass the anon mapping to the function and alloc the anon folio with the mapping. It's worth noticing that handling CoW PTEs also needs this theoretically. This should be done in next few patches. Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mapping.rs | 6 +++++- src/kernel/mem/mm_list/mm_area.rs | 31 +++++++++++++++++++------------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index ecf8c98c..8eda80e1 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -2,7 +2,7 @@ use alloc::sync::Arc; use eonix_mm::paging::{Folio as _, PFN}; -use crate::kernel::mem::{Folio, PageCache, PageOffset}; +use crate::kernel::mem::{Folio, FolioOwned, PageCache, PageOffset}; #[derive(Debug, Clone)] pub struct FileMapping { @@ -43,6 +43,10 @@ impl AnonMapping { fn split(&self, _offset: PageOffset) -> (Self, Self) { (Self::new(), Self::new()) } + + pub fn alloc_folio(&self) -> FolioOwned { + FolioOwned::alloc() + } } impl FileMapping { diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 2727c013..c4aa398c 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -14,7 +14,7 @@ use intrusive_collections::{ }; use super::Mapping; -use crate::kernel::mem::mm_list::mapping::add_mapping; +use crate::kernel::mem::mm_list::mapping::{add_mapping, AnonMapping}; use crate::kernel::mem::mm_list::page_table::KernelPageTable; use crate::kernel::mem::mm_list::{ remove_mapping, FileMapping, MemListLock, PageTableExt, @@ -462,6 +462,7 @@ impl MemArea { async fn missing_file( &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: PageOffset, write: bool, file_mapping: &FileMapping, + anon_mapping: Option<&AnonMapping>, ) -> KResult<()> { assert!( offset.byte_count() < file_mapping.length, @@ -479,7 +480,7 @@ impl MemArea { return; } - if self.is_shared() { + let Some(anon_mapping) = anon_mapping else { // We don't process dirty flags in write faults. // Simply assume that page will eventually be dirtied. // So here we can set the dirty flag now. @@ -487,7 +488,7 @@ impl MemArea { attr.insert(PageAttribute::WRITE); *pfn = cache_page.add_mapping(); return; - } + }; if !write { // Delay the copy-on-write until write fault happens. @@ -496,9 +497,8 @@ impl MemArea { return; } - // XXX: Change this. Let's handle mapped pages before CoW pages. // Nah, we are writing to a mapped private mapping... - let mut new_page = FolioOwned::alloc(); + let mut new_page = anon_mapping.alloc_folio(); new_page .as_bytes_mut() .copy_from_slice(cache_page.lock().as_bytes()); @@ -525,8 +525,11 @@ impl MemArea { Ok(()) } - fn missing_anon(&self, pfn: &mut PFN, attr: &mut PageAttribute) { - let mut folio = FolioOwned::alloc(); + fn missing_anon( + &self, pfn: &mut PFN, attr: &mut PageAttribute, + anon_mapping: &AnonMapping, + ) { + let mut folio = anon_mapping.alloc_folio(); folio.as_bytes_mut().fill(0); *pfn = add_mapping(folio.share()); @@ -558,12 +561,16 @@ impl MemArea { attr.insert(PageAttribute::USER); match &self.mapping { - Mapping::Anonymous(_) => { - self.missing_anon(pfn, attr); + Mapping::Anonymous(anon_mapping) => { + self.missing_anon(pfn, attr, anon_mapping); + } + Mapping::PrivateFile { file, anon } => { + self.missing_file(pfn, attr, offset, write, file, Some(anon)) + .await?; } - Mapping::PrivateFile { file: mapping, .. } - | Mapping::SharedFile(mapping) => { - self.missing_file(pfn, attr, offset, write, mapping).await?; + Mapping::SharedFile(file_mapping) => { + self.missing_file(pfn, attr, offset, write, file_mapping, None) + .await?; } } From d56bf6fa0a2d0a073c0995951b19aac9a31709e5 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 26 Mar 2026 01:06:08 +0800 Subject: [PATCH 143/153] mm: use the anon mapping to alloc folio in CoW handling Signed-off-by: greatbridf --- src/kernel/mem/mm_list/mapping.rs | 4 ++-- src/kernel/mem/mm_list/mm_area.rs | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 8eda80e1..2df83f17 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -44,8 +44,8 @@ impl AnonMapping { (Self::new(), Self::new()) } - pub fn alloc_folio(&self) -> FolioOwned { - FolioOwned::alloc() + pub fn add_folio(&self, folio: FolioOwned) -> Folio { + folio.share() } } diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index c4aa398c..0163632d 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -445,18 +445,24 @@ impl MemArea { return; } - let mut new_page = FolioOwned::alloc(); + let anon_mapping = match &self.mapping { + Mapping::Anonymous(anon) => anon, + Mapping::PrivateFile { anon, .. } => anon, + Mapping::SharedFile(_) => unreachable!("CoW PTE of shared areas?"), + }; + + let mut new_folio = FolioOwned::alloc(); unsafe { // SAFETY: `folio` is CoW, which means that others won't write to it. let old_page_data = folio.get_bytes_ptr().as_ref(); - let new_page_data = new_page.as_bytes_mut(); + let new_page_data = new_folio.as_bytes_mut(); new_page_data.copy_from_slice(old_page_data); }; attr.remove(PageAttribute::ACCESSED); - *pfn = add_mapping(new_page.share()); + *pfn = add_mapping(anon_mapping.add_folio(new_folio)); } async fn missing_file( @@ -498,13 +504,13 @@ impl MemArea { } // Nah, we are writing to a mapped private mapping... - let mut new_page = anon_mapping.alloc_folio(); - new_page + let mut new_folio = FolioOwned::alloc(); + new_folio .as_bytes_mut() .copy_from_slice(cache_page.lock().as_bytes()); attr.insert(PageAttribute::WRITE); - *pfn = add_mapping(new_page.share()); + *pfn = add_mapping(anon_mapping.add_folio(new_folio)); }; file_mapping @@ -529,10 +535,10 @@ impl MemArea { &self, pfn: &mut PFN, attr: &mut PageAttribute, anon_mapping: &AnonMapping, ) { - let mut folio = anon_mapping.alloc_folio(); + let mut folio = FolioOwned::alloc(); folio.as_bytes_mut().fill(0); - *pfn = add_mapping(folio.share()); + *pfn = add_mapping(anon_mapping.add_folio(folio)); attr.insert(PageAttribute::PRESENT); From 92563ea4e0b18fa779ccb5a68fc3e773ed58c31f Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 27 Mar 2026 00:44:09 +0800 Subject: [PATCH 144/153] mm, page_cache: remove alloc_zeroed() This function was used by some fs to allocate blank pages. Remove this since it's no longer used. Signed-off-by: greatbridf --- src/kernel/mem/page_cache.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 432581fd..ee18eb59 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -6,7 +6,7 @@ use eonix_mm::paging::{PAGE_SIZE, PFN}; use eonix_sync::{atomic, Mutex}; use super::page_alloc::PageFlags; -use super::{Folio, FolioOwned}; +use super::Folio; use crate::io::{Buffer, Stream}; use crate::kernel::constants::EINVAL; use crate::kernel::mem::mm_list::add_mapping; @@ -28,15 +28,6 @@ impl CachePage { CachePage(Folio::alloc()) } - pub fn new_zeroed() -> Self { - CachePage({ - let mut folio = FolioOwned::alloc(); - folio.as_bytes_mut().fill(0); - - folio.share() - }) - } - pub fn is_dirty(&self) -> bool { atomic!(self.flags, has, PageFlags::DIRTY) } From 42c3291a69e2a8975415e882e55c4afea16841d6 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 9 Apr 2026 23:38:57 +0800 Subject: [PATCH 145/153] mm: introduce the AnonFolio For anonymous pages mapped to user space, introduce a new wrapper around the struct Folio named AnonFolio, which is also a mappable folio type similar to folios for some PageCache. Before mapping a page to some area, associate the folio with the AnonMapping using `add_folio()`. Signed-off-by: greatbridf --- src/kernel/mem.rs | 2 ++ src/kernel/mem/mapped.rs | 3 +++ src/kernel/mem/mapped/anon.rs | 19 +++++++++++++++++++ src/kernel/mem/mm_list/mapping.rs | 6 +++--- src/kernel/mem/mm_list/mm_area.rs | 6 +++--- 5 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 src/kernel/mem/mapped.rs create mode 100644 src/kernel/mem/mapped/anon.rs diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index e9b48390..71f2b07f 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -4,6 +4,7 @@ mod access; mod address; mod allocator; mod folio; +mod mapped; mod mm_list; mod page_alloc; mod page_cache; @@ -11,6 +12,7 @@ mod types; pub use access::PhysAccess; pub use folio::{Folio, FolioOwned, LockedFolio}; +pub use mapped::AnonFolio; pub use mm_list::{handle_kernel_page_fault, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache}; diff --git a/src/kernel/mem/mapped.rs b/src/kernel/mem/mapped.rs new file mode 100644 index 00000000..064050c2 --- /dev/null +++ b/src/kernel/mem/mapped.rs @@ -0,0 +1,3 @@ +mod anon; + +pub use anon::AnonFolio; diff --git a/src/kernel/mem/mapped/anon.rs b/src/kernel/mem/mapped/anon.rs new file mode 100644 index 00000000..5ca92b5c --- /dev/null +++ b/src/kernel/mem/mapped/anon.rs @@ -0,0 +1,19 @@ +use eonix_macros::TransparentDeref; +use eonix_mm::paging::PFN; + +use crate::kernel::mem::mm_list::add_mapping; +use crate::kernel::mem::{Folio, FolioOwned}; + +#[repr(transparent)] +#[derive(TransparentDeref)] +pub struct AnonFolio(Folio); + +impl AnonFolio { + pub fn new(folio: FolioOwned) -> Self { + Self(folio.share()) + } + + pub fn add_mapping(self) -> PFN { + add_mapping(self.0) + } +} diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 2df83f17..0c227d51 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -2,7 +2,7 @@ use alloc::sync::Arc; use eonix_mm::paging::{Folio as _, PFN}; -use crate::kernel::mem::{Folio, FolioOwned, PageCache, PageOffset}; +use crate::kernel::mem::{AnonFolio, Folio, FolioOwned, PageCache, PageOffset}; #[derive(Debug, Clone)] pub struct FileMapping { @@ -44,8 +44,8 @@ impl AnonMapping { (Self::new(), Self::new()) } - pub fn add_folio(&self, folio: FolioOwned) -> Folio { - folio.share() + pub fn add_folio(&self, folio: FolioOwned) -> AnonFolio { + AnonFolio::new(folio) } } diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 0163632d..93aa123e 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -462,7 +462,7 @@ impl MemArea { }; attr.remove(PageAttribute::ACCESSED); - *pfn = add_mapping(anon_mapping.add_folio(new_folio)); + *pfn = anon_mapping.add_folio(new_folio).add_mapping(); } async fn missing_file( @@ -510,7 +510,7 @@ impl MemArea { .copy_from_slice(cache_page.lock().as_bytes()); attr.insert(PageAttribute::WRITE); - *pfn = add_mapping(anon_mapping.add_folio(new_folio)); + *pfn = anon_mapping.add_folio(new_folio).add_mapping(); }; file_mapping @@ -538,7 +538,7 @@ impl MemArea { let mut folio = FolioOwned::alloc(); folio.as_bytes_mut().fill(0); - *pfn = add_mapping(anon_mapping.add_folio(folio)); + *pfn = anon_mapping.add_folio(folio).add_mapping(); attr.insert(PageAttribute::PRESENT); From 34d60c0ea5daf0d4f296241126dce6a1470d7164 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 9 Apr 2026 23:49:27 +0800 Subject: [PATCH 146/153] style, vfs: reformat fat32.rs, file.rs and inode.rs No functional change. Signed-off-by: greatbridf --- src/fs/fat32.rs | 87 +++++++++++++++++++---------------- src/fs/tmpfs/file.rs | 78 +++++++++++-------------------- src/kernel/vfs/inode/inode.rs | 5 +- 3 files changed, 76 insertions(+), 94 deletions(-) diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index c1feebdf..428e4c98 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -39,7 +39,10 @@ impl RawCluster { match self.0 { ..Self::START | Self::EOC..Self::INVL => None, Self::INVL.. => { - unreachable!("invalid cluster number: RawCluster({:#08x})", self.0) + unreachable!( + "invalid cluster number: RawCluster({:#08x})", + self.0 + ) } no => Some(Cluster(no)), } @@ -113,7 +116,9 @@ struct FatFs { impl SuperBlock for FatFs {} impl FatFs { - async fn read_cluster(&self, mut cluster: Cluster, buf: &Folio) -> KResult<()> { + async fn read_cluster( + &self, mut cluster: Cluster, buf: &Folio, + ) -> KResult<()> { cluster = cluster.normalized(); let rq = BlockDeviceRequest::Read { @@ -137,7 +142,8 @@ impl FatFs { let info = info.assume_filled_ref()?; let mut fat = Box::new_uninit_slice( - 512 * info.sectors_per_fat as usize / core::mem::size_of::(), + 512 * info.sectors_per_fat as usize + / core::mem::size_of::(), ); device @@ -151,8 +157,8 @@ impl FatFs { let sectors_per_cluster = info.sectors_per_cluster; let rootdir_cluster = info.root_cluster.parse().ok_or(EINVAL)?; - let data_start_sector = - info.reserved_sectors as u64 + info.fat_copies as u64 * info.sectors_per_fat as u64; + let data_start_sector = info.reserved_sectors as u64 + + info.fat_copies as u64 * info.sectors_per_fat as u64; let volume_label = { let end = info @@ -168,8 +174,10 @@ impl FatFs { let fat = unsafe { fat.assume_init() }; - let rootdir_cluster_count = ClusterIterator::new(fat.as_ref(), rootdir_cluster).count(); - let rootdir_size = rootdir_cluster_count as u32 * sectors_per_cluster as u32 * 512; + let rootdir_cluster_count = + ClusterIterator::new(fat.as_ref(), rootdir_cluster).count(); + let rootdir_size = + rootdir_cluster_count as u32 * sectors_per_cluster as u32 * 512; let fatfs = SbUse::new( SuperBlockInfo { @@ -243,26 +251,21 @@ impl InodeOps for FileInode { type SuperBlock = FatFs; async fn read( - &self, - _: SbUse, - inode: &InodeUse, - buffer: &mut dyn Buffer, - offset: usize, + &self, _: SbUse, inode: &InodeUse, + buffer: &mut dyn Buffer, offset: usize, ) -> KResult { inode.get_page_cache().read(buffer, offset).await } async fn read_page( - &self, - sb: SbUse, - inode: &InodeUse, - page: &mut CachePage, - offset: PageOffset, + &self, sb: SbUse, inode: &InodeUse, + page: &mut CachePage, offset: PageOffset, ) -> KResult<()> { let fs = &sb.backend; let fat = sb.backend.fat.read().await; - if offset >= PageOffset::from_byte_ceil(inode.info.lock().size as usize) { + if offset >= PageOffset::from_byte_ceil(inode.info.lock().size as usize) + { unreachable!("read_page called with offset beyond file size"); } @@ -272,10 +275,11 @@ impl InodeOps for FileInode { } // XXX: Ugly and inefficient O(n^2) algorithm for sequential file read. - let cluster = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)) - .skip(offset.page_count()) - .next() - .ok_or(EIO)?; + let cluster = + ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)) + .skip(offset.page_count()) + .next() + .ok_or(EIO)?; fs.read_cluster(cluster, &page).await?; @@ -316,7 +320,9 @@ impl DirInode { ) } - async fn read_dir_pages(&self, sb: &SbUse, inode: &InodeUse) -> KResult<()> { + async fn read_dir_pages( + &self, sb: &SbUse, inode: &InodeUse, + ) -> KResult<()> { let mut dir_pages = self.dir_pages.write().await; if !dir_pages.is_empty() { return Ok(()); @@ -325,7 +331,8 @@ impl DirInode { let fs = &sb.backend; let fat = fs.fat.read().await; - let clusters = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)); + let clusters = + ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)); for cluster in clusters { let page = FolioOwned::alloc(); @@ -338,9 +345,7 @@ impl DirInode { } async fn get_dir_pages( - &self, - sb: &SbUse, - inode: &InodeUse, + &self, sb: &SbUse, inode: &InodeUse, ) -> KResult> + use<'_>> { { let dir_pages = self.dir_pages.read().await; @@ -363,9 +368,7 @@ impl InodeOps for DirInode { type SuperBlock = FatFs; async fn lookup( - &self, - sb: SbUse, - inode: &InodeUse, + &self, sb: SbUse, inode: &InodeUse, dentry: &Arc, ) -> KResult> { let dir_pages = self.get_dir_pages(&sb, inode).await?; @@ -389,9 +392,13 @@ impl InodeOps for DirInode { let sbref = SbRef::from(&sb); if entry.is_directory { - return Ok(Some(DirInode::new(entry.cluster, sbref, entry.size) as _)); + return Ok(Some( + DirInode::new(entry.cluster, sbref, entry.size) as _, + )); } else { - return Ok(Some(FileInode::new(entry.cluster, sbref, entry.size) as _)); + return Ok(Some( + FileInode::new(entry.cluster, sbref, entry.size) as _, + )); } } @@ -399,10 +406,7 @@ impl InodeOps for DirInode { } async fn readdir( - &self, - sb: SbUse, - inode: &InodeUse, - offset: usize, + &self, sb: SbUse, inode: &InodeUse, offset: usize, callback: &mut (dyn FnMut(&[u8], Ino) -> KResult + Send), ) -> KResult> { let fs = &sb.backend; @@ -412,7 +416,8 @@ impl InodeOps for DirInode { let cluster_offset = offset / cluster_size; let inner_offset = offset % cluster_size; - let inner_raw_dirent_offset = inner_offset / core::mem::size_of::(); + let inner_raw_dirent_offset = + inner_offset / core::mem::size_of::(); let dir_data = dir_pages .iter() @@ -448,13 +453,17 @@ struct FatMountCreator; impl MountCreator for FatMountCreator { fn check_signature(&self, mut first_block: &[u8]) -> KResult { match first_block.split_off(82..) { - Some([b'F', b'A', b'T', b'3', b'2', b' ', b' ', b' ', ..]) => Ok(true), + Some([b'F', b'A', b'T', b'3', b'2', b' ', b' ', b' ', ..]) => { + Ok(true) + } Some(..) => Ok(false), None => Err(EIO), } } - async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc) -> KResult { + async fn create_mount( + &self, _source: &str, _flags: u64, mp: &Arc, + ) -> KResult { let (fatfs, root_inode) = FatFs::create(DeviceId::new(8, 1)).await?; Mount::new(mp, fatfs, root_inode) diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index aafae539..666c73dd 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -5,7 +5,9 @@ use super::TmpFs; use crate::io::{Buffer, Stream}; use crate::kernel::mem::{CachePage, PageCache, PageOffset}; use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse, WriteOffset}; +use crate::kernel::vfs::inode::{ + Ino, InodeInfo, InodeOps, InodeUse, WriteOffset, +}; use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; use crate::kernel::vfs::{SbRef, SbUse}; use crate::prelude::KResult; @@ -13,7 +15,9 @@ use crate::prelude::KResult; pub struct FileInode; impl FileInode { - pub fn new(ino: Ino, sb: SbRef, size: usize, perm: Permission) -> InodeUse { + pub fn new( + ino: Ino, sb: SbRef, size: usize, perm: Permission, + ) -> InodeUse { let now = Instant::now(); InodeUse::new( @@ -39,22 +43,16 @@ impl InodeOps for FileInode { type SuperBlock = TmpFs; async fn read( - &self, - _: SbUse, - inode: &InodeUse, - buffer: &mut dyn Buffer, - offset: usize, + &self, _: SbUse, inode: &InodeUse, + buffer: &mut dyn Buffer, offset: usize, ) -> KResult { let _lock = inode.rwsem.read().await; inode.get_page_cache().read(buffer, offset).await } async fn write( - &self, - _: SbUse, - inode: &InodeUse, - stream: &mut dyn Stream, - offset: WriteOffset<'_>, + &self, _: SbUse, inode: &InodeUse, + stream: &mut dyn Stream, offset: WriteOffset<'_>, ) -> KResult { let _lock = inode.rwsem.write().await; @@ -88,10 +86,7 @@ impl InodeOps for FileInode { } async fn truncate( - &self, - _: SbUse, - inode: &InodeUse, - length: usize, + &self, _: SbUse, inode: &InodeUse, length: usize, ) -> KResult<()> { let _lock = inode.rwsem.write().await; @@ -105,10 +100,7 @@ impl InodeOps for FileInode { } async fn chmod( - &self, - _sb: SbUse, - inode: &InodeUse, - perm: Permission, + &self, _sb: SbUse, inode: &InodeUse, perm: Permission, ) -> KResult<()> { let mut info = inode.info.lock(); @@ -119,10 +111,7 @@ impl InodeOps for FileInode { } async fn read_page( - &self, - _: SbUse, - _: &InodeUse, - page: &mut CachePage, + &self, _: SbUse, _: &InodeUse, page: &mut CachePage, _: PageOffset, ) -> KResult<()> { page.lock().as_bytes_mut().fill(0); @@ -130,10 +119,7 @@ impl InodeOps for FileInode { } async fn write_page( - &self, - _: SbUse, - _: &InodeUse, - _: &mut CachePage, + &self, _: SbUse, _: &InodeUse, _: &mut CachePage, _: PageOffset, ) -> KResult<()> { // XXX: actually we should refuse to do the writeback. @@ -142,13 +128,9 @@ impl InodeOps for FileInode { } async fn write_begin<'a>( - &self, - _: SbUse, - _: &InodeUse, - page_cache: &PageCache, - pages: &'a mut BTreeMap, - offset: usize, - _: usize, + &self, _: SbUse, _: &InodeUse, + page_cache: &PageCache, pages: &'a mut BTreeMap, + offset: usize, _: usize, ) -> KResult<&'a mut CachePage> { // TODO: Remove dependency on `page_cache`. page_cache @@ -157,13 +139,8 @@ impl InodeOps for FileInode { } async fn write_end( - &self, - _: SbUse, - inode: &InodeUse, - _: &PageCache, - _: &mut BTreeMap, - offset: usize, - _: usize, + &self, _: SbUse, inode: &InodeUse, _: &PageCache, + _: &mut BTreeMap, offset: usize, _: usize, copied: usize, ) -> KResult<()> { let now = Instant::now(); @@ -181,7 +158,9 @@ pub struct DeviceInode { } impl DeviceInode { - pub fn new(ino: Ino, sb: SbRef, mode: Mode, devid: DeviceId) -> InodeUse { + pub fn new( + ino: Ino, sb: SbRef, mode: Mode, devid: DeviceId, + ) -> InodeUse { let now = Instant::now(); InodeUse::new( @@ -207,10 +186,7 @@ impl InodeOps for DeviceInode { type SuperBlock = TmpFs; async fn chmod( - &self, - _sb: SbUse, - inode: &InodeUse, - perm: Permission, + &self, _sb: SbUse, inode: &InodeUse, perm: Permission, ) -> KResult<()> { let mut info = inode.info.lock(); info.perm = perm; @@ -219,7 +195,9 @@ impl InodeOps for DeviceInode { Ok(()) } - fn devid(&self, _: SbUse, _: &InodeUse) -> KResult { + fn devid( + &self, _: SbUse, _: &InodeUse, + ) -> KResult { Ok(self.devid) } } @@ -255,9 +233,7 @@ impl InodeOps for SymlinkInode { type SuperBlock = TmpFs; async fn readlink( - &self, - _sb: SbUse, - _inode: &InodeUse, + &self, _sb: SbUse, _inode: &InodeUse, buffer: &mut dyn Buffer, ) -> KResult { buffer diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs index 5f0b98c2..ab69b337 100644 --- a/src/kernel/vfs/inode/inode.rs +++ b/src/kernel/vfs/inode/inode.rs @@ -282,10 +282,7 @@ pub struct InodeUse(Arc); impl InodeUse { pub fn new( - sb: SbRef, - ino: Ino, - format: Format, - info: InodeInfo, + sb: SbRef, ino: Ino, format: Format, info: InodeInfo, ops: impl InodeOps, ) -> Self { let inode = Inode { From f9aa5aa5973dd2c330977bce20de8cafa75a0d29 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Thu, 9 Apr 2026 23:50:01 +0800 Subject: [PATCH 147/153] vfs, inode: remove {read,write}_direct The two functions are unused for long so remove them. If they are needed later, add them back manually then. Signed-off-by: greatbridf --- src/kernel/vfs/inode/inode.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs index ab69b337..6c2ab0cb 100644 --- a/src/kernel/vfs/inode/inode.rs +++ b/src/kernel/vfs/inode/inode.rs @@ -190,10 +190,6 @@ define_inode_ops! { Err(EINVAL) } - async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult { - Err(EINVAL) - } - async fn write( &self, stream: &mut dyn Stream, @@ -202,14 +198,6 @@ define_inode_ops! { Err(EINVAL) } - async fn write_direct( - &self, - stream: &mut dyn Stream, - offset: usize, - ) -> KResult { - Err(EINVAL) - } - async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult { Err(EINVAL) } From 99598a2a2ec13d5c1ed6e27729927dbe63acadad Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 10 Apr 2026 00:04:34 +0800 Subject: [PATCH 148/153] mm, vfs: change read_page() arg type to FolioOwned It doesn't make sense for read_page() to accept a &mut CachePage. All it needs to do is to read and copy data from the underlying filesystem to the given folio. So change that to FolioOwned, and allocate and pass a FolioOwned to the function. After the change, we don't need to create CachePages outsides the page cache module. Change CachePage::new() to be private. Signed-off-by: greatbridf --- src/fs/fat32.rs | 5 ++--- src/fs/tmpfs/file.rs | 6 +++--- src/kernel/mem/page_cache.rs | 10 +++++----- src/kernel/vfs/inode/inode.rs | 4 ++-- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 428e4c98..7bc57fde 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -12,7 +12,7 @@ use itertools::Itertools; use crate::io::{Buffer, ByteBuffer, UninitBuffer}; use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::{CachePage, Folio, FolioOwned, PageOffset}; +use crate::kernel::mem::{Folio, FolioOwned, PageOffset}; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; @@ -259,7 +259,7 @@ impl InodeOps for FileInode { async fn read_page( &self, sb: SbUse, inode: &InodeUse, - page: &mut CachePage, offset: PageOffset, + page: &mut FolioOwned, offset: PageOffset, ) -> KResult<()> { let fs = &sb.backend; let fat = sb.backend.fat.read().await; @@ -285,7 +285,6 @@ impl InodeOps for FileInode { let real_len = (inode.info.lock().size as usize) - offset.byte_count(); if real_len < PAGE_SIZE { - let mut page = page.lock(); page.as_bytes_mut()[real_len..].fill(0); } diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs index 666c73dd..7322ca4a 100644 --- a/src/fs/tmpfs/file.rs +++ b/src/fs/tmpfs/file.rs @@ -3,7 +3,7 @@ use alloc::sync::Arc; use super::TmpFs; use crate::io::{Buffer, Stream}; -use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::mem::{CachePage, FolioOwned, PageCache, PageOffset}; use crate::kernel::timer::Instant; use crate::kernel::vfs::inode::{ Ino, InodeInfo, InodeOps, InodeUse, WriteOffset, @@ -111,10 +111,10 @@ impl InodeOps for FileInode { } async fn read_page( - &self, _: SbUse, _: &InodeUse, page: &mut CachePage, + &self, _: SbUse, _: &InodeUse, page: &mut FolioOwned, _: PageOffset, ) -> KResult<()> { - page.lock().as_bytes_mut().fill(0); + page.as_bytes_mut().fill(0); Ok(()) } diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index ee18eb59..156c475f 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -10,7 +10,7 @@ use super::Folio; use crate::io::{Buffer, Stream}; use crate::kernel::constants::EINVAL; use crate::kernel::mem::mm_list::add_mapping; -use crate::kernel::mem::PageOffset; +use crate::kernel::mem::{FolioOwned, PageOffset}; use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; @@ -24,8 +24,8 @@ pub struct PageCache { pub struct CachePage(Folio); impl CachePage { - pub fn new() -> Self { - CachePage(Folio::alloc()) + fn new(folio: FolioOwned) -> Self { + CachePage(folio.share()) } pub fn is_dirty(&self) -> bool { @@ -62,10 +62,10 @@ impl PageCache { match pages.entry(pgoff) { Entry::Occupied(ent) => Ok(ent.into_mut()), Entry::Vacant(vacant_entry) => { - let mut new_page = CachePage::new(); + let mut new_page = FolioOwned::alloc(); self.inode.read_page(&mut new_page, pgoff).await?; - Ok(vacant_entry.insert(new_page)) + Ok(vacant_entry.insert(CachePage::new(new_page))) } } } diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs index 6c2ab0cb..90aab649 100644 --- a/src/kernel/vfs/inode/inode.rs +++ b/src/kernel/vfs/inode/inode.rs @@ -11,7 +11,7 @@ use eonix_sync::{RwLock, Spin}; use super::{Ino, RenameData, WriteOffset}; use crate::io::{Buffer, Stream}; use crate::kernel::constants::{EINVAL, EPERM}; -use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::mem::{CachePage, FolioOwned, PageCache, PageOffset}; use crate::kernel::timer::Instant; use crate::kernel::vfs::dentry::Dentry; use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; @@ -215,7 +215,7 @@ define_inode_ops! { } // PAGE CACHE OPERATIONS - async fn read_page(&self, page: &mut CachePage, offset: PageOffset) -> KResult<()> { + async fn read_page(&self, page: &mut FolioOwned, offset: PageOffset) -> KResult<()> { Err(EINVAL) } From 3d5ef32e64156833cf563ecd53d2414597ddab3b Mon Sep 17 00:00:00 2001 From: greatbridf Date: Fri, 10 Apr 2026 00:44:03 +0800 Subject: [PATCH 149/153] hal, riscv: support user bit of table PTEs The standard says that PA_U bit should be cleared and reserved for future use, so we used to ignore the USER flag for table attributes. This results in memory leaks of page table folios as they're never released (we only release page table folios that is both present and for userspace usage, but USER bit is ignored when set). The workaround is to reuse the PA_COW bit to indicate the entry is for userspace usage. An entry can be CoW only if it's a leaf entry. So both PA_COW and PA_MMAP can be reused. We'll take the CoW bit as it has a smaller number... Signed-off-by: greatbridf --- crates/eonix_hal/src/arch/riscv64/mm.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 731cfc28..53ff2bf2 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -33,6 +33,7 @@ pub const PA_D: u64 = 0b1 << 7; // in RSW pub const PA_COW: u64 = 0b1 << 8; +pub const PA_U_TABLE: u64 = 0b1 << 8; pub const PA_MMAP: u64 = 0b1 << 9; #[allow(dead_code)] @@ -125,12 +126,9 @@ impl RawAttribute for PageAttribute64 { if self.0 & PA_G != 0 { table_attr |= TableAttribute::GLOBAL; } - if self.0 & PA_U != 0 { + if self.0 & PA_U_TABLE != 0 { table_attr |= TableAttribute::USER; } - if self.0 & PA_A != 0 { - table_attr |= TableAttribute::ACCESSED; - } Some(table_attr) } @@ -220,7 +218,8 @@ impl From for PageAttribute64 { match attr { TableAttribute::PRESENT => raw_attr |= PA_V, TableAttribute::GLOBAL => raw_attr |= PA_G, - TableAttribute::USER | TableAttribute::ACCESSED => {} + TableAttribute::USER => raw_attr |= PA_U_TABLE, + TableAttribute::ACCESSED => {} _ => unreachable!("Invalid table attribute"), } } From cf4805a91f005b0bb9e05c779828e78a48d94ab6 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Sat, 11 Apr 2026 16:51:32 +0800 Subject: [PATCH 150/153] mm: refactor set PTE CoW logic, removing PageTable helpers Signed-off-by: greatbridf --- src/kernel/mem/mm_list.rs | 40 ------------------------------- src/kernel/mem/mm_list/mm_area.rs | 38 +++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 5cc3c6f4..0cf7207d 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -593,23 +593,7 @@ impl fmt::Debug for MMList { } } -trait PageTableExt { - fn set_copy_on_write(&self, from: &Self, range: VRange); -} - -impl PageTableExt for KernelPageTable { - fn set_copy_on_write(&self, from: &Self, range: VRange) { - let to_iter = self.iter_user(range); - let from_iter = from.iter_user(range); - - for (to, from) in to_iter.zip(from_iter) { - to.set_copy_on_write(from); - } - } -} - trait PTEExt { - fn set_copy_on_write(&mut self, from: &mut Self); fn take_if_present(&mut self) -> Option; fn release(&mut self); } @@ -618,30 +602,6 @@ impl PTEExt for T where T: PTE, { - fn set_copy_on_write(&mut self, from: &mut Self) { - let (pfn, raw_attr) = from.get(); - let mut attr = raw_attr.as_page_attr().expect("Not a page attribute"); - - if !attr.contains(PageAttribute::PRESENT) { - // Copy non-installed mapped PTEs directly to the new PTE and delay - // its handling till the page fault. - self.set(pfn, attr.into()); - return; - } - - attr.remove(PageAttribute::WRITE | PageAttribute::DIRTY); - attr.insert(PageAttribute::COPY_ON_WRITE); - - let pfn = unsafe { - // SAFETY: We get the pfn from a valid page table entry, which - // should have been created via `add_mapping`. - duplicate_mapping(pfn) - }; - - self.set(pfn, T::Attr::from(attr & !PageAttribute::ACCESSED)); - from.set_attr(T::Attr::from(attr)); - } - fn take_if_present(&mut self) -> Option { let (pfn, raw_attr) = self.take(); let attr = raw_attr.as_page_attr().expect("Not a page"); diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 93aa123e..54f32a81 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -17,7 +17,7 @@ use super::Mapping; use crate::kernel::mem::mm_list::mapping::{add_mapping, AnonMapping}; use crate::kernel::mem::mm_list::page_table::KernelPageTable; use crate::kernel::mem::mm_list::{ - remove_mapping, FileMapping, MemListLock, PageTableExt, + duplicate_mapping, remove_mapping, FileMapping, MemListLock, }; use crate::kernel::mem::{CachePage, FolioOwned, PageOffset, Permission}; use crate::prelude::KResult; @@ -680,7 +680,14 @@ fn dup_area_private<'a, 'b: 'a>( ) { list.insert(area.clone(from_lock)); - to_pgtable.set_copy_on_write(from_pgtable, *area.range.as_ref(from_lock)); + let range = area.range.as_ref(from_lock); + + let to_iter = to_pgtable.iter_user(*range); + let from_iter = from_pgtable.iter_user(*range); + + for (to, from) in to_iter.zip(from_iter) { + set_pte_cow(to, from); + } } pub fn dup_area_to_list<'a, 'b: 'a>( @@ -694,3 +701,30 @@ pub fn dup_area_to_list<'a, 'b: 'a>( dup_area_private(area, list, from_lock, from_pgtable, to_pgtable); } + +fn set_pte_cow(to: &mut Ent, from: &mut Ent) +where + Ent: PTE, +{ + let (pfn, raw_attr) = from.get(); + let mut attr = raw_attr.as_page_attr().expect("Not a page attribute"); + + if !attr.contains(PageAttribute::PRESENT) { + // Copy non-installed mapped PTEs directly to the new PTE and delay + // its handling till the page fault. + to.set(pfn, attr.into()); + return; + } + + attr.remove(PageAttribute::WRITE | PageAttribute::DIRTY); + attr.insert(PageAttribute::COPY_ON_WRITE); + + let pfn = unsafe { + // SAFETY: We get the pfn from a valid page table entry, which + // should have been created via `add_mapping`. + duplicate_mapping(pfn) + }; + + to.set(pfn, Ent::Attr::from(attr & !PageAttribute::ACCESSED)); + from.set_attr(Ent::Attr::from(attr)); +} From 6feabcf2d130f76282db0e4b7f83d7a2b01ff731 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 9 Sep 2026 00:33:21 +0800 Subject: [PATCH 151/153] mm: introduce MapFolio for folios mapped to page tables Introduce MapFolio, a wrapper around Folio for folios whose contents can be mapped into page tables, along with the per-folio data backing it. That data, currently a mapcount counting how many page-table entries reference the folio, is stored inline in RawPage's PageData union and is guarded by the new MAPPED page flag. A folio is turned mappable by FolioOwned::into_mappable(), which initializes the data and publishes the flag with release semantics; Folio::drop() tears the data down again on the last reference. AnonFolio and CachePage now wrap MapFolio instead of a bare Folio, so anonymous and file-cache folios become mappable at creation time. The free add_mapping()/duplicate_mapping()/remove_mapping() helpers of mm_list::mapping are folded into MapFolio methods that bump or drop the mapcount while handing a refcounted PFN to (or retrieving one from) the page tables. The CoW path in MemArea::handle_cow() now decides whether the faulted page must be copied by checking whether its mapcount is one (MapFolio::is_exclusive()) instead of reading the shared refcount after removing the mapping. The refcount conflated page-table mappings with the folio's other owners, and the old check had to remove the mapping, inspect the refcount, and re-add it; mapcount tracks mappings directly, so an exclusive page can simply be made writable in place without refcount churn. Checking is not yet race-free against concurrent faults on the same page -- locking the folio is left as an XXX comment. Signed-off-by: greatbridf --- src/kernel/mem.rs | 2 +- src/kernel/mem/folio.rs | 4 + src/kernel/mem/mapped.rs | 2 + src/kernel/mem/mapped/anon.rs | 9 +- src/kernel/mem/mapped/map_folio.rs | 128 ++++++++++++++++++ src/kernel/mem/mm_list.rs | 7 +- src/kernel/mem/mm_list/mapping.rs | 36 +---- src/kernel/mem/mm_list/mm_area.rs | 32 +++-- src/kernel/mem/page_alloc/raw_page.rs | 6 + .../mem/page_alloc/raw_page/map_page.rs | 97 +++++++++++++ src/kernel/mem/page_cache.rs | 10 +- 11 files changed, 269 insertions(+), 64 deletions(-) create mode 100644 src/kernel/mem/mapped/map_folio.rs create mode 100644 src/kernel/mem/page_alloc/raw_page/map_page.rs diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index 71f2b07f..61512cbf 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -12,7 +12,7 @@ mod types; pub use access::PhysAccess; pub use folio::{Folio, FolioOwned, LockedFolio}; -pub use mapped::AnonFolio; +pub use mapped::{AnonFolio, MapFolio}; pub use mm_list::{handle_kernel_page_fault, MMList, Mapping, Permission}; pub use page_alloc::{GlobalPageAlloc, RawPage}; pub use page_cache::{CachePage, PageCache}; diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs index e6f893be..c855459e 100644 --- a/src/kernel/mem/folio.rs +++ b/src/kernel/mem/folio.rs @@ -147,6 +147,10 @@ impl Drop for Folio { match self.refcount.fetch_sub(1, Ordering::AcqRel) { 0 => unreachable!("Refcount for an in-use page is 0"), 1 => unsafe { + if self.is_map() { + Self::_drop_map(self); + } + GlobalPageAlloc::GLOBAL.dealloc_raw(self.0.as_mut()) }, _ => {} diff --git a/src/kernel/mem/mapped.rs b/src/kernel/mem/mapped.rs index 064050c2..b7ed460c 100644 --- a/src/kernel/mem/mapped.rs +++ b/src/kernel/mem/mapped.rs @@ -1,3 +1,5 @@ mod anon; +mod map_folio; pub use anon::AnonFolio; +pub use map_folio::{MapFolio, MapFolioData}; diff --git a/src/kernel/mem/mapped/anon.rs b/src/kernel/mem/mapped/anon.rs index 5ca92b5c..03c23b6f 100644 --- a/src/kernel/mem/mapped/anon.rs +++ b/src/kernel/mem/mapped/anon.rs @@ -1,19 +1,18 @@ use eonix_macros::TransparentDeref; use eonix_mm::paging::PFN; -use crate::kernel::mem::mm_list::add_mapping; -use crate::kernel::mem::{Folio, FolioOwned}; +use crate::kernel::mem::{FolioOwned, MapFolio}; #[repr(transparent)] #[derive(TransparentDeref)] -pub struct AnonFolio(Folio); +pub struct AnonFolio(MapFolio); impl AnonFolio { pub fn new(folio: FolioOwned) -> Self { - Self(folio.share()) + Self(folio.into_mappable()) } pub fn add_mapping(self) -> PFN { - add_mapping(self.0) + self.0.clone().add_mapping() } } diff --git a/src/kernel/mem/mapped/map_folio.rs b/src/kernel/mem/mapped/map_folio.rs new file mode 100644 index 00000000..ab5cd8d6 --- /dev/null +++ b/src/kernel/mem/mapped/map_folio.rs @@ -0,0 +1,128 @@ +use core::sync::atomic::AtomicUsize; + +use eonix_macros::TransparentDeref; +use eonix_mm::paging::{Folio as _, PFN}; +use eonix_sync::atomic; + +use crate::kernel::mem::{Folio, FolioOwned}; + +pub struct MapFolioData { + mapcount: AtomicUsize, +} + +#[repr(transparent)] +#[derive(Clone, TransparentDeref)] +pub struct MapFolio(Folio); + +impl MapFolioData { + pub const fn new() -> Self { + Self { + mapcount: AtomicUsize::new(0), + } + } +} + +impl FolioOwned { + /// Make the folio a mappable one. + pub fn into_mappable(self) -> MapFolio { + assert!(!self.is_map(), "Already mappable: {:?}", &self as &Folio); + + unsafe { + // SAFETY: We own the folio. + Folio::_make_map(&self); + } + + MapFolio(self.share()) + } +} + +impl MapFolio { + /// Borrow the mappable folio from page table. + /// + /// # Safety + /// `pfn` must be previously created via [`Self::add_mapping`] and the + /// mapping must not be removed during the function call. + /// + /// Otherwise this is undefined behavior. + pub unsafe fn with_mapping( + pfn: PFN, func: impl FnOnce(&Self) -> O, + ) -> O { + unsafe { + // SAFETY: `add_mapping()` calls `Folio::into_raw()` + Folio::with_raw(pfn, |folio| { + // SAFETY: MapFolio is repr(transparent) + let map_folio = &*(folio as *const Folio as *const MapFolio); + + func(map_folio) + }) + } + } + + fn map_data(&self) -> &MapFolioData { + unsafe { + // SAFETY: Guaranteed by the invariance. + self._map_data_unchecked() + } + } + + /// Turn the folio together with its refcount into a raw [`PFN`] that can be + /// mapped into some page table. + pub fn add_mapping(self) -> PFN { + let map_data = self.map_data(); + + atomic!(@Relaxed, map_data.mapcount, fetch_add, 1); + + unsafe { + // SAFETY: MapFolio is repr(transparent) + core::mem::transmute::(self).into_raw() + } + } + + /// Duplicate the mapping in page tables. + /// + /// # Safety + /// `pfn` must be previously created via [`Self::add_mapping`]. + /// + /// Otherwise this is undefined behavior. + pub unsafe fn duplicate_mapping(pfn: PFN) -> PFN { + unsafe { + // SAFETY: See above. + Self::with_mapping(pfn, |map_folio| { + Self(map_folio.0.clone()).add_mapping() + }) + } + } + + /// Remove the mapping from the page table. + /// + /// # Safety + /// `pfn` must be previously created via [`Self::add_mapping`]. + /// + /// Otherwise this is undefined behavior. + pub unsafe fn remove_mapping(pfn: PFN) -> Self { + let folio = unsafe { + // SAFETY: `pfn` is created via `add_mapping`, which uses `into_raw`. + Folio::from_raw(pfn) + }; + + let map_folio = Self(folio); + let map_data = map_folio.map_data(); + + atomic!(@AcqRel, map_data.mapcount, fetch_sub, 1); + + map_folio + } + + pub fn is_exclusive(&self) -> bool { + let map_data = self.map_data(); + let mapcount = atomic!(@Acquire, map_data.mapcount, load); + + mapcount == 1 + } + + pub fn into_inner(self) -> Folio { + let Self(folio) = self; + + folio + } +} diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index 0cf7207d..b8e3901c 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -19,9 +19,7 @@ use eonix_sync::Mutex; use mm_area::AreaList; use page_table::KernelPageTable; -pub use self::mapping::{ - add_mapping, duplicate_mapping, remove_mapping, FileMapping, Mapping, -}; +pub use self::mapping::{FileMapping, Mapping}; pub use self::mm_area::{AreaFlags, MemArea}; pub use self::page_fault::handle_kernel_page_fault; use super::address::{VAddrExt as _, VRangeExt as _}; @@ -29,6 +27,7 @@ use super::Folio; use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; use crate::kernel::mem::mm_list::brk::ProgramBreak; use crate::kernel::mem::mm_list::mm_area::dup_area_to_list; +use crate::kernel::mem::MapFolio; use crate::prelude::*; use crate::sync::ArcSwap; @@ -608,7 +607,7 @@ where attr.contains(PageAttribute::PRESENT).then(|| unsafe { // SAFETY: Present PTEs are always created via `add_mapping`. - remove_mapping(pfn) + MapFolio::remove_mapping(pfn).into_inner() }) } diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 0c227d51..1b57c914 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,8 +1,6 @@ use alloc::sync::Arc; -use eonix_mm::paging::{Folio as _, PFN}; - -use crate::kernel::mem::{AnonFolio, Folio, FolioOwned, PageCache, PageOffset}; +use crate::kernel::mem::{AnonFolio, FolioOwned, PageCache, PageOffset}; #[derive(Debug, Clone)] pub struct FileMapping { @@ -127,35 +125,3 @@ impl Mapping { } } } - -/// Turn the folio together with its refcount into a raw [`PFN`] that can be -/// mapped into some page table. -pub fn add_mapping(folio: Folio) -> PFN { - folio.into_raw() -} - -/// Duplicate the mapping in page tables. -/// -/// # Safety -/// `pfn` must be previously created via [`add_mapping`]. -/// -/// Otherwise this is undefined behavior. -pub unsafe fn duplicate_mapping(pfn: PFN) -> PFN { - unsafe { - // SAFETY: `pfn` is created via `add_mapping`, which uses `into_raw`. - Folio::with_raw(pfn, |folio| add_mapping(folio.clone())) - } -} - -/// Remove mappings in page tables. -/// -/// # Safety -/// `pfn` must be previously created via [`add_mapping`]. -/// -/// Otherwise this is undefined behavior. -pub unsafe fn remove_mapping(pfn: PFN) -> Folio { - unsafe { - // SAFETY: `pfn` is created via `add_mapping`, which uses `into_raw`. - Folio::from_raw(pfn) - } -} diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs index 54f32a81..e895ef2e 100644 --- a/src/kernel/mem/mm_list/mm_area.rs +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -14,12 +14,12 @@ use intrusive_collections::{ }; use super::Mapping; -use crate::kernel::mem::mm_list::mapping::{add_mapping, AnonMapping}; +use crate::kernel::mem::mm_list::mapping::AnonMapping; use crate::kernel::mem::mm_list::page_table::KernelPageTable; -use crate::kernel::mem::mm_list::{ - duplicate_mapping, remove_mapping, FileMapping, MemListLock, +use crate::kernel::mem::mm_list::{FileMapping, MemListLock}; +use crate::kernel::mem::{ + CachePage, FolioOwned, MapFolio, PageOffset, Permission, }; -use crate::kernel::mem::{CachePage, FolioOwned, PageOffset, Permission}; use crate::prelude::KResult; bitflags::bitflags! { @@ -431,20 +431,26 @@ impl MemArea { attr.remove(PageAttribute::COPY_ON_WRITE); attr.set(PageAttribute::WRITE, self.can_write()); - let folio = unsafe { + // XXX: Lock the page here to exclude concurrent faulting threads. + // Multiple threads could be here even if the page's mapcount is 1. + let is_exclusive = unsafe { // SAFETY: `pfn` is a mapping taken from some page table. - remove_mapping(*pfn) + MapFolio::with_mapping(*pfn, |map_folio| map_folio.is_exclusive()) }; - // XXX: Change me!!! - if folio.refcount.load(Ordering::Relaxed) == 1 { - // SAFETY: This is actually safe. If we read `1` here and we have `MMList` lock - // held, there couldn't be neither other processes sharing the page, nor other - // threads making the page COW at the same time. - *pfn = add_mapping(folio); + if is_exclusive { + // SAFETY: This is actually safe. If we read `1` here and we have + // `MMList` lock held, there couldn't be neither other + // processes sharing the page, nor other threads making the + // page CoW at the same time. return; } + let folio = unsafe { + // SAFETY: `pfn` is a mapping taken from some page table. + MapFolio::remove_mapping(*pfn) + }; + let anon_mapping = match &self.mapping { Mapping::Anonymous(anon) => anon, Mapping::PrivateFile { anon, .. } => anon, @@ -722,7 +728,7 @@ where let pfn = unsafe { // SAFETY: We get the pfn from a valid page table entry, which // should have been created via `add_mapping`. - duplicate_mapping(pfn) + MapFolio::duplicate_mapping(pfn) }; to.set(pfn, Ent::Attr::from(attr & !PageAttribute::ACCESSED)); diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 24f7d561..916c4582 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,3 +1,6 @@ +mod map_page; + +use core::mem::ManuallyDrop; use core::ptr::NonNull; use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; @@ -9,6 +12,7 @@ use eonix_sync::atomic; use intrusive_list::{container_of, Link, List}; use slab_allocator::{SlabPage, SlabPageAlloc, SlabSlot}; +use self::map_page::MapRawPageData; use super::zones::ZONE; use super::{GlobalPageAlloc, PerCpuPage}; use crate::kernel::mem::PhysAccess; @@ -33,6 +37,7 @@ impl SlabPageData { #[repr(C)] union PageData { slab: SlabPageData, + map: ManuallyDrop, } pub struct RawPage { @@ -55,6 +60,7 @@ impl PageFlags { pub const SLAB: u32 = 1 << 3; pub const DIRTY: u32 = 1 << 4; pub const LOCAL: u32 = 1 << 6; + pub const MAPPED: u32 = 1 << 7; pub fn has(&self, flag: u32, order: Ordering) -> bool { (self.0.load(order) & flag) == flag diff --git a/src/kernel/mem/page_alloc/raw_page/map_page.rs b/src/kernel/mem/page_alloc/raw_page/map_page.rs new file mode 100644 index 00000000..f98ed5f8 --- /dev/null +++ b/src/kernel/mem/page_alloc/raw_page/map_page.rs @@ -0,0 +1,97 @@ +use core::cell::UnsafeCell; + +use eonix_macros::TransparentDeref; +use eonix_sync::atomic; + +use crate::kernel::mem::mapped::MapFolioData; +use crate::kernel::mem::page_alloc::PageFlags; +use crate::kernel::mem::{Folio, RawPage}; + +pub struct MapRawPageData(UnsafeCell); + +/// # Invariant +/// A MapPage always has MAPPED bit set and has initialized MapPageData. +#[repr(transparent)] +#[derive(TransparentDeref)] +struct MapRawPage(RawPage); + +impl MapRawPage { + fn data(&self) -> &MapFolioData { + unsafe { + // SAFETY: Guaranteed by the type invariant. Sync with writers by + // the atomic operations of MAPPED bit. + &*self.shared_data.map.0.get() + } + } +} + +impl Folio { + /// Get the folio's map data without synchronization overhead. + /// + /// # Safety + /// The folio must have been previously initialized as mappable through + /// [`Self::_make_map`] and synchronized with the one initializing it. + pub unsafe fn _map_data_unchecked(&self) -> &MapFolioData { + let raw_page: &RawPage = self; + + let map_raw_page = unsafe { + // SAFETY: `MapPage` is `repr(transparent)`. + &*(raw_page as *const RawPage as *const MapRawPage) + }; + + map_raw_page.data() + } + + /// Get the folio's map data. + pub fn _map_data(&self) -> &MapFolioData { + assert!(self.is_map(), "Not mappable: {:?}", self); + + unsafe { + // SAFETY: Synchronized using acquire semantics. + self._map_data_unchecked() + } + } + + /// Check whether the folio is intialized as mappable with acquire semantics. + pub fn is_map(&self) -> bool { + atomic!(@Acquire, self.flags, has, PageFlags::MAPPED) + } + + /// Make the folio a mappable one. + /// + /// # Safety + /// The caller must exclude concurrent calls. + /// Otherwise, it is an undefined behavior. + pub unsafe fn _make_map(folio: &Self) { + debug_assert!( + !atomic!(@Relaxed, folio.flags, has, + PageFlags::SLAB | PageFlags::MAPPED), + "Conflict flags" + ); + + unsafe { + // SAFETY: We have the page lock held to exclude concurrent + // make_mapping callers. And readers sync with us through + // acquiring the MAPPED bit. + folio.shared_data.map.0.get().write(MapFolioData::new()); + } + + atomic!(@Release, folio.flags, set, PageFlags::MAPPED); + } + + /// Make the folio not mappable. + /// + /// # Safety + /// The caller must exclude concurrent calls and must not use it as mappable + /// after calling this function. + /// + /// Otherwise, it is an undefined behavior. + pub unsafe fn _drop_map(folio: &mut Self) { + let _map_data = unsafe { + // SAFETY: We are dropping the mapping data this folio holds. + folio.shared_data.map.0.get().read() + }; + + atomic!(@Release, folio.flags, clear, PageFlags::MAPPED); + } +} diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 156c475f..b3268970 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -6,11 +6,9 @@ use eonix_mm::paging::{PAGE_SIZE, PFN}; use eonix_sync::{atomic, Mutex}; use super::page_alloc::PageFlags; -use super::Folio; use crate::io::{Buffer, Stream}; use crate::kernel::constants::EINVAL; -use crate::kernel::mem::mm_list::add_mapping; -use crate::kernel::mem::{FolioOwned, PageOffset}; +use crate::kernel::mem::{FolioOwned, MapFolio, PageOffset}; use crate::kernel::vfs::inode::InodeUse; use crate::prelude::KResult; @@ -21,11 +19,11 @@ pub struct PageCache { #[repr(transparent)] #[derive(TransparentDeref)] -pub struct CachePage(Folio); +pub struct CachePage(MapFolio); impl CachePage { fn new(folio: FolioOwned) -> Self { - CachePage(folio.share()) + CachePage(folio.into_mappable()) } pub fn is_dirty(&self) -> bool { @@ -41,7 +39,7 @@ impl CachePage { } pub fn add_mapping(&self) -> PFN { - add_mapping(self.0.clone()) + self.0.clone().add_mapping() } } From 59041aa0f7a0ff9e7cccab0caf7f5eb07b8ad8b1 Mon Sep 17 00:00:00 2001 From: greatbridf Date: Wed, 9 Sep 2026 00:41:45 +0800 Subject: [PATCH 152/153] chore: update gitignore for vim coc settings Signed-off-by: greatbridf --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4684b698..4449bb62 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/ .idea/ .vscode/settings.json +.vim/ test/ From bf9f2784df12aac3e86b0b0d06d8fa732b21146f Mon Sep 17 00:00:00 2001 From: greatbridf Date: Mon, 19 Oct 2026 23:32:47 +0800 Subject: [PATCH 153/153] fix/futex: forget to .await in `futex_exit` Testcase `pthread_test` freezes when waiting for clear tid futex wakeups. It turns out to be that we forget to call .await in `futex_exit()` so `do_futex_exit()` is never called. Fix by appending the missing `.await`. Fixes: b6d54d6a0ff1 ("mm, proc: add an exited thread reaper") Signed-off-by: greatbridf --- src/kernel/task/futex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel/task/futex.rs b/src/kernel/task/futex.rs index 4dd57615..3c0f342b 100644 --- a/src/kernel/task/futex.rs +++ b/src/kernel/task/futex.rs @@ -334,7 +334,7 @@ async fn do_futex_exit(thread: &Thread) -> KResult<()> { pub async fn futex_exit(thread: &Thread) { // We don't care about any error happened inside. // If they've set up a wrong pointer, good luck to them... - let _ = do_futex_exit(thread); + let _ = do_futex_exit(thread).await; } async fn do_futex_exec(thread: &Thread) -> KResult<()> {