core.cxx File Reference

Virtual Memory Engine — core operations. More...

import std;
import lib.format;
import lib.rbtree;
import lib.error;
import zxfoundation.scoms.metadata.core;
import zxfoundation.scoms.kobject.ref;
import zxfoundation.scoms.kobject.types;
import arch.s390x.cpu.skey;
import zxfoundation.memory.page_cache;
import arch.s390x.cpu.irq;
import zxfoundation.memory.objects;
import zxfoundation.base.typestate;
import zxfoundation.memory.kmalloc;
import zxfoundation.dgp.isolation.types;
import zxfoundation.scoms.kobject.base;
import zxfoundation.memory.pmm;
import zxfoundation.memory.rmap;
import zxfoundation.dgp.cap.core;
import arch.s390x.mmu;
import zxfoundation.memory.page_cache.types;
import zxfoundation.dgp.memory_contract;
import zxfoundation.dgp.cap.types;
import zxfoundation.memory.pmm.types;
import zxfoundation.sync.rcu;
import zxfoundation.scoms.table;
import zxfoundation.memory.vm.types;
import zxfoundation.memory.numa.types;
import arch.s390x.cpu.lowcore;
import zxfoundation.base.types;
import zxfoundation.sys.syschk.core;
import zxfoundation.sync.qspinlock.core;
import zxfoundation.sync.lockable;
import zxfoundation.memory.hhdm;
import zxfoundation.sys.printk.core;
import lib.container_of;
import zxfoundation.sys.printk.types;

Include dependency graph for core.cxx:

Data Structures
struct	zxfoundation::memory::vm::detail::rb_insert_plan

Typedefs
using	zxfoundation::memory::vm::region_visitor = bool (*)(vm_region&) noexcept
	Walk every active vm_region in the system via callback.

Functions
auto	zxfoundation::memory::vm::detail::add_committed (vm_region &rgn, u64 delta) noexcept -> void
	Atomically add to committed.
auto	zxfoundation::memory::vm::detail::add_resident (vm_region &rgn, u64 delta) noexcept -> void
	Atomically add to resident.
auto	zxfoundation::memory::vm::detail::alloc_region (u64 va_start, u64 va_end, vm_prot prot, vm_flags flags, vm_backing backing, u64 phys_base) noexcept -> std::expected< vm_region *, lib::kernel_error >
	Allocate a vm_region from the SCOMS table, initialize it, and return.
auto	zxfoundation::memory::vm::detail::apply_vm_invalidation (u64 root_phys, const vm_region &rgn, zxfoundation::dgp::memory_contract::invalidation_scope scope) noexcept -> void
	Apply the invalidation scope attached to a VM/DGP policy result.
auto	zxfoundation::memory::vm::detail::attach_region_owner (vm_region &rgn, const vm_space &space) noexcept -> std::expected< void, lib::kernel_error >
auto	zxfoundation::memory::vm::detail::bind_region_owner (vm_region &rgn, u32 owner_domain) noexcept -> std::expected< void, lib::kernel_error >
auto	zxfoundation::memory::vm::detail::build_contract_request (const vm_space &space, u64 va_start, u64 length, vm_prot prot, vm_flags flags, vm_backing backing, u64 phys_base, u8 storage_key, u8 numa_node) noexcept -> zxfoundation::dgp::memory_contract::contract_request
	Build a complete DGP memory-contract request for a VM operation.
auto	zxfoundation::memory::vm::detail::commit_contract_generation (vm_space &space, const zxfoundation::dgp::memory_contract::contract_result &contract) noexcept -> void
auto	zxfoundation::memory::vm::detail::commit_region_policy (vm_space &space, vm_region &rgn, const zxfoundation::dgp::memory_contract::contract_result &contract) noexcept -> std::expected< void, lib::kernel_error >
auto	zxfoundation::memory::vm::detail::contract_metadata_compatible (const vm_region &lhs, const vm_region &rhs) noexcept -> bool
auto	zxfoundation::memory::vm::detail::create_region_mapping_object (vm_space &space, vm_region &rgn, objects::memory_object_ref grant, const objects::memory_object_policy &policy) noexcept -> std::expected< void, lib::kernel_error >
auto	zxfoundation::memory::vm::detail::create_region_memory_objects (vm_space &space, vm_region &rgn, const objects::memory_object_policy &policy) noexcept -> std::expected< void, lib::kernel_error >
auto	zxfoundation::memory::vm::detail::destroy_region_memory_objects (vm_region &rgn) noexcept -> void
auto	zxfoundation::memory::vm::detail::dgp_access_from_vm (vm_prot prot, vm_flags flags, vm_backing backing) noexcept -> zxfoundation::dgp::memory_contract::memory_access
	Convert VM protection and sharing flags into DGP access authority.
auto	zxfoundation::memory::vm::detail::dgp_numa_policy_from_vm (vm_flags flags) noexcept -> zxfoundation::dgp::memory_contract::numa_policy
	Convert VM placement flags into the current DGP NUMA policy enum.
auto	zxfoundation::memory::vm::detail::dgp_scope_from_vm (vm_flags flags, vm_backing backing) noexcept -> zxfoundation::dgp::memory_contract::memory_scope
	Convert VM sharing flags into a DGP memory-contract scope.
auto	zxfoundation::memory::vm::detail::map_physical_range (u64 root_phys, const vm_region &rgn, u64 phys_base, arch::s390x::mmu::page_prot pp) noexcept -> std::expected< void, lib::kernel_error >
	Map physical pages for a region [va_start, va_end) using identity phys_base.
auto	zxfoundation::memory::vm::detail::next_policy_from_contract (const vm_space &space, const zxfoundation::dgp::memory_contract::contract_result &contract) noexcept -> objects::memory_object_policy
auto	zxfoundation::memory::vm::detail::policy_from_contract (const zxfoundation::dgp::memory_contract::contract_result &contract, u64 mapping_generation, u32 storage_key_generation) noexcept -> objects::memory_object_policy
auto	zxfoundation::memory::vm::detail::policy_from_region (const vm_region &rgn) noexcept -> objects::memory_object_policy
auto	zxfoundation::memory::vm::detail::prot_to_mmu (vm_prot prot) noexcept -> arch::s390x::mmu::page_prot
	Convert VME protection to MMU page_prot.
auto	zxfoundation::memory::vm::detail::rb_commit_insert (vm_space &space, vm_region &rgn, const rb_insert_plan &plan) noexcept -> void
auto	zxfoundation::memory::vm::detail::rb_discard_split_region (vm_space &space, vm_region *rgn) noexcept -> void
	Discard a temporary split region without changing mapped bytes. Used only to reverse an interval split before the enclosing operation has removed any mapping or VMA state.
auto	zxfoundation::memory::vm::detail::rb_find_rcu (const vm_space &space, u64 va) noexcept -> vm_region *
	RCU-safe tree walk — each pointer load uses dereference.
auto	zxfoundation::memory::vm::detail::rb_insert_region (vm_space &space, vm_region *rgn) noexcept -> bool
	Insert a vm_region into the rb-tree, keyed by va_start.
auto	zxfoundation::memory::vm::detail::rb_plan_insert (vm_space &space, const vm_region &rgn, const vm_region *excluded=nullptr) noexcept -> std::expected< rb_insert_plan, lib::kernel_error >
	Find a non-overlapping insertion position for an interval while treating excluded as absent from the tree.
auto	zxfoundation::memory::vm::detail::rb_remove_and_free (vm_space &space, vm_region *rgn) noexcept -> void
	Remove a vm_region from the rb-tree and defer freeing via RCU.
auto	zxfoundation::memory::vm::detail::rcu_free_region_cb (sync::rcu::rcu_head *head) noexcept -> void
	RCU callback: free a vm_region back to the SCOMS table after a grace period.
auto	zxfoundation::memory::vm::detail::read_committed (const vm_region &rgn) noexcept -> u64
	Read committed count from a vm_region (handles atomic).
auto	zxfoundation::memory::vm::detail::read_resident (const vm_region &rgn) noexcept -> u64
	Read resident count from a vm_region (handles atomic).
auto	zxfoundation::memory::vm::detail::region_of (const lib::rb_node *node) noexcept -> const vm_region *
auto	zxfoundation::memory::vm::detail::region_of (lib::rb_node *node) noexcept -> vm_region *
auto	zxfoundation::memory::vm::detail::region_unlink_rmap (vm_region &rgn) noexcept -> void
	Unlink every rmap entry belonging to rgn's resident resident slots.
auto	zxfoundation::memory::vm::detail::register_vm_cap_lookup () noexcept -> void
auto	zxfoundation::memory::vm::detail::release_slot_mapping (vm_page_slot &slot) noexcept -> bool
	Release the pin owned by one resident VM slot.
constexpr auto	zxfoundation::memory::vm::detail::same_memory_ref (objects::memory_object_ref lhs, objects::memory_object_ref rhs) noexcept -> bool
auto	zxfoundation::memory::vm::detail::store_committed (vm_region &rgn, u64 val) noexcept -> void
	Store committed count in a vm_region.
auto	zxfoundation::memory::vm::detail::store_resident (vm_region &rgn, u64 val) noexcept -> void
	Store resident count in a vm_region.
auto	zxfoundation::memory::vm::detail::table_policy_from_region (const vm_region &rgn) noexcept -> pmm::allocation_request
	Build a page-table allocation request from VM/DGP placement metadata.
auto	zxfoundation::memory::vm::detail::teardown_region_slots (vm_region &rgn) noexcept -> u32
auto	zxfoundation::memory::vm::detail::unmap_range_pages (u64 root_phys, u64 va_start, u64 va_end) noexcept -> void
	Remove PTEs without guessing whether the mapped frame is PMM-owned.
auto	zxfoundation::memory::vm::detail::update_region_memory_object_ranges (vm_region &rgn) noexcept -> void
auto	zxfoundation::memory::vm::vm_destroy_space (vm_space &space) noexcept -> void
	Destroy all regions in a VME space, freeing region descriptors.
auto	zxfoundation::memory::vm::vm_drain_phys_range (u64 phys_start, u64 phys_end) noexcept -> u32
	Drain resident mappings whose phys lies in [phys_start,phys_end].
auto	zxfoundation::memory::vm::vm_find (const vm_space &space, u64 va) noexcept -> vm_region *
	RCU-protected lookup: find the vm_region containing VA.
auto	zxfoundation::memory::vm::vm_find_gap (const vm_space &space, u64 size, u64 align) noexcept -> std::expected< u64, lib::kernel_error >
	Top-down scan from mmap_base for a gap of at least size bytes.
auto	zxfoundation::memory::vm::vm_find_grant_mapping (vm_space &space, objects::memory_object_ref grant) noexcept -> vm_region *
	Find one live mapping view that consumes grant.
auto	zxfoundation::memory::vm::vm_for_each_region (region_visitor callback) noexcept -> void
auto	zxfoundation::memory::vm::vm_init () noexcept -> void
	Initialize the VME subsystem (create SCOMS table for vm_region).
auto	zxfoundation::memory::vm::detail::vm_join_split (vm_space &space, vm_region &lower, vm_region &upper) noexcept -> void
	Undo one successful vm_split before either half is mutated.
auto	zxfoundation::memory::vm::vm_map (vm_space &space, u64 size, vm_prot prot, vm_flags flags) noexcept -> std::expected< u64, lib::kernel_error >
	Map a region of size bytes with given protection and flags.
auto	zxfoundation::memory::vm::vm_map_extent (vm_space &space, u64 root_phys, u64 va, u64 length, vm_prot prot, objects::memory_object_ref extent_ref, u64 extent_offset) noexcept -> std::expected< void, lib::kernel_error >
	Map a checked subrange of a capability-bound physical extent. The caller supplies an offset, never an arbitrary physical address.
auto	zxfoundation::memory::vm::vm_map_fixed (vm_space &space, u64 va, u64 size, vm_prot prot, vm_flags flags, vm_backing backing, u64 phys_base, objects::memory_object_ref mapped_extent) noexcept -> std::expected< void, lib::kernel_error >
	Map at an exact VA with specified backing.
auto	zxfoundation::memory::vm::vm_map_shared_region (vm_space &target, u64 target_root_phys, u64 target_va, u64 length, vm_region &source, vm_prot prot) noexcept -> std::expected< void, lib::kernel_error >
auto	zxfoundation::memory::vm::vm_protect (vm_space &space, u64 root_phys, u64 va, u64 size, vm_prot new_prot) noexcept -> std::expected< void, lib::kernel_error >
	Change protection bits for [va, va+size).
auto	zxfoundation::memory::vm::vm_region_find_by_id (u32 id) noexcept -> vm_region *
	Look up a vm_region by SCOMS ID.
auto	zxfoundation::memory::vm::vm_region_maps_phys (const vm_region &rgn, u64 phys) noexcept -> bool
	Test if phys is mapped-resident in rgn.
auto	zxfoundation::memory::vm::vm_rekey_region (vm_space &space, vm_region &rgn, u64 new_start) noexcept -> bool
auto	zxfoundation::memory::vm::vm_space_drain_phys_range (vm_space &space, u64 phys_start, u64 phys_end) noexcept -> u32
	Drop all resident page mappings in space whose backing physical address lies in [phys_start, phys_end).
auto	zxfoundation::memory::vm::vm_space_init (vm_space &space) noexcept -> void
	Initialize a per-domain VME address space.
auto	zxfoundation::memory::vm::vm_space_set_commit_limit (vm_space &space, u64 limit_pages) noexcept -> std::expected< void, lib::kernel_error >
	Set a domain VM-space committed-page budget.
auto	zxfoundation::memory::vm::vm_space_try_charge_commit (vm_space &space, u64 pages) noexcept -> bool
	Atomically reserve committed pages for a caller that cannot retain space.lock through its fallible page-install work.
auto	zxfoundation::memory::vm::vm_space_uncharge_commit (vm_space &space, u64 pages) noexcept -> void
	Return a page charge obtained from vm_space_try_charge_commit.
auto	zxfoundation::memory::vm::vm_split (vm_space &space, vm_region *rgn, u64 split_va) noexcept -> std::expected< vm_region *, lib::kernel_error >
	Split rgn at split_va into [rgn->va_start, split_va) and [split_va, old_end).
auto	zxfoundation::memory::vm::vm_try_merge (vm_space &space, vm_region *rgn) noexcept -> void
	Try to merge rgn with its predecessor and/or successor.
auto	zxfoundation::memory::vm::vm_unmap (vm_space &space, u64 root_phys, u64 va, u64 size) noexcept -> std::expected< void, lib::kernel_error >
	Unmap pages in [va, va+size), splitting partially-overlapping regions, removing fully-covered regions, and freeing physical pages.

Variables
no_export scoms::scoms_table< vm_region, MAX_VM_REGIONS >	zxfoundation::memory::vm::detail::g_region_table {}
no_export bool	zxfoundation::memory::vm::detail::g_vme_initialized {false}

Detailed Description

Virtual Memory Engine — core operations.

SPDX-License-Identifier: Apache-2.0

Typedef Documentation

region_visitor

typedef bool(* zxfoundation::memory::vm::region_visitor)(vm_region &) noexcept = bool (*)(vm_region&) noexcept

export

Walk every active vm_region in the system via callback.

Parameters

[in]

callback

Function pointer invoked per region

Function Documentation

rb_insert_region()

auto zxfoundation::memory::vm::detail::rb_insert_region ( vm_space & space, vm_region * rgn )->bool

noexcept

Insert a vm_region into the rb-tree, keyed by va_start.

Returns

true on success, false if an overlapping region was found.

Here is the caller graph for this function:

vm_drain_phys_range()

auto zxfoundation::memory::vm::vm_drain_phys_range ( u64 phys_start, u64 phys_end )->u32

nodiscardexportnoexcept

Drain resident mappings whose phys lies in [phys_start,phys_end].

Parameters

[in]	phys_start	Inclusive start of physical range.
[in]	phys_end	Exclusive end of physical range.

Returns

Number of physical frames freed.

Here is the call graph for this function:

vm_find()

auto zxfoundation::memory::vm::vm_find ( const vm_space & space, u64 va )->vm_region *

nodiscardexportnoexcept

RCU-protected lookup: find the vm_region containing VA.

Returns

Pointer to the region, or nullptr if no region contains va.

vm_find_gap()

auto zxfoundation::memory::vm::vm_find_gap ( const vm_space & space, u64 size, u64 align )->std::expected< u64, lib::kernel_error >

nodiscardexportnoexcept

Top-down scan from mmap_base for a gap of at least size bytes.

Parameters

[in]	space	The address space to scan.
[in]	size	Required gap size (must be page-aligned).
[in]	align	Alignment requirement (must be power of 2).

Returns

The base VA of the gap on success.

vm_map()

auto zxfoundation::memory::vm::vm_map ( vm_space & space, u64 size, vm_prot prot, vm_flags flags )->std::expected< u64, lib::kernel_error >

nodiscardexportnoexcept

Map a region of size bytes with given protection and flags.

Returns

The virtual address of the mapped region on success.

vm_region_find_by_id()

auto zxfoundation::memory::vm::vm_region_find_by_id ( u32 id ) ->vm_region *

nodiscardexportnoexcept

Look up a vm_region by SCOMS ID.

Returns

Raw pointer, or nullptr if not found.

vm_rekey_region()

auto zxfoundation::memory::vm::vm_rekey_region ( vm_space & space, vm_region & rgn, u64 new_start )->bool

nodiscardexportnoexcept

Parameters

[in,out]	space	Address space whose tree is locked by the caller.
[in,out]	rgn	Region whose start key changes.
[in]	new_start	New page-aligned inclusive start address.

Returns

Success when the new interval can be reinserted.

Here is the call graph for this function:

vm_space_drain_phys_range()

auto zxfoundation::memory::vm::vm_space_drain_phys_range ( vm_space & space, u64 phys_start, u64 phys_end )->u32

nodiscardexportnoexcept

Drop all resident page mappings in space whose backing physical address lies in [phys_start, phys_end).

Parameters

[in,out]	space	Address space to sweep (locked internally).
[in]	phys_start	Inclusive start of physical range.
[in]	phys_end	Exclusive end of physical range.

Returns

Number of physical frames freed (not slots dropped).

Here is the call graph for this function:

vm_space_set_commit_limit()

auto zxfoundation::memory::vm::vm_space_set_commit_limit ( vm_space & space, u64 limit_pages )->std::expected< void, lib::kernel_error >

nodiscardexportnoexcept

Set a domain VM-space committed-page budget.

Note

A live budget may not be reduced below pages already committed.

vm_split()

auto zxfoundation::memory::vm::vm_split ( vm_space & space, vm_region * rgn, u64 split_va )->std::expected< vm_region *, lib::kernel_error >

nodiscardexportnoexcept

Split rgn at split_va into [rgn->va_start, split_va) and [split_va, old_end).

Returns

Pointer to the new region [split_va, old_end) on success.