Physical Memory Manager. More...

import std;
import arch.s390x.cpu.skey;
import zxfoundation.memory.numa.types;
import zxfoundation.memory.pmm.types;
import zxfoundation.base.transaction;
import zxfoundation.memory.hhdm;
import zxfoundation.memory.numa;
import zxfoundation.scoms.kobject.types;
import lib.error;
import zxfoundation.sync.lockable;
import zxfoundation.base.types;
import arch.s390x.cpu.lowcore;
import arch.s390x.init.zxfl.protocol;
import zxfoundation.sync.qspinlock.core;
import zxfoundation.base.linksym;
import lib.static_vector;
import arch.s390x.cpu.irq;
import zxfoundation.sched.context.core;
import zxfoundation.sys.syschk.core;

Include dependency graph for core.cxx:

Data Structures
struct	zxfoundation::memory::pmm::detail::buddy_zone
	Per-zone buddy allocator state. More...
struct	zxfoundation::memory::pmm::detail::free_node
	Intrusive doubly-linked free-list node, written into the first 16 bytes of each free buddy block via HHDM. More...
struct	zxfoundation::memory::pmm::detail::node_allocation
struct	zxfoundation::memory::pmm::detail::numa_zone_set
	Per-node allocator state. More...
struct	zxfoundation::memory::pmm::detail::pcp_node
	Intrusive singly-linked PCP node, written into the first 8 bytes of each PCP-cached page. More...
struct	zxfoundation::memory::pmm::detail::resv_entry

Functions
static constexpr auto	zxfoundation::memory::pmm::detail::align_resv (resv_entry r) noexcept -> resv_entry
auto	zxfoundation::memory::pmm::alloc (const allocation_request &request) noexcept -> std::expected< allocation_result, lib::kernel_error >
	Allocate a single folio.
auto	zxfoundation::memory::pmm::alloc (u32 order, zone z, purpose p, page_init init) noexcept -> std::expected< folio_embryo, lib::kernel_error >
	Allocate a single folio.
auto	zxfoundation::memory::pmm::detail::alloc_locked (u32 order, zone z, u8 node_id) noexcept -> u64
	Allocate a block of the given order from a node-owned zone.
auto	zxfoundation::memory::pmm::detail::alloc_node_aware (u32 order, zone z, const numa::allocation_policy &policy, u8 local_node) noexcept -> node_allocation
auto	zxfoundation::memory::pmm::detail::assert_allocatable_frame (u64 phys) noexcept -> void
	Fail-closed validation of a frame handed back by the buddy / PCP layer.
auto	zxfoundation::memory::pmm::detail::boot_region_node_id (const arch::s390x::init::zxfl::mem_region &region, u64 phys) noexcept -> u8
auto	zxfoundation::memory::pmm::detail::desc_of (u64 phys) noexcept -> std::expected< frame_desc *, lib::kernel_error >
	Access the frame_desc for a given physical address.
auto	zxfoundation::memory::pmm::desc_of_checked (u64 phys) noexcept -> std::expected< frame_desc *, lib::kernel_error >
	Fail-closed frame-descriptor accessor.
auto	zxfoundation::memory::pmm::frame_clear_mapping_metadata (u64 phys) noexcept -> std::expected< void, lib::kernel_error >
	Clear the authority identity stamped on one physical frame.
auto	zxfoundation::memory::pmm::frame_desc_of (u64 phys) noexcept -> frame_desc *
	Return a pointer to the frame_desc for a physical address.
auto	zxfoundation::memory::pmm::detail::frame_node_id (u64 phys) noexcept -> u8
auto	zxfoundation::memory::pmm::frame_node_of (u64 phys) noexcept -> std::expected< u8, lib::kernel_error >
	Return the NUMA node id recorded for a frame.
auto	zxfoundation::memory::pmm::frame_object_generation_of (u64 phys) noexcept -> std::expected< u32, lib::kernel_error >
	Resolve the SCOMS generation paired with a frame authority id.
auto	zxfoundation::memory::pmm::frame_object_id_of (u64 phys) noexcept -> std::expected< u32, lib::kernel_error >
	Resolve the SCOMS memory_object_id that authorizes a frame.
auto	zxfoundation::memory::pmm::frame_ref (u64 phys) noexcept -> std::expected< u16, lib::kernel_error >
	Atomically increment a frame's refcount.
auto	zxfoundation::memory::pmm::frame_refcount (u64 phys) noexcept -> std::expected< u16, lib::kernel_error >
	Return the current atomic reference count of a frame.
auto	zxfoundation::memory::pmm::frame_release_pin (u64 phys) noexcept -> std::expected< bool, lib::kernel_error >
	Release one order-zero mapping pin and return the frame.
auto	zxfoundation::memory::pmm::frame_set_mapping_metadata (u64 phys, u32 object_id, u32 object_generation, bool writable) noexcept -> std::expected< void, lib::kernel_error >
	Stamp the SCOMS memory_object id that authorizes this frame.
auto	zxfoundation::memory::pmm::frame_unref (u64 phys) noexcept -> std::expected< u16, lib::kernel_error >
	Atomically decrement a frame's refcount.
auto	zxfoundation::memory::pmm::frame_validate_mapping (u64 phys_start, u64 length, u32 object_id, u32 object_generation) noexcept -> std::expected< void, lib::kernel_error >
	Validate that a physical range is owned by one SCOMS object.
auto	zxfoundation::memory::pmm::free (folio_active f) noexcept -> void
	Return a folio to the allocator.
auto	zxfoundation::memory::pmm::detail::free_locked (u64 phys, u32 order, zone z, u8 node_id) noexcept -> void
	Return a block to a node-owned zone, coalescing with same-node free buddies.
auto	zxfoundation::memory::pmm::free_page_count () noexcept -> u64
	Advisory free page count (sum across all zones).
auto	zxfoundation::memory::pmm::init (const arch::s390x::init::zxfl::protocol &proto) noexcept -> void
	Initialize the PMM from the ZXFL boot protocol.
auto	zxfoundation::memory::pmm::install_allocation_retry (allocation_retry_fn callback) noexcept -> std::expected< void, lib::kernel_error >
	Install the allocation-pressure reclaim callback.
auto	zxfoundation::memory::pmm::detail::local_node_id () noexcept -> u8
auto	zxfoundation::memory::pmm::max_pfn () noexcept -> u64
static auto	zxfoundation::memory::pmm::detail::next_resv_start (u64 phys, u64 limit, const resv_entry *resv, u32 count) noexcept -> u64
auto	zxfoundation::memory::pmm::detail::node_is_allocatable (u8 node_id) noexcept -> bool
constexpr auto	zxfoundation::memory::pmm::detail::normalize_node_id (u8 node_id) noexcept -> u8
constexpr auto	zxfoundation::memory::pmm::detail::owner_for_purpose (purpose p) noexcept -> frame_owner
auto	zxfoundation::memory::pmm::detail::pcp_drain (pcp_list &lst, u32 ord, zone z, u8 node_id) noexcept -> void
	Drain PCP_BATCH/2 pages from a list back to the buddy. Takes the zone spinlock internally (IRQs already disabled by caller).
auto	zxfoundation::memory::pmm::detail::pcp_pop (pcp_list &lst) noexcept -> u64
	Pop one page of order ord from a PCP list.
auto	zxfoundation::memory::pmm::detail::pcp_push (pcp_list &lst, u64 phys) noexcept -> void
	Push one page onto a PCP list head.
auto	zxfoundation::memory::pmm::detail::pcp_refill (pcp_list &lst, u32 ord, zone z, u8 node_id) noexcept -> void
	Refill a PCP list by bulk-allocating PCP_BATCH pages from buddy.
auto	zxfoundation::memory::pmm::detail::quarantine_failed_allocation (u64 phys, u32 order, u8 node_id) noexcept -> void
	Retire an allocation that failed post-allocation validation.
auto	zxfoundation::memory::pmm::reserve (u64 phys, u32 order) noexcept -> void
	Mark a physical range as permanently reserved (runtime call).
static auto	zxfoundation::memory::pmm::detail::resv_skip_end (u64 phys, const resv_entry *resv, u32 count) noexcept -> u64
static auto	zxfoundation::memory::pmm::detail::sort_merge_reservations (resv_entry *resv, u32 &count) noexcept -> void
auto	zxfoundation::memory::pmm::total_page_count () noexcept -> u64
	Total managed page count (sum across all zones).
auto	zxfoundation::memory::pmm::detail::try_alloc_node (u32 order, zone z, u8 node_id) noexcept -> node_allocation
auto	zxfoundation::memory::pmm::detail::validate_page_table_allocation (u64 phys, u32 order, zone z, u8 node_id) noexcept -> std::expected< void, lib::kernel_error >
constexpr auto	zxfoundation::memory::pmm::detail::zone_of (u64 phys) noexcept -> zone

Variables
allocation_retry_fn	zxfoundation::memory::pmm::detail::g_allocation_retry {nullptr}
static frame_desc *	zxfoundation::memory::pmm::detail::g_frame_table {nullptr}
	Flat frame descriptor table.
static bool	zxfoundation::memory::pmm::detail::g_initialized {false}
static u64	zxfoundation::memory::pmm::detail::g_max_pfn {0}
	Highest PFN ever observed in the boot memory map.
static numa_zone_set	zxfoundation::memory::pmm::detail::g_node_zones [numa::MAX_NUMA_NODES] {}
	Node-owned buddy, lock, and accounting state.
std::atomic< u32 >	zxfoundation::memory::pmm::detail::g_reclaim_active {0U}
static u64	zxfoundation::memory::pmm::detail::g_retired_pages {0}
	Frames deliberately retired after validation or MCCK failure.
constexpr u32	zxfoundation::memory::pmm::detail::MAX_RESERVATIONS

Detailed Description

Physical Memory Manager.

SPDX-License-Identifier: Apache-2.0

Function Documentation

◆ alloc() [1/2]

auto zxfoundation::memory::pmm::alloc ( const allocation_request & request ) ->std::expected< allocation_result, lib::kernel_error >

nodiscardexportnoexcept

Allocate a single folio.

Parameters

[in] request Node-aware allocation request.

Returns: Allocation result containing the folio and resolved placement metadata.

Note: Order-zero through PCP_MAX_ORDER allocations attempt the current CPU's node-local PCP first.

Warning: Allocation is forbidden in MCCK context and multi-page allocation is forbidden in hard IRQ context.

◆ alloc() [2/2]

auto zxfoundation::memory::pmm::alloc	(	u32	order,
		zone	z,
		purpose	p,
		page_init	init )->std::expected< folio_embryo, lib::kernel_error >

nodiscardexportnoexcept

Allocate a single folio.

Parameters

[in]	order	Buddy order (0–MAX_ORDER).
[in]	z	Zone constraint.
[in]	p	Semantic purpose annotation.
[in]	init	Initialization policy.

Returns: folio_embryo on success, kernel_error on failure.

◆ alloc_locked()

auto zxfoundation::memory::pmm::detail::alloc_locked	(	u32	order,
		zone	z,
		u8	node_id )->u64

noexcept

Allocate a block of the given order from a node-owned zone.

Precondition: Caller holds the selected node-zone lock.

Returns: Physical base address of the allocated block, or 0 on failure.

◆ assert_allocatable_frame()

auto zxfoundation::memory::pmm::detail::assert_allocatable_frame ( u64 phys ) ->void

noexcept

Fail-closed validation of a frame handed back by the buddy / PCP layer.

Parameters

[in] phys Candidate physical base address taken from a free list.

Warning: Triggers syschk_fatal on any inconsistency and does not return in that case.

◆ desc_of()

auto zxfoundation::memory::pmm::detail::desc_of ( u64 phys ) ->std::expected< frame_desc *, lib::kernel_error >

nodiscardnoexcept

Access the frame_desc for a given physical address.

Warning: Only valid after pmm::init() has set g_frame_table.

◆ desc_of_checked()

auto zxfoundation::memory::pmm::desc_of_checked ( u64 phys ) ->std::expected< frame_desc *, lib::kernel_error >

nodiscardexportnoexcept

Fail-closed frame-descriptor accessor.

Parameters

[in] phys Physical address (lower PAGE_SHIFT bits are ignored).

Returns: Pointer into the frame descriptor table, or a kernel_error when the PMM is uninitialized or phys is outside the managed physical range.

◆ frame_desc_of()

auto zxfoundation::memory::pmm::frame_desc_of ( u64 phys ) ->frame_desc *

nodiscardexportnoexcept

Return a pointer to the frame_desc for a physical address.

Parameters

[in] phys Physical address (lower PAGE_SHIFT bits are ignored).

Returns: Pointer into the frame descriptor table, or nullptr if the address is out of range or init() has not been called.

◆ frame_node_of()

auto zxfoundation::memory::pmm::frame_node_of ( u64 phys ) ->std::expected< u8, lib::kernel_error >

nodiscardexportnoexcept

Return the NUMA node id recorded for a frame.

Parameters

[in] phys Physical frame address.

Returns: NUMA node id, or a checked physical-address error.

◆ frame_object_id_of()

auto zxfoundation::memory::pmm::frame_object_id_of ( u64 phys ) ->std::expected< u32, lib::kernel_error >

nodiscardexportnoexcept

Resolve the SCOMS memory_object_id that authorizes a frame.

Parameters

[in] phys Physical frame address.

Returns: SCOMS id, or KOBJECT_ID_INVALID when the frame is not bound to a memory authority (buddy-owned, reserved, page-table).

◆ frame_ref()

auto zxfoundation::memory::pmm::frame_ref ( u64 phys ) ->std::expected< u16, lib::kernel_error >

nodiscardexportnoexcept

Atomically increment a frame's refcount.

Parameters

[in] phys Physical base address of the frame.

Returns: New refcount on success, kernel_error on invalid address.

◆ frame_refcount()

auto zxfoundation::memory::pmm::frame_refcount ( u64 phys ) ->std::expected< u16, lib::kernel_error >

nodiscardexportnoexcept

Return the current atomic reference count of a frame.

Parameters

[in] phys Physical frame address.

Returns: Reference count or a checked physical-address error.

◆ frame_release_pin()

auto zxfoundation::memory::pmm::frame_release_pin ( u64 phys ) ->std::expected< bool, lib::kernel_error >

nodiscardexportnoexcept

Release one order-zero mapping pin and return the frame.

Returns: True when this call returned the frame to buddy/PCP, false when another pin still owns it.

Note: The frame descriptor supplies the NUMA node and physical zone

Returns: True when this call returned the frame to buddy/PCP, false when another pin still owns it.

◆ frame_set_mapping_metadata()

auto zxfoundation::memory::pmm::frame_set_mapping_metadata	(	u64	phys,
		u32	object_id,
		u32	object_generation,
		bool	writable )->std::expected< void, lib::kernel_error >

nodiscardexportnoexcept

Stamp the SCOMS memory_object id that authorizes this frame.

Parameters

[in]	phys	Physical frame address.
[in]	object_id	SCOMS id of the authoritative memory_object (typically a vm_region's storage_object grant, or KOBJECT_ID_INVALID to clear).
[in]	object_generation	SCOMS generation paired with `object_id`.
[in]	writable	True when the frame backs a writable mapping.

Returns: Success or a checked physical-address error.

◆ frame_unref()

auto zxfoundation::memory::pmm::frame_unref ( u64 phys ) ->std::expected< u16, lib::kernel_error >

nodiscardexportnoexcept

Atomically decrement a frame's refcount.

Parameters

[in] phys Physical base address of the frame.

Returns: New refcount on success, kernel_error on invalid address.

◆ frame_validate_mapping()

auto zxfoundation::memory::pmm::frame_validate_mapping	(	u64	phys_start,
		u64	length,
		u32	object_id,
		u32	object_generation )->std::expected< void, lib::kernel_error >

nodiscardexportnoexcept

Validate that a physical range is owned by one SCOMS object.

Parameters

[in]	phys_start	Page-aligned physical range start.
[in]	length	Range length in bytes.
[in]	object_id	Required SCOMS id (must match frame_desc::object_id).
[in]	object_generation	Required paired SCOMS generation.

Returns: Success only when every page carries the exact live object identity and generation, and is not MCCK-offline.

◆ free()

auto zxfoundation::memory::pmm::free ( folio_active f ) ->void

exportnoexcept

Return a folio to the allocator.

Parameters

[in] f The folio to free. Must not be empty.

◆ free_locked()

auto zxfoundation::memory::pmm::detail::free_locked	(	u64	phys,
		u32	order,
		zone	z,
		u8	node_id )->void

noexcept

Return a block to a node-owned zone, coalescing with same-node free buddies.

Precondition: Caller holds the selected node-zone lock.

◆ init()

auto zxfoundation::memory::pmm::init ( const arch::s390x::init::zxfl::protocol & proto ) ->void

exportnoexcept

Initialize the PMM from the ZXFL boot protocol.

Parameters

[in] proto Validated ZXFL boot protocol.

◆ pcp_pop()

auto zxfoundation::memory::pmm::detail::pcp_pop ( pcp_list & lst ) ->u64

nodiscardnoexcept

Pop one page of order ord from a PCP list.

Returns: Physical base address, or 0 if list is empty.

◆ reserve()

auto zxfoundation::memory::pmm::reserve	(	u64	phys,
		u32	order )->void

exportnoexcept

Mark a physical range as permanently reserved (runtime call).

Parameters

[in]	phys	Start of the range (page-aligned).
[in]	order	log2 of the number of pages to reserve.

Variable Documentation

◆ MAX_RESERVATIONS

u32 zxfoundation::memory::pmm::detail::MAX_RESERVATIONS

constexpr

Initial value:

=

arch::s390x::init::zxfl::ZXFL_MEM_MAP_MAX + 5U

ZXFoundation™ 26h2

Data Structures

Functions

Variables

Detailed Description

Function Documentation

◆ alloc() [1/2]

◆ alloc() [2/2]

◆ alloc_locked()

◆ assert_allocatable_frame()

◆ desc_of()

◆ desc_of_checked()

◆ frame_desc_of()

◆ frame_node_of()

◆ frame_object_id_of()

◆ frame_ref()

◆ frame_refcount()

◆ frame_release_pin()

◆ frame_set_mapping_metadata()

◆ frame_unref()

◆ frame_validate_mapping()

◆ free()

◆ free_locked()

◆ init()

◆ pcp_pop()

◆ reserve()

Variable Documentation

◆ MAX_RESERVATIONS