core.cxxm

Go to the documentation of this file.

/// SPDX-License-Identifier: Apache-2.0
/// @file arch/s390x/mmu/core.cxxm
/// @brief MMU engine — transactional, type-safe DAT table mapper.
 
export module arch.s390x.mmu.core;
import arch.s390x.mmu.types;
import arch.s390x.mmu.consts;
import arch.s390x.mmu.tlb;
import zxfoundation.base.types;
import zxfoundation.memory.pmm;
import zxfoundation.memory.pmm.types;
import zxfoundation.base.transaction;
import lib.error;
import std;
 
namespace arch::s390x::mmu::detail {
 
    using namespace zxfoundation::memory;
 
    from_global(reference) bool g_has_edat1;
    from_global(reference) bool g_has_edat2;
 
    /// @brief Build the default PMM request used for DAT table pages.
    [[nodiscard]] auto default_table_request() noexcept -> pmm::allocation_request;
 
    /// @brief Fill a freshly-allocated table with empty (invalid) entries.
    [[nodiscard]] auto init_table(dat_level lvl, u64 phys) noexcept -> ::std::expected<void, ::lib::kernel_error>;
 
    /// @brief Map a single 4 KiB page using the two-phase protocol.
    /// @return  On success, empty expected.
    auto map_page_impl(
        u64       root_phys,
        dat_level root_level,
        u64       virt,
        u64       phys,
        page_prot prot,
        const pmm::allocation_request& table_policy
    ) noexcept -> ::std::expected<void, ::lib::kernel_error>;
 
    /// @brief Unmap whatever exists at @p virt (4 KiB, 1 MiB, or 2 GiB).
    /// @return Size of the mapping removed, or kernel_error if nothing was mapped.
    auto unmap_at_impl(
        u64       root_phys,
        dat_level root_level,
        u64       virt
    ) noexcept -> ::std::expected<u64, ::lib::kernel_error>;
 
    /// @brief Walk the page tables to resolve virt → phys.
    auto walk_impl(
        u64       root_phys,
        dat_level root_level,
        u64       virt
    ) noexcept -> ::std::expected<u64, ::lib::kernel_error>;
 
    /// @brief Change protection bits on an existing mapping.
    auto protect_impl(
        u64       root_phys,
        dat_level root_level,
        u64       virt,
        page_prot new_prot
    ) noexcept -> ::std::expected<void, ::lib::kernel_error>;
 
    /// @brief Map a 1 MiB large page (EDAT-1) via segment entry.
    auto map_large_1m_impl(
        u64       root_phys,
        dat_level root_level,
        u64       virt,
        u64       phys,
        page_prot prot
    ) noexcept -> ::std::expected<void, ::lib::kernel_error>;
 
    /// @brief Map a 2 GiB large page (EDAT-2) via region-3 entry.
    auto map_large_2g_impl(
        u64       root_phys,
        dat_level root_level,
        u64       virt,
        u64       phys,
        page_prot prot
    ) noexcept -> ::std::expected<void, ::lib::kernel_error>;
 
} // namespace arch::s390x::mmu::detail
 
export {
 
namespace arch::s390x::mmu {
 
    /// @brief Initialize the MMU engine.
    auto init() noexcept -> void;
 
    /// @brief Query EDAT-1 (1 MiB large page) availability.
    [[nodiscard]] auto has_edat1() noexcept -> bool;
 
    /// @brief Query EDAT-2 (2 GiB large page) availability.
    [[nodiscard]] auto has_edat2() noexcept -> bool;
 
    /// @brief Allocate and initialize a CRST table for the given level.
    template <dat_level Level>
        requires (Level != dat_level::page)
    [[nodiscard]] auto crst_alloc() noexcept
        -> ::std::expected<dat_table<Level>, ::lib::kernel_error>
    {
        auto result = zxfoundation::memory::pmm::alloc(
            CRST_ALLOC_ORDER,
            zxfoundation::memory::pmm::zone::normal,
            zxfoundation::memory::pmm::purpose::page_table);
        if (!result) [[unlikely]]
            return ::std::unexpected(result.error());
        const u64 phys = result->phys_base;
        auto init_result = detail::init_table(Level, phys);
        if (!init_result) [[unlikely]] {
            zxfoundation::memory::pmm::folio_active f{
                phys,
                CRST_ALLOC_ORDER,
                zxfoundation::memory::pmm::purpose::page_table,
                zxfoundation::memory::pmm::zone::normal};
            zxfoundation::memory::pmm::free(::std::move(f));
            return ::std::unexpected(init_result.error());
        }
        return dat_table<Level>(phys);
    }
 
    /// @brief Allocate and initialize a page table.
    [[nodiscard]] auto pte_table_alloc() noexcept
        -> ::std::expected<dat_table<dat_level::page>, ::lib::kernel_error>;
 
    /// @brief Free a previously allocated table.
    template <dat_level Level>
    auto table_free(dat_table<Level> tbl) noexcept -> void {
        if (!tbl.valid()) return;
        constexpr u32 order = (Level == dat_level::page)
            ? PTE_ALLOC_ORDER : CRST_ALLOC_ORDER;
        zxfoundation::memory::pmm::folio_active f{
            tbl.origin_phys, order,
            zxfoundation::memory::pmm::purpose::page_table,
            zxfoundation::memory::pmm::zone::normal};
        zxfoundation::memory::pmm::free(::std::move(f));
    }
 
    /// @brief Build a typed ASCE for a kernel address space.
    template <dat_level Root>
        requires (Root != dat_level::page)
    [[nodiscard]] auto asce_build(dat_table<Root> root) noexcept -> typed_asce<Root> {
        const u64 val = (root.origin_phys & ASCE_ORIGIN_MASK)
                      | asce_type_code<Root>()
                      | ASCE_LENGTH_MASK;
        return typed_asce<Root>(val);
    }
 
    /// @brief Map a single 4 KiB page.  Side-effect-free on failure.
    /// @pre  Caller holds exclusive access to the root table.
    template <dat_level Root>
    [[nodiscard]] auto map_page(
        dat_table<Root> root,
        u64             virt,
        u64             phys,
        page_prot       prot
    ) noexcept -> ::std::expected<void, ::lib::kernel_error> {
        const auto table_policy = detail::default_table_request();
        auto result = detail::map_page_impl(
            root.origin_phys, Root, virt, phys, prot, table_policy);
        return result;
    }
 
    /// @brief Map a single 4 KiB page with explicit page-table allocation placement.
    /// @param[in] root Root DAT table.
    /// @param[in] virt Virtual address to map.
    /// @param[in] phys Physical frame to install.
    /// @param[in] prot Hardware protection bits.
    /// @param[in] table_policy PMM allocation request used for newly allocated DAT tables.
    /// @return Success or kernel error.
    /// @note Existing child tables are reused; the policy applies only to tables allocated during this map walk.
    template <dat_level Root>
    [[nodiscard]] auto map_page_with_policy(
        dat_table<Root> root,
        u64             virt,
        u64             phys,
        page_prot       prot,
        const zxfoundation::memory::pmm::allocation_request& table_policy
    ) noexcept -> ::std::expected<void, ::lib::kernel_error> {
        auto result = detail::map_page_impl(
            root.origin_phys, Root, virt, phys, prot, table_policy);
        return result;
    }
 
    /// @brief Unmap a single page/large-page at @p virt.
    /// @return  Physical address of the unmapped frame, or kernel_error.
    /// @pre     Caller holds exclusive access to the root table.
    template <dat_level Root>
    [[nodiscard]] auto unmap_page(
        dat_table<Root> root,
        u64             virt
    ) noexcept -> ::std::expected<u64, ::lib::kernel_error> {
        // First resolve the physical address (walk).
        auto phys_result = detail::walk_impl(
            root.origin_phys, Root, virt);
        if (!phys_result)
            return ::std::unexpected(phys_result.error());
        const u64 old_phys = *phys_result;
        // Now unmap.
        auto size_result = detail::unmap_at_impl(
            root.origin_phys, Root, virt);
        if (!size_result)
            return ::std::unexpected(size_result.error());
        return old_phys;
    }
 
    /// @brief Runtime-dispatched unmap keyed on an ASCE value.
    /// @param[in] asce_val ASCE value (root table + type + length).
    /// @param[in] virt     Virtual address whose PTE to invalidate.
    /// @return The physical address of the unmapped frame, or
    ///         kernel_error (not_present if the address was already
    ///         unmapped, invalid_address if the root level is bogus).
    [[nodiscard]] auto unmap_page_dispatch(
        u64 asce_val, u64 virt
    ) noexcept -> ::std::expected<u64, ::lib::kernel_error>;
 
    /// @brief Walk the page tables to resolve virt → phys.
    /// @return  Physical address corresponding to @p virt, or kernel_error.
    template <dat_level Root>
    [[nodiscard]] auto walk(
        dat_table<Root> root,
        u64             virt
    ) noexcept -> ::std::expected<u64, ::lib::kernel_error> {
        return detail::walk_impl(root.origin_phys, Root, virt);
    }
 
    /// @brief Change protection bits on an existing mapping.
    /// @pre  Caller holds exclusive access to the root table.
    template <dat_level Root>
    [[nodiscard]] auto protect(
        dat_table<Root> root,
        u64             virt,
        page_prot       new_prot
    ) noexcept -> ::std::expected<void, ::lib::kernel_error> {
        return detail::protect_impl(
            root.origin_phys, Root, virt, new_prot);
    }
 
    /// @brief Transactional wrapper for batched MMU operations.
    class mmu_transaction
        : public zxfoundation::base::transaction<mmu_transaction> {
        friend class zxfoundation::base::transaction<mmu_transaction>;
 
        u64       root_phys_{0};
        dat_level root_level_{dat_level::region_1};
        u64       asce_val_{0};
        u64       range_start_{0};
        u64       range_cursor_{0};
 
    public:
        mmu_transaction() noexcept = default;
 
        mmu_transaction(
            u64       root_phys,
            dat_level root_level,
            u64       asce_val,
            u64       range_start
        ) noexcept
            : root_phys_(root_phys)
            , root_level_(root_level)
            , asce_val_(asce_val)
            , range_start_(range_start)
            , range_cursor_(range_start)
        {}
 
        mmu_transaction(const mmu_transaction&)            = delete;
        auto operator=(const mmu_transaction&)             = delete;
 
        /// @brief Advance the cursor after a successful mapping.
        auto advance(u64 bytes) noexcept -> void;
 
        /// @brief Commit the transaction and flush TLB.
        auto commit_and_flush() noexcept -> void;
 
    private:
        /// @brief Rollback: unmap all successfully-mapped entries, then flush.
        auto on_rollback() const noexcept -> void;
    };
 
    /// @brief Map a contiguous range [virt, virt+len) → [phys, phys+len).
    /// @return  Total bytes mapped on success, or kernel_error.
    /// @pre     Caller holds exclusive access to the root table.
    /// @pre     @p virt and @p phys are PAGE_SIZE_4K-aligned.
    /// @pre     @p len is PAGE_SIZE_4K-aligned and > 0.
    template <dat_level Root>
    [[nodiscard]] auto map_range(
        dat_table<Root> root,
        u64             virt,
        u64             phys,
        u64             len,
        page_prot       prot
    ) noexcept -> ::std::expected<u64, ::lib::kernel_error> {
        const u64 asce_val = asce_build(root).raw();
        mmu_transaction txn(root.origin_phys, Root, asce_val, virt);
 
        u64 va = virt;
        u64 pa = phys;
        u64 remaining = len;
 
        while (remaining > 0) {
            u64 step;
 
            // Try 2 GiB large page (EDAT-2).
            if (detail::g_has_edat2
                && remaining >= REGION3_SIZE
                && (va & (REGION3_SIZE - 1)) == 0
                && (pa & (REGION3_SIZE - 1)) == 0) {
                auto r = detail::map_large_2g_impl(
                    root.origin_phys, Root, va, pa, prot);
                if (!r)
                    return ::std::unexpected(r.error());
                step = REGION3_SIZE;
            }
            // Try 1 MiB large page (EDAT-1).
            else if (detail::g_has_edat1
                     && remaining >= SEGMENT_SIZE
                     && (va & (SEGMENT_SIZE - 1)) == 0
                     && (pa & (SEGMENT_SIZE - 1)) == 0) {
                auto r = detail::map_large_1m_impl(
                    root.origin_phys, Root, va, pa, prot);
                if (!r)
                    return ::std::unexpected(r.error());
                step = SEGMENT_SIZE;
            }
            // Fall back to 4 KiB.
            else {
                auto r = detail::map_page_impl(
                    root.origin_phys, Root, va, pa, prot);
                if (!r)
                    return ::std::unexpected(r.error());
                step = PAGE_SIZE_4K;
            }
 
            txn.advance(step);
            va        += step;
            pa        += step;
            remaining -= step;
        }
 
        txn.commit_and_flush();
        return len;
    }
 
    /// @brief Unmap a contiguous range [virt, virt+len).
    /// @return  Total bytes unmapped, or kernel_error.
    /// @pre     Caller holds exclusive access to the root table.
    template <dat_level Root>
    [[nodiscard]] auto unmap_range(
        dat_table<Root> root,
        u64             virt,
        u64             len
    ) noexcept -> ::std::expected<u64, ::lib::kernel_error> {
        u64 va = virt;
        u64 remaining = len;
        u64 unmapped = 0;
 
        while (remaining > 0) {
            auto r = detail::unmap_at_impl(
                root.origin_phys, Root, va);
            u64 step;
            if (r) {
                step = *r;
                unmapped += step;
            } else {
                // Nothing mapped here — skip a page.
                step = PAGE_SIZE_4K;
            }
            va        += step;
            remaining  = (remaining >= step) ? remaining - step : 0;
        }
 
        // Single IDTE flush for the entire address space.
        if (unmapped > 0) {
            const u64 asce_val = asce_build(root).raw();
            tlb_flush_asce(asce_val);
        }
        return unmapped;
    }
 
} // namespace arch::s390x::mmu
 
} // end export