refactor!: convert shadow ranges to memory ranges

Author: edubart

.gitignore

@@ -6,6 +6,7 @@
 *.a
 *.lib
 *.wasm
+*.TODO
 
 build
 pkg

src/Makefile

@@ -151,7 +151,8 @@ HASH_LIBS=
 
 #DEFS+= -DMT_ALL_DIRTY
 
-WARNS=-Wall -Wextra -Wpedantic
+C_WARNS=-Wall -Wextra -Wpedantic
+CXX_WARNS=$(C_WARNS) -Wno-missing-field-initializers
 
 # Place our include directories before the system's
 INCS+= \
@@ -305,8 +306,8 @@ else
 DEFS+=-DNO_THREADS
 endif
 
-CXXFLAGS+=$(OPTFLAGS) -std=gnu++20 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(WARNS)
-CFLAGS+=$(OPTFLAGS) -std=gnu99 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(WARNS)
+CXXFLAGS+=$(OPTFLAGS) -std=gnu++20 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(CXX_WARNS)
+CFLAGS+=$(OPTFLAGS) -std=gnu99 -fvisibility=hidden -MMD $(PICCFLAGS) $(CC_MARCH) $(INCS) $(GCFLAGS) $(UBFLAGS) $(DEFS) $(C_WARNS)
 LDFLAGS+=$(UBFLAGS)
 
 ifeq ($(coverage),yes)
@@ -357,14 +358,12 @@ LIBCARTESI_OBJS:= \
 	machine.o \
 	memory-address-range.o \
 	os.o \
+	os-mmap.o \
 	plic-address-range.o \
 	pristine-merkle-tree.o \
 	replay-step-state-access-interop.o \
 	send-cmio-response.o \
 	sha3.o \
-	shadow-state-address-range.o \
-	shadow-tlb-address-range.o \
-	shadow-uarch-state-address-range.o \
 	uarch-pristine-hash.o \
 	uarch-pristine-ram.o \
 	uarch-pristine-state-hash.o \

src/address-range-constants.h

@@ -25,14 +25,16 @@ namespace cartesi {
 
 /// \brief Fixed address ranges.
 enum AR_ranges : uint64_t {
-    AR_SHADOW_STATE_START = EXPAND_UINT64_C(AR_SHADOW_STATE_START_DEF),   ///< Start of shadow state range
-    AR_SHADOW_STATE_LENGTH = EXPAND_UINT64_C(AR_SHADOW_STATE_LENGTH_DEF), ///< Length of shadow state range
-    AR_PMAS_START = EXPAND_UINT64_C(AR_PMAS_START_DEF),                   ///< Start of PMAS list range
-    AR_PMAS_LENGTH = EXPAND_UINT64_C(AR_PMAS_LENGTH_DEF),                 ///< Length of PMAS list range
-    AR_DTB_START = EXPAND_UINT64_C(AR_DTB_START_DEF),                     ///< Start of DTB range
-    AR_DTB_LENGTH = EXPAND_UINT64_C(AR_DTB_LENGTH_DEF),                   ///< Length of DTB range
-    AR_SHADOW_TLB_START = EXPAND_UINT64_C(AR_SHADOW_TLB_START_DEF),       ///< Start of shadow TLB range
-    AR_SHADOW_TLB_LENGTH = EXPAND_UINT64_C(AR_SHADOW_TLB_LENGTH_DEF),     ///< Length of shadow TLB range
+    AR_SHADOW_STATE_START = EXPAND_UINT64_C(AR_SHADOW_STATE_START_DEF),           ///< Start of shadow state range
+    AR_SHADOW_STATE_LENGTH = EXPAND_UINT64_C(AR_SHADOW_STATE_LENGTH_DEF),         ///< Length of shadow state range
+    AR_SHADOW_REGISTERS_START = EXPAND_UINT64_C(AR_SHADOW_REGISTERS_START_DEF),   ///< Start of shadow registers range
+    AR_SHADOW_REGISTERS_LENGTH = EXPAND_UINT64_C(AR_SHADOW_REGISTERS_LENGTH_DEF), ///< Length of shadow registers range
+    AR_SHADOW_TLB_START = EXPAND_UINT64_C(AR_SHADOW_TLB_START_DEF),               ///< Start of shadow TLB range
+    AR_SHADOW_TLB_LENGTH = EXPAND_UINT64_C(AR_SHADOW_TLB_LENGTH_DEF),             ///< Length of shadow TLB range
+    AR_PMAS_START = EXPAND_UINT64_C(AR_PMAS_START_DEF),                           ///< Start of PMAS list range
+    AR_PMAS_LENGTH = EXPAND_UINT64_C(AR_PMAS_LENGTH_DEF),                         ///< Length of PMAS list range
+    AR_DTB_START = EXPAND_UINT64_C(AR_DTB_START_DEF),                             ///< Start of DTB range
+    AR_DTB_LENGTH = EXPAND_UINT64_C(AR_DTB_LENGTH_DEF),                           ///< Length of DTB range
     AR_SHADOW_UARCH_STATE_START =
         EXPAND_UINT64_C(AR_SHADOW_UARCH_STATE_START_DEF), ///< Start of uarch shadow state range
     AR_SHADOW_UARCH_STATE_LENGTH =
@@ -61,6 +63,10 @@ enum AR_ranges : uint64_t {
     AR_RAM_START = EXPAND_UINT64_C(AR_RAM_START_DEF), ///< Start of RAM range
 };
 
+static_assert(AR_SHADOW_STATE_LENGTH >= AR_SHADOW_REGISTERS_LENGTH + AR_SHADOW_TLB_LENGTH);
+static_assert(AR_SHADOW_TLB_START == AR_SHADOW_REGISTERS_START + AR_SHADOW_REGISTERS_LENGTH);
+static_assert(AR_SHADOW_STATE_START == AR_SHADOW_REGISTERS_START);
+
 /// \brief PMA constants.
 enum AR_constants : uint64_t {
     AR_PAGE_SIZE_LOG2 = EXPAND_UINT64_C(AR_PAGE_SIZE_LOG2_DEF), ///< Log<sub>2</sub> of physical memory page size.

src/address-range-defines.h

@@ -19,11 +19,13 @@
 
 // NOLINTBEGIN(cppcoreguidelines-macro-usage,cppcoreguidelines-macro-to-enum,modernize-macro-to-enum)
 #define AR_SHADOW_STATE_START_DEF 0x0            ///< Shadow start address
-#define AR_SHADOW_STATE_LENGTH_DEF 0x1000        ///< Shadow length in bytes
+#define AR_SHADOW_STATE_LENGTH_DEF 0x8000        ///< Shadow length in bytes
+#define AR_SHADOW_REGISTERS_START_DEF 0x0        ///< Shadow registers start address
+#define AR_SHADOW_REGISTERS_LENGTH_DEF 0x1000    ///< Shadow registers length in bytes
+#define AR_SHADOW_TLB_START_DEF 0x1000           ///< Shadow TLB start address
+#define AR_SHADOW_TLB_LENGTH_DEF 0x6000          ///< Shadow TLB length in bytes
 #define AR_PMAS_START_DEF 0x10000                ///< PMA Array start address
 #define AR_PMAS_LENGTH_DEF 0x1000                ///< PMA Array length in bytes
-#define AR_SHADOW_TLB_START_DEF 0x1000           ///< TLB start address
-#define AR_SHADOW_TLB_LENGTH_DEF 0x6000          ///< TLB length in bytes
 #define AR_SHADOW_UARCH_STATE_START_DEF 0x400000 ///< microarchitecture shadow state start address
 #define AR_SHADOW_UARCH_STATE_LENGTH_DEF 0x1000  ///< microarchitecture shadow state length
 #define AR_UARCH_RAM_START_DEF 0x600000          ///< microarchitecture RAM start address

src/address-range.h

@@ -220,6 +220,20 @@ class address_range {
         return m_flags.IW;
     }
 
+    /// \brief Tests if range can be replaced
+    /// \returns True if and only if range can be replaced
+    bool is_replaceable() const noexcept {
+        switch (m_flags.DID) {
+            case PMA_ISTART_DID::memory:
+            case PMA_ISTART_DID::flash_drive:
+            case PMA_ISTART_DID::cmio_rx_buffer:
+            case PMA_ISTART_DID::cmio_tx_buffer:
+                return true;
+            default:
+                return false;
+        }
+    }
+
     /// \brief Returns driver ID associated to range
     /// \returns Teh driver ID
     PMA_ISTART_DID get_driver_id() const noexcept {
@@ -327,6 +341,18 @@ class address_range {
         return do_is_page_marked_dirty(offset);
     }
 
+    /// \brief Returns true if the mapped memory is read-only on the host
+    /// \returns True if the memory is read-only in the host
+    bool is_host_read_only() const noexcept {
+        return do_is_host_read_only();
+    }
+
+    /// \brief Returns true if pages marked as dirty once are uncleanable
+    /// \returns True if and only if dirty pages are uncleanable
+    bool is_page_uncleanable() const noexcept {
+        return do_is_page_uncleanable();
+    }
+
 private:
     // Default implementation of peek() always fails
     virtual bool do_peek(const machine & /*m*/, uint64_t /*offset*/, uint64_t /*length*/,
@@ -371,6 +397,14 @@ class address_range {
     virtual bool do_is_page_marked_dirty(uint64_t /*offset*/) const noexcept {
         return true;
     }
+
+    virtual bool do_is_host_read_only() const noexcept {
+        return false;
+    }
+
+    virtual bool do_is_page_uncleanable() const noexcept {
+        return false;
+    }
 };
 
 template <size_t N>

src/cartesi-machine.lua

@@ -107,6 +107,9 @@ where options are:
         data_filename:<filename>
         dht_filename:<filename>
         shared
+        create
+        truncate
+        read_only
         mount:<string>
         user:<string>
 
@@ -137,6 +140,18 @@ where options are:
         target modifications to flash drive modify the memory and hash tree files.
         by default, image files are not modified and changes are lost.
 
+        create (optional)
+        create the backing storage file, shared must also be true.
+
+        truncate (optional)
+        truncate the memory length to match memory lengths different from the backing storage,
+        in case of shared flash drive, then it also truncates the underlying backing file.
+        by default, when a length is present it must also match the backing storage length.
+
+        read_only (optional)
+        mark flash drive as read-only, disallowing write attempts from the host or the guest.
+        by default, flash drives are not read-only, thus writable.
+
         mount (optional)
         whether the flash drive should be mounted automatically in init.
         by default, the drive is mounted if there is an image file backing it,
@@ -171,11 +186,12 @@ where options are:
 
   --ram=<key>:<value>[,<key>:<value>[,...]...]
   --dtb=<key>:<value>[,<key>:<value>[,...]...]
-  --tlb=<key>:<value>[,<key>:<value>[,...]...]
+  --processor=<key>:<value>[,<key>:<value>[,...]...]
   --cmio-rx-buffer=<key>:<value>[,<key>:<value>[,...]...]
   --cmio-tx-buffer=<key>:<value>[,<key>:<value>[,...]...]
   --pmas=<key>:<value>[,<key>:<value>[,...]...]
   --uarch-ram=<key>:<value>[,<key>:<value>[,...]...]
+  --uarch-processor=<key>:<value>[,<key>:<value>[,...]...]
     configures file storage for other memory ranges in the machine
 
     <key>:<value> is one of
@@ -471,6 +487,31 @@ where options are:
   --load=<directory>
     load machine previously stored in <directory>.
 
+  --init-shared=<directory>
+    initialize a machine sharing snapshot created in <directory>.
+    writable address ranges are shared, thus runtime modifications persist in the snapshot.
+    writable address ranges use reflinks on copy-on-write filesystems for efficient copying.
+    read-only address ranges use hard links to avoid unnecessary copying.
+    address ranges sparsity is preserved to minimize host storage usage.
+
+    MUST BE USED WITH --no-rollback
+
+  --load-shared=<directory>
+    load an existing machine sharing snapshot from <directory>,
+    writable address ranges are shared, thus runtime modifications persists in the snapshot.
+    address ranges sparsity is preserved to minimize host storage usage.
+
+    MUST BE USED WITH --no-rollback
+
+  --clone-shared=<source_directory>,<destination_directory>
+    clones a snapshot from source directory to destination directory and load a machine sharing it.
+    writable address ranges are shared, thus runtime modifications persist in the snapshot.
+    writable address ranges use reflinks on copy-on-write filesystems for efficient copying.
+    read-only address ranges use hard links to avoid unnecessary copying.
+    address ranges sparsity is preserved to minimize host storage usage.
+
+    MUST BE USED WITH --no-rollback
+
   --initial-hash
     print initial state hash before running machine.
 
@@ -614,6 +655,9 @@ local flash_data_filename = { root = images_path .. "rootfs.ext2" }
 local flash_dht_filename = {}
 local flash_label_order = { "root" }
 local flash_shared = {}
+local flash_create = {}
+local flash_truncate = {}
+local flash_read_only = {}
 local flash_mount = {}
 local flash_user = {}
 local flash_start = {}
@@ -1012,13 +1056,6 @@ local options = {
             return true
         end,
     },
-    {
-        "^(%-%-tlb%=(.+))$",
-        function(all, opts)
-            tlb.backing_store = parse_backing_store(opts, "tlb", all, tlb.backing_store)
-            return true
-        end,
-    },
     {
         "^(%-%-pmas%=(.+))$",
         function(all, opts)
@@ -1146,6 +1183,9 @@ local options = {
                 data_filename = true,
                 dht_filename = true,
                 shared = true,
+                create = true,
+                truncate = true,
+                read_only = true,
                 mount = true,
                 user = true,
                 length = true,
@@ -1155,6 +1195,9 @@ local options = {
             if f.data_filename == true then f.data_filename = "" end
             if f.dht_filename == true then f.dht_filename = "" end
             assert(not f.shared or f.shared == true, "invalid flash drive shared value in " .. all)
+            assert(not f.create or f.create == true, "invalid flash drive create value in " .. all)
+            assert(not f.truncate or f.truncate == true, "invalid flash drive read_only value in " .. all)
+            assert(not f.read_only or f.read_only == true, "invalid flash drive truncate value in " .. all)
             if f.mount == nil then
                 -- mount only if there is a file backing
                 if f.data_filename and f.data_filename ~= "" then
@@ -1179,8 +1222,14 @@ local options = {
             flash_start[d] = f.start or flash_start[d]
             flash_length[d] = f.length or flash_length[d]
             flash_shared[d] = f.shared or flash_shared[d]
+            flash_create[d] = f.create or flash_create[d]
+            flash_truncate[d] = f.truncate or flash_truncate[d]
+            flash_read_only[d] = f.read_only or flash_read_only[d]
             flash_mount[d] = f.mount or flash_mount[d]
             flash_user[d] = f.user or flash_user[d]
+            if d == "root" and f.read_only then -- Mount root filesystem as read-only
+                dtb.bootargs = dtb.bootargs:gsub("rw", "ro")
+            end
             return true
         end,
     },
@@ -1335,6 +1384,9 @@ local options = {
             flash_start.root = nil
             flash_length.root = nil
             flash_shared.root = nil
+            flash_create.root = nil
+            flash_truncate.root = nil
+            flash_read_only.root = nil
             table.remove(flash_label_order, 1)
             dtb.bootargs = dtb.bootargs:gsub(" root=$", "")
             return true
@@ -1846,7 +1898,10 @@ echo "
                 data_filename = flash_data_filename[label],
                 dht_filename = flash_dht_filename[label],
                 shared = flash_shared[label],
+                create = flash_create[label],
+                truncate = flash_truncate[label],
             },
+            read_only = flash_read_only[label],
             start = flash_start[label],
             length = flash_length[label] or -1,
         }

src/hot-tlb.h

@@ -0,0 +1,53 @@
+// Copyright Cartesi and individual authors (see AUTHORS)
+// SPDX-License-Identifier: LGPL-3.0-or-later
+//
+// This program is free software: you can redistribute it and/or modify it under
+// the terms of the GNU Lesser General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option) any
+// later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT ANY
+// WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+// PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License along
+// with this program (see COPYING). If not, see <https://www.gnu.org/licenses/>.
+//
+
+#ifndef HOT_TLB_H
+#define HOT_TLB_H
+
+/// \file
+/// \brief TLB definitions
+/// \details The Translation Lookaside Buffer is a small cache used to speed up translation between address spaces.
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+
+#include "host-addr.h"
+#include "shadow-tlb.h"
+
+namespace cartesi {
+
+/// \brief Hot TLB slot.
+/// \details
+/// Given a target virtual address vaddr within a page matching vaddr_page in TLB slot, the corresponding host address
+/// haddr = vaddr + vh_offset.
+struct hot_tlb_slot final {
+    uint64_t vaddr_page{TLB_INVALID_PAGE}; ///< Target virtual address of start of page
+    host_addr vh_offset{0};                ///< Offset from target virtual address in the same page to host address
+};
+
+using hot_tlb_set = std::array<hot_tlb_slot, TLB_SET_SIZE>;
+using hot_tlb_state = std::array<hot_tlb_set, TLB_NUM_SETS_>;
+
+static_assert(sizeof(uint64_t) >= sizeof(uintptr_t), "TLB expects host pointer fit in 64 bits");
+
+// We need to ensure TLB state sizes are fixed across different platforms
+static_assert(sizeof(hot_tlb_state) == 3 * TLB_SET_SIZE * 2 * sizeof(uint64_t), "unexpected hot TLB state size");
+static_assert(alignof(hot_tlb_state) == sizeof(uint64_t), "unexpected hot TLB state alignment");
+
+} // namespace cartesi
+
+#endif

src/i-prefer-shadow-state.h

@@ -25,8 +25,8 @@
 #include <type_traits>
 
 #include "meta.h"
-#include "shadow-state.h"
-#include "tlb.h"
+#include "shadow-registers.h"
+#include "shadow-tlb.h"
 
 namespace cartesi {
 
@@ -48,12 +48,12 @@ class i_prefer_shadow_state { // CRTP
     }
 
 public:
-    uint64_t read_shadow_state(shadow_state_what what) const {
-        return derived().do_read_shadow_state(what);
+    uint64_t read_shadow_register(shadow_registers_what what) const {
+        return derived().do_read_shadow_register(what);
     }
 
-    void write_shadow_state(shadow_state_what what, uint64_t val) const {
-        derived().do_write_shadow_state(what, val);
+    void write_shadow_register(shadow_registers_what what, uint64_t val) const {
+        derived().do_write_shadow_register(what, val);
     }
 };
 

src/i-prefer-shadow-uarch-state.h

@@ -25,8 +25,8 @@
 #include <type_traits>
 
 #include "meta.h"
+#include "shadow-tlb.h"
 #include "shadow-uarch-state.h"
-#include "tlb.h"
 
 namespace cartesi {