api: implement API for resident and zero memory

Implement API /memory which returns two bitmaps: resident and empty.
`resident` tracks whether a guest page is in the resident set and `empty`
tracks whether it's actually all 0s.

Both bitmaps are structures as vectors of u64, so their length is:
total_number_of_pages.div_ceil(64).

Pages are ordered in the order of pages as reported by/memory/mappings.

Signed-off-by: Babis Chalios <[email protected]>

Author: bchalios

src/firecracker/src/api_server/parsed_request.rs

@@ -199,6 +199,7 @@ impl ParsedRequest {
                 ),
                 VmmData::FullVmConfig(config) => Self::success_response_with_data(config),
                 VmmData::MemoryMappings(mappings) => Self::success_response_with_data(mappings),
+                VmmData::Memory(meminfo) => Self::success_response_with_data(meminfo),
             },
             Err(vmm_action_error) => {
                 let mut response = match vmm_action_error {
@@ -616,6 +617,9 @@ pub mod tests {
                 VmmData::MemoryMappings(mappings) => {
                     http_response(&serde_json::to_string(mappings).unwrap(), 200)
                 }
+                VmmData::Memory(meminfo) => {
+                    http_response(&serde_json::to_string(meminfo).unwrap(), 200)
+                }
             };
             let response = ParsedRequest::convert_to_response(&data);
             response.write_all(&mut buf).unwrap();

src/firecracker/src/api_server/request/memory_info.rs

@@ -1,4 +1,4 @@
-use micro_http::{Method, StatusCode};
+use micro_http::Method;
 use vmm::rpc_interface::VmmAction;
 
 use crate::api_server::parsed_request::{ParsedRequest, RequestError};
@@ -13,9 +13,6 @@ where
             format!("/memory/{}", unknown_path),
             Method::Get,
         )),
-        None => Err(RequestError::Generic(
-            StatusCode::BadRequest,
-            "Missing memory info type.".to_string(),
-        )),
+        None => Ok(ParsedRequest::new_sync(VmmAction::GetMemory)),
     }
 }

src/vmm/src/builder.rs

@@ -318,6 +318,7 @@ pub fn build_microvm_for_boot(
         vcpus_handles: Vec::new(),
         vcpus_exit_evt,
         device_manager,
+        page_size: vm_resources.machine_config.huge_pages.page_size(),
     };
     let vmm = Arc::new(Mutex::new(vmm));
 
@@ -518,6 +519,7 @@ pub fn build_microvm_from_snapshot(
         vcpus_handles: Vec::new(),
         vcpus_exit_evt,
         device_manager,
+        page_size: vm_resources.machine_config.huge_pages.page_size(),
     };
 
     // Move vcpus to their own threads and start their state machine in the 'Paused' state.
@@ -751,6 +753,7 @@ pub(crate) mod tests {
     use vmm_sys_util::tempfile::TempFile;
 
     use super::*;
+    use crate::arch::host_page_size;
     use crate::device_manager::tests::default_device_manager;
     use crate::devices::virtio::block::CacheType;
     use crate::devices::virtio::generated::virtio_ids;
@@ -836,6 +839,7 @@ pub(crate) mod tests {
             vcpus_handles: Vec::new(),
             vcpus_exit_evt,
             device_manager: default_device_manager(),
+            page_size: host_page_size(),
         }
     }
 

src/vmm/src/lib.rs

@@ -152,6 +152,7 @@ use crate::vstate::memory::{GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
 use crate::vstate::vcpu::VcpuState;
 pub use crate::vstate::vcpu::{Vcpu, VcpuConfig, VcpuEvent, VcpuHandle, VcpuResponse};
 pub use crate::vstate::vm::Vm;
+use crate::vstate::vm::mincore_bitmap;
 
 /// Shorthand type for the EventManager flavour used by Firecracker.
 pub type EventManager = BaseEventManager<Arc<Mutex<dyn MutEventSubscriber>>>;
@@ -314,6 +315,8 @@ pub struct Vmm {
     vcpus_exit_evt: EventFd,
     // Device manager
     device_manager: DeviceManager,
+    /// Page size used for backing guest memory
+    pub page_size: usize,
 }
 
 impl Vmm {
@@ -697,21 +700,80 @@ impl Vmm {
         let mut mappings = vec![];
         let mut offset = 0;
 
-        for region in self.vm.guest_memory().iter() {
+        for region in self
+            .vm
+            .guest_memory()
+            .iter()
+            .flat_map(|region| region.plugged_slots())
+        {
+            let size = region.slice.len();
             #[allow(deprecated)]
             mappings.push(GuestRegionUffdMapping {
-                base_host_virt_addr: region.as_ptr() as u64,
-                size: region.size(),
+                base_host_virt_addr: region.slice.ptr_guard_mut().as_ptr() as u64,
+                size,
                 offset,
                 page_size,
                 page_size_kib: page_size,
             });
 
-            offset += usize_to_u64(region.size());
+            offset += usize_to_u64(size);
         }
 
         mappings
     }
+
+    /// Get info regarding resident and empty pages for guest memory
+    pub fn guest_memory_info(&self, page_size: usize) -> Result<(Vec<u64>, Vec<u64>), VmmError> {
+        let mut resident = vec![];
+        let mut empty = vec![];
+        let zero_page = vec![0u8; page_size];
+
+        for mem_slot in self
+            .vm
+            .guest_memory()
+            .iter()
+            .flat_map(|region| region.plugged_slots())
+        {
+            debug_assert!(mem_slot.slice.len().is_multiple_of(page_size));
+            debug_assert!(
+                (mem_slot.slice.ptr_guard_mut().as_ptr() as usize).is_multiple_of(page_size)
+            );
+
+            let len = mem_slot.slice.len();
+            let nr_pages = len / page_size;
+            let addr = mem_slot.slice.ptr_guard_mut().as_ptr();
+            let mut curr_empty = vec![0u64; nr_pages.div_ceil(64)];
+            let curr_resident = mincore_bitmap(addr, mem_slot.slice.len(), page_size)?;
+
+            for page_idx in 0..nr_pages {
+                if (curr_resident[page_idx / 64] & (1u64 << (page_idx % 64))) == 0 {
+                    continue;
+                }
+
+                // SAFETY: `addr` points to a memory region that is `nr_pages * page_size` long.
+                let curr_addr = unsafe { addr.add(page_idx * page_size) };
+
+                // SAFETY: both addresses are valid and they point to a memory region
+                // that is (at least) `page_size` long
+                let ret = unsafe {
+                    libc::memcmp(
+                        curr_addr.cast::<libc::c_void>(),
+                        zero_page.as_ptr().cast::<libc::c_void>(),
+                        page_size,
+                    )
+                };
+
+                if ret == 0 {
+                    curr_empty[page_idx / 64] |= 1u64 << (page_idx % 64);
+                }
+            }
+
+            resident.extend_from_slice(&curr_resident);
+            empty.extend_from_slice(&curr_empty);
+        }
+
+        Ok((resident, empty))
+    }
 }
 
 /// Process the content of the MPIDR_EL1 register in order to be able to pass it to KVM

src/vmm/src/persist.rs

@@ -162,8 +162,8 @@ pub fn create_snapshot(
     snapshot_state_to_file(&microvm_state, &params.snapshot_path)?;
 
     if let Some(mem_file_path) = params.mem_file_path.as_ref() {
-    vmm.vm
-        .snapshot_memory_to_file(mem_file_path, params.snapshot_type)?;
+        vmm.vm
+            .snapshot_memory_to_file(mem_file_path, params.snapshot_type, vmm.page_size)?;
     }
 
     // We need to mark queues as dirty again for all activated devices. The reason we

src/vmm/src/rpc_interface.rs

@@ -28,9 +28,9 @@ use crate::vmm_config::balloon::{
 use crate::vmm_config::boot_source::{BootSourceConfig, BootSourceConfigError};
 use crate::vmm_config::drive::{BlockDeviceConfig, BlockDeviceUpdateConfig, DriveError};
 use crate::vmm_config::entropy::{EntropyDeviceConfig, EntropyDeviceError};
-use crate::vmm_config::instance_info::InstanceInfo;
+use crate::vmm_config::instance_info::{InstanceInfo, VmState};
 use crate::vmm_config::machine_config::{MachineConfig, MachineConfigError, MachineConfigUpdate};
-use crate::vmm_config::meminfo::MemoryMapingsResponse;
+use crate::vmm_config::meminfo::{MemoryMapingsResponse, MemoryResponse};
 use crate::vmm_config::memory_hotplug::{
     MemoryHotplugConfig, MemoryHotplugConfigError, MemoryHotplugSizeUpdate,
 };
@@ -149,6 +149,8 @@ pub enum VmmAction {
     UpdateMachineConfiguration(MachineConfigUpdate),
     /// Get the guest memory mappings to host memory
     GetMemoryMappings,
+    /// Get guest memory resident and empty pages information
+    GetMemory,
 }
 
 /// Wrapper for all errors associated with VMM actions.
@@ -200,6 +202,8 @@ pub enum VmmActionError {
     OperationNotSupportedPostBoot,
     /// The requested operation is not supported before starting the microVM.
     OperationNotSupportedPreBoot,
+    /// The requested operation is not supported while the microVM is running.
+    OperationNotSupportedWhileRunning,
     /// Start microvm error: {0}
     StartMicrovm(#[from] StartMicrovmError),
     /// Vsock config error: {0}
@@ -233,6 +237,8 @@ pub enum VmmData {
     HintingStatus(HintingStatus),
     /// The guest memory mapping information.
     MemoryMappings(MemoryMapingsResponse),
+    /// The guest memory resident and empty pages information
+    Memory(MemoryResponse),
 }
 
 /// Trait used for deduplicating the MMDS request handling across the two ApiControllers.
@@ -501,7 +507,8 @@ impl<'a> PrebootApiController<'a> {
             | StartFreePageHinting(_)
             | GetFreePageHintingStatus
             | StopFreePageHinting
-            | GetMemoryMappings => Err(VmmActionError::OperationNotSupportedPreBoot),
+            | GetMemoryMappings
+            | GetMemory => Err(VmmActionError::OperationNotSupportedPreBoot),
             #[cfg(target_arch = "x86_64")]
             SendCtrlAltDel => Err(VmmActionError::OperationNotSupportedPreBoot),
         }
@@ -778,6 +785,7 @@ impl RuntimeApiController {
                 .map(|_| VmmData::Empty)
                 .map_err(VmmActionError::MemoryHotplugUpdate),
             GetMemoryMappings => self.get_guest_memory_mappings(),
+            GetMemory => self.get_guest_memory_info(),
             // Operations not allowed post-boot.
             ConfigureBootSource(_)
             | ConfigureLogger(_)
@@ -957,6 +965,25 @@ impl RuntimeApiController {
         info!("'get memory mappings' VMM action took {elapsed_time_us} us.");
         Ok(VmmData::MemoryMappings(MemoryMapingsResponse { mappings }))
     }
+
+    /// Get resident and empty pages information for guest memory
+    fn get_guest_memory_info(&self) -> Result<VmmData, VmmActionError> {
+        let start_us = get_time_us(ClockType::Monotonic);
+        let vmm = self.vmm.lock().expect("Poisoned lock");
+
+        // Check if VM is paused
+        if vmm.instance_info.state != VmState::Paused {
+            return Err(VmmActionError::OperationNotSupportedWhileRunning);
+        }
+
+        let page_size = self.vm_resources.machine_config.huge_pages.page_size();
+        let (resident, empty) = vmm.guest_memory_info(page_size)?;
+
+        let elapsed_time_us = get_time_us(ClockType::Monotonic) - start_us;
+        info!("'get memory info' VMM action took {elapsed_time_us} us.");
+
+        Ok(VmmData::Memory(MemoryResponse { resident, empty }))
+    }
 }
 
 #[cfg(test)]

src/vmm/src/vmm_config/meminfo.rs

@@ -9,3 +9,13 @@ pub struct MemoryMapingsResponse {
     pub mappings: Vec<GuestRegionUffdMapping>,
 }
 
+/// Information about guest memory resident pages and pages that are all-0s
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct MemoryResponse {
+    /// Bitmap for resident pages. The bitmap is encoded as a vector of u64 values.
+    /// Each bit represents whether a page is present in the resident memory set
+    pub resident: Vec<u64>,
+    /// Bitmap for empty pages. The bitmap is encoded as a vector of u64 values.
+    /// Each bit represents whether a page is empty (all 0s).
+    pub empty: Vec<u64>,
+}

src/vmm/src/vstate/vm.rs

@@ -305,7 +305,7 @@ impl Vm {
     }
 
     /// Retrieves the KVM dirty bitmap for each of the guest's memory regions.
-    pub fn get_dirty_bitmap(&self) -> Result<DirtyBitmap, VmError> {
+    pub fn get_dirty_bitmap(&self, page_size: usize) -> Result<DirtyBitmap, VmError> {
         self.guest_memory()
             .iter()
             .flat_map(|region| region.plugged_slots())
@@ -318,6 +318,7 @@ impl Vm {
                     None => mincore_bitmap(
                         mem_slot.slice.ptr_guard_mut().as_ptr(),
                         mem_slot.slice.len(),
+                        page_size,
                     )?,
                 };
                 Ok((mem_slot.slot, bitmap))
@@ -335,6 +336,7 @@ impl Vm {
         &self,
         mem_file_path: &Path,
         snapshot_type: SnapshotType,
+        page_size: usize,
     ) -> Result<(), CreateSnapshotError> {
         use self::CreateSnapshotError::*;
 
@@ -377,7 +379,7 @@ impl Vm {
 
         match snapshot_type {
             SnapshotType::Diff => {
-                let dirty_bitmap = self.get_dirty_bitmap()?;
+                let dirty_bitmap = self.get_dirty_bitmap(page_size)?;
                 self.guest_memory().dump_dirty(&mut file, &dirty_bitmap)?;
             }
             SnapshotType::Full => {
@@ -503,7 +505,11 @@ impl Vm {
 
 /// Use `mincore(2)` to overapproximate the dirty bitmap for the given memslot. To be used
 /// if a diff snapshot is requested, but dirty page tracking wasn't enabled.
-fn mincore_bitmap(addr: *mut u8, len: usize) -> Result<Vec<u64>, VmError> {
+pub(crate) fn mincore_bitmap(
+    addr: *mut u8,
+    len: usize,
+    page_size: usize,
+) -> Result<Vec<u64>, VmError> {
     // TODO: Once Host 5.10 goes out of support, we can make this more robust and work on
     // swap-enabled systems, by doing mlock2(MLOCK_ONFAULT)/munlock() in this function (to
     // force swapped-out pages to get paged in, so that mincore will consider them incore).
@@ -513,8 +519,11 @@ fn mincore_bitmap(addr: *mut u8, len: usize) -> Result<Vec<u64>, VmError> {
     // Mincore always works at PAGE_SIZE granularity, even if the VMA we are dealing with
     // is a hugetlbfs VMA (e.g. to report a single hugepage as "present", mincore will
     // give us 512 4k markers with the lowest bit set).
-    let page_size = host_page_size();
-    let mut mincore_bitmap = vec![0u8; len / page_size];
+    let host_page_size = host_page_size();
+    let mut mincore_bitmap = vec![0u8; len / host_page_size];
+    // The bitmap we return though tracks pages in terms of the actually used page size. In
+    // the case of a hugetlbfs VMA, we just need to check if the first of the reported pages
+    // is present.
     let mut bitmap = vec![0u64; (len / page_size).div_ceil(64)];
 
     // SAFETY: The safety invariants of GuestRegionMmap ensure that region.as_ptr() is a valid
@@ -529,7 +538,8 @@ fn mincore_bitmap(addr: *mut u8, len: usize) -> Result<Vec<u64>, VmError> {
         return Err(VmError::Mincore(vmm_sys_util::errno::Error::last()));
     }
 
-    for (page_idx, b) in mincore_bitmap.iter().enumerate() {
+    let step = page_size / host_page_size;
+    for (page_idx, b) in mincore_bitmap.iter().step_by(step).enumerate() {
         bitmap[page_idx / 64] |= (*b as u64 & 0x1) << (page_idx as u64 % 64);
     }