VirtualMemory.c

Go to the documentation of this file.

 
#include <Register/Intel/Cpuid.h>
#include "DxeIpl.h"
#include "VirtualMemory.h"
 
//
// Global variable to keep track current available memory used as page table.
//
PAGE_TABLE_POOL  *mPageTablePool = NULL;
 
VOID
ClearFirst4KPage (
  IN  VOID  *HobStart
  )
{
  EFI_PEI_HOB_POINTERS  RscHob;
  EFI_PEI_HOB_POINTERS  MemHob;
  BOOLEAN               DoClear;
 
  RscHob.Raw = HobStart;
  MemHob.Raw = HobStart;
  DoClear    = FALSE;
 
  //
  // Check if page 0 exists and free
  //
  while ((RscHob.Raw = GetNextHob (
                         EFI_HOB_TYPE_RESOURCE_DESCRIPTOR,
                         RscHob.Raw
                         )) != NULL)
  {
    if ((RscHob.ResourceDescriptor->ResourceType == EFI_RESOURCE_SYSTEM_MEMORY) &&
        (RscHob.ResourceDescriptor->PhysicalStart == 0))
    {
      DoClear = TRUE;
      //
      // Make sure memory at 0-4095 has not been allocated.
      //
      while ((MemHob.Raw = GetNextHob (
                             EFI_HOB_TYPE_MEMORY_ALLOCATION,
                             MemHob.Raw
                             )) != NULL)
      {
        if (MemHob.MemoryAllocation->AllocDescriptor.MemoryBaseAddress
            < EFI_PAGE_SIZE)
        {
          DoClear = FALSE;
          break;
        }
 
        MemHob.Raw = GET_NEXT_HOB (MemHob);
      }
 
      break;
    }
 
    RscHob.Raw = GET_NEXT_HOB (RscHob);
  }
 
  if (DoClear) {
    DEBUG ((DEBUG_INFO, "Clearing first 4K-page!\r\n"));
    SetMem (NULL, EFI_PAGE_SIZE, 0);
  }
 
  return;
}
 
BOOLEAN
IsNullDetectionEnabled (
  VOID
  )
{
  return ((PcdGet8 (PcdNullPointerDetectionPropertyMask) & BIT0) != 0);
}
 
BOOLEAN
IsExecuteDisableBitAvailable (
  VOID
  )
{
  UINT32   RegEax;
  UINT32   RegEdx;
  BOOLEAN  Available;
 
  Available = FALSE;
  AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
  if (RegEax >= 0x80000001) {
    AsmCpuid (0x80000001, NULL, NULL, NULL, &RegEdx);
    if ((RegEdx & BIT20) != 0) {
      //
      // Bit 20: Execute Disable Bit available.
      //
      Available = TRUE;
    }
  }
 
  return Available;
}
 
BOOLEAN
IsEnableNonExecNeeded (
  VOID
  )
{
  if (!IsExecuteDisableBitAvailable ()) {
    return FALSE;
  }
 
  //
  // XD flag (BIT63) in page table entry is only valid if IA32_EFER.NXE is set.
  // Features controlled by Following PCDs need this feature to be enabled.
  //
  return (PcdGetBool (PcdSetNxForStack) ||
          PcdGet64 (PcdDxeNxMemoryProtectionPolicy) != 0 ||
          PcdGet32 (PcdImageProtectionPolicy) != 0);
}
 
VOID
EnableExecuteDisableBit (
  VOID
  )
{
  UINT64  MsrRegisters;
 
  MsrRegisters = AsmReadMsr64 (0xC0000080);
  if ((MsrRegisters & BIT11) == 0) {
    MsrRegisters |= BIT11;
    AsmWriteMsr64 (0xC0000080, MsrRegisters);
  }
}
 
BOOLEAN
ToSplitPageTable (
  IN EFI_PHYSICAL_ADDRESS  Address,
  IN UINTN                 Size,
  IN EFI_PHYSICAL_ADDRESS  StackBase,
  IN UINTN                 StackSize,
  IN EFI_PHYSICAL_ADDRESS  GhcbBase,
  IN UINTN                 GhcbSize
  )
{
  if (IsNullDetectionEnabled () && (Address == 0)) {
    return TRUE;
  }
 
  if (PcdGetBool (PcdCpuStackGuard)) {
    if ((StackBase >= Address) && (StackBase < (Address + Size))) {
      return TRUE;
    }
  }
 
  if (PcdGetBool (PcdSetNxForStack)) {
    if ((Address < StackBase + StackSize) && ((Address + Size) > StackBase)) {
      return TRUE;
    }
  }
 
  if (GhcbBase != 0) {
    if ((Address < GhcbBase + GhcbSize) && ((Address + Size) > GhcbBase)) {
      return TRUE;
    }
  }
 
  return FALSE;
}
 
BOOLEAN
InitializePageTablePool (
  IN UINTN  PoolPages
  )
{
  VOID  *Buffer;
 
  //
  // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
  // header.
  //
  PoolPages += 1;   // Add one page for header.
  PoolPages  = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
               PAGE_TABLE_POOL_UNIT_PAGES;
  Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
  if (Buffer == NULL) {
    DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
    return FALSE;
  }
 
  //
  // Link all pools into a list for easier track later.
  //
  if (mPageTablePool == NULL) {
    mPageTablePool           = Buffer;
    mPageTablePool->NextPool = mPageTablePool;
  } else {
    ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
    mPageTablePool->NextPool              = Buffer;
    mPageTablePool                        = Buffer;
  }
 
  //
  // Reserve one page for pool header.
  //
  mPageTablePool->FreePages = PoolPages - 1;
  mPageTablePool->Offset    = EFI_PAGES_TO_SIZE (1);
 
  return TRUE;
}
 
VOID *
AllocatePageTableMemory (
  IN UINTN  Pages
  )
{
  VOID  *Buffer;
 
  if (Pages == 0) {
    return NULL;
  }
 
  //
  // Renew the pool if necessary.
  //
  if ((mPageTablePool == NULL) ||
      (Pages > mPageTablePool->FreePages))
  {
    if (!InitializePageTablePool (Pages)) {
      return NULL;
    }
  }
 
  Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
 
  mPageTablePool->Offset    += EFI_PAGES_TO_SIZE (Pages);
  mPageTablePool->FreePages -= Pages;
 
  return Buffer;
}
 
VOID
Split2MPageTo4K (
  IN EFI_PHYSICAL_ADDRESS  PhysicalAddress,
  IN OUT UINT64            *PageEntry2M,
  IN EFI_PHYSICAL_ADDRESS  StackBase,
  IN UINTN                 StackSize,
  IN EFI_PHYSICAL_ADDRESS  GhcbBase,
  IN UINTN                 GhcbSize
  )
{
  EFI_PHYSICAL_ADDRESS  PhysicalAddress4K;
  UINTN                 IndexOfPageTableEntries;
  PAGE_TABLE_4K_ENTRY   *PageTableEntry;
  UINT64                AddressEncMask;
 
  //
  // Make sure AddressEncMask is contained to smallest supported address field
  //
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 
  PageTableEntry = AllocatePageTableMemory (1);
  ASSERT (PageTableEntry != NULL);
 
  //
  // Fill in 2M page entry.
  //
  *PageEntry2M = (UINT64)(UINTN)PageTableEntry | AddressEncMask | IA32_PG_P | IA32_PG_RW;
 
  PhysicalAddress4K = PhysicalAddress;
  for (IndexOfPageTableEntries = 0; IndexOfPageTableEntries < 512; IndexOfPageTableEntries++, PageTableEntry++, PhysicalAddress4K += SIZE_4KB) {
    //
    // Fill in the Page Table entries
    //
    PageTableEntry->Uint64 = (UINT64)PhysicalAddress4K;
 
    //
    // The GHCB range consists of two pages per CPU, the GHCB and a
    // per-CPU variable page. The GHCB page needs to be mapped as an
    // unencrypted page while the per-CPU variable page needs to be
    // mapped encrypted. These pages alternate in assignment.
    //
    if (  (GhcbBase == 0)
       || (PhysicalAddress4K < GhcbBase)
       || (PhysicalAddress4K >= GhcbBase + GhcbSize)
       || (((PhysicalAddress4K - GhcbBase) & SIZE_4KB) != 0))
    {
      PageTableEntry->Uint64 |= AddressEncMask;
    }
 
    PageTableEntry->Bits.ReadWrite = 1;
 
    if ((IsNullDetectionEnabled () && (PhysicalAddress4K == 0)) ||
        (PcdGetBool (PcdCpuStackGuard) && (PhysicalAddress4K == StackBase)))
    {
      PageTableEntry->Bits.Present = 0;
    } else {
      PageTableEntry->Bits.Present = 1;
    }
 
    if (  PcdGetBool (PcdSetNxForStack)
       && (PhysicalAddress4K >= StackBase)
       && (PhysicalAddress4K < StackBase + StackSize))
    {
      //
      // Set Nx bit for stack.
      //
      PageTableEntry->Bits.Nx = 1;
    }
  }
}
 
VOID
Split1GPageTo2M (
  IN EFI_PHYSICAL_ADDRESS  PhysicalAddress,
  IN OUT UINT64            *PageEntry1G,
  IN EFI_PHYSICAL_ADDRESS  StackBase,
  IN UINTN                 StackSize,
  IN EFI_PHYSICAL_ADDRESS  GhcbBase,
  IN UINTN                 GhcbSize
  )
{
  EFI_PHYSICAL_ADDRESS  PhysicalAddress2M;
  UINTN                 IndexOfPageDirectoryEntries;
  PAGE_TABLE_ENTRY      *PageDirectoryEntry;
  UINT64                AddressEncMask;
 
  //
  // Make sure AddressEncMask is contained to smallest supported address field
  //
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 
  PageDirectoryEntry = AllocatePageTableMemory (1);
  ASSERT (PageDirectoryEntry != NULL);
 
  //
  // Fill in 1G page entry.
  //
  *PageEntry1G = (UINT64)(UINTN)PageDirectoryEntry | AddressEncMask | IA32_PG_P | IA32_PG_RW;
 
  PhysicalAddress2M = PhysicalAddress;
  for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectoryEntry++, PhysicalAddress2M += SIZE_2MB) {
    if (ToSplitPageTable (PhysicalAddress2M, SIZE_2MB, StackBase, StackSize, GhcbBase, GhcbSize)) {
      //
      // Need to split this 2M page that covers NULL or stack range.
      //
      Split2MPageTo4K (PhysicalAddress2M, (UINT64 *)PageDirectoryEntry, StackBase, StackSize, GhcbBase, GhcbSize);
    } else {
      //
      // Fill in the Page Directory entries
      //
      PageDirectoryEntry->Uint64         = (UINT64)PhysicalAddress2M | AddressEncMask;
      PageDirectoryEntry->Bits.ReadWrite = 1;
      PageDirectoryEntry->Bits.Present   = 1;
      PageDirectoryEntry->Bits.MustBe1   = 1;
    }
  }
}
 
VOID
SetPageTablePoolReadOnly (
  IN  UINTN                 PageTableBase,
  IN  EFI_PHYSICAL_ADDRESS  Address,
  IN  BOOLEAN               Level4Paging
  )
{
  UINTN                 Index;
  UINTN                 EntryIndex;
  UINT64                AddressEncMask;
  EFI_PHYSICAL_ADDRESS  PhysicalAddress;
  UINT64                *PageTable;
  UINT64                *NewPageTable;
  UINT64                PageAttr;
  UINT64                LevelSize[5];
  UINT64                LevelMask[5];
  UINTN                 LevelShift[5];
  UINTN                 Level;
  UINT64                PoolUnitSize;
 
  ASSERT (PageTableBase != 0);
 
  //
  // Since the page table is always from page table pool, which is always
  // located at the boundary of PcdPageTablePoolAlignment, we just need to
  // set the whole pool unit to be read-only.
  //
  Address = Address & PAGE_TABLE_POOL_ALIGN_MASK;
 
  LevelShift[1] = PAGING_L1_ADDRESS_SHIFT;
  LevelShift[2] = PAGING_L2_ADDRESS_SHIFT;
  LevelShift[3] = PAGING_L3_ADDRESS_SHIFT;
  LevelShift[4] = PAGING_L4_ADDRESS_SHIFT;
 
  LevelMask[1] = PAGING_4K_ADDRESS_MASK_64;
  LevelMask[2] = PAGING_2M_ADDRESS_MASK_64;
  LevelMask[3] = PAGING_1G_ADDRESS_MASK_64;
  LevelMask[4] = PAGING_1G_ADDRESS_MASK_64;
 
  LevelSize[1] = SIZE_4KB;
  LevelSize[2] = SIZE_2MB;
  LevelSize[3] = SIZE_1GB;
  LevelSize[4] = SIZE_512GB;
 
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) &
                   PAGING_1G_ADDRESS_MASK_64;
  PageTable    = (UINT64 *)(UINTN)PageTableBase;
  PoolUnitSize = PAGE_TABLE_POOL_UNIT_SIZE;
 
  for (Level = (Level4Paging) ? 4 : 3; Level > 0; --Level) {
    Index  = ((UINTN)RShiftU64 (Address, LevelShift[Level]));
    Index &= PAGING_PAE_INDEX_MASK;
 
    PageAttr = PageTable[Index];
    if ((PageAttr & IA32_PG_PS) == 0) {
      //
      // Go to next level of table.
      //
      PageTable = (UINT64 *)(UINTN)(PageAttr & ~AddressEncMask &
                                    PAGING_4K_ADDRESS_MASK_64);
      continue;
    }
 
    if (PoolUnitSize >= LevelSize[Level]) {
      //
      // Clear R/W bit if current page granularity is not larger than pool unit
      // size.
      //
      if ((PageAttr & IA32_PG_RW) != 0) {
        while (PoolUnitSize > 0) {
          //
          // PAGE_TABLE_POOL_UNIT_SIZE and PAGE_TABLE_POOL_ALIGNMENT are fit in
          // one page (2MB). Then we don't need to update attributes for pages
          // crossing page directory. ASSERT below is for that purpose.
          //
          ASSERT (Index < EFI_PAGE_SIZE/sizeof (UINT64));
 
          PageTable[Index] &= ~(UINT64)IA32_PG_RW;
          PoolUnitSize     -= LevelSize[Level];
 
          ++Index;
        }
      }
 
      break;
    } else {
      //
      // The smaller granularity of page must be needed.
      //
      ASSERT (Level > 1);
 
      NewPageTable = AllocatePageTableMemory (1);
      ASSERT (NewPageTable != NULL);
 
      PhysicalAddress = PageAttr & LevelMask[Level];
      for (EntryIndex = 0;
           EntryIndex < EFI_PAGE_SIZE/sizeof (UINT64);
           ++EntryIndex)
      {
        NewPageTable[EntryIndex] = PhysicalAddress  | AddressEncMask |
                                   IA32_PG_P | IA32_PG_RW;
        if (Level > 2) {
          NewPageTable[EntryIndex] |= IA32_PG_PS;
        }
 
        PhysicalAddress += LevelSize[Level - 1];
      }
 
      PageTable[Index] = (UINT64)(UINTN)NewPageTable | AddressEncMask |
                         IA32_PG_P | IA32_PG_RW;
      PageTable = NewPageTable;
    }
  }
}
 
VOID
EnablePageTableProtection (
  IN  UINTN    PageTableBase,
  IN  BOOLEAN  Level4Paging
  )
{
  PAGE_TABLE_POOL       *HeadPool;
  PAGE_TABLE_POOL       *Pool;
  UINT64                PoolSize;
  EFI_PHYSICAL_ADDRESS  Address;
 
  if (mPageTablePool == NULL) {
    return;
  }
 
  //
  // No need to clear CR0.WP since PageTableBase has't been written to CR3 yet.
  // SetPageTablePoolReadOnly might update mPageTablePool. It's safer to
  // remember original one in advance.
  //
  HeadPool = mPageTablePool;
  Pool     = HeadPool;
  do {
    Address  = (EFI_PHYSICAL_ADDRESS)(UINTN)Pool;
    PoolSize = Pool->Offset + EFI_PAGES_TO_SIZE (Pool->FreePages);
 
    //
    // The size of one pool must be multiple of PAGE_TABLE_POOL_UNIT_SIZE, which
    // is one of page size of the processor (2MB by default). Let's apply the
    // protection to them one by one.
    //
    while (PoolSize > 0) {
      SetPageTablePoolReadOnly (PageTableBase, Address, Level4Paging);
      Address  += PAGE_TABLE_POOL_UNIT_SIZE;
      PoolSize -= PAGE_TABLE_POOL_UNIT_SIZE;
    }
 
    Pool = Pool->NextPool;
  } while (Pool != HeadPool);
 
  //
  // Enable write protection, after page table attribute updated.
  //
  AsmWriteCr0 (AsmReadCr0 () | CR0_WP);
}
 
UINTN
CreateIdentityMappingPageTables (
  IN EFI_PHYSICAL_ADDRESS  StackBase,
  IN UINTN                 StackSize,
  IN EFI_PHYSICAL_ADDRESS  GhcbBase,
  IN UINTN                 GhcbSize
  )
{
  UINT32                                       RegEax;
  CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_ECX  EcxFlags;
  UINT32                                       RegEdx;
  UINT8                                        PhysicalAddressBits;
  EFI_PHYSICAL_ADDRESS                         PageAddress;
  UINTN                                        IndexOfPml5Entries;
  UINTN                                        IndexOfPml4Entries;
  UINTN                                        IndexOfPdpEntries;
  UINTN                                        IndexOfPageDirectoryEntries;
  UINT32                                       NumberOfPml5EntriesNeeded;
  UINT32                                       NumberOfPml4EntriesNeeded;
  UINT32                                       NumberOfPdpEntriesNeeded;
  PAGE_MAP_AND_DIRECTORY_POINTER               *PageMapLevel5Entry;
  PAGE_MAP_AND_DIRECTORY_POINTER               *PageMapLevel4Entry;
  PAGE_MAP_AND_DIRECTORY_POINTER               *PageMap;
  PAGE_MAP_AND_DIRECTORY_POINTER               *PageDirectoryPointerEntry;
  PAGE_TABLE_ENTRY                             *PageDirectoryEntry;
  UINTN                                        TotalPagesNum;
  UINTN                                        BigPageAddress;
  VOID                                         *Hob;
  BOOLEAN                                      Page5LevelEnabled;
  BOOLEAN                                      Page1GSupport;
  PAGE_TABLE_1G_ENTRY                          *PageDirectory1GEntry;
  UINT64                                       AddressEncMask;
  IA32_CR4                                     Cr4;
 
  //
  // Set PageMapLevel5Entry to suppress incorrect compiler/analyzer warnings
  //
  PageMapLevel5Entry = NULL;
 
  //
  // Make sure AddressEncMask is contained to smallest supported address field
  //
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 
  Page1GSupport = FALSE;
  if (PcdGetBool (PcdUse1GPageTable)) {
    AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
    if (RegEax >= 0x80000001) {
      AsmCpuid (0x80000001, NULL, NULL, NULL, &RegEdx);
      if ((RegEdx & BIT26) != 0) {
        Page1GSupport = TRUE;
      }
    }
  }
 
  //
  // Get physical address bits supported.
  //
  Hob = GetFirstHob (EFI_HOB_TYPE_CPU);
  if (Hob != NULL) {
    PhysicalAddressBits = ((EFI_HOB_CPU *)Hob)->SizeOfMemorySpace;
  } else {
    AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
    if (RegEax >= 0x80000008) {
      AsmCpuid (0x80000008, &RegEax, NULL, NULL, NULL);
      PhysicalAddressBits = (UINT8)RegEax;
    } else {
      PhysicalAddressBits = 36;
    }
  }
 
  if (sizeof (UINTN) == sizeof (UINT64)) {
    //
    // If cpu has already run in 64bit long mode PEI, Page table Level in DXE must align with previous level.
    //
    Cr4.UintN         = AsmReadCr4 ();
    Page5LevelEnabled = (Cr4.Bits.LA57 != 0);
    if (Page5LevelEnabled) {
      ASSERT (PcdGetBool (PcdUse5LevelPageTable));
    }
  } else {
    //
    // If cpu runs in 32bit protected mode PEI, Page table Level in DXE is decided by PCD and feature capability.
    //
    Page5LevelEnabled = FALSE;
    if (PcdGetBool (PcdUse5LevelPageTable)) {
      AsmCpuidEx (
        CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS,
        CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS_SUB_LEAF_INFO,
        NULL,
        NULL,
        &EcxFlags.Uint32,
        NULL
        );
      if (EcxFlags.Bits.FiveLevelPage != 0) {
        Page5LevelEnabled = TRUE;
      }
    }
  }
 
  DEBUG ((DEBUG_INFO, "AddressBits=%u 5LevelPaging=%u 1GPage=%u\n", PhysicalAddressBits, Page5LevelEnabled, Page1GSupport));
 
  //
  // IA-32e paging translates 48-bit linear addresses to 52-bit physical addresses
  //  when 5-Level Paging is disabled,
  //  due to either unsupported by HW, or disabled by PCD.
  //
  ASSERT (PhysicalAddressBits <= 52);
  if (!Page5LevelEnabled && (PhysicalAddressBits > 48)) {
    PhysicalAddressBits = 48;
  }
 
  //
  // Calculate the table entries needed.
  //
  NumberOfPml5EntriesNeeded = 1;
  if (PhysicalAddressBits > 48) {
    NumberOfPml5EntriesNeeded = (UINT32)LShiftU64 (1, PhysicalAddressBits - 48);
    PhysicalAddressBits       = 48;
  }
 
  NumberOfPml4EntriesNeeded = 1;
  if (PhysicalAddressBits > 39) {
    NumberOfPml4EntriesNeeded = (UINT32)LShiftU64 (1, PhysicalAddressBits - 39);
    PhysicalAddressBits       = 39;
  }
 
  NumberOfPdpEntriesNeeded = 1;
  ASSERT (PhysicalAddressBits > 30);
  NumberOfPdpEntriesNeeded = (UINT32)LShiftU64 (1, PhysicalAddressBits - 30);
 
  //
  // Pre-allocate big pages to avoid later allocations.
  //
  if (!Page1GSupport) {
    TotalPagesNum = ((NumberOfPdpEntriesNeeded + 1) * NumberOfPml4EntriesNeeded + 1) * NumberOfPml5EntriesNeeded + 1;
  } else {
    TotalPagesNum = (NumberOfPml4EntriesNeeded + 1) * NumberOfPml5EntriesNeeded + 1;
  }
 
  //
  // Substract the one page occupied by PML5 entries if 5-Level Paging is disabled.
  //
  if (!Page5LevelEnabled) {
    TotalPagesNum--;
  }
 
  DEBUG ((
    DEBUG_INFO,
    "Pml5=%u Pml4=%u Pdp=%u TotalPage=%Lu\n",
    NumberOfPml5EntriesNeeded,
    NumberOfPml4EntriesNeeded,
    NumberOfPdpEntriesNeeded,
    (UINT64)TotalPagesNum
    ));
 
  BigPageAddress = (UINTN)AllocatePageTableMemory (TotalPagesNum);
  ASSERT (BigPageAddress != 0);
 
  //
  // By architecture only one PageMapLevel4 exists - so lets allocate storage for it.
  //
  PageMap = (VOID *)BigPageAddress;
  if (Page5LevelEnabled) {
    //
    // By architecture only one PageMapLevel5 exists - so lets allocate storage for it.
    //
    PageMapLevel5Entry = PageMap;
    BigPageAddress    += SIZE_4KB;
  }
 
  PageAddress = 0;
 
  for ( IndexOfPml5Entries = 0
        ; IndexOfPml5Entries < NumberOfPml5EntriesNeeded
        ; IndexOfPml5Entries++)
  {
    //
    // Each PML5 entry points to a page of PML4 entires.
    // So lets allocate space for them and fill them in in the IndexOfPml4Entries loop.
    // When 5-Level Paging is disabled, below allocation happens only once.
    //
    PageMapLevel4Entry = (VOID *)BigPageAddress;
    BigPageAddress    += SIZE_4KB;
 
    if (Page5LevelEnabled) {
      //
      // Make a PML5 Entry
      //
      PageMapLevel5Entry->Uint64         = (UINT64)(UINTN)PageMapLevel4Entry | AddressEncMask;
      PageMapLevel5Entry->Bits.ReadWrite = 1;
      PageMapLevel5Entry->Bits.Present   = 1;
      PageMapLevel5Entry++;
    }
 
    for ( IndexOfPml4Entries = 0
          ; IndexOfPml4Entries < (NumberOfPml5EntriesNeeded == 1 ? NumberOfPml4EntriesNeeded : 512)
          ; IndexOfPml4Entries++, PageMapLevel4Entry++)
    {
      //
      // Each PML4 entry points to a page of Page Directory Pointer entires.
      // So lets allocate space for them and fill them in in the IndexOfPdpEntries loop.
      //
      PageDirectoryPointerEntry = (VOID *)BigPageAddress;
      BigPageAddress           += SIZE_4KB;
 
      //
      // Make a PML4 Entry
      //
      PageMapLevel4Entry->Uint64         = (UINT64)(UINTN)PageDirectoryPointerEntry | AddressEncMask;
      PageMapLevel4Entry->Bits.ReadWrite = 1;
      PageMapLevel4Entry->Bits.Present   = 1;
 
      if (Page1GSupport) {
        PageDirectory1GEntry = (VOID *)PageDirectoryPointerEntry;
 
        for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectory1GEntry++, PageAddress += SIZE_1GB) {
          if (ToSplitPageTable (PageAddress, SIZE_1GB, StackBase, StackSize, GhcbBase, GhcbSize)) {
            Split1GPageTo2M (PageAddress, (UINT64 *)PageDirectory1GEntry, StackBase, StackSize, GhcbBase, GhcbSize);
          } else {
            //
            // Fill in the Page Directory entries
            //
            PageDirectory1GEntry->Uint64         = (UINT64)PageAddress | AddressEncMask;
            PageDirectory1GEntry->Bits.ReadWrite = 1;
            PageDirectory1GEntry->Bits.Present   = 1;
            PageDirectory1GEntry->Bits.MustBe1   = 1;
          }
        }
      } else {
        for ( IndexOfPdpEntries = 0
              ; IndexOfPdpEntries < (NumberOfPml4EntriesNeeded == 1 ? NumberOfPdpEntriesNeeded : 512)
              ; IndexOfPdpEntries++, PageDirectoryPointerEntry++)
        {
          //
          // Each Directory Pointer entries points to a page of Page Directory entires.
          // So allocate space for them and fill them in in the IndexOfPageDirectoryEntries loop.
          //
          PageDirectoryEntry = (VOID *)BigPageAddress;
          BigPageAddress    += SIZE_4KB;
 
          //
          // Fill in a Page Directory Pointer Entries
          //
          PageDirectoryPointerEntry->Uint64         = (UINT64)(UINTN)PageDirectoryEntry | AddressEncMask;
          PageDirectoryPointerEntry->Bits.ReadWrite = 1;
          PageDirectoryPointerEntry->Bits.Present   = 1;
 
          for (IndexOfPageDirectoryEntries = 0; IndexOfPageDirectoryEntries < 512; IndexOfPageDirectoryEntries++, PageDirectoryEntry++, PageAddress += SIZE_2MB) {
            if (ToSplitPageTable (PageAddress, SIZE_2MB, StackBase, StackSize, GhcbBase, GhcbSize)) {
              //
              // Need to split this 2M page that covers NULL or stack range.
              //
              Split2MPageTo4K (PageAddress, (UINT64 *)PageDirectoryEntry, StackBase, StackSize, GhcbBase, GhcbSize);
            } else {
              //
              // Fill in the Page Directory entries
              //
              PageDirectoryEntry->Uint64         = (UINT64)PageAddress | AddressEncMask;
              PageDirectoryEntry->Bits.ReadWrite = 1;
              PageDirectoryEntry->Bits.Present   = 1;
              PageDirectoryEntry->Bits.MustBe1   = 1;
            }
          }
        }
 
        //
        // Fill with null entry for unused PDPTE
        //
        ZeroMem (PageDirectoryPointerEntry, (512 - IndexOfPdpEntries) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER));
      }
    }
 
    //
    // For the PML4 entries we are not using fill in a null entry.
    //
    ZeroMem (PageMapLevel4Entry, (512 - IndexOfPml4Entries) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER));
  }
 
  if (Page5LevelEnabled) {
    Cr4.UintN     = AsmReadCr4 ();
    Cr4.Bits.LA57 = 1;
    AsmWriteCr4 (Cr4.UintN);
    //
    // For the PML5 entries we are not using fill in a null entry.
    //
    ZeroMem (PageMapLevel5Entry, (512 - IndexOfPml5Entries) * sizeof (PAGE_MAP_AND_DIRECTORY_POINTER));
  }
 
  //
  // Protect the page table by marking the memory used for page table to be
  // read-only.
  //
  EnablePageTableProtection ((UINTN)PageMap, TRUE);
 
  //
  // Set IA32_EFER.NXE if necessary.
  //
  if (IsEnableNonExecNeeded ()) {
    EnableExecuteDisableBit ();
  }
 
  return (UINTN)PageMap;
}