CpuPaging.c

Go to the documentation of this file.

 
#include <Register/Intel/Cpuid.h>
#include <Register/Intel/Msr.h>
#include <Library/MemoryAllocationLib.h>
#include <Library/CpuLib.h>
#include <Library/BaseLib.h>
#include <Guid/MigratedFvInfo.h>
 
#include "CpuMpPei.h"
 
#define IA32_PG_P   BIT0
#define IA32_PG_RW  BIT1
#define IA32_PG_U   BIT2
#define IA32_PG_A   BIT5
#define IA32_PG_D   BIT6
#define IA32_PG_PS  BIT7
#define IA32_PG_NX  BIT63
 
#define PAGE_ATTRIBUTE_BITS  (IA32_PG_RW | IA32_PG_P)
#define PAGE_PROGATE_BITS    (IA32_PG_D | IA32_PG_A | IA32_PG_NX | IA32_PG_U | \
                               PAGE_ATTRIBUTE_BITS)
 
#define PAGING_PAE_INDEX_MASK        0x1FF
#define PAGING_4K_ADDRESS_MASK_64    0x000FFFFFFFFFF000ull
#define PAGING_2M_ADDRESS_MASK_64    0x000FFFFFFFE00000ull
#define PAGING_1G_ADDRESS_MASK_64    0x000FFFFFC0000000ull
#define PAGING_512G_ADDRESS_MASK_64  0x000FFF8000000000ull
 
typedef enum {
  PageNone = 0,
  PageMin  = 1,
  Page4K   = PageMin,
  Page2M   = 2,
  Page1G   = 3,
  Page512G = 4,
  PageMax  = Page512G
} PAGE_ATTRIBUTE;
 
typedef struct {
  PAGE_ATTRIBUTE    Attribute;
  UINT64            Length;
  UINT64            AddressMask;
  UINTN             AddressBitOffset;
  UINTN             AddressBitLength;
} PAGE_ATTRIBUTE_TABLE;
 
PAGE_ATTRIBUTE_TABLE  mPageAttributeTable[] = {
  { PageNone, 0,          0,                           0,  0 },
  { Page4K,   SIZE_4KB,   PAGING_4K_ADDRESS_MASK_64,   12, 9 },
  { Page2M,   SIZE_2MB,   PAGING_2M_ADDRESS_MASK_64,   21, 9 },
  { Page1G,   SIZE_1GB,   PAGING_1G_ADDRESS_MASK_64,   30, 9 },
  { Page512G, SIZE_512GB, PAGING_512G_ADDRESS_MASK_64, 39, 9 },
};
 
EFI_PEI_NOTIFY_DESCRIPTOR  mPostMemNotifyList[] = {
  {
    (EFI_PEI_PPI_DESCRIPTOR_NOTIFY_CALLBACK | EFI_PEI_PPI_DESCRIPTOR_TERMINATE_LIST),
    &gEfiPeiMemoryDiscoveredPpiGuid,
    MemoryDiscoveredPpiNotifyCallback
  }
};
 
BOOLEAN
IsIa32PaeSupported (
  VOID
  )
{
  UINT32                  RegEax;
  CPUID_VERSION_INFO_EDX  RegEdx;
 
  AsmCpuid (CPUID_SIGNATURE, &RegEax, NULL, NULL, NULL);
  if (RegEax >= CPUID_VERSION_INFO) {
    AsmCpuid (CPUID_VERSION_INFO, NULL, NULL, NULL, &RegEdx.Uint32);
    if (RegEdx.Bits.PAE != 0) {
      return TRUE;
    }
  }
 
  return FALSE;
}
 
VOID *
AllocatePageTableMemory (
  IN UINTN  Pages
  )
{
  VOID  *Address;
 
  Address = AllocatePages (Pages);
  if (Address != NULL) {
    ZeroMem (Address, EFI_PAGES_TO_SIZE (Pages));
  }
 
  return Address;
}
 
PAGE_ATTRIBUTE
GetPageTableTopLevelType (
  VOID
  )
{
  MSR_IA32_EFER_REGISTER  MsrEfer;
 
  MsrEfer.Uint64 = AsmReadMsr64 (MSR_CORE_IA32_EFER);
 
  return (MsrEfer.Bits.LMA == 1) ? Page512G : Page1G;
}
 
VOID *
GetPageTableEntry (
  IN  PHYSICAL_ADDRESS  Address,
  OUT PAGE_ATTRIBUTE    *PageAttribute
  )
{
  INTN    Level;
  UINTN   Index;
  UINT64  *PageTable;
  UINT64  AddressEncMask;
 
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask);
  PageTable      = (UINT64 *)(UINTN)(AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
  for (Level = (INTN)GetPageTableTopLevelType (); Level > 0; --Level) {
    Index  = (UINTN)RShiftU64 (Address, mPageAttributeTable[Level].AddressBitOffset);
    Index &= PAGING_PAE_INDEX_MASK;
 
    //
    // No mapping?
    //
    if (PageTable[Index] == 0) {
      *PageAttribute = PageNone;
      return NULL;
    }
 
    //
    // Page memory?
    //
    if (((PageTable[Index] & IA32_PG_PS) != 0) || (Level == PageMin)) {
      *PageAttribute = (PAGE_ATTRIBUTE)Level;
      return &PageTable[Index];
    }
 
    //
    // Page directory or table
    //
    PageTable = (UINT64 *)(UINTN)(PageTable[Index] &
                                  ~AddressEncMask &
                                  PAGING_4K_ADDRESS_MASK_64);
  }
 
  *PageAttribute = PageNone;
  return NULL;
}
 
RETURN_STATUS
SplitPage (
  IN  UINT64          *PageEntry,
  IN  PAGE_ATTRIBUTE  PageAttribute,
  IN  PAGE_ATTRIBUTE  SplitAttribute,
  IN  BOOLEAN         Recursively
  )
{
  UINT64          BaseAddress;
  UINT64          *NewPageEntry;
  UINTN           Index;
  UINT64          AddressEncMask;
  PAGE_ATTRIBUTE  SplitTo;
 
  if ((SplitAttribute == PageNone) || (SplitAttribute >= PageAttribute)) {
    ASSERT (SplitAttribute != PageNone);
    ASSERT (SplitAttribute < PageAttribute);
    return RETURN_INVALID_PARAMETER;
  }
 
  NewPageEntry = AllocatePageTableMemory (1);
  if (NewPageEntry == NULL) {
    ASSERT (NewPageEntry != NULL);
    return RETURN_OUT_OF_RESOURCES;
  }
 
  //
  // One level down each step to achieve more compact page table.
  //
  SplitTo        = PageAttribute - 1;
  AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) &
                   mPageAttributeTable[SplitTo].AddressMask;
  BaseAddress = *PageEntry &
                ~PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) &
                mPageAttributeTable[PageAttribute].AddressMask;
  for (Index = 0; Index < SIZE_4KB / sizeof (UINT64); Index++) {
    NewPageEntry[Index] = BaseAddress | AddressEncMask |
                          ((*PageEntry) & PAGE_PROGATE_BITS);
 
    if (SplitTo != PageMin) {
      NewPageEntry[Index] |= IA32_PG_PS;
    }
 
    if (Recursively && (SplitTo > SplitAttribute)) {
      SplitPage (&NewPageEntry[Index], SplitTo, SplitAttribute, Recursively);
    }
 
    BaseAddress += mPageAttributeTable[SplitTo].Length;
  }
 
  (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | PAGE_ATTRIBUTE_BITS;
 
  return RETURN_SUCCESS;
}
 
RETURN_STATUS
EFIAPI
ConvertMemoryPageAttributes (
  IN  PHYSICAL_ADDRESS  BaseAddress,
  IN  UINT64            Length,
  IN  UINT64            Attributes
  )
{
  UINT64                *PageEntry;
  PAGE_ATTRIBUTE        PageAttribute;
  RETURN_STATUS         Status;
  EFI_PHYSICAL_ADDRESS  MaximumAddress;
 
  if ((Length == 0) ||
      ((BaseAddress & (SIZE_4KB - 1)) != 0) ||
      ((Length & (SIZE_4KB - 1)) != 0))
  {
    ASSERT (Length > 0);
    ASSERT ((BaseAddress & (SIZE_4KB - 1)) == 0);
    ASSERT ((Length & (SIZE_4KB - 1)) == 0);
 
    return RETURN_INVALID_PARAMETER;
  }
 
  MaximumAddress = (EFI_PHYSICAL_ADDRESS)MAX_UINT32;
  if ((BaseAddress > MaximumAddress) ||
      (Length > MaximumAddress) ||
      (BaseAddress > MaximumAddress - (Length - 1)))
  {
    return RETURN_UNSUPPORTED;
  }
 
  //
  // Below logic is to check 2M/4K page to make sure we do not waste memory.
  //
  while (Length != 0) {
    PageEntry = GetPageTableEntry (BaseAddress, &PageAttribute);
    if (PageEntry == NULL) {
      return RETURN_UNSUPPORTED;
    }
 
    if (PageAttribute != Page4K) {
      Status = SplitPage (PageEntry, PageAttribute, Page4K, FALSE);
      if (RETURN_ERROR (Status)) {
        return Status;
      }
 
      //
      // Do it again until the page is 4K.
      //
      continue;
    }
 
    //
    // Just take care of 'present' bit for Stack Guard.
    //
    if ((Attributes & IA32_PG_P) != 0) {
      *PageEntry |= (UINT64)IA32_PG_P;
    } else {
      *PageEntry &= ~((UINT64)IA32_PG_P);
    }
 
    //
    // Convert success, move to next
    //
    BaseAddress += SIZE_4KB;
    Length      -= SIZE_4KB;
  }
 
  return RETURN_SUCCESS;
}
 
EFI_STATUS
EnablePaePageTable (
  VOID
  )
{
  EFI_STATUS  Status;
 
  UINTN               PageTable;
  VOID                *Buffer;
  UINTN               BufferSize;
  IA32_MAP_ATTRIBUTE  MapAttribute;
  IA32_MAP_ATTRIBUTE  MapMask;
 
  PageTable                   = 0;
  Buffer                      = NULL;
  BufferSize                  = 0;
  MapAttribute.Uint64         = 0;
  MapMask.Uint64              = MAX_UINT64;
  MapAttribute.Bits.Present   = 1;
  MapAttribute.Bits.ReadWrite = 1;
 
  //
  // 1:1 map 4GB in 32bit mode
  //
  Status = PageTableMap (&PageTable, PagingPae, 0, &BufferSize, 0, SIZE_4GB, &MapAttribute, &MapMask, NULL);
  ASSERT (Status == EFI_BUFFER_TOO_SMALL);
  if (Status != EFI_BUFFER_TOO_SMALL) {
    return Status;
  }
 
  //
  // Allocate required Buffer.
  //
  Buffer = AllocatePageTableMemory (EFI_SIZE_TO_PAGES (BufferSize));
  ASSERT (Buffer != NULL);
  if (Buffer == NULL) {
    return EFI_OUT_OF_RESOURCES;
  }
 
  Status = PageTableMap (&PageTable, PagingPae, Buffer, &BufferSize, 0, SIZE_4GB, &MapAttribute, &MapMask, NULL);
  ASSERT_EFI_ERROR (Status);
  if (EFI_ERROR (Status) || (PageTable == 0)) {
    return EFI_OUT_OF_RESOURCES;
  }
 
  //
  // Write the Pagetable to CR3.
  //
  AsmWriteCr3 (PageTable);
 
  //
  // Enable CR4.PAE
  //
  AsmWriteCr4 (AsmReadCr4 () | BIT5);
 
  //
  // Enable CR0.PG
  //
  AsmWriteCr0 (AsmReadCr0 () | BIT31);
 
  DEBUG ((
    DEBUG_INFO,
    "EnablePaePageTable: Created PageTable = 0x%x, BufferSize = %x\n",
    PageTable,
    BufferSize
    ));
 
  return Status;
}
 
VOID
EFIAPI
GetStackBase (
  IN OUT VOID  *Buffer
  )
{
  EFI_PHYSICAL_ADDRESS  StackBase;
 
  StackBase  = (EFI_PHYSICAL_ADDRESS)(UINTN)&StackBase;
  StackBase += BASE_4KB;
  StackBase &= ~((EFI_PHYSICAL_ADDRESS)BASE_4KB - 1);
  StackBase -= PcdGet32 (PcdCpuApStackSize);
 
  *(EFI_PHYSICAL_ADDRESS *)Buffer = StackBase;
}
 
VOID
SetupStackGuardPage (
  VOID
  )
{
  EFI_PEI_HOB_POINTERS  Hob;
  EFI_PHYSICAL_ADDRESS  StackBase;
  UINTN                 NumberOfProcessors;
  UINTN                 Bsp;
  UINTN                 Index;
  EFI_STATUS            Status;
 
  //
  // One extra page at the bottom of the stack is needed for Guard page.
  //
  if (PcdGet32 (PcdCpuApStackSize) <= EFI_PAGE_SIZE) {
    DEBUG ((DEBUG_ERROR, "PcdCpuApStackSize is not big enough for Stack Guard!\n"));
    ASSERT (FALSE);
  }
 
  Status = MpInitLibGetNumberOfProcessors (&NumberOfProcessors, NULL);
  ASSERT_EFI_ERROR (Status);
 
  if (EFI_ERROR (Status)) {
    NumberOfProcessors = 1;
  }
 
  MpInitLibWhoAmI (&Bsp);
  for (Index = 0; Index < NumberOfProcessors; ++Index) {
    StackBase = 0;
 
    if (Index == Bsp) {
      Hob.Raw = GetHobList ();
      while ((Hob.Raw = GetNextHob (EFI_HOB_TYPE_MEMORY_ALLOCATION, Hob.Raw)) != NULL) {
        if (CompareGuid (
              &gEfiHobMemoryAllocStackGuid,
              &(Hob.MemoryAllocationStack->AllocDescriptor.Name)
              ))
        {
          StackBase = Hob.MemoryAllocationStack->AllocDescriptor.MemoryBaseAddress;
          break;
        }
 
        Hob.Raw = GET_NEXT_HOB (Hob);
      }
    } else {
      //
      // Ask AP to return is stack base address.
      //
      MpInitLibStartupThisAP (GetStackBase, Index, NULL, 0, (VOID *)&StackBase, NULL);
    }
 
    ASSERT (StackBase != 0);
    //
    // Set Guard page at stack base address.
    //
    ConvertMemoryPageAttributes (StackBase, EFI_PAGE_SIZE, 0);
    DEBUG ((
      DEBUG_INFO,
      "Stack Guard set at %lx [cpu%lu]!\n",
      (UINT64)StackBase,
      (UINT64)Index
      ));
  }
 
  //
  // Publish the changes of page table.
  //
  CpuFlushTlb ();
}
 
EFI_STATUS
EFIAPI
MemoryDiscoveredPpiNotifyCallback (
  IN EFI_PEI_SERVICES           **PeiServices,
  IN EFI_PEI_NOTIFY_DESCRIPTOR  *NotifyDescriptor,
  IN VOID                       *Ppi
  )
{
  EFI_STATUS              Status;
  BOOLEAN                 InitStackGuard;
  EDKII_MIGRATED_FV_INFO  *MigratedFvInfo;
  EFI_PEI_HOB_POINTERS    Hob;
  IA32_CR0                Cr0;
 
  //
  // Paging must be setup first. Otherwise the exception TSS setup during MP
  // initialization later will not contain paging information and then fail
  // the task switch (for the sake of stack switch).
  //
  InitStackGuard = FALSE;
  Hob.Raw        = NULL;
  if (IsIa32PaeSupported ()) {
    Hob.Raw        = GetFirstGuidHob (&gEdkiiMigratedFvInfoGuid);
    InitStackGuard = PcdGetBool (PcdCpuStackGuard);
  }
 
  //
  // Some security features depend on the page table enabling. So, here
  // is to enable paging if it is not enabled (only in 32bit mode).
  //
  Cr0.UintN = AsmReadCr0 ();
  if ((Cr0.Bits.PG == 0) && (InitStackGuard || (Hob.Raw != NULL))) {
    ASSERT (sizeof (UINTN) == sizeof (UINT32));
 
    Status = EnablePaePageTable ();
    if (EFI_ERROR (Status)) {
      DEBUG ((DEBUG_ERROR, "MemoryDiscoveredPpiNotifyCallback: Failed to enable PAE page table: %r.\n", Status));
      CpuDeadLoop ();
    }
  }
 
  Status = InitializeCpuMpWorker ((CONST EFI_PEI_SERVICES **)PeiServices);
  ASSERT_EFI_ERROR (Status);
 
  if (InitStackGuard) {
    SetupStackGuardPage ();
  }
 
  while (Hob.Raw != NULL) {
    MigratedFvInfo = GET_GUID_HOB_DATA (Hob);
 
    //
    // Enable #PF exception, so if the code access SPI after disable NEM, it will generate
    // the exception to avoid potential vulnerability.
    //
    ConvertMemoryPageAttributes (MigratedFvInfo->FvOrgBase, MigratedFvInfo->FvLength, 0);
 
    Hob.Raw = GET_NEXT_HOB (Hob);
    Hob.Raw = GetNextGuidHob (&gEdkiiMigratedFvInfoGuid, Hob.Raw);
  }
 
  CpuFlushTlb ();
 
  return Status;
}