IoMmuBuffer.c

Go to the documentation of this file.

 
#include <Library/BaseLib.h>
#include <Library/BaseMemoryLib.h>
#include <Library/DebugLib.h>
#include <Library/MemoryAllocationLib.h>
#include <Library/MemEncryptSevLib.h>
#include <Library/MemEncryptTdxLib.h>
#include <Library/PcdLib.h>
#include <Library/SynchronizationLib.h>
#include <Library/UefiBootServicesTableLib.h>
#include "IoMmuInternal.h"
 
extern BOOLEAN  mReservedSharedMemSupported;
 
#define SIZE_OF_MEM_RANGE(MemRange)  (MemRange->HeaderSize + MemRange->DataSize)
 
#define RESERVED_MEM_BITMAP_4K_MASK    0xf
#define RESERVED_MEM_BITMAP_32K_MASK   0xff0
#define RESERVED_MEM_BITMAP_128K_MASK  0x3000
#define RESERVED_MEM_BITMAP_1M_MASK    0x40000
#define RESERVED_MEM_BITMAP_2M_MASK    0x180000
#define RESERVED_MEM_BITMAP_MASK       0x1fffff
 
STATIC IOMMU_RESERVED_MEM_RANGE  mReservedMemRanges[] = {
  { RESERVED_MEM_BITMAP_4K_MASK,   0,  4, SIZE_4KB,   SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_32K_MASK,  4,  8, SIZE_32KB,  SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_128K_MASK, 12, 2, SIZE_128KB, SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_1M_MASK,   14, 1, SIZE_1MB,   SIZE_4KB, 0 },
  { RESERVED_MEM_BITMAP_2M_MASK,   15, 2, SIZE_2MB,   SIZE_4KB, 0 },
};
 
//
// Bitmap of the allocation of reserved memory.
//
STATIC UINT32  mReservedMemBitmap = 0;
 
//
// Start address of the reserved memory region.
//
STATIC EFI_PHYSICAL_ADDRESS  mReservedSharedMemAddress = 0;
 
//
// Total size of the reserved memory region.
//
STATIC UINT32  mReservedSharedMemSize = 0;
 
STATIC
UINT32
CalcuateReservedMemSize (
  VOID
  )
{
  UINT32                    Index;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;
 
  if (mReservedSharedMemSize != 0) {
    return mReservedSharedMemSize;
  }
 
  for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) {
    MemRange                = &mReservedMemRanges[Index];
    mReservedSharedMemSize += (SIZE_OF_MEM_RANGE (MemRange) * MemRange->Slots);
  }
 
  return mReservedSharedMemSize;
}
 
EFI_STATUS
IoMmuInitReservedSharedMem (
  VOID
  )
{
  EFI_STATUS                Status;
  UINT32                    Index1, Index2;
  UINTN                     TotalPages;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;
  EFI_PHYSICAL_ADDRESS      PhysicalAddress;
  UINT64                    SharedAddress;
 
  if (!mReservedSharedMemSupported) {
    return EFI_UNSUPPORTED;
  }
 
  TotalPages = EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ());
 
  PhysicalAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocatePages (TotalPages);
  DEBUG ((
    DEBUG_VERBOSE,
    "%a: ReservedMem (%d pages) address = 0x%llx\n",
    __func__,
    TotalPages,
    PhysicalAddress
    ));
 
  mReservedMemBitmap        = 0;
  mReservedSharedMemAddress = PhysicalAddress;
 
  for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) {
    MemRange                         = &mReservedMemRanges[Index1];
    MemRange->StartAddressOfMemRange = PhysicalAddress;
 
    for (Index2 = 0; Index2 < MemRange->Slots; Index2++) {
      SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize);
 
      if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptSevClearPageEncMask (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptTdxSetPageSharedBit (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else {
        ASSERT (FALSE);
      }
    }
 
    PhysicalAddress += (MemRange->Slots * SIZE_OF_MEM_RANGE (MemRange));
  }
 
  return EFI_SUCCESS;
}
 
EFI_STATUS
IoMmuReleaseReservedSharedMem (
  BOOLEAN  MemoryMapLocked
  )
{
  EFI_STATUS                Status;
  UINT32                    Index1, Index2;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;
  UINT64                    SharedAddress;
 
  if (!mReservedSharedMemSupported) {
    return EFI_SUCCESS;
  }
 
  for (Index1 = 0; Index1 < ARRAY_SIZE (mReservedMemRanges); Index1++) {
    MemRange = &mReservedMemRanges[Index1];
    for (Index2 = 0; Index2 < MemRange->Slots; Index2++) {
      SharedAddress = (UINT64)(UINTN)(MemRange->StartAddressOfMemRange + Index2 * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize);
 
      if (CC_GUEST_IS_SEV (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptSevSetPageEncMask (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr))) {
        Status = MemEncryptTdxClearPageSharedBit (
                   0,
                   SharedAddress,
                   EFI_SIZE_TO_PAGES (MemRange->DataSize)
                   );
        ASSERT (!EFI_ERROR (Status));
      } else {
        ASSERT (FALSE);
      }
    }
  }
 
  if (!MemoryMapLocked) {
    FreePages ((VOID *)(UINTN)mReservedSharedMemAddress, EFI_SIZE_TO_PAGES (CalcuateReservedMemSize ()));
    mReservedSharedMemAddress = 0;
    mReservedMemBitmap        = 0;
  }
 
  mReservedSharedMemSupported = FALSE;
 
  return EFI_SUCCESS;
}
 
STATIC
EFI_STATUS
InternalAllocateBuffer (
  IN  EFI_ALLOCATE_TYPE        Type,
  IN  EFI_MEMORY_TYPE          MemoryType,
  IN  UINTN                    Pages,
  OUT UINT32                   *ReservedMemBit,
  IN OUT EFI_PHYSICAL_ADDRESS  *PhysicalAddress
  )
{
  UINT32                    MemBitmap;
  UINT32                    ReservedMemBitmap;
  UINT8                     Index;
  IOMMU_RESERVED_MEM_RANGE  *MemRange;
  UINTN                     PagesOfLastMemRange;
 
  *ReservedMemBit = 0;
 
  if (Pages == 0) {
    ASSERT (FALSE);
    return EFI_INVALID_PARAMETER;
  }
 
  if (!mReservedSharedMemSupported) {
    goto LegacyAllocateBuffer;
  }
 
  if (mReservedSharedMemAddress == 0) {
    goto LegacyAllocateBuffer;
  }
 
  PagesOfLastMemRange = 0;
 
  for (Index = 0; Index < ARRAY_SIZE (mReservedMemRanges); Index++) {
    if ((Pages > PagesOfLastMemRange) && (Pages <= EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize))) {
      break;
    }
 
    PagesOfLastMemRange = EFI_SIZE_TO_PAGES (mReservedMemRanges[Index].DataSize);
  }
 
  if (Index == ARRAY_SIZE (mReservedMemRanges)) {
    // There is no suitable size of reserved memory. Turn to legacy allocate.
    goto LegacyAllocateBuffer;
  }
 
  MemRange = &mReservedMemRanges[Index];
 
  do {
    ReservedMemBitmap = mReservedMemBitmap;
 
    if ((ReservedMemBitmap & MemRange->BitmapMask) == MemRange->BitmapMask) {
      // The reserved memory is exhausted. Turn to legacy allocate.
      goto LegacyAllocateBuffer;
    }
 
    MemBitmap = (ReservedMemBitmap & MemRange->BitmapMask) >> MemRange->Shift;
 
    for (Index = 0; Index < MemRange->Slots; Index++) {
      if ((MemBitmap & (UINT8)(1<<Index)) == 0) {
        break;
      }
    }
 
    ASSERT (Index != MemRange->Slots);
 
    *PhysicalAddress = MemRange->StartAddressOfMemRange + Index * SIZE_OF_MEM_RANGE (MemRange) + MemRange->HeaderSize;
    *ReservedMemBit  = (UINT32)(1 << (Index + MemRange->Shift));
  } while (ReservedMemBitmap != InterlockedCompareExchange32 (
                                  &mReservedMemBitmap,
                                  ReservedMemBitmap,
                                  ReservedMemBitmap | *ReservedMemBit
                                  ));
 
  DEBUG ((
    DEBUG_VERBOSE,
    "%a: range-size: %lx, start-address=0x%llx, pages=0x%llx, bits=0x%lx, bitmap: %lx => %lx\n",
    __func__,
    MemRange->DataSize,
    *PhysicalAddress,
    Pages,
    *ReservedMemBit,
    ReservedMemBitmap,
    ReservedMemBitmap | *ReservedMemBit
    ));
 
  return EFI_SUCCESS;
 
LegacyAllocateBuffer:
 
  *ReservedMemBit = 0;
  return gBS->AllocatePages (Type, MemoryType, Pages, PhysicalAddress);
}
 
EFI_STATUS
IoMmuAllocateBounceBuffer (
  IN     EFI_ALLOCATE_TYPE  Type,
  IN     EFI_MEMORY_TYPE    MemoryType,
  IN OUT MAP_INFO           *MapInfo
  )
{
  EFI_STATUS  Status;
 
  Status = InternalAllocateBuffer (
             Type,
             MemoryType,
             MapInfo->NumberOfPages,
             &MapInfo->ReservedMemBitmap,
             &MapInfo->PlainTextAddress
             );
  ASSERT (Status == EFI_SUCCESS);
 
  return Status;
}
 
STATIC
VOID
ClearReservedMemBit (
  IN  UINT32  ReservedMemBit
  )
{
  UINT32  ReservedMemBitmap;
 
  do {
    ReservedMemBitmap = mReservedMemBitmap;
  } while (ReservedMemBitmap != InterlockedCompareExchange32 (
                                  &mReservedMemBitmap,
                                  ReservedMemBitmap,
                                  ReservedMemBitmap & ~ReservedMemBit
                                  ));
}
 
EFI_STATUS
IoMmuFreeBounceBuffer (
  IN OUT     MAP_INFO  *MapInfo
  )
{
  if (MapInfo->ReservedMemBitmap == 0) {
    gBS->FreePages (MapInfo->PlainTextAddress, MapInfo->NumberOfPages);
  } else {
    DEBUG ((
      DEBUG_VERBOSE,
      "%a: PlainTextAddress=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n",
      __func__,
      MapInfo->PlainTextAddress,
      MapInfo->ReservedMemBitmap,
      mReservedMemBitmap,
      mReservedMemBitmap & ((UINT32)(~MapInfo->ReservedMemBitmap))
      ));
    ClearReservedMemBit (MapInfo->ReservedMemBitmap);
    MapInfo->PlainTextAddress  = 0;
    MapInfo->ReservedMemBitmap = 0;
  }
 
  return EFI_SUCCESS;
}
 
EFI_STATUS
IoMmuAllocateCommonBuffer (
  IN EFI_MEMORY_TYPE        MemoryType,
  IN UINTN                  CommonBufferPages,
  OUT EFI_PHYSICAL_ADDRESS  *PhysicalAddress,
  OUT UINT32                *ReservedMemBitmap
  )
{
  EFI_STATUS  Status;
 
  Status = InternalAllocateBuffer (
             AllocateMaxAddress,
             MemoryType,
             CommonBufferPages,
             ReservedMemBitmap,
             PhysicalAddress
             );
  ASSERT (Status == EFI_SUCCESS);
 
  if (*ReservedMemBitmap != 0) {
    *PhysicalAddress -= SIZE_4KB;
  }
 
  return Status;
}
 
EFI_STATUS
IoMmuFreeCommonBuffer (
  IN COMMON_BUFFER_HEADER  *CommonBufferHeader,
  IN UINTN                 CommonBufferPages
  )
{
  if (!mReservedSharedMemSupported) {
    goto LegacyFreeCommonBuffer;
  }
 
  if (CommonBufferHeader->ReservedMemBitmap == 0) {
    goto LegacyFreeCommonBuffer;
  }
 
  DEBUG ((
    DEBUG_VERBOSE,
    "%a: CommonBuffer=0x%Lx, bits=0x%Lx, bitmap: %Lx => %Lx\n",
    __func__,
    (UINT64)(UINTN)CommonBufferHeader + SIZE_4KB,
    CommonBufferHeader->ReservedMemBitmap,
    mReservedMemBitmap,
    mReservedMemBitmap & ((UINT32)(~CommonBufferHeader->ReservedMemBitmap))
    ));
 
  ClearReservedMemBit (CommonBufferHeader->ReservedMemBitmap);
  return EFI_SUCCESS;
 
LegacyFreeCommonBuffer:
  return gBS->FreePages ((UINTN)CommonBufferHeader, CommonBufferPages);
}