VmgExitVcHandler.c

Go to the documentation of this file.

 
#include <Base.h>
#include <Uefi.h>
#include <Library/BaseMemoryLib.h>
#include <Library/LocalApicLib.h>
#include <Library/MemEncryptSevLib.h>
#include <Library/VmgExitLib.h>
#include <Register/Amd/Msr.h>
#include <Register/Intel/Cpuid.h>
#include <IndustryStandard/InstructionParsing.h>
 
#include "VmgExitVcHandler.h"
// #include <Library/MemEncryptSevLib.h>
 
//
// Instruction execution mode definition
//
typedef enum {
  LongMode64Bit = 0,
  LongModeCompat32Bit,
  LongModeCompat16Bit,
} SEV_ES_INSTRUCTION_MODE;
 
//
// Instruction size definition (for operand and address)
//
typedef enum {
  Size8Bits = 0,
  Size16Bits,
  Size32Bits,
  Size64Bits,
} SEV_ES_INSTRUCTION_SIZE;
 
//
// Intruction segment definition
//
typedef enum {
  SegmentEs = 0,
  SegmentCs,
  SegmentSs,
  SegmentDs,
  SegmentFs,
  SegmentGs,
} SEV_ES_INSTRUCTION_SEGMENT;
 
//
// Instruction rep function definition
//
typedef enum {
  RepNone = 0,
  RepZ,
  RepNZ,
} SEV_ES_INSTRUCTION_REP;
 
typedef struct {
  UINT8    Rm;
  UINT8    Reg;
  UINT8    Mod;
} SEV_ES_INSTRUCTION_MODRM_EXT;
 
typedef struct {
  UINT8    Base;
  UINT8    Index;
  UINT8    Scale;
} SEV_ES_INSTRUCTION_SIB_EXT;
 
//
// Instruction opcode definition
//
typedef struct {
  SEV_ES_INSTRUCTION_MODRM_EXT    ModRm;
 
  SEV_ES_INSTRUCTION_SIB_EXT      Sib;
 
  UINTN                           RegData;
  UINTN                           RmData;
} SEV_ES_INSTRUCTION_OPCODE_EXT;
 
//
// Instruction parsing context definition
//
typedef struct {
  GHCB                             *Ghcb;
 
  SEV_ES_INSTRUCTION_MODE          Mode;
  SEV_ES_INSTRUCTION_SIZE          DataSize;
  SEV_ES_INSTRUCTION_SIZE          AddrSize;
  BOOLEAN                          SegmentSpecified;
  SEV_ES_INSTRUCTION_SEGMENT       Segment;
  SEV_ES_INSTRUCTION_REP           RepMode;
 
  UINT8                            *Begin;
  UINT8                            *End;
 
  UINT8                            *Prefixes;
  UINT8                            *OpCodes;
  UINT8                            *Displacement;
  UINT8                            *Immediate;
 
  INSTRUCTION_REX_PREFIX           RexPrefix;
 
  BOOLEAN                          ModRmPresent;
  INSTRUCTION_MODRM                ModRm;
 
  BOOLEAN                          SibPresent;
  INSTRUCTION_SIB                  Sib;
 
  UINTN                            PrefixSize;
  UINTN                            OpCodeSize;
  UINTN                            DisplacementSize;
  UINTN                            ImmediateSize;
 
  SEV_ES_INSTRUCTION_OPCODE_EXT    Ext;
} SEV_ES_INSTRUCTION_DATA;
 
//
// Non-automatic Exit function prototype
//
typedef
UINT64
(*NAE_EXIT) (
  GHCB                     *Ghcb,
  EFI_SYSTEM_CONTEXT_X64   *Regs,
  SEV_ES_INSTRUCTION_DATA  *InstructionData
  );
 
//
// SEV-SNP Cpuid table entry/function
//
typedef PACKED struct {
  UINT32    EaxIn;
  UINT32    EcxIn;
  UINT64    Unused;
  UINT64    Unused2;
  UINT32    Eax;
  UINT32    Ebx;
  UINT32    Ecx;
  UINT32    Edx;
  UINT64    Reserved;
} SEV_SNP_CPUID_FUNCTION;
 
//
// SEV-SNP Cpuid page format
//
typedef PACKED struct {
  UINT32                    Count;
  UINT32                    Reserved1;
  UINT64                    Reserved2;
  SEV_SNP_CPUID_FUNCTION    function[0];
} SEV_SNP_CPUID_INFO;
 
STATIC
UINT64 *
GetRegisterPointer (
  IN EFI_SYSTEM_CONTEXT_X64  *Regs,
  IN UINT8                   Register
  )
{
  UINT64  *Reg;
 
  switch (Register) {
    case 0:
      Reg = &Regs->Rax;
      break;
    case 1:
      Reg = &Regs->Rcx;
      break;
    case 2:
      Reg = &Regs->Rdx;
      break;
    case 3:
      Reg = &Regs->Rbx;
      break;
    case 4:
      Reg = &Regs->Rsp;
      break;
    case 5:
      Reg = &Regs->Rbp;
      break;
    case 6:
      Reg = &Regs->Rsi;
      break;
    case 7:
      Reg = &Regs->Rdi;
      break;
    case 8:
      Reg = &Regs->R8;
      break;
    case 9:
      Reg = &Regs->R9;
      break;
    case 10:
      Reg = &Regs->R10;
      break;
    case 11:
      Reg = &Regs->R11;
      break;
    case 12:
      Reg = &Regs->R12;
      break;
    case 13:
      Reg = &Regs->R13;
      break;
    case 14:
      Reg = &Regs->R14;
      break;
    case 15:
      Reg = &Regs->R15;
      break;
    default:
      Reg = NULL;
  }
 
  ASSERT (Reg != NULL);
 
  return Reg;
}
 
STATIC
VOID
UpdateForDisplacement (
  IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
  IN     UINTN                    Size
  )
{
  InstructionData->DisplacementSize = Size;
  InstructionData->Immediate       += Size;
  InstructionData->End             += Size;
}
 
STATIC
BOOLEAN
IsRipRelative (
  IN SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 
  Ext = &InstructionData->Ext;
 
  return ((InstructionData->Mode == LongMode64Bit) &&
          (Ext->ModRm.Mod == 0) &&
          (Ext->ModRm.Rm == 5)  &&
          (InstructionData->SibPresent == FALSE));
}
 
STATIC
UINT64
GetEffectiveMemoryAddress (
  IN EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
  UINT64                         EffectiveAddress;
 
  Ext              = &InstructionData->Ext;
  EffectiveAddress = 0;
 
  if (IsRipRelative (InstructionData)) {
    //
    // RIP-relative displacement is a 32-bit signed value
    //
    INT32  RipRelative;
 
    RipRelative = *(INT32 *)InstructionData->Displacement;
 
    UpdateForDisplacement (InstructionData, 4);
 
    //
    // Negative displacement is handled by standard UINT64 wrap-around.
    //
    return Regs->Rip + (UINT64)RipRelative;
  }
 
  switch (Ext->ModRm.Mod) {
    case 1:
      UpdateForDisplacement (InstructionData, 1);
      EffectiveAddress += (UINT64)(*(INT8 *)(InstructionData->Displacement));
      break;
    case 2:
      switch (InstructionData->AddrSize) {
        case Size16Bits:
          UpdateForDisplacement (InstructionData, 2);
          EffectiveAddress += (UINT64)(*(INT16 *)(InstructionData->Displacement));
          break;
        default:
          UpdateForDisplacement (InstructionData, 4);
          EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
          break;
      }
 
      break;
  }
 
  if (InstructionData->SibPresent) {
    INT64  Displacement;
 
    if (Ext->Sib.Index != 4) {
      CopyMem (
        &Displacement,
        GetRegisterPointer (Regs, Ext->Sib.Index),
        sizeof (Displacement)
        );
      Displacement *= (INT64)(1 << Ext->Sib.Scale);
 
      //
      // Negative displacement is handled by standard UINT64 wrap-around.
      //
      EffectiveAddress += (UINT64)Displacement;
    }
 
    if ((Ext->Sib.Base != 5) || Ext->ModRm.Mod) {
      EffectiveAddress += *GetRegisterPointer (Regs, Ext->Sib.Base);
    } else {
      UpdateForDisplacement (InstructionData, 4);
      EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
    }
  } else {
    EffectiveAddress += *GetRegisterPointer (Regs, Ext->ModRm.Rm);
  }
 
  return EffectiveAddress;
}
 
STATIC
VOID
DecodeModRm (
  IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
  INSTRUCTION_REX_PREFIX         *RexPrefix;
  INSTRUCTION_MODRM              *ModRm;
  INSTRUCTION_SIB                *Sib;
 
  RexPrefix = &InstructionData->RexPrefix;
  Ext       = &InstructionData->Ext;
  ModRm     = &InstructionData->ModRm;
  Sib       = &InstructionData->Sib;
 
  InstructionData->ModRmPresent = TRUE;
  ModRm->Uint8                  = *(InstructionData->End);
 
  InstructionData->Displacement++;
  InstructionData->Immediate++;
  InstructionData->End++;
 
  Ext->ModRm.Mod = ModRm->Bits.Mod;
  Ext->ModRm.Reg = (RexPrefix->Bits.BitR << 3) | ModRm->Bits.Reg;
  Ext->ModRm.Rm  = (RexPrefix->Bits.BitB << 3) | ModRm->Bits.Rm;
 
  Ext->RegData = *GetRegisterPointer (Regs, Ext->ModRm.Reg);
 
  if (Ext->ModRm.Mod == 3) {
    Ext->RmData = *GetRegisterPointer (Regs, Ext->ModRm.Rm);
  } else {
    if (ModRm->Bits.Rm == 4) {
      InstructionData->SibPresent = TRUE;
      Sib->Uint8                  = *(InstructionData->End);
 
      InstructionData->Displacement++;
      InstructionData->Immediate++;
      InstructionData->End++;
 
      Ext->Sib.Scale = Sib->Bits.Scale;
      Ext->Sib.Index = (RexPrefix->Bits.BitX << 3) | Sib->Bits.Index;
      Ext->Sib.Base  = (RexPrefix->Bits.BitB << 3) | Sib->Bits.Base;
    }
 
    Ext->RmData = GetEffectiveMemoryAddress (Regs, InstructionData);
  }
}
 
STATIC
VOID
DecodePrefixes (
  IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  SEV_ES_INSTRUCTION_MODE  Mode;
  SEV_ES_INSTRUCTION_SIZE  ModeDataSize;
  SEV_ES_INSTRUCTION_SIZE  ModeAddrSize;
  UINT8                    *Byte;
 
  //
  // Always in 64-bit mode
  //
  Mode         = LongMode64Bit;
  ModeDataSize = Size32Bits;
  ModeAddrSize = Size64Bits;
 
  InstructionData->Mode     = Mode;
  InstructionData->DataSize = ModeDataSize;
  InstructionData->AddrSize = ModeAddrSize;
 
  InstructionData->Prefixes = InstructionData->Begin;
 
  Byte = InstructionData->Prefixes;
  for ( ; ; Byte++, InstructionData->PrefixSize++) {
    //
    // Check the 0x40 to 0x4F range using an if statement here since some
    // compilers don't like the "case 0x40 ... 0x4F:" syntax. This avoids
    // 16 case statements below.
    //
    if ((*Byte >= REX_PREFIX_START) && (*Byte <= REX_PREFIX_STOP)) {
      InstructionData->RexPrefix.Uint8 = *Byte;
      if ((*Byte & REX_64BIT_OPERAND_SIZE_MASK) != 0) {
        InstructionData->DataSize = Size64Bits;
      }
 
      continue;
    }
 
    switch (*Byte) {
      case OVERRIDE_SEGMENT_CS:
      case OVERRIDE_SEGMENT_DS:
      case OVERRIDE_SEGMENT_ES:
      case OVERRIDE_SEGMENT_SS:
        if (Mode != LongMode64Bit) {
          InstructionData->SegmentSpecified = TRUE;
          InstructionData->Segment          = (*Byte >> 3) & 3;
        }
 
        break;
 
      case OVERRIDE_SEGMENT_FS:
      case OVERRIDE_SEGMENT_GS:
        InstructionData->SegmentSpecified = TRUE;
        InstructionData->Segment          = *Byte & 7;
        break;
 
      case OVERRIDE_OPERAND_SIZE:
        if (InstructionData->RexPrefix.Uint8 == 0) {
          InstructionData->DataSize =
            (Mode == LongMode64Bit)       ? Size16Bits :
            (Mode == LongModeCompat32Bit) ? Size16Bits :
            (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
        }
 
        break;
 
      case OVERRIDE_ADDRESS_SIZE:
        InstructionData->AddrSize =
          (Mode == LongMode64Bit)       ? Size32Bits :
          (Mode == LongModeCompat32Bit) ? Size16Bits :
          (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
        break;
 
      case LOCK_PREFIX:
        break;
 
      case REPZ_PREFIX:
        InstructionData->RepMode = RepZ;
        break;
 
      case REPNZ_PREFIX:
        InstructionData->RepMode = RepNZ;
        break;
 
      default:
        InstructionData->OpCodes    = Byte;
        InstructionData->OpCodeSize = (*Byte == TWO_BYTE_OPCODE_ESCAPE) ? 2 : 1;
 
        InstructionData->End          = Byte + InstructionData->OpCodeSize;
        InstructionData->Displacement = InstructionData->End;
        InstructionData->Immediate    = InstructionData->End;
        return;
    }
  }
}
 
STATIC
UINT64
InstructionLength (
  IN SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  return (UINT64)(InstructionData->End - InstructionData->Begin);
}
 
STATIC
VOID
InitInstructionData (
  IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
  IN     GHCB                     *Ghcb,
  IN     EFI_SYSTEM_CONTEXT_X64   *Regs
  )
{
  SetMem (InstructionData, sizeof (*InstructionData), 0);
  InstructionData->Ghcb  = Ghcb;
  InstructionData->Begin = (UINT8 *)Regs->Rip;
  InstructionData->End   = (UINT8 *)Regs->Rip;
 
  DecodePrefixes (Regs, InstructionData);
}
 
STATIC
UINT64
UnsupportedExit (
  IN GHCB                     *Ghcb,
  IN EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  Status;
 
  Status = VmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
  if (Status == 0) {
    GHCB_EVENT_INJECTION  Event;
 
    Event.Uint64          = 0;
    Event.Elements.Vector = GP_EXCEPTION;
    Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
    Event.Elements.Valid  = 1;
 
    Status = Event.Uint64;
  }
 
  return Status;
}
 
STATIC
UINT64
ValidateMmioMemory (
  IN GHCB   *Ghcb,
  IN UINTN  MemoryAddress,
  IN UINTN  MemoryLength
  )
{
  MEM_ENCRYPT_SEV_ADDRESS_RANGE_STATE  State;
  GHCB_EVENT_INJECTION                 GpEvent;
  UINTN                                Address;
 
  //
  // Allow APIC accesses (which will have the encryption bit set during
  // SEC and PEI phases).
  //
  Address = MemoryAddress & ~(SIZE_4KB - 1);
  if (Address == GetLocalApicBaseAddress ()) {
    return 0;
  }
 
  State = MemEncryptSevGetAddressRangeState (
            0,
            MemoryAddress,
            MemoryLength
            );
  if (State == MemEncryptSevAddressRangeUnencrypted) {
    return 0;
  }
 
  //
  // Any state other than unencrypted is an error, issue a #GP.
  //
  DEBUG ((
    DEBUG_ERROR,
    "MMIO using encrypted memory: %lx\n",
    (UINT64)MemoryAddress
    ));
  GpEvent.Uint64          = 0;
  GpEvent.Elements.Vector = GP_EXCEPTION;
  GpEvent.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
  GpEvent.Elements.Valid  = 1;
 
  return GpEvent.Uint64;
}
 
STATIC
UINT64
MmioExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  ExitInfo1, ExitInfo2, Status;
  UINTN   Bytes;
  UINT64  *Register;
  UINT8   OpCode, SignByte;
  UINTN   Address;
 
  Bytes = 0;
 
  OpCode = *(InstructionData->OpCodes);
  if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
    OpCode = *(InstructionData->OpCodes + 1);
  }
 
  switch (OpCode) {
    //
    // MMIO write (MOV reg/memX, regX)
    //
    case 0x88:
      Bytes = 1;
    //
    // fall through
    //
    case 0x89:
      DecodeModRm (Regs, InstructionData);
      Bytes = ((Bytes != 0) ? Bytes :
               (InstructionData->DataSize == Size16Bits) ? 2 :
               (InstructionData->DataSize == Size32Bits) ? 4 :
               (InstructionData->DataSize == Size64Bits) ? 8 :
               0);
 
      if (InstructionData->Ext.ModRm.Mod == 3) {
        //
        // NPF on two register operands???
        //
        return UnsupportedExit (Ghcb, Regs, InstructionData);
      }
 
      Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
      if (Status != 0) {
        return Status;
      }
 
      ExitInfo1 = InstructionData->Ext.RmData;
      ExitInfo2 = Bytes;
      CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      break;
 
    //
    // MMIO write (MOV moffsetX, aX)
    //
    case 0xA2:
      Bytes = 1;
    //
    // fall through
    //
    case 0xA3:
      Bytes = ((Bytes != 0) ? Bytes :
               (InstructionData->DataSize == Size16Bits) ? 2 :
               (InstructionData->DataSize == Size32Bits) ? 4 :
               (InstructionData->DataSize == Size64Bits) ? 8 :
               0);
 
      InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
      InstructionData->End          += InstructionData->ImmediateSize;
 
      //
      // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
      // Use a STATIC_ASSERT to be certain the code is being built as X64.
      //
      STATIC_ASSERT (
        sizeof (UINTN) == sizeof (UINT64),
        "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
        );
 
      Address = 0;
      CopyMem (
        &Address,
        InstructionData->Immediate,
        InstructionData->ImmediateSize
        );
 
      Status = ValidateMmioMemory (Ghcb, Address, Bytes);
      if (Status != 0) {
        return Status;
      }
 
      ExitInfo1 = Address;
      ExitInfo2 = Bytes;
      CopyMem (Ghcb->SharedBuffer, &Regs->Rax, Bytes);
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      break;
 
    //
    // MMIO write (MOV reg/memX, immX)
    //
    case 0xC6:
      Bytes = 1;
    //
    // fall through
    //
    case 0xC7:
      DecodeModRm (Regs, InstructionData);
      Bytes = ((Bytes != 0) ? Bytes :
               (InstructionData->DataSize == Size16Bits) ? 2 :
               (InstructionData->DataSize == Size32Bits) ? 4 :
               0);
 
      InstructionData->ImmediateSize = Bytes;
      InstructionData->End          += Bytes;
 
      Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
      if (Status != 0) {
        return Status;
      }
 
      ExitInfo1 = InstructionData->Ext.RmData;
      ExitInfo2 = Bytes;
      CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      break;
 
    //
    // MMIO read (MOV regX, reg/memX)
    //
    case 0x8A:
      Bytes = 1;
    //
    // fall through
    //
    case 0x8B:
      DecodeModRm (Regs, InstructionData);
      Bytes = ((Bytes != 0) ? Bytes :
               (InstructionData->DataSize == Size16Bits) ? 2 :
               (InstructionData->DataSize == Size32Bits) ? 4 :
               (InstructionData->DataSize == Size64Bits) ? 8 :
               0);
      if (InstructionData->Ext.ModRm.Mod == 3) {
        //
        // NPF on two register operands???
        //
        return UnsupportedExit (Ghcb, Regs, InstructionData);
      }
 
      Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
      if (Status != 0) {
        return Status;
      }
 
      ExitInfo1 = InstructionData->Ext.RmData;
      ExitInfo2 = Bytes;
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
      if (Bytes == 4) {
        //
        // Zero-extend for 32-bit operation
        //
        *Register = 0;
      }
 
      CopyMem (Register, Ghcb->SharedBuffer, Bytes);
      break;
 
    //
    // MMIO read (MOV aX, moffsetX)
    //
    case 0xA0:
      Bytes = 1;
    //
    // fall through
    //
    case 0xA1:
      Bytes = ((Bytes != 0) ? Bytes :
               (InstructionData->DataSize == Size16Bits) ? 2 :
               (InstructionData->DataSize == Size32Bits) ? 4 :
               (InstructionData->DataSize == Size64Bits) ? 8 :
               0);
 
      InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
      InstructionData->End          += InstructionData->ImmediateSize;
 
      //
      // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
      // Use a STATIC_ASSERT to be certain the code is being built as X64.
      //
      STATIC_ASSERT (
        sizeof (UINTN) == sizeof (UINT64),
        "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
        );
 
      Address = 0;
      CopyMem (
        &Address,
        InstructionData->Immediate,
        InstructionData->ImmediateSize
        );
 
      Status = ValidateMmioMemory (Ghcb, Address, Bytes);
      if (Status != 0) {
        return Status;
      }
 
      ExitInfo1 = Address;
      ExitInfo2 = Bytes;
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      if (Bytes == 4) {
        //
        // Zero-extend for 32-bit operation
        //
        Regs->Rax = 0;
      }
 
      CopyMem (&Regs->Rax, Ghcb->SharedBuffer, Bytes);
      break;
 
    //
    // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
    //
    case 0xB6:
      Bytes = 1;
    //
    // fall through
    //
    case 0xB7:
      DecodeModRm (Regs, InstructionData);
      Bytes = (Bytes != 0) ? Bytes : 2;
 
      Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
      if (Status != 0) {
        return Status;
      }
 
      ExitInfo1 = InstructionData->Ext.RmData;
      ExitInfo2 = Bytes;
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
      SetMem (Register, (UINTN)(1 << InstructionData->DataSize), 0);
      CopyMem (Register, Ghcb->SharedBuffer, Bytes);
      break;
 
    //
    // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
    //
    case 0xBE:
      Bytes = 1;
    //
    // fall through
    //
    case 0xBF:
      DecodeModRm (Regs, InstructionData);
      Bytes = (Bytes != 0) ? Bytes : 2;
 
      Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
      if (Status != 0) {
        return Status;
      }
 
      ExitInfo1 = InstructionData->Ext.RmData;
      ExitInfo2 = Bytes;
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      if (Bytes == 1) {
        UINT8  *Data;
 
        Data     = (UINT8 *)Ghcb->SharedBuffer;
        SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
      } else {
        UINT16  *Data;
 
        Data     = (UINT16 *)Ghcb->SharedBuffer;
        SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
      }
 
      Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
      SetMem (Register, (UINTN)(1 << InstructionData->DataSize), SignByte);
      CopyMem (Register, Ghcb->SharedBuffer, Bytes);
      break;
 
    default:
      DEBUG ((DEBUG_ERROR, "Invalid MMIO opcode (%x)\n", OpCode));
      Status = GP_EXCEPTION;
      ASSERT (FALSE);
  }
 
  return Status;
}
 
STATIC
UINT64
MwaitExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  DecodeModRm (Regs, InstructionData);
 
  Ghcb->SaveArea.Rax = Regs->Rax;
  VmgSetOffsetValid (Ghcb, GhcbRax);
  Ghcb->SaveArea.Rcx = Regs->Rcx;
  VmgSetOffsetValid (Ghcb, GhcbRcx);
 
  return VmgExit (Ghcb, SVM_EXIT_MWAIT, 0, 0);
}
 
STATIC
UINT64
MonitorExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  DecodeModRm (Regs, InstructionData);
 
  Ghcb->SaveArea.Rax = Regs->Rax;  // Identity mapped, so VA = PA
  VmgSetOffsetValid (Ghcb, GhcbRax);
  Ghcb->SaveArea.Rcx = Regs->Rcx;
  VmgSetOffsetValid (Ghcb, GhcbRcx);
  Ghcb->SaveArea.Rdx = Regs->Rdx;
  VmgSetOffsetValid (Ghcb, GhcbRdx);
 
  return VmgExit (Ghcb, SVM_EXIT_MONITOR, 0, 0);
}
 
STATIC
UINT64
WbinvdExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  return VmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
}
 
STATIC
UINT64
RdtscpExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  Status;
 
  DecodeModRm (Regs, InstructionData);
 
  Status = VmgExit (Ghcb, SVM_EXIT_RDTSCP, 0, 0);
  if (Status != 0) {
    return Status;
  }
 
  if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
      !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
      !VmgIsOffsetValid (Ghcb, GhcbRdx))
  {
    return UnsupportedExit (Ghcb, Regs, InstructionData);
  }
 
  Regs->Rax = Ghcb->SaveArea.Rax;
  Regs->Rcx = Ghcb->SaveArea.Rcx;
  Regs->Rdx = Ghcb->SaveArea.Rdx;
 
  return 0;
}
 
STATIC
UINT64
VmmCallExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  Status;
 
  DecodeModRm (Regs, InstructionData);
 
  Ghcb->SaveArea.Rax = Regs->Rax;
  VmgSetOffsetValid (Ghcb, GhcbRax);
  Ghcb->SaveArea.Cpl = (UINT8)(Regs->Cs & 0x3);
  VmgSetOffsetValid (Ghcb, GhcbCpl);
 
  Status = VmgExit (Ghcb, SVM_EXIT_VMMCALL, 0, 0);
  if (Status != 0) {
    return Status;
  }
 
  if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
    return UnsupportedExit (Ghcb, Regs, InstructionData);
  }
 
  Regs->Rax = Ghcb->SaveArea.Rax;
 
  return 0;
}
 
STATIC
UINT64
MsrExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  ExitInfo1, Status;
 
  ExitInfo1 = 0;
 
  switch (*(InstructionData->OpCodes + 1)) {
    case 0x30: // WRMSR
      ExitInfo1          = 1;
      Ghcb->SaveArea.Rax = Regs->Rax;
      VmgSetOffsetValid (Ghcb, GhcbRax);
      Ghcb->SaveArea.Rdx = Regs->Rdx;
      VmgSetOffsetValid (Ghcb, GhcbRdx);
    //
    // fall through
    //
    case 0x32: // RDMSR
      Ghcb->SaveArea.Rcx = Regs->Rcx;
      VmgSetOffsetValid (Ghcb, GhcbRcx);
      break;
    default:
      return UnsupportedExit (Ghcb, Regs, InstructionData);
  }
 
  Status = VmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
  if (Status != 0) {
    return Status;
  }
 
  if (ExitInfo1 == 0) {
    if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
        !VmgIsOffsetValid (Ghcb, GhcbRdx))
    {
      return UnsupportedExit (Ghcb, Regs, InstructionData);
    }
 
    Regs->Rax = Ghcb->SaveArea.Rax;
    Regs->Rdx = Ghcb->SaveArea.Rdx;
  }
 
  return 0;
}
 
STATIC
UINT64
IoioExitInfo (
  IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  ExitInfo;
 
  ExitInfo = 0;
 
  switch (*(InstructionData->OpCodes)) {
    //
    // INS opcodes
    //
    case 0x6C:
    case 0x6D:
      ExitInfo |= IOIO_TYPE_INS;
      ExitInfo |= IOIO_SEG_ES;
      ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
      break;
 
    //
    // OUTS opcodes
    //
    case 0x6E:
    case 0x6F:
      ExitInfo |= IOIO_TYPE_OUTS;
      ExitInfo |= IOIO_SEG_DS;
      ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
      break;
 
    //
    // IN immediate opcodes
    //
    case 0xE4:
    case 0xE5:
      InstructionData->ImmediateSize = 1;
      InstructionData->End++;
      ExitInfo |= IOIO_TYPE_IN;
      ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
      break;
 
    //
    // OUT immediate opcodes
    //
    case 0xE6:
    case 0xE7:
      InstructionData->ImmediateSize = 1;
      InstructionData->End++;
      ExitInfo |= IOIO_TYPE_OUT;
      ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
      break;
 
    //
    // IN register opcodes
    //
    case 0xEC:
    case 0xED:
      ExitInfo |= IOIO_TYPE_IN;
      ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
      break;
 
    //
    // OUT register opcodes
    //
    case 0xEE:
    case 0xEF:
      ExitInfo |= IOIO_TYPE_OUT;
      ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
      break;
 
    default:
      return 0;
  }
 
  switch (*(InstructionData->OpCodes)) {
    //
    // Single-byte opcodes
    //
    case 0x6C:
    case 0x6E:
    case 0xE4:
    case 0xE6:
    case 0xEC:
    case 0xEE:
      ExitInfo |= IOIO_DATA_8;
      break;
 
    //
    // Length determined by instruction parsing
    //
    default:
      ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
                                                          : IOIO_DATA_32;
  }
 
  switch (InstructionData->AddrSize) {
    case Size16Bits:
      ExitInfo |= IOIO_ADDR_16;
      break;
 
    case Size32Bits:
      ExitInfo |= IOIO_ADDR_32;
      break;
 
    case Size64Bits:
      ExitInfo |= IOIO_ADDR_64;
      break;
 
    default:
      break;
  }
 
  if (InstructionData->RepMode != 0) {
    ExitInfo |= IOIO_REP;
  }
 
  return ExitInfo;
}
 
STATIC
UINT64
IoioExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64   ExitInfo1, ExitInfo2, Status;
  BOOLEAN  IsString;
 
  ExitInfo1 = IoioExitInfo (Regs, InstructionData);
  if (ExitInfo1 == 0) {
    return UnsupportedExit (Ghcb, Regs, InstructionData);
  }
 
  IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
  if (IsString) {
    UINTN  IoBytes, VmgExitBytes;
    UINTN  GhcbCount, OpCount;
 
    Status = 0;
 
    IoBytes   = IOIO_DATA_BYTES (ExitInfo1);
    GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
 
    OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
    while (OpCount != 0) {
      ExitInfo2    = MIN (OpCount, GhcbCount);
      VmgExitBytes = ExitInfo2 * IoBytes;
 
      if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
        CopyMem (Ghcb->SharedBuffer, (VOID *)Regs->Rsi, VmgExitBytes);
        Regs->Rsi += VmgExitBytes;
      }
 
      Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
      VmgSetOffsetValid (Ghcb, GhcbSwScratch);
      Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
      if (Status != 0) {
        return Status;
      }
 
      if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
        CopyMem ((VOID *)Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
        Regs->Rdi += VmgExitBytes;
      }
 
      if ((ExitInfo1 & IOIO_REP) != 0) {
        Regs->Rcx -= ExitInfo2;
      }
 
      OpCount -= ExitInfo2;
    }
  } else {
    if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
      Ghcb->SaveArea.Rax = 0;
    } else {
      CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
    }
 
    VmgSetOffsetValid (Ghcb, GhcbRax);
 
    Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
    if (Status != 0) {
      return Status;
    }
 
    if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
      if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
        return UnsupportedExit (Ghcb, Regs, InstructionData);
      }
 
      CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
    }
  }
 
  return 0;
}
 
STATIC
UINT64
InvdExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  return VmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
}
 
STATIC
BOOLEAN
GetCpuidHyp (
  IN OUT GHCB     *Ghcb,
  IN     UINT32   EaxIn,
  IN     UINT32   EcxIn,
  IN     UINT64   XCr0,
  IN OUT UINT32   *Eax,
  IN OUT UINT32   *Ebx,
  IN OUT UINT32   *Ecx,
  IN OUT UINT32   *Edx,
  IN OUT UINT64   *Status,
  IN OUT BOOLEAN  *UnsupportedExit
  )
{
  *UnsupportedExit   = FALSE;
  Ghcb->SaveArea.Rax = EaxIn;
  VmgSetOffsetValid (Ghcb, GhcbRax);
  Ghcb->SaveArea.Rcx = EcxIn;
  VmgSetOffsetValid (Ghcb, GhcbRcx);
  if (EaxIn == CPUID_EXTENDED_STATE) {
    Ghcb->SaveArea.XCr0 = XCr0;
    VmgSetOffsetValid (Ghcb, GhcbXCr0);
  }
 
  *Status = VmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
  if (*Status != 0) {
    return FALSE;
  }
 
  if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
      !VmgIsOffsetValid (Ghcb, GhcbRbx) ||
      !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
      !VmgIsOffsetValid (Ghcb, GhcbRdx))
  {
    *UnsupportedExit = TRUE;
    return FALSE;
  }
 
  if (Eax) {
    *Eax = (UINT32)(UINTN)Ghcb->SaveArea.Rax;
  }
 
  if (Ebx) {
    *Ebx = (UINT32)(UINTN)Ghcb->SaveArea.Rbx;
  }
 
  if (Ecx) {
    *Ecx = (UINT32)(UINTN)Ghcb->SaveArea.Rcx;
  }
 
  if (Edx) {
    *Edx = (UINT32)(UINTN)Ghcb->SaveArea.Rdx;
  }
 
  return TRUE;
}
 
STATIC
BOOLEAN
SnpEnabled (
  VOID
  )
{
  MSR_SEV_STATUS_REGISTER  Msr;
 
  Msr.Uint32 = AsmReadMsr32 (MSR_SEV_STATUS);
 
  return !!Msr.Bits.SevSnpBit;
}
 
STATIC
BOOLEAN
GetCpuidXSaveSize (
  IN     UINT64   XFeaturesEnabled,
  IN     UINT32   XSaveBaseSize,
  IN OUT UINT32   *XSaveSize,
  IN     BOOLEAN  Compacted
  )
{
  SEV_SNP_CPUID_INFO  *CpuidInfo;
  UINT64              XFeaturesFound = 0;
  UINT32              Idx;
 
  *XSaveSize = XSaveBaseSize;
  CpuidInfo  = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
 
  for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
    SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
 
    if (!((CpuidFn->EaxIn == 0xD) &&
          ((CpuidFn->EcxIn == 0) || (CpuidFn->EcxIn == 1))))
    {
      continue;
    }
 
    if (XFeaturesFound & (1ULL << CpuidFn->EcxIn) ||
        !(XFeaturesEnabled & (1ULL << CpuidFn->EcxIn)))
    {
      continue;
    }
 
    XFeaturesFound |= (1ULL << CpuidFn->EcxIn);
    if (Compacted) {
      *XSaveSize += CpuidFn->Eax;
    } else {
      *XSaveSize = MAX (*XSaveSize, CpuidFn->Eax + CpuidFn->Ebx);
    }
  }
 
  /*
   * Either the guest set unsupported XCR0/XSS bits, or the corresponding
   * entries in the CPUID table were not present. This is an invalid state.
   */
  if (XFeaturesFound != (XFeaturesEnabled & ~3UL)) {
    return FALSE;
  }
 
  return TRUE;
}
 
STATIC
BOOLEAN
IsFunctionIndexed (
  IN     UINT32  EaxIn
  )
{
  switch (EaxIn) {
    case CPUID_CACHE_PARAMS:
    case CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS:
    case CPUID_EXTENDED_TOPOLOGY:
    case CPUID_EXTENDED_STATE:
    case CPUID_INTEL_RDT_MONITORING:
    case CPUID_INTEL_RDT_ALLOCATION:
    case CPUID_INTEL_SGX:
    case CPUID_INTEL_PROCESSOR_TRACE:
    case CPUID_DETERMINISTIC_ADDRESS_TRANSLATION_PARAMETERS:
    case CPUID_V2_EXTENDED_TOPOLOGY:
    case 0x8000001D: /* Cache Topology Information */
      return TRUE;
  }
 
  return FALSE;
}
 
STATIC
BOOLEAN
GetCpuidFw (
  IN OUT GHCB     *Ghcb,
  IN     UINT32   EaxIn,
  IN     UINT32   EcxIn,
  IN     UINT64   XCr0,
  IN OUT UINT32   *Eax,
  IN OUT UINT32   *Ebx,
  IN OUT UINT32   *Ecx,
  IN OUT UINT32   *Edx,
  IN OUT UINT64   *Status,
  IN OUT BOOLEAN  *Unsupported
  )
{
  SEV_SNP_CPUID_INFO  *CpuidInfo;
  BOOLEAN             Found;
  UINT32              Idx;
 
  CpuidInfo = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
  Found     = FALSE;
 
  for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
    SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
 
    if (CpuidFn->EaxIn != EaxIn) {
      continue;
    }
 
    if (IsFunctionIndexed (CpuidFn->EaxIn) && (CpuidFn->EcxIn != EcxIn)) {
      continue;
    }
 
    *Eax = CpuidFn->Eax;
    *Ebx = CpuidFn->Ebx;
    *Ecx = CpuidFn->Ecx;
    *Edx = CpuidFn->Edx;
 
    Found = TRUE;
    break;
  }
 
  if (!Found) {
    *Eax = *Ebx = *Ecx = *Edx = 0;
    goto Out;
  }
 
  if (EaxIn == CPUID_VERSION_INFO) {
    IA32_CR4  Cr4;
    UINT32    Ebx2;
    UINT32    Edx2;
 
    if (!GetCpuidHyp (
           Ghcb,
           EaxIn,
           EcxIn,
           XCr0,
           NULL,
           &Ebx2,
           NULL,
           &Edx2,
           Status,
           Unsupported
           ))
    {
      return FALSE;
    }
 
    /* initial APIC ID */
    *Ebx = (*Ebx & 0x00FFFFFF) | (Ebx2 & 0xFF000000);
    /* APIC enabled bit */
    *Edx = (*Edx & ~BIT9) | (Edx2 & BIT9);
    /* OSXSAVE enabled bit */
    Cr4.UintN = AsmReadCr4 ();
    *Ecx      = (Cr4.Bits.OSXSAVE) ? (*Ecx & ~BIT27) | (*Ecx & BIT27)
                              : (*Ecx & ~BIT27);
  } else if (EaxIn == CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS) {
    IA32_CR4  Cr4;
 
    Cr4.UintN = AsmReadCr4 ();
    /* OSPKE enabled bit */
    *Ecx = (Cr4.Bits.PKE) ? (*Ecx | BIT4) : (*Ecx & ~BIT4);
  } else if (EaxIn == CPUID_EXTENDED_TOPOLOGY) {
    if (!GetCpuidHyp (
           Ghcb,
           EaxIn,
           EcxIn,
           XCr0,
           NULL,
           NULL,
           NULL,
           Edx,
           Status,
           Unsupported
           ))
    {
      return FALSE;
    }
  } else if ((EaxIn == CPUID_EXTENDED_STATE) && ((EcxIn == 0) || (EcxIn == 1))) {
    MSR_IA32_XSS_REGISTER  XssMsr;
    BOOLEAN                Compacted;
    UINT32                 XSaveSize;
 
    XssMsr.Uint64 = 0;
    Compacted     = FALSE;
    if (EcxIn == 1) {
      /*
       * The PPR and APM aren't clear on what size should be encoded in
       * 0xD:0x1:EBX when compaction is not enabled by either XSAVEC or
       * XSAVES, as these are generally fixed to 1 on real CPUs. Report
       * this undefined case as an error.
       */
      if (!(*Eax & (BIT3 | BIT1))) {
        /* (XSAVES | XSAVEC) */
        return FALSE;
      }
 
      Compacted     = TRUE;
      XssMsr.Uint64 = AsmReadMsr64 (MSR_IA32_XSS);
    }
 
    if (!GetCpuidXSaveSize (
           XCr0 | XssMsr.Uint64,
           *Ebx,
           &XSaveSize,
           Compacted
           ))
    {
      return FALSE;
    }
 
    *Ebx = XSaveSize;
  } else if (EaxIn == 0x8000001E) {
    UINT32  Ebx2;
    UINT32  Ecx2;
 
    /* extended APIC ID */
    if (!GetCpuidHyp (
           Ghcb,
           EaxIn,
           EcxIn,
           XCr0,
           Eax,
           &Ebx2,
           &Ecx2,
           NULL,
           Status,
           Unsupported
           ))
    {
      return FALSE;
    }
 
    /* compute ID */
    *Ebx = (*Ebx & 0xFFFFFF00) | (Ebx2 & 0x000000FF);
    /* node ID */
    *Ecx = (*Ecx & 0xFFFFFF00) | (Ecx2 & 0x000000FF);
  }
 
Out:
  *Status      = 0;
  *Unsupported = FALSE;
  return TRUE;
}
 
STATIC
UINT64
CpuidExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  BOOLEAN  Unsupported;
  UINT64   Status;
  UINT32   EaxIn;
  UINT32   EcxIn;
  UINT64   XCr0;
  UINT32   Eax;
  UINT32   Ebx;
  UINT32   Ecx;
  UINT32   Edx;
 
  EaxIn = (UINT32)(UINTN)Regs->Rax;
  EcxIn = (UINT32)(UINTN)Regs->Rcx;
 
  if (EaxIn == CPUID_EXTENDED_STATE) {
    IA32_CR4  Cr4;
 
    Cr4.UintN           = AsmReadCr4 ();
    Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
    XCr0                = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
  }
 
  if (SnpEnabled ()) {
    if (!GetCpuidFw (
           Ghcb,
           EaxIn,
           EcxIn,
           XCr0,
           &Eax,
           &Ebx,
           &Ecx,
           &Edx,
           &Status,
           &Unsupported
           ))
    {
      goto CpuidFail;
    }
  } else {
    if (!GetCpuidHyp (
           Ghcb,
           EaxIn,
           EcxIn,
           XCr0,
           &Eax,
           &Ebx,
           &Ecx,
           &Edx,
           &Status,
           &Unsupported
           ))
    {
      goto CpuidFail;
    }
  }
 
  Regs->Rax = Eax;
  Regs->Rbx = Ebx;
  Regs->Rcx = Ecx;
  Regs->Rdx = Edx;
 
  return 0;
 
CpuidFail:
  if (Unsupported) {
    return UnsupportedExit (Ghcb, Regs, InstructionData);
  }
 
  return Status;
}
 
STATIC
UINT64
RdpmcExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  Status;
 
  Ghcb->SaveArea.Rcx = Regs->Rcx;
  VmgSetOffsetValid (Ghcb, GhcbRcx);
 
  Status = VmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
  if (Status != 0) {
    return Status;
  }
 
  if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
      !VmgIsOffsetValid (Ghcb, GhcbRdx))
  {
    return UnsupportedExit (Ghcb, Regs, InstructionData);
  }
 
  Regs->Rax = Ghcb->SaveArea.Rax;
  Regs->Rdx = Ghcb->SaveArea.Rdx;
 
  return 0;
}
 
STATIC
UINT64
RdtscExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  UINT64  Status;
 
  Status = VmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
  if (Status != 0) {
    return Status;
  }
 
  if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
      !VmgIsOffsetValid (Ghcb, GhcbRdx))
  {
    return UnsupportedExit (Ghcb, Regs, InstructionData);
  }
 
  Regs->Rax = Ghcb->SaveArea.Rax;
  Regs->Rdx = Ghcb->SaveArea.Rdx;
 
  return 0;
}
 
STATIC
UINT64
Dr7WriteExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
  SEV_ES_PER_CPU_DATA            *SevEsData;
  UINT64                         *Register;
  UINT64                         Status;
 
  Ext       = &InstructionData->Ext;
  SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
 
  DecodeModRm (Regs, InstructionData);
 
  //
  // MOV DRn always treats MOD == 3 no matter how encoded
  //
  Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
 
  //
  // Using a value of 0 for ExitInfo1 means RAX holds the value
  //
  Ghcb->SaveArea.Rax = *Register;
  VmgSetOffsetValid (Ghcb, GhcbRax);
 
  Status = VmgExit (Ghcb, SVM_EXIT_DR7_WRITE, 0, 0);
  if (Status != 0) {
    return Status;
  }
 
  SevEsData->Dr7       = *Register;
  SevEsData->Dr7Cached = 1;
 
  return 0;
}
 
STATIC
UINT64
Dr7ReadExit (
  IN OUT GHCB                     *Ghcb,
  IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
  IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
  )
{
  SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
  SEV_ES_PER_CPU_DATA            *SevEsData;
  UINT64                         *Register;
 
  Ext       = &InstructionData->Ext;
  SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
 
  DecodeModRm (Regs, InstructionData);
 
  //
  // MOV DRn always treats MOD == 3 no matter how encoded
  //
  Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
 
  //
  // If there is a cached valued for DR7, return that. Otherwise return the
  // DR7 standard reset value of 0x400 (no debug breakpoints set).
  //
  *Register = (SevEsData->Dr7Cached == 1) ? SevEsData->Dr7 : 0x400;
 
  return 0;
}
 
EFI_STATUS
EFIAPI
InternalVmgExitHandleVc (
  IN OUT GHCB                *Ghcb,
  IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
  IN OUT EFI_SYSTEM_CONTEXT  SystemContext
  )
{
  EFI_SYSTEM_CONTEXT_X64   *Regs;
  NAE_EXIT                 NaeExit;
  SEV_ES_INSTRUCTION_DATA  InstructionData;
  UINT64                   ExitCode, Status;
  EFI_STATUS               VcRet;
  BOOLEAN                  InterruptState;
 
  VcRet = EFI_SUCCESS;
 
  Regs = SystemContext.SystemContextX64;
 
  VmgInit (Ghcb, &InterruptState);
 
  ExitCode = Regs->ExceptionData;
  switch (ExitCode) {
    case SVM_EXIT_DR7_READ:
      NaeExit = Dr7ReadExit;
      break;
 
    case SVM_EXIT_DR7_WRITE:
      NaeExit = Dr7WriteExit;
      break;
 
    case SVM_EXIT_RDTSC:
      NaeExit = RdtscExit;
      break;
 
    case SVM_EXIT_RDPMC:
      NaeExit = RdpmcExit;
      break;
 
    case SVM_EXIT_CPUID:
      NaeExit = CpuidExit;
      break;
 
    case SVM_EXIT_INVD:
      NaeExit = InvdExit;
      break;
 
    case SVM_EXIT_IOIO_PROT:
      NaeExit = IoioExit;
      break;
 
    case SVM_EXIT_MSR:
      NaeExit = MsrExit;
      break;
 
    case SVM_EXIT_VMMCALL:
      NaeExit = VmmCallExit;
      break;
 
    case SVM_EXIT_RDTSCP:
      NaeExit = RdtscpExit;
      break;
 
    case SVM_EXIT_WBINVD:
      NaeExit = WbinvdExit;
      break;
 
    case SVM_EXIT_MONITOR:
      NaeExit = MonitorExit;
      break;
 
    case SVM_EXIT_MWAIT:
      NaeExit = MwaitExit;
      break;
 
    case SVM_EXIT_NPF:
      NaeExit = MmioExit;
      break;
 
    default:
      NaeExit = UnsupportedExit;
  }
 
  InitInstructionData (&InstructionData, Ghcb, Regs);
 
  Status = NaeExit (Ghcb, Regs, &InstructionData);
  if (Status == 0) {
    Regs->Rip += InstructionLength (&InstructionData);
  } else {
    GHCB_EVENT_INJECTION  Event;
 
    Event.Uint64 = Status;
    if (Event.Elements.ErrorCodeValid != 0) {
      Regs->ExceptionData = Event.Elements.ErrorCode;
    } else {
      Regs->ExceptionData = 0;
    }
 
    *ExceptionType = Event.Elements.Vector;
 
    VcRet = EFI_PROTOCOL_ERROR;
  }
 
  VmgDone (Ghcb, InterruptState);
 
  return VcRet;
}
 
VOID
EFIAPI
VmgExitIssueAssert (
  IN OUT SEV_ES_PER_CPU_DATA  *SevEsData
  )
{
  //
  // Progress will be halted, so set VcCount to allow for ASSERT output
  // to be seen.
  //
  SevEsData->VcCount = 0;
 
  ASSERT (FALSE);
  CpuDeadLoop ();
}