NvmExpressPassthru.c

Go to the documentation of this file.

 
#include "NvmExpress.h"
 
VOID
NvmeDumpStatus (
  IN NVME_CQ  *Cq
  )
{
  DEBUG ((DEBUG_VERBOSE, "Dump NVMe Completion Entry Status from [0x%x]:\n", Cq));
 
  DEBUG ((DEBUG_VERBOSE, "  SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n", Cq->Sqid, Cq->Pt, Cq->Cid));
 
  DEBUG ((DEBUG_VERBOSE, "  NVMe Cmd Execution Result - "));
 
  switch (Cq->Sct) {
    case 0x0:
      switch (Cq->Sc) {
        case 0x0:
          DEBUG ((DEBUG_VERBOSE, "Successful Completion\n"));
          break;
        case 0x1:
          DEBUG ((DEBUG_VERBOSE, "Invalid Command Opcode\n"));
          break;
        case 0x2:
          DEBUG ((DEBUG_VERBOSE, "Invalid Field in Command\n"));
          break;
        case 0x3:
          DEBUG ((DEBUG_VERBOSE, "Command ID Conflict\n"));
          break;
        case 0x4:
          DEBUG ((DEBUG_VERBOSE, "Data Transfer Error\n"));
          break;
        case 0x5:
          DEBUG ((DEBUG_VERBOSE, "Commands Aborted due to Power Loss Notification\n"));
          break;
        case 0x6:
          DEBUG ((DEBUG_VERBOSE, "Internal Device Error\n"));
          break;
        case 0x7:
          DEBUG ((DEBUG_VERBOSE, "Command Abort Requested\n"));
          break;
        case 0x8:
          DEBUG ((DEBUG_VERBOSE, "Command Aborted due to SQ Deletion\n"));
          break;
        case 0x9:
          DEBUG ((DEBUG_VERBOSE, "Command Aborted due to Failed Fused Command\n"));
          break;
        case 0xA:
          DEBUG ((DEBUG_VERBOSE, "Command Aborted due to Missing Fused Command\n"));
          break;
        case 0xB:
          DEBUG ((DEBUG_VERBOSE, "Invalid Namespace or Format\n"));
          break;
        case 0xC:
          DEBUG ((DEBUG_VERBOSE, "Command Sequence Error\n"));
          break;
        case 0xD:
          DEBUG ((DEBUG_VERBOSE, "Invalid SGL Last Segment Descriptor\n"));
          break;
        case 0xE:
          DEBUG ((DEBUG_VERBOSE, "Invalid Number of SGL Descriptors\n"));
          break;
        case 0xF:
          DEBUG ((DEBUG_VERBOSE, "Data SGL Length Invalid\n"));
          break;
        case 0x10:
          DEBUG ((DEBUG_VERBOSE, "Metadata SGL Length Invalid\n"));
          break;
        case 0x11:
          DEBUG ((DEBUG_VERBOSE, "SGL Descriptor Type Invalid\n"));
          break;
        case 0x80:
          DEBUG ((DEBUG_VERBOSE, "LBA Out of Range\n"));
          break;
        case 0x81:
          DEBUG ((DEBUG_VERBOSE, "Capacity Exceeded\n"));
          break;
        case 0x82:
          DEBUG ((DEBUG_VERBOSE, "Namespace Not Ready\n"));
          break;
        case 0x83:
          DEBUG ((DEBUG_VERBOSE, "Reservation Conflict\n"));
          break;
      }
 
      break;
 
    case 0x1:
      switch (Cq->Sc) {
        case 0x0:
          DEBUG ((DEBUG_VERBOSE, "Completion Queue Invalid\n"));
          break;
        case 0x1:
          DEBUG ((DEBUG_VERBOSE, "Invalid Queue Identifier\n"));
          break;
        case 0x2:
          DEBUG ((DEBUG_VERBOSE, "Maximum Queue Size Exceeded\n"));
          break;
        case 0x3:
          DEBUG ((DEBUG_VERBOSE, "Abort Command Limit Exceeded\n"));
          break;
        case 0x5:
          DEBUG ((DEBUG_VERBOSE, "Asynchronous Event Request Limit Exceeded\n"));
          break;
        case 0x6:
          DEBUG ((DEBUG_VERBOSE, "Invalid Firmware Slot\n"));
          break;
        case 0x7:
          DEBUG ((DEBUG_VERBOSE, "Invalid Firmware Image\n"));
          break;
        case 0x8:
          DEBUG ((DEBUG_VERBOSE, "Invalid Interrupt Vector\n"));
          break;
        case 0x9:
          DEBUG ((DEBUG_VERBOSE, "Invalid Log Page\n"));
          break;
        case 0xA:
          DEBUG ((DEBUG_VERBOSE, "Invalid Format\n"));
          break;
        case 0xB:
          DEBUG ((DEBUG_VERBOSE, "Firmware Application Requires Conventional Reset\n"));
          break;
        case 0xC:
          DEBUG ((DEBUG_VERBOSE, "Invalid Queue Deletion\n"));
          break;
        case 0xD:
          DEBUG ((DEBUG_VERBOSE, "Feature Identifier Not Saveable\n"));
          break;
        case 0xE:
          DEBUG ((DEBUG_VERBOSE, "Feature Not Changeable\n"));
          break;
        case 0xF:
          DEBUG ((DEBUG_VERBOSE, "Feature Not Namespace Specific\n"));
          break;
        case 0x10:
          DEBUG ((DEBUG_VERBOSE, "Firmware Application Requires NVM Subsystem Reset\n"));
          break;
        case 0x80:
          DEBUG ((DEBUG_VERBOSE, "Conflicting Attributes\n"));
          break;
        case 0x81:
          DEBUG ((DEBUG_VERBOSE, "Invalid Protection Information\n"));
          break;
        case 0x82:
          DEBUG ((DEBUG_VERBOSE, "Attempted Write to Read Only Range\n"));
          break;
      }
 
      break;
 
    case 0x2:
      switch (Cq->Sc) {
        case 0x80:
          DEBUG ((DEBUG_VERBOSE, "Write Fault\n"));
          break;
        case 0x81:
          DEBUG ((DEBUG_VERBOSE, "Unrecovered Read Error\n"));
          break;
        case 0x82:
          DEBUG ((DEBUG_VERBOSE, "End-to-end Guard Check Error\n"));
          break;
        case 0x83:
          DEBUG ((DEBUG_VERBOSE, "End-to-end Application Tag Check Error\n"));
          break;
        case 0x84:
          DEBUG ((DEBUG_VERBOSE, "End-to-end Reference Tag Check Error\n"));
          break;
        case 0x85:
          DEBUG ((DEBUG_VERBOSE, "Compare Failure\n"));
          break;
        case 0x86:
          DEBUG ((DEBUG_VERBOSE, "Access Denied\n"));
          break;
      }
 
      break;
 
    default:
      break;
  }
}
 
VOID *
NvmeCreatePrpList (
  IN     EFI_PCI_IO_PROTOCOL   *PciIo,
  IN     EFI_PHYSICAL_ADDRESS  PhysicalAddr,
  IN     UINTN                 Pages,
  OUT VOID                     **PrpListHost,
  IN OUT UINTN                 *PrpListNo,
  OUT VOID                     **Mapping
  )
{
  UINTN                 PrpEntryNo;
  UINT64                PrpListBase;
  UINTN                 PrpListIndex;
  UINTN                 PrpEntryIndex;
  UINT64                Remainder;
  EFI_PHYSICAL_ADDRESS  PrpListPhyAddr;
  UINTN                 Bytes;
  EFI_STATUS            Status;
 
  //
  // The number of Prp Entry in a memory page.
  //
  PrpEntryNo = EFI_PAGE_SIZE / sizeof (UINT64);
 
  //
  // Calculate total PrpList number.
  //
  *PrpListNo = (UINTN)DivU64x64Remainder ((UINT64)Pages, (UINT64)PrpEntryNo - 1, &Remainder);
  if (*PrpListNo == 0) {
    *PrpListNo = 1;
  } else if ((Remainder != 0) && (Remainder != 1)) {
    *PrpListNo += 1;
  } else if (Remainder == 1) {
    Remainder = PrpEntryNo;
  } else if (Remainder == 0) {
    Remainder = PrpEntryNo - 1;
  }
 
  Status = PciIo->AllocateBuffer (
                    PciIo,
                    AllocateAnyPages,
                    EfiBootServicesData,
                    *PrpListNo,
                    PrpListHost,
                    0
                    );
 
  if (EFI_ERROR (Status)) {
    return NULL;
  }
 
  Bytes  = EFI_PAGES_TO_SIZE (*PrpListNo);
  Status = PciIo->Map (
                    PciIo,
                    EfiPciIoOperationBusMasterCommonBuffer,
                    *PrpListHost,
                    &Bytes,
                    &PrpListPhyAddr,
                    Mapping
                    );
 
  if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (*PrpListNo))) {
    DEBUG ((DEBUG_ERROR, "NvmeCreatePrpList: create PrpList failure!\n"));
    goto EXIT;
  }
 
  //
  // Fill all PRP lists except of last one.
  //
  ZeroMem (*PrpListHost, Bytes);
  for (PrpListIndex = 0; PrpListIndex < *PrpListNo - 1; ++PrpListIndex) {
    PrpListBase = *(UINT64 *)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
 
    for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
      if (PrpEntryIndex != PrpEntryNo - 1) {
        //
        // Fill all PRP entries except of last one.
        //
        *((UINT64 *)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
        PhysicalAddr                                   += EFI_PAGE_SIZE;
      } else {
        //
        // Fill last PRP entries with next PRP List pointer.
        //
        *((UINT64 *)(UINTN)PrpListBase + PrpEntryIndex) = PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE;
      }
    }
  }
 
  //
  // Fill last PRP list.
  //
  PrpListBase = *(UINT64 *)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
  for (PrpEntryIndex = 0; PrpEntryIndex < Remainder; ++PrpEntryIndex) {
    *((UINT64 *)(UINTN)PrpListBase + PrpEntryIndex) = PhysicalAddr;
    PhysicalAddr                                   += EFI_PAGE_SIZE;
  }
 
  return (VOID *)(UINTN)PrpListPhyAddr;
 
EXIT:
  PciIo->FreeBuffer (PciIo, *PrpListNo, *PrpListHost);
  return NULL;
}
 
EFI_STATUS
AbortAsyncPassThruTasks (
  IN NVME_CONTROLLER_PRIVATE_DATA  *Private
  )
{
  EFI_PCI_IO_PROTOCOL       *PciIo;
  LIST_ENTRY                *Link;
  LIST_ENTRY                *NextLink;
  NVME_BLKIO2_SUBTASK       *Subtask;
  NVME_BLKIO2_REQUEST       *BlkIo2Request;
  NVME_PASS_THRU_ASYNC_REQ  *AsyncRequest;
  EFI_BLOCK_IO2_TOKEN       *Token;
  EFI_TPL                   OldTpl;
  EFI_STATUS                Status;
 
  PciIo  = Private->PciIo;
  OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
 
  //
  // Cancel the unsubmitted subtasks.
  //
  for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
       !IsNull (&Private->UnsubmittedSubtasks, Link);
       Link = NextLink)
  {
    NextLink      = GetNextNode (&Private->UnsubmittedSubtasks, Link);
    Subtask       = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
    BlkIo2Request = Subtask->BlockIo2Request;
    Token         = BlkIo2Request->Token;
 
    BlkIo2Request->UnsubmittedSubtaskNum--;
    if (Subtask->IsLast) {
      BlkIo2Request->LastSubtaskSubmitted = TRUE;
    }
 
    Token->TransactionStatus = EFI_ABORTED;
 
    RemoveEntryList (Link);
    InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
    gBS->SignalEvent (Subtask->Event);
  }
 
  //
  // Cleanup the resources for the asynchronous PassThru requests.
  //
  for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
       !IsNull (&Private->AsyncPassThruQueue, Link);
       Link = NextLink)
  {
    NextLink     = GetNextNode (&Private->AsyncPassThruQueue, Link);
    AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
 
    if (AsyncRequest->MapData != NULL) {
      PciIo->Unmap (PciIo, AsyncRequest->MapData);
    }
 
    if (AsyncRequest->MapMeta != NULL) {
      PciIo->Unmap (PciIo, AsyncRequest->MapMeta);
    }
 
    if (AsyncRequest->MapPrpList != NULL) {
      PciIo->Unmap (PciIo, AsyncRequest->MapPrpList);
    }
 
    if (AsyncRequest->PrpListHost != NULL) {
      PciIo->FreeBuffer (
               PciIo,
               AsyncRequest->PrpListNo,
               AsyncRequest->PrpListHost
               );
    }
 
    RemoveEntryList (Link);
    gBS->SignalEvent (AsyncRequest->CallerEvent);
    FreePool (AsyncRequest);
  }
 
  if (IsListEmpty (&Private->AsyncPassThruQueue) &&
      IsListEmpty (&Private->UnsubmittedSubtasks))
  {
    Status = EFI_SUCCESS;
  } else {
    Status = EFI_DEVICE_ERROR;
  }
 
  gBS->RestoreTPL (OldTpl);
 
  return Status;
}
 
EFI_STATUS
EFIAPI
NvmExpressPassThru (
  IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL        *This,
  IN     UINT32                                    NamespaceId,
  IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET  *Packet,
  IN     EFI_EVENT                                 Event OPTIONAL
  )
{
  NVME_CONTROLLER_PRIVATE_DATA   *Private;
  EFI_STATUS                     Status;
  EFI_STATUS                     PreviousStatus;
  EFI_PCI_IO_PROTOCOL            *PciIo;
  NVME_SQ                        *Sq;
  volatile NVME_CQ               *Cq;
  UINT16                         QueueId;
  UINT16                         QueueSize;
  UINT32                         Bytes;
  UINT16                         Offset;
  EFI_EVENT                      TimerEvent;
  EFI_PCI_IO_PROTOCOL_OPERATION  Flag;
  EFI_PHYSICAL_ADDRESS           PhyAddr;
  VOID                           *MapData;
  VOID                           *MapMeta;
  VOID                           *MapPrpList;
  UINTN                          MapLength;
  UINT64                         *Prp;
  VOID                           *PrpListHost;
  UINTN                          PrpListNo;
  UINT32                         Attributes;
  UINT32                         IoAlign;
  UINT32                         MaxTransLen;
  UINT32                         Data;
  NVME_PASS_THRU_ASYNC_REQ       *AsyncRequest;
  EFI_TPL                        OldTpl;
 
  //
  // check the data fields in Packet parameter.
  //
  if ((This == NULL) || (Packet == NULL)) {
    return EFI_INVALID_PARAMETER;
  }
 
  if ((Packet->NvmeCmd == NULL) || (Packet->NvmeCompletion == NULL)) {
    return EFI_INVALID_PARAMETER;
  }
 
  if ((Packet->QueueType != NVME_ADMIN_QUEUE) && (Packet->QueueType != NVME_IO_QUEUE)) {
    return EFI_INVALID_PARAMETER;
  }
 
  //
  // 'Attributes' with neither EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL nor
  // EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL set is an illegal
  // configuration.
  //
  Attributes = This->Mode->Attributes;
  if ((Attributes & (EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_PHYSICAL |
                     EFI_NVM_EXPRESS_PASS_THRU_ATTRIBUTES_LOGICAL)) == 0)
  {
    return EFI_INVALID_PARAMETER;
  }
 
  //
  // Buffer alignment check for TransferBuffer & MetadataBuffer.
  //
  IoAlign = This->Mode->IoAlign;
  if ((IoAlign > 0) && (((UINTN)Packet->TransferBuffer & (IoAlign - 1)) != 0)) {
    return EFI_INVALID_PARAMETER;
  }
 
  if ((IoAlign > 0) && (((UINTN)Packet->MetadataBuffer & (IoAlign - 1)) != 0)) {
    return EFI_INVALID_PARAMETER;
  }
 
  Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 
  //
  // Check NamespaceId is valid or not.
  //
  if ((NamespaceId > Private->ControllerData->Nn) &&
      (NamespaceId != (UINT32)-1))
  {
    return EFI_INVALID_PARAMETER;
  }
 
  //
  // Check whether TransferLength exceeds the maximum data transfer size.
  //
  if (Private->ControllerData->Mdts != 0) {
    MaxTransLen = (1 << (Private->ControllerData->Mdts)) *
                  (1 << (Private->Cap.Mpsmin + 12));
    if (Packet->TransferLength > MaxTransLen) {
      Packet->TransferLength = MaxTransLen;
      return EFI_BAD_BUFFER_SIZE;
    }
  }
 
  PciIo       = Private->PciIo;
  MapData     = NULL;
  MapMeta     = NULL;
  MapPrpList  = NULL;
  PrpListHost = NULL;
  PrpListNo   = 0;
  Prp         = NULL;
  TimerEvent  = NULL;
  Status      = EFI_SUCCESS;
  QueueSize   = MIN (NVME_ASYNC_CSQ_SIZE, Private->Cap.Mqes) + 1;
 
  if (Packet->QueueType == NVME_ADMIN_QUEUE) {
    QueueId = 0;
  } else {
    if (Event == NULL) {
      QueueId = 1;
    } else {
      QueueId = 2;
 
      //
      // Submission queue full check.
      //
      if ((Private->SqTdbl[QueueId].Sqt + 1) % QueueSize ==
          Private->AsyncSqHead)
      {
        return EFI_NOT_READY;
      }
    }
  }
 
  Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
  Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
 
  if (Packet->NvmeCmd->Nsid != NamespaceId) {
    return EFI_INVALID_PARAMETER;
  }
 
  ZeroMem (Sq, sizeof (NVME_SQ));
  Sq->Opc  = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
  Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
  Sq->Cid  = Private->Cid[QueueId]++;
  Sq->Nsid = Packet->NvmeCmd->Nsid;
 
  //
  // Currently we only support PRP for data transfer, SGL is NOT supported.
  //
  ASSERT (Sq->Psdt == 0);
  if (Sq->Psdt != 0) {
    DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: doesn't support SGL mechanism\n"));
    return EFI_UNSUPPORTED;
  }
 
  Sq->Prp[0] = (UINT64)(UINTN)Packet->TransferBuffer;
  if ((Packet->QueueType == NVME_ADMIN_QUEUE) &&
      ((Sq->Opc == NVME_ADMIN_CRIOCQ_CMD) || (Sq->Opc == NVME_ADMIN_CRIOSQ_CMD)))
  {
    //
    // Currently, we only use the IO Completion/Submission queues created internally
    // by this driver during controller initialization. Any other IO queues created
    // will not be consumed here. The value is little to accept external IO queue
    // creation requests, so here we will return EFI_UNSUPPORTED for external IO
    // queue creation request.
    //
    if (!Private->CreateIoQueue) {
      DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: Does not support external IO queues creation request.\n"));
      return EFI_UNSUPPORTED;
    }
  } else if ((Sq->Opc & (BIT0 | BIT1)) != 0) {
    //
    // If the NVMe cmd has data in or out, then mapping the user buffer to the PCI controller specific addresses.
    //
    if (((Packet->TransferLength != 0) && (Packet->TransferBuffer == NULL)) ||
        ((Packet->TransferLength == 0) && (Packet->TransferBuffer != NULL)))
    {
      return EFI_INVALID_PARAMETER;
    }
 
    if ((Sq->Opc & BIT0) != 0) {
      Flag = EfiPciIoOperationBusMasterRead;
    } else {
      Flag = EfiPciIoOperationBusMasterWrite;
    }
 
    if ((Packet->TransferLength != 0) && (Packet->TransferBuffer != NULL)) {
      MapLength = Packet->TransferLength;
      Status    = PciIo->Map (
                           PciIo,
                           Flag,
                           Packet->TransferBuffer,
                           &MapLength,
                           &PhyAddr,
                           &MapData
                           );
      if (EFI_ERROR (Status) || (Packet->TransferLength != MapLength)) {
        return EFI_OUT_OF_RESOURCES;
      }
 
      Sq->Prp[0] = PhyAddr;
      Sq->Prp[1] = 0;
    }
 
    if ((Packet->MetadataLength != 0) && (Packet->MetadataBuffer != NULL)) {
      MapLength = Packet->MetadataLength;
      Status    = PciIo->Map (
                           PciIo,
                           Flag,
                           Packet->MetadataBuffer,
                           &MapLength,
                           &PhyAddr,
                           &MapMeta
                           );
      if (EFI_ERROR (Status) || (Packet->MetadataLength != MapLength)) {
        PciIo->Unmap (
                 PciIo,
                 MapData
                 );
 
        return EFI_OUT_OF_RESOURCES;
      }
 
      Sq->Mptr = PhyAddr;
    }
  }
 
  //
  // If the buffer size spans more than two memory pages (page size as defined in CC.Mps),
  // then build a PRP list in the second PRP submission queue entry.
  //
  Offset = ((UINT16)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
  Bytes  = Packet->TransferLength;
 
  if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
    //
    // Create PrpList for remaining data buffer.
    //
    PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
    Prp     = NvmeCreatePrpList (PciIo, PhyAddr, EFI_SIZE_TO_PAGES (Offset + Bytes) - 1, &PrpListHost, &PrpListNo, &MapPrpList);
    if (Prp == NULL) {
      Status = EFI_OUT_OF_RESOURCES;
      goto EXIT;
    }
 
    Sq->Prp[1] = (UINT64)(UINTN)Prp;
  } else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
    Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
  }
 
  if (Packet->NvmeCmd->Flags & CDW2_VALID) {
    Sq->Rsvd2 = (UINT64)Packet->NvmeCmd->Cdw2;
  }
 
  if (Packet->NvmeCmd->Flags & CDW3_VALID) {
    Sq->Rsvd2 |= LShiftU64 ((UINT64)Packet->NvmeCmd->Cdw3, 32);
  }
 
  if (Packet->NvmeCmd->Flags & CDW10_VALID) {
    Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
  }
 
  if (Packet->NvmeCmd->Flags & CDW11_VALID) {
    Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
  }
 
  if (Packet->NvmeCmd->Flags & CDW12_VALID) {
    Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
  }
 
  if (Packet->NvmeCmd->Flags & CDW13_VALID) {
    Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
  }
 
  if (Packet->NvmeCmd->Flags & CDW14_VALID) {
    Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
  }
 
  if (Packet->NvmeCmd->Flags & CDW15_VALID) {
    Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
  }
 
  //
  // Ring the submission queue doorbell.
  //
  if ((Event != NULL) && (QueueId != 0)) {
    Private->SqTdbl[QueueId].Sqt =
      (Private->SqTdbl[QueueId].Sqt + 1) % QueueSize;
  } else {
    Private->SqTdbl[QueueId].Sqt ^= 1;
  }
 
  Data   = ReadUnaligned32 ((UINT32 *)&Private->SqTdbl[QueueId]);
  Status = PciIo->Mem.Write (
                        PciIo,
                        EfiPciIoWidthUint32,
                        NVME_BAR,
                        NVME_SQTDBL_OFFSET (QueueId, Private->Cap.Dstrd),
                        1,
                        &Data
                        );
 
  if (EFI_ERROR (Status)) {
    goto EXIT;
  }
 
  //
  // For non-blocking requests, return directly if the command is placed
  // in the submission queue.
  //
  if ((Event != NULL) && (QueueId != 0)) {
    AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
    if (AsyncRequest == NULL) {
      Status = EFI_DEVICE_ERROR;
      goto EXIT;
    }
 
    AsyncRequest->Signature   = NVME_PASS_THRU_ASYNC_REQ_SIG;
    AsyncRequest->Packet      = Packet;
    AsyncRequest->CommandId   = Sq->Cid;
    AsyncRequest->CallerEvent = Event;
    AsyncRequest->MapData     = MapData;
    AsyncRequest->MapMeta     = MapMeta;
    AsyncRequest->MapPrpList  = MapPrpList;
    AsyncRequest->PrpListNo   = PrpListNo;
    AsyncRequest->PrpListHost = PrpListHost;
 
    OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
    InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
    gBS->RestoreTPL (OldTpl);
 
    return EFI_SUCCESS;
  }
 
  Status = gBS->CreateEvent (
                  EVT_TIMER,
                  TPL_CALLBACK,
                  NULL,
                  NULL,
                  &TimerEvent
                  );
  if (EFI_ERROR (Status)) {
    goto EXIT;
  }
 
  Status = gBS->SetTimer (TimerEvent, TimerRelative, Packet->CommandTimeout);
 
  if (EFI_ERROR (Status)) {
    goto EXIT;
  }
 
  //
  // Wait for completion queue to get filled in.
  //
  Status = EFI_TIMEOUT;
  while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
    if (Cq->Pt != Private->Pt[QueueId]) {
      Status = EFI_SUCCESS;
      break;
    }
  }
 
  //
  // Check the NVMe cmd execution result
  //
  if (Status != EFI_TIMEOUT) {
    if ((Cq->Sct == 0) && (Cq->Sc == 0)) {
      Status = EFI_SUCCESS;
    } else {
      Status = EFI_DEVICE_ERROR;
      //
      // Dump every completion entry status for debugging.
      //
      DEBUG_CODE_BEGIN ();
      NvmeDumpStatus ((NVME_CQ *)Cq);
      DEBUG_CODE_END ();
    }
 
    //
    // Copy the Respose Queue entry for this command to the callers response buffer
    //
    CopyMem (Packet->NvmeCompletion, (VOID *)Cq, sizeof (EFI_NVM_EXPRESS_COMPLETION));
  } else {
    ReportStatusCode ((EFI_ERROR_MAJOR | EFI_ERROR_CODE), (EFI_IO_BUS_SCSI | EFI_IOB_EC_INTERFACE_ERROR));
 
    //
    // Timeout occurs for an NVMe command. Reset the controller to abort the
    // outstanding commands.
    //
    DEBUG ((DEBUG_ERROR, "NvmExpressPassThru: Timeout occurs for an NVMe command.\n"));
 
    //
    // Disable the timer to trigger the process of async transfers temporarily.
    //
    Status = gBS->SetTimer (Private->TimerEvent, TimerCancel, 0);
    if (EFI_ERROR (Status)) {
      goto EXIT;
    }
 
    //
    // Reset the NVMe controller.
    //
    Status = NvmeControllerInit (Private);
    if (!EFI_ERROR (Status)) {
      Status = AbortAsyncPassThruTasks (Private);
      if (!EFI_ERROR (Status)) {
        //
        // Re-enable the timer to trigger the process of async transfers.
        //
        Status = gBS->SetTimer (Private->TimerEvent, TimerPeriodic, NVME_HC_ASYNC_TIMER);
        if (!EFI_ERROR (Status)) {
          //
          // Return EFI_TIMEOUT to indicate a timeout occurs for NVMe PassThru command.
          //
          Status = EFI_TIMEOUT;
        }
      }
    } else {
      Status = EFI_DEVICE_ERROR;
    }
 
    goto EXIT;
  }
 
  if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
    Private->Pt[QueueId] ^= 1;
  }
 
  Data           = ReadUnaligned32 ((UINT32 *)&Private->CqHdbl[QueueId]);
  PreviousStatus = Status;
  Status         = PciIo->Mem.Write (
                                PciIo,
                                EfiPciIoWidthUint32,
                                NVME_BAR,
                                NVME_CQHDBL_OFFSET (QueueId, Private->Cap.Dstrd),
                                1,
                                &Data
                                );
  // The return status of PciIo->Mem.Write should not override
  // previous status if previous status contains error.
  Status = EFI_ERROR (PreviousStatus) ? PreviousStatus : Status;
 
  //
  // For now, the code does not support the non-blocking feature for admin queue.
  // If Event is not NULL for admin queue, signal the caller's event here.
  //
  if (Event != NULL) {
    ASSERT (QueueId == 0);
    gBS->SignalEvent (Event);
  }
 
EXIT:
  if (MapData != NULL) {
    PciIo->Unmap (
             PciIo,
             MapData
             );
  }
 
  if (MapMeta != NULL) {
    PciIo->Unmap (
             PciIo,
             MapMeta
             );
  }
 
  if (MapPrpList != NULL) {
    PciIo->Unmap (
             PciIo,
             MapPrpList
             );
  }
 
  if (Prp != NULL) {
    PciIo->FreeBuffer (PciIo, PrpListNo, PrpListHost);
  }
 
  if (TimerEvent != NULL) {
    gBS->CloseEvent (TimerEvent);
  }
 
  return Status;
}
 
EFI_STATUS
EFIAPI
NvmExpressGetNextNamespace (
  IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *This,
  IN OUT UINT32                              *NamespaceId
  )
{
  NVME_CONTROLLER_PRIVATE_DATA  *Private;
  NVME_ADMIN_NAMESPACE_DATA     *NamespaceData;
  UINT32                        NextNamespaceId;
  EFI_STATUS                    Status;
 
  if ((This == NULL) || (NamespaceId == NULL)) {
    return EFI_INVALID_PARAMETER;
  }
 
  NamespaceData = NULL;
  Status        = EFI_NOT_FOUND;
 
  Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
  //
  // If the NamespaceId input value is 0xFFFFFFFF, then get the first valid namespace ID
  //
  if (*NamespaceId == 0xFFFFFFFF) {
    //
    // Start with the first namespace ID
    //
    NextNamespaceId = 1;
    //
    // Allocate buffer for Identify Namespace data.
    //
    NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
 
    if (NamespaceData == NULL) {
      return EFI_NOT_FOUND;
    }
 
    Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
    if (EFI_ERROR (Status)) {
      goto Done;
    }
 
    *NamespaceId = NextNamespaceId;
  } else {
    if (*NamespaceId > Private->ControllerData->Nn) {
      return EFI_INVALID_PARAMETER;
    }
 
    NextNamespaceId = *NamespaceId + 1;
    if (NextNamespaceId > Private->ControllerData->Nn) {
      return EFI_NOT_FOUND;
    }
 
    //
    // Allocate buffer for Identify Namespace data.
    //
    NamespaceData = (NVME_ADMIN_NAMESPACE_DATA *)AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
    if (NamespaceData == NULL) {
      return EFI_NOT_FOUND;
    }
 
    Status = NvmeIdentifyNamespace (Private, NextNamespaceId, NamespaceData);
    if (EFI_ERROR (Status)) {
      goto Done;
    }
 
    *NamespaceId = NextNamespaceId;
  }
 
Done:
  if (NamespaceData != NULL) {
    FreePool (NamespaceData);
  }
 
  return Status;
}
 
EFI_STATUS
EFIAPI
NvmExpressGetNamespace (
  IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *This,
  IN     EFI_DEVICE_PATH_PROTOCOL            *DevicePath,
  OUT UINT32                                 *NamespaceId
  )
{
  NVME_NAMESPACE_DEVICE_PATH    *Node;
  NVME_CONTROLLER_PRIVATE_DATA  *Private;
 
  if ((This == NULL) || (DevicePath == NULL) || (NamespaceId == NULL)) {
    return EFI_INVALID_PARAMETER;
  }
 
  if (DevicePath->Type != MESSAGING_DEVICE_PATH) {
    return EFI_UNSUPPORTED;
  }
 
  Node    = (NVME_NAMESPACE_DEVICE_PATH *)DevicePath;
  Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 
  if (DevicePath->SubType == MSG_NVME_NAMESPACE_DP) {
    if (DevicePathNodeLength (DevicePath) != sizeof (NVME_NAMESPACE_DEVICE_PATH)) {
      return EFI_NOT_FOUND;
    }
 
    //
    // Check NamespaceId in the device path node is valid or not.
    //
    if ((Node->NamespaceId == 0) ||
        (Node->NamespaceId > Private->ControllerData->Nn))
    {
      return EFI_NOT_FOUND;
    }
 
    *NamespaceId = Node->NamespaceId;
 
    return EFI_SUCCESS;
  } else {
    return EFI_UNSUPPORTED;
  }
}
 
EFI_STATUS
EFIAPI
NvmExpressBuildDevicePath (
  IN     EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL  *This,
  IN     UINT32                              NamespaceId,
  IN OUT EFI_DEVICE_PATH_PROTOCOL            **DevicePath
  )
{
  NVME_NAMESPACE_DEVICE_PATH    *Node;
  NVME_CONTROLLER_PRIVATE_DATA  *Private;
  EFI_STATUS                    Status;
  NVME_ADMIN_NAMESPACE_DATA     *NamespaceData;
 
  //
  // Validate parameters
  //
  if ((This == NULL) || (DevicePath == NULL)) {
    return EFI_INVALID_PARAMETER;
  }
 
  Status  = EFI_SUCCESS;
  Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (This);
 
  //
  // Check NamespaceId is valid or not.
  //
  if ((NamespaceId == 0) ||
      (NamespaceId > Private->ControllerData->Nn))
  {
    return EFI_NOT_FOUND;
  }
 
  Node = (NVME_NAMESPACE_DEVICE_PATH *)AllocateZeroPool (sizeof (NVME_NAMESPACE_DEVICE_PATH));
  if (Node == NULL) {
    return EFI_OUT_OF_RESOURCES;
  }
 
  Node->Header.Type    = MESSAGING_DEVICE_PATH;
  Node->Header.SubType = MSG_NVME_NAMESPACE_DP;
  SetDevicePathNodeLength (&Node->Header, sizeof (NVME_NAMESPACE_DEVICE_PATH));
  Node->NamespaceId = NamespaceId;
 
  //
  // Allocate a buffer for Identify Namespace data.
  //
  NamespaceData = NULL;
  NamespaceData = AllocateZeroPool (sizeof (NVME_ADMIN_NAMESPACE_DATA));
  if (NamespaceData == NULL) {
    Status = EFI_OUT_OF_RESOURCES;
    goto Exit;
  }
 
  //
  // Get UUID from specified Identify Namespace data.
  //
  Status = NvmeIdentifyNamespace (
             Private,
             NamespaceId,
             (VOID *)NamespaceData
             );
 
  if (EFI_ERROR (Status)) {
    goto Exit;
  }
 
  Node->NamespaceUuid = NamespaceData->Eui64;
 
  *DevicePath = (EFI_DEVICE_PATH_PROTOCOL *)Node;
 
Exit:
  if (NamespaceData != NULL) {
    FreePool (NamespaceData);
  }
 
  if (EFI_ERROR (Status)) {
    FreePool (Node);
  }
 
  return Status;
}