45 PrpEntryNo = EFI_PAGE_SIZE /
sizeof (UINT64);
55 if (PrpListNo > NVME_PRP_SIZE) {
58 "%a: The implementation only supports PrpList number up to 4."
59 " But %d are needed here.\n",
66 PrpListHost = (VOID *)(
UINTN)NVME_PRP_BASE (Private);
69 PrpListPhyAddr = (UINT64)(
UINTN)(PrpListHost);
75 for (PrpListIndex = 0; PrpListIndex < PrpListNo - 1; ++PrpListIndex) {
76 PrpListBase = (
UINTN)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
78 for (PrpEntryIndex = 0; PrpEntryIndex < PrpEntryNo; ++PrpEntryIndex) {
79 PrpEntry = (UINT8 *)(
UINTN)(PrpListBase + PrpEntryIndex *
sizeof (UINT64));
80 if (PrpEntryIndex != PrpEntryNo - 1) {
84 CopyMem (PrpEntry, (VOID *)(
UINTN)(&PhysicalAddr),
sizeof (UINT64));
85 PhysicalAddr += EFI_PAGE_SIZE;
90 NewPhyAddr = (PrpListPhyAddr + (PrpListIndex + 1) * EFI_PAGE_SIZE);
91 CopyMem (PrpEntry, (VOID *)(
UINTN)(&NewPhyAddr),
sizeof (UINT64));
99 PrpListBase = (
UINTN)PrpListHost + PrpListIndex * EFI_PAGE_SIZE;
100 for (PrpEntryIndex = 0; PrpEntryIndex < ((Remainder != 0) ? Remainder : PrpEntryNo); ++PrpEntryIndex) {
101 PrpEntry = (UINT8 *)(
UINTN)(PrpListBase + PrpEntryIndex *
sizeof (UINT64));
102 CopyMem (PrpEntry, (VOID *)(
UINTN)(&PhysicalAddr),
sizeof (UINT64));
104 PhysicalAddr += EFI_PAGE_SIZE;
107 return PrpListPhyAddr;
121 if ((Cq->Sct == 0x0) && (Cq->Sc == 0x0)) {
125 DEBUG ((DEBUG_INFO,
"Dump NVMe Completion Entry Status from [0x%x]:\n", (
UINTN)Cq));
128 " SQ Identifier : [0x%x], Phase Tag : [%d], Cmd Identifier : [0x%x]\n",
133 DEBUG ((DEBUG_INFO,
" Status Code Type : [0x%x], Status Code : [0x%x]\n", Cq->Sct, Cq->Sc));
134 DEBUG ((DEBUG_INFO,
" NVMe Cmd Execution Result - "));
140 DEBUG ((DEBUG_INFO,
"Successful Completion\n"));
143 DEBUG ((DEBUG_INFO,
"Invalid Command Opcode\n"));
146 DEBUG ((DEBUG_INFO,
"Invalid Field in Command\n"));
149 DEBUG ((DEBUG_INFO,
"Command ID Conflict\n"));
152 DEBUG ((DEBUG_INFO,
"Data Transfer Error\n"));
155 DEBUG ((DEBUG_INFO,
"Commands Aborted due to Power Loss Notification\n"));
158 DEBUG ((DEBUG_INFO,
"Internal Device Error\n"));
161 DEBUG ((DEBUG_INFO,
"Command Abort Requested\n"));
164 DEBUG ((DEBUG_INFO,
"Command Aborted due to SQ Deletion\n"));
167 DEBUG ((DEBUG_INFO,
"Command Aborted due to Failed Fused Command\n"));
170 DEBUG ((DEBUG_INFO,
"Command Aborted due to Missing Fused Command\n"));
173 DEBUG ((DEBUG_INFO,
"Invalid Namespace or Format\n"));
176 DEBUG ((DEBUG_INFO,
"Command Sequence Error\n"));
179 DEBUG ((DEBUG_INFO,
"Invalid SGL Last Segment Descriptor\n"));
182 DEBUG ((DEBUG_INFO,
"Invalid Number of SGL Descriptors\n"));
185 DEBUG ((DEBUG_INFO,
"Data SGL Length Invalid\n"));
188 DEBUG ((DEBUG_INFO,
"Metadata SGL Length Invalid\n"));
191 DEBUG ((DEBUG_INFO,
"SGL Descriptor Type Invalid\n"));
194 DEBUG ((DEBUG_INFO,
"LBA Out of Range\n"));
197 DEBUG ((DEBUG_INFO,
"Capacity Exceeded\n"));
200 DEBUG ((DEBUG_INFO,
"Namespace Not Ready\n"));
203 DEBUG ((DEBUG_INFO,
"Reservation Conflict\n"));
212 DEBUG ((DEBUG_INFO,
"Completion Queue Invalid\n"));
215 DEBUG ((DEBUG_INFO,
"Invalid Queue Identifier\n"));
218 DEBUG ((DEBUG_INFO,
"Maximum Queue Size Exceeded\n"));
221 DEBUG ((DEBUG_INFO,
"Abort Command Limit Exceeded\n"));
224 DEBUG ((DEBUG_INFO,
"Asynchronous Event Request Limit Exceeded\n"));
227 DEBUG ((DEBUG_INFO,
"Invalid Firmware Slot\n"));
230 DEBUG ((DEBUG_INFO,
"Invalid Firmware Image\n"));
233 DEBUG ((DEBUG_INFO,
"Invalid Interrupt Vector\n"));
236 DEBUG ((DEBUG_INFO,
"Invalid Log Page\n"));
239 DEBUG ((DEBUG_INFO,
"Invalid Format\n"));
242 DEBUG ((DEBUG_INFO,
"Firmware Application Requires Conventional Reset\n"));
245 DEBUG ((DEBUG_INFO,
"Invalid Queue Deletion\n"));
248 DEBUG ((DEBUG_INFO,
"Feature Identifier Not Saveable\n"));
251 DEBUG ((DEBUG_INFO,
"Feature Not Changeable\n"));
254 DEBUG ((DEBUG_INFO,
"Feature Not Namespace Specific\n"));
257 DEBUG ((DEBUG_INFO,
"Firmware Application Requires NVM Subsystem Reset\n"));
260 DEBUG ((DEBUG_INFO,
"Conflicting Attributes\n"));
263 DEBUG ((DEBUG_INFO,
"Invalid Protection Information\n"));
266 DEBUG ((DEBUG_INFO,
"Attempted Write to Read Only Range\n"));
275 DEBUG ((DEBUG_INFO,
"Write Fault\n"));
278 DEBUG ((DEBUG_INFO,
"Unrecovered Read Error\n"));
281 DEBUG ((DEBUG_INFO,
"End-to-end Guard Check Error\n"));
284 DEBUG ((DEBUG_INFO,
"End-to-end Application Tag Check Error\n"));
287 DEBUG ((DEBUG_INFO,
"End-to-end Reference Tag Check Error\n"));
290 DEBUG ((DEBUG_INFO,
"Compare Failure\n"));
293 DEBUG ((DEBUG_INFO,
"Access Denied\n"));
300 DEBUG ((DEBUG_INFO,
"Unknown error\n"));
304 return EFI_DEVICE_ERROR;
341 IN UINT32 NamespaceId,
364 if (Packet ==
NULL) {
367 "%a, Invalid parameter: Packet(%lx)\n",
371 return EFI_INVALID_PARAMETER;
374 if ((Packet->NvmeCmd ==
NULL) || (Packet->NvmeCompletion ==
NULL)) {
377 "%a, Invalid parameter: NvmeCmd (%lx)/NvmeCompletion(%lx)\n",
379 (
UINTN)Packet->NvmeCmd,
380 (
UINTN)Packet->NvmeCompletion
382 return EFI_INVALID_PARAMETER;
385 if ((Packet->QueueType != NVME_ADMIN_QUEUE) && (Packet->QueueType != NVME_IO_QUEUE)) {
388 "%a, Invalid parameter: QueueId(%lx)\n",
390 (
UINTN)Packet->QueueType
392 return EFI_INVALID_PARAMETER;
395 QueueId = Packet->QueueType;
396 Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
397 Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
398 if (QueueId == NVME_ADMIN_QUEUE) {
399 SqSize = NVME_ASQ_SIZE + 1;
400 CqSize = NVME_ACQ_SIZE + 1;
402 SqSize = NVME_CSQ_SIZE + 1;
403 CqSize = NVME_CCQ_SIZE + 1;
406 if (Packet->NvmeCmd->Nsid != NamespaceId) {
409 "%a: Nsid mismatch (%x, %x)\n",
411 Packet->NvmeCmd->Nsid,
414 return EFI_INVALID_PARAMETER;
418 Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
419 Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
420 Sq->Cid = Private->Cid[QueueId]++;
421 Sq->Nsid = Packet->NvmeCmd->Nsid;
426 ASSERT (Sq->Psdt == 0);
428 DEBUG ((DEBUG_ERROR,
"%a: Does not support SGL mechanism.\n", __func__));
429 return EFI_UNSUPPORTED;
432 Sq->Prp[0] = (UINT64)(
UINTN)Packet->TransferBuffer;
441 if ((Sq->Opc & (BIT0 | BIT1)) != 0) {
442 if (((Packet->TransferLength != 0) && (Packet->TransferBuffer ==
NULL)) ||
443 ((Packet->TransferLength == 0) && (Packet->TransferBuffer !=
NULL)))
445 return EFI_INVALID_PARAMETER;
452 if ((Packet->QueueType == NVME_ADMIN_QUEUE) &&
453 ((Sq->Opc == NVME_ADMIN_CRIOCQ_CMD) || (Sq->Opc == NVME_ADMIN_CRIOSQ_CMD)))
455 if ((Packet->TransferBuffer != Private->SqBuffer[NVME_IO_QUEUE]) &&
456 (Packet->TransferBuffer != Private->CqBuffer[NVME_IO_QUEUE]))
460 "%a: Does not support external IO queues creation request.\n",
463 return EFI_UNSUPPORTED;
466 if ((Sq->Opc & BIT0) != 0) {
472 if ((Packet->TransferLength != 0) && (Packet->TransferBuffer !=
NULL)) {
473 MapLength = Packet->TransferLength;
476 Packet->TransferBuffer,
481 if (EFI_ERROR (Status) || (MapLength != Packet->TransferLength)) {
482 Status = EFI_OUT_OF_RESOURCES;
483 DEBUG ((DEBUG_ERROR,
"%a: Fail to map data buffer.\n", __func__));
487 Sq->Prp[0] = PhyAddr;
490 if ((Packet->MetadataLength != 0) && (Packet->MetadataBuffer !=
NULL)) {
491 MapLength = Packet->MetadataLength;
494 Packet->MetadataBuffer,
499 if (EFI_ERROR (Status) || (MapLength != Packet->MetadataLength)) {
500 Status = EFI_OUT_OF_RESOURCES;
501 DEBUG ((DEBUG_ERROR,
"%a: Fail to map meta data buffer.\n", __func__));
514 Offset = ((UINT32)Sq->Prp[0]) & (EFI_PAGE_SIZE - 1);
515 Bytes = Packet->TransferLength;
517 if ((Offset + Bytes) > (EFI_PAGE_SIZE * 2)) {
521 PhyAddr = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
527 if (Sq->Prp[1] == 0) {
528 Status = EFI_OUT_OF_RESOURCES;
529 DEBUG ((DEBUG_ERROR,
"%a: Create PRP list fail, Status - %r\n", __func__, Status));
532 }
else if ((Offset + Bytes) > EFI_PAGE_SIZE) {
533 Sq->Prp[1] = (Sq->Prp[0] + EFI_PAGE_SIZE) & ~(EFI_PAGE_SIZE - 1);
536 if (Packet->NvmeCmd->Flags & CDW10_VALID) {
537 Sq->Payload.Raw.Cdw10 = Packet->NvmeCmd->Cdw10;
540 if (Packet->NvmeCmd->Flags & CDW11_VALID) {
541 Sq->Payload.Raw.Cdw11 = Packet->NvmeCmd->Cdw11;
544 if (Packet->NvmeCmd->Flags & CDW12_VALID) {
545 Sq->Payload.Raw.Cdw12 = Packet->NvmeCmd->Cdw12;
548 if (Packet->NvmeCmd->Flags & CDW13_VALID) {
549 Sq->Payload.Raw.Cdw13 = Packet->NvmeCmd->Cdw13;
552 if (Packet->NvmeCmd->Flags & CDW14_VALID) {
553 Sq->Payload.Raw.Cdw14 = Packet->NvmeCmd->Cdw14;
556 if (Packet->NvmeCmd->Flags & CDW15_VALID) {
557 Sq->Payload.Raw.Cdw15 = Packet->NvmeCmd->Cdw15;
563 Private->SqTdbl[QueueId].Sqt++;
564 if (Private->SqTdbl[QueueId].Sqt == SqSize) {
565 Private->SqTdbl[QueueId].Sqt = 0;
569 Status = NVME_SET_SQTDBL (Private, QueueId, &Data32);
570 if (EFI_ERROR (Status)) {
571 DEBUG ((DEBUG_ERROR,
"%a: NVME_SET_SQTDBL fail, Status - %r\n", __func__, Status));
578 Status = EFI_TIMEOUT;
580 while (Timer < Packet->CommandTimeout) {
581 if (Cq->Pt != Private->Pt[QueueId]) {
587 Timer += NVME_POLL_INTERVAL;
590 if (Status == EFI_TIMEOUT) {
594 DEBUG ((DEBUG_ERROR,
"%a: Timeout occurs for the PassThru command.\n", __func__));
596 if (EFI_ERROR (Status)) {
597 Status = EFI_DEVICE_ERROR;
602 Status = EFI_TIMEOUT;
611 Private->CqHdbl[QueueId].Cqh++;
612 if (Private->CqHdbl[QueueId].Cqh == CqSize) {
613 Private->CqHdbl[QueueId].Cqh = 0;
614 Private->Pt[QueueId] ^= 1;
626 NVME_SET_CQHDBL (Private, QueueId, &Private->CqHdbl[QueueId]);
629 if (MapMeta !=
NULL) {
633 if (MapData !=
NULL) {
667 if ((This ==
NULL) || (DevicePathLength ==
NULL) || (DevicePath ==
NULL)) {
668 return EFI_INVALID_PARAMETER;
671 Private = GET_NVME_PEIM_HC_PRIVATE_DATA_FROM_THIS_NVME_PASSTHRU (This);
673 *DevicePathLength = Private->DevicePathLength;
674 *DevicePath =
AllocateCopyPool (Private->DevicePathLength, Private->DevicePath);
675 if (*DevicePath ==
NULL) {
676 *DevicePathLength = 0;
677 return EFI_OUT_OF_RESOURCES;
723 IN OUT UINT32 *NamespaceId
730 if ((This ==
NULL) || (NamespaceId ==
NULL)) {
731 return EFI_INVALID_PARAMETER;
734 Private = GET_NVME_PEIM_HC_PRIVATE_DATA_FROM_THIS_NVME_PASSTHRU (This);
736 Status = EFI_NOT_FOUND;
741 if (Private->ActiveNamespaceNum == 0) {
742 return EFI_NOT_FOUND;
748 if (*NamespaceId == 0xFFFFFFFF) {
752 *NamespaceId = Private->NamespaceInfo[0].NamespaceId;
755 if (*NamespaceId > Private->ControllerData->Nn) {
756 return EFI_INVALID_PARAMETER;
759 if ((*NamespaceId + 1) > Private->ControllerData->Nn) {
760 return EFI_NOT_FOUND;
763 for (DeviceIndex = 0; DeviceIndex < Private->ActiveNamespaceNum; DeviceIndex++) {
764 if (*NamespaceId == Private->NamespaceInfo[DeviceIndex].NamespaceId) {
765 if ((DeviceIndex + 1) < Private->ActiveNamespaceNum) {
766 *NamespaceId = Private->NamespaceInfo[DeviceIndex + 1].NamespaceId;
813 IN UINT32 NamespaceId,
820 if ((This ==
NULL) || (Packet ==
NULL)) {
821 return EFI_INVALID_PARAMETER;
824 Private = GET_NVME_PEIM_HC_PRIVATE_DATA_FROM_THIS_NVME_PASSTHRU (This);
828 if ((NamespaceId > Private->ControllerData->Nn) &&
829 (NamespaceId != (UINT32)-1))
831 return EFI_INVALID_PARAMETER;
EFI_STATUS IoMmuUnmap(IN VOID *Mapping)
EFI_STATUS IoMmuMap(IN EDKII_IOMMU_OPERATION Operation, IN VOID *HostAddress, IN OUT UINTN *NumberOfBytes, OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, OUT VOID **Mapping)
UINTN EFIAPI MicroSecondDelay(IN UINTN MicroSeconds)
UINT64 EFIAPI DivU64x64Remainder(IN UINT64 Dividend, IN UINT64 Divisor, OUT UINT64 *Remainder OPTIONAL)
UINT32 EFIAPI ReadUnaligned32(IN CONST UINT32 *Buffer)
VOID *EFIAPI CopyMem(OUT VOID *DestinationBuffer, IN CONST VOID *SourceBuffer, IN UINTN Length)
VOID *EFIAPI ZeroMem(OUT VOID *Buffer, IN UINTN Length)
VOID *EFIAPI AllocateCopyPool(IN UINTN AllocationSize, IN CONST VOID *Buffer)
#define DEBUG(Expression)
EFI_STATUS NvmeControllerInit(IN NVME_CONTROLLER_PRIVATE_DATA *Private)
EFI_STATUS EFIAPI NvmePassThruGetDevicePath(IN EDKII_PEI_NVM_EXPRESS_PASS_THRU_PPI *This, OUT UINTN *DevicePathLength, OUT EFI_DEVICE_PATH_PROTOCOL **DevicePath)
EFI_STATUS EFIAPI NvmePassThruGetNextNameSpace(IN EDKII_PEI_NVM_EXPRESS_PASS_THRU_PPI *This, IN OUT UINT32 *NamespaceId)
EFI_STATUS NvmePassThruExecute(IN PEI_NVME_CONTROLLER_PRIVATE_DATA *Private, IN UINT32 NamespaceId, IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet)
EFI_STATUS NvmeCheckCqStatus(IN volatile NVME_CQ *Cq)
UINT64 NvmeCreatePrpList(IN PEI_NVME_CONTROLLER_PRIVATE_DATA *Private, IN EFI_PHYSICAL_ADDRESS PhysicalAddr, IN UINTN Pages)
EFI_STATUS EFIAPI NvmePassThru(IN EDKII_PEI_NVM_EXPRESS_PASS_THRU_PPI *This, IN UINT32 NamespaceId, IN OUT EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet)
@ EdkiiIoMmuOperationBusMasterWrite
@ EdkiiIoMmuOperationBusMasterRead
VOID EFIAPI Exit(IN EFI_STATUS Status)
UINT64 EFI_PHYSICAL_ADDRESS
#define EFI_PAGES_TO_SIZE(Pages)
#define EFI_SIZE_TO_PAGES(Size)