derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev-amdgpu: Implement PCIe indirect read/write

Browse Source 
PCIe can read/write to any 32-bit address using the PCI index/index2
registers as an address and then reading/writing the corresponding
data/data2 register.

This commit adds this functionality and removes one magic value being
written to support GPU POST. This feature is disabled for Vega10 which
relies on an MMIO trace for too many values to implement in the MMIO
interface.

Change-Id: Iacfdd1294a7652fc3e60304b57df536d318c847b
This commit is contained in:
Matthew Poremba
2024-02-13 17:38:20 -06:00
parent 047c194780
commit 39153cd234
2 changed files with 39 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/dev/amdgpu/amdgpu_nbio.cc
View File

@@ -53,7 +53,21 @@ AMDGPUNbio::setGPUDevice(AMDGPUDevice *gpu_device)
void
AMDGPUNbio::readMMIO(PacketPtr pkt, Addr offset)
{
// For Vega10 we rely on the golden values in an MMIO trace. Return
// immediately as to not clobber those values.
if (gpuDevice->getGfxVersion() == GfxVersion::gfx900) {
if (offset == AMDGPU_PCIE_DATA || offset == AMDGPU_PCIE_DATA2) {
return;
}
}
switch (offset) {
// PCIE_DATA, PCIE_DATA2, PCIE_INDEX, and PCIE_INDEX2 handle "indirect
// "register reads/writes from the driver. This provides a way to read
// any register by providing a 32-bit address to one of the two INDEX
// registers and then reading the corresponding DATA register. See:
// https://github.com/ROCm/ROCK-Kernel-Driver/blob/roc-6.0.x/drivers/
// gpu/drm/amd/amdgpu/amdgpu_device.c#L459
case AMDGPU_PCIE_DATA:
{
uint32_t value = gpuDevice->getRegVal(pcie_index_reg);
@@ -62,19 +76,20 @@ AMDGPUNbio::readMMIO(PacketPtr pkt, Addr offset)
pkt->setLE<uint32_t>(value);
}
break;
// This is a PCIe status register. At some point during driver init
// the driver checks that interrupts are enabled. This is only
// checked once, so if the MMIO trace does not exactly line up with
// what the driver is doing in gem5, this may still have the first
// bit zero causing driver to fail. Therefore, we always set this
// bit to one as there is no harm to do so.
case AMDGPU_PCIE_DATA2:
{
uint32_t value = pkt->getLE<uint32_t>() | 0x1;
DPRINTF(AMDGPUDevice, "Marking interrupts enabled: %#lx\n", value);
uint32_t value = gpuDevice->getRegVal(pcie_index2_reg);
DPRINTF(AMDGPUDevice, "Read PCIe index2 %lx data2 %x\n",
pcie_index2_reg, value);
pkt->setLE<uint32_t>(value);
}
break;
case AMDGPU_PCIE_INDEX:
pkt->setLE<uint32_t>(pcie_index_reg);
break;
case AMDGPU_PCIE_INDEX2:
pkt->setLE<uint32_t>(pcie_index2_reg);
break;
case AMDGPU_MM_DATA:
pkt->setLE<uint32_t>(gpuDevice->getRegVal(mm_index_reg));
break;
@@ -130,14 +145,24 @@ AMDGPUNbio::writeMMIO(PacketPtr pkt, Addr offset)
DPRINTF(AMDGPUDevice, "MM write to reg %#lx data %#lx\n",
mm_index_reg, pkt->getLE<uint32_t>());
gpuDevice->setRegVal(AMDGPU_MM_DATA, pkt->getLE<uint32_t>());
// PCIE_DATA, PCIE_DATA2, PCIE_INDEX, and PCIE_INDEX2 handle "indirect
// "register reads/writes from the driver. This provides a way to read
// any register by providing a 32-bit address to one of the two INDEX
// registers and then reading the corresponding DATA register. See:
// https://github.com/ROCm/ROCK-Kernel-Driver/blob/roc-6.0.x/drivers/
// gpu/drm/amd/amdgpu/amdgpu_device.c#L459
} else if (offset == AMDGPU_PCIE_INDEX) {
assert(pkt->getSize() == 4);
pcie_index_reg = insertBits(pcie_index_reg, 31, 0,
pkt->getLE<uint32_t>());
pcie_index_reg = pkt->getLE<uint32_t>();
} else if (offset == AMDGPU_PCIE_DATA) {
assert(pkt->getSize() == 4);
gpuDevice->setRegVal(pcie_index_reg, pkt->getLE<uint32_t>());
} else if (offset == AMDGPU_PCIE_INDEX2) {
assert(pkt->getSize() == 4);
pcie_index_reg = insertBits(pcie_index_reg, 63, 32,
pkt->getLE<uint32_t>());
pcie_index2_reg = pkt->getLE<uint32_t>();
} else if (offset == AMDGPU_PCIE_DATA2) {
assert(pkt->getSize() == 4);
gpuDevice->setRegVal(pcie_index2_reg, pkt->getLE<uint32_t>());
} else if (offset == AMDGPU_MP0_SMN_C2PMSG_35) {
// See psp_v3_1_bootloader_load_sos in amdgpu driver code.
if (pkt->getLE<uint32_t>() == 0x10000) {

3
src/dev/amdgpu/amdgpu_nbio.hh
View File

@@ -110,7 +110,8 @@ class AMDGPUNbio
* Driver initialization sequence helper variables.
*/
uint64_t mm_index_reg = 0;
uint64_t pcie_index_reg = 0;
uint32_t pcie_index_reg = 0;
uint32_t pcie_index2_reg = 0;
std::unordered_map<uint32_t, uint32_t> triggered_reads;
/*