derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev-amdgpu: Implement GPU clock MMIOs

Browse Source 
The ROCr runtime uses a combination of HSA signal timestamps and
hardware MMIOs to calculate profiling times. At the beginning of an
application a timestamp is read from the GPU using MMIOs. The clock
MMIOs reside in the GFX MMIO region, so a new AMDGPUGfx class is added
to handle these MMIOs.

The timestamp value is expected to be in nanoseconds, so we simply use
the gem5 tick converted to ns.

Change-Id: I7d1cba40d5042a7f7a81fd4d132402dc11b71bd4
This commit is contained in:
Matthew Poremba
2023-10-06 13:02:31 -05:00
parent 6a4b2bb096
commit 75a7f30dfb
5 changed files with 157 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/dev/amdgpu/SConscript
View File

@@ -39,6 +39,7 @@ SimObject('AMDGPU.py', sim_objects=['AMDGPUDevice', 'AMDGPUInterruptHandler',
tags='x86 isa')
Source('amdgpu_device.cc', tags='x86 isa')
Source('amdgpu_gfx.cc', tags='x86 isa')
Source('amdgpu_nbio.cc', tags='x86 isa')
Source('amdgpu_vm.cc', tags='x86 isa')
Source('interrupt_handler.cc', tags='x86 isa')

6
src/dev/amdgpu/amdgpu_device.cc
View File

@@ -379,6 +379,9 @@ AMDGPUDevice::readMMIO(PacketPtr pkt, Addr offset)
case GRBM_BASE:
gpuvm.readMMIO(pkt, aperture_offset >> GRBM_OFFSET_SHIFT);
break;
case GFX_BASE:
gfx.readMMIO(pkt, aperture_offset);
break;
case MMHUB_BASE:
gpuvm.readMMIO(pkt, aperture_offset >> MMHUB_OFFSET_SHIFT);
break;
@@ -507,6 +510,9 @@ AMDGPUDevice::writeMMIO(PacketPtr pkt, Addr offset)
case NBIO_BASE:
nbio.writeMMIO(pkt, aperture_offset);
break;
case GFX_BASE:
gfx.writeMMIO(pkt, aperture_offset);
break;
default:
DPRINTF(AMDGPUDevice, "Unknown MMIO aperture for %#x\n", offset);
break;

2
src/dev/amdgpu/amdgpu_device.hh
View File

@@ -36,6 +36,7 @@
#include "base/bitunion.hh"
#include "dev/amdgpu/amdgpu_defines.hh"
#include "dev/amdgpu/amdgpu_gfx.hh"
#include "dev/amdgpu/amdgpu_nbio.hh"
#include "dev/amdgpu/amdgpu_vm.hh"
#include "dev/amdgpu/memory_manager.hh"
@@ -109,6 +110,7 @@ class AMDGPUDevice : public PciDevice
* Blocks of the GPU
*/
AMDGPUNbio nbio;
AMDGPUGfx gfx;
AMDGPUMemoryManager *gpuMemMgr;
AMDGPUInterruptHandler *deviceIH;
AMDGPUVM gpuvm;

73
src/dev/amdgpu/amdgpu_gfx.cc Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright (c) 2023 Advanced Micro Devices, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "dev/amdgpu/amdgpu_gfx.hh"
#include "mem/packet_access.hh"
#include "sim/core.hh"
namespace gem5
{
void
AMDGPUGfx::readMMIO(PacketPtr pkt, Addr offset)
{
switch (offset) {
case AMDGPU_MM_RLC_GPU_CLOCK_COUNT_LSB:
pkt->setLE<uint32_t>(captured_clock_count);
break;
case AMDGPU_MM_RLC_GPU_CLOCK_COUNT_MSB:
pkt->setLE<uint32_t>(captured_clock_count >> 32);
break;
default:
break;
}
}
void
AMDGPUGfx::writeMMIO(PacketPtr pkt, Addr offset)
{
switch (offset) {
case AMDGPU_MM_RLC_CAPTURE_GPU_CLOCK_COUNT:
// Use gem5 Ticks in nanoseconds are the counter. The first capture
// is expected to return zero.
if (captured_clock_count == 1) {
captured_clock_count = 0;
} else {
captured_clock_count = curTick() / sim_clock::as_int::ns;
}
break;
default:
break;
}
}
} // namespace gem5

75
src/dev/amdgpu/amdgpu_gfx.hh Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) 2023 Advanced Micro Devices, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __DEV_AMDGPU_AMDGPU_GFX_HH__
#define __DEV_AMDGPU_AMDGPU_GFX_HH__
#include "base/types.hh"
#include "mem/packet.hh"
/**
* MMIO offsets for GFX. This class handles MMIO reads/writes to the GFX_BASE
* aperture which are generally read/written by the gfx driver source here:
*
* drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
* https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver/blob/master/
* drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
*
* The MMIO addresses in the file are dword addresses. Here they are converted
* to byte addresses so gem5 does not need to shift the values.
*/
// Registers used to read GPU clock count used in profiling
#define AMDGPU_MM_RLC_GPU_CLOCK_COUNT_LSB 0x13090
#define AMDGPU_MM_RLC_GPU_CLOCK_COUNT_MSB 0x13094
#define AMDGPU_MM_RLC_CAPTURE_GPU_CLOCK_COUNT 0x13098
namespace gem5
{
class AMDGPUGfx
{
public:
AMDGPUGfx() { }
void readMMIO(PacketPtr pkt, Addr offset);
void writeMMIO(PacketPtr pkt, Addr offset);
private:
/*
* GPU clock count at the time capture MMIO is received.
*/
uint64_t captured_clock_count = 1;
};
} // namespace gem5
#endif // __DEV_AMDGPU_AMDGPU_GFX_HH__