derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

configs, gpu-compute: Add option to specify gfx version

Browse Source 
Currently uses gfx801, gfx803, gfx900 for Carrizo, Fiji,
and Vega respectively

Change-Id: I62758914b6a60f16dd4f2141a23c0a9141a4e1a0
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/42217
Maintainer: Matthew Poremba <matthew.poremba@amd.com>
Maintainer: Matt Sinclair <mattdsinclair@gmail.com>
Reviewed-by: Matt Sinclair <mattdsinclair@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>
This commit is contained in:
Kyle Roarty
2019-06-07 17:37:50 -04:00
committed by Matthew Poremba
parent ad43083bb3
commit eb09361eef
5 changed files with 248 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
configs/example/apu_se.py
View File

@@ -188,6 +188,9 @@ parser.add_argument("--m-type", type='int', default=5,
"the driver on a per-page basis. Valid values are "
"between 0-7")
parser.add_argument("--gfx-version", type="string", default='gfx801',
help="Gfx version for gpu: gfx801, gfx803, gfx900")
Ruby.define_options(parser)
# add TLB options to the parser
@@ -430,6 +433,7 @@ if args.dgpu:
# HSA kernel mode driver
gpu_driver = GPUComputeDriver(filename = "kfd", isdGPU = args.dgpu,
gfxVersion = args.gfx_version,
dGPUPoolID = 1, m_type = args.m_type)
# Creating the GPU kernel launching components: that is the HSA
@@ -667,8 +671,15 @@ root = Root(system=system, full_system=False)
# Create the /sys/devices filesystem for the simulator so that the HSA Runtime
# knows what type of GPU hardware we are simulating
if args.dgpu:
hsaTopology.createFijiTopology(args)
assert (args.gfx_version in ['gfx803', 'gfx900']),\
"Incorrect gfx version for dGPU"
if args.gfx_version == 'gfx803':
hsaTopology.createFijiTopology(args)
elif args.gfx_version == 'gfx900':
hsaTopology.createVegaTopology(args)
else:
assert (args.gfx_version in ['gfx801']),\
"Incorrect gfx version for APU"
hsaTopology.createCarrizoTopology(args)
m5.ticks.setGlobalFrequency('1THz')

177
configs/example/hsaTopology.py
View File

@@ -49,6 +49,183 @@ def remake_dir(path):
rmtree(path)
makedirs(path)
# This fakes out a dGPU setup so the runtime operates correctly. The spoofed
# system has a single dGPU and a single socket CPU. Note that more complex
# topologies (multi-GPU, multi-socket CPUs) need to have a different setup
# here or the runtime won't be able to issue Memcpies from one node to another.
#
# TODO: There is way too much hardcoded here. It doesn't effect anything in
# our current ROCm stack (1.6), but it is highly possible that it will in the
# future. We might need to scrub through this and extract the appropriate
# fields from the simulator in the future.
def createVegaTopology(options):
topology_dir = joinpath(m5.options.outdir, \
'fs/sys/devices/virtual/kfd/kfd/topology')
remake_dir(topology_dir)
amdgpu_dir = joinpath(m5.options.outdir, \
'fs/sys/module/amdgpu/parameters')
remake_dir(amdgpu_dir)
pci_ids_dir = joinpath(m5.options.outdir, \
'fs/usr/share/hwdata/')
remake_dir(pci_ids_dir)
# Vega reported VM size in GB. Used to reserve an allocation from CPU
# to implement SVM (i.e. GPUVM64 pointers and X86 pointers agree)
file_append((amdgpu_dir, 'vm_size'), 256)
# Ripped from real Vega platform to appease KMT version checks
file_append((topology_dir, 'generation_id'), 2)
# Set up system properties. Regiter as ast-rocm server
sys_prop = 'platform_oem 35498446626881\n' + \
'platform_id 71791775140929\n' + \
'platform_rev 2\n'
file_append((topology_dir, 'system_properties'), sys_prop)
# Populate the topology tree
# Our dGPU system is two nodes. Node 0 is a CPU and Node 1 is a dGPU
node_dir = joinpath(topology_dir, 'nodes/0')
remake_dir(node_dir)
# Register as a CPU
file_append((node_dir, 'gpu_id'), 0)
file_append((node_dir, 'name'), '')
# CPU links. Only thing that matters is we tell the runtime that GPU is
# connected through PCIe to CPU socket 0.
io_links = 1
io_dir = joinpath(node_dir, 'io_links/0')
remake_dir(io_dir)
io_prop = 'type 2\n' + \
'version_major 0\n' + \
'version_minor 0\n' + \
'node_from 0\n' + \
'node_to 1\n' + \
'weight 20\n' + \
'min_latency 0\n' + \
'max_latency 0\n' + \
'min_bandwidth 0\n' + \
'max_bandwidth 0\n' + \
'recommended_transfer_size 0\n' + \
'flags 13\n'
file_append((io_dir, 'properties'), io_prop)
# Populate CPU node properties
node_prop = 'cpu_cores_count %s\n' % options.num_cpus + \
'simd_count 0\n' + \
'mem_banks_count 1\n' + \
'caches_count 0\n' + \
'io_links_count %s\n' % io_links + \
'cpu_core_id_base 0\n' + \
'simd_id_base 0\n' + \
'max_waves_per_simd 0\n' + \
'lds_size_in_kb 0\n' + \
'gds_size_in_kb 0\n' + \
'wave_front_size 64\n' + \
'array_count 0\n' + \
'simd_arrays_per_engine 0\n' + \
'cu_per_simd_array 0\n' + \
'simd_per_cu 0\n' + \
'max_slots_scratch_cu 0\n' + \
'vendor_id 0\n' + \
'device_id 0\n' + \
'location_id 0\n' + \
'drm_render_minor 0\n' + \
'max_engine_clk_ccompute 3400\n'
file_append((node_dir, 'properties'), node_prop)
# CPU memory reporting
mem_dir = joinpath(node_dir, 'mem_banks/0')
remake_dir(mem_dir)
mem_prop = 'heap_type %s\n' % HsaHeaptype.HSA_HEAPTYPE_SYSTEM.value + \
'size_in_bytes 33704329216\n' + \
'flags 0\n' + \
'width 72\n' + \
'mem_clk_max 2400\n'
file_append((mem_dir, 'properties'), mem_prop)
# Build the GPU node
node_dir = joinpath(topology_dir, 'nodes/1')
remake_dir(node_dir)
# Register as a Vega
file_append((node_dir, 'gpu_id'), 22124)
file_append((node_dir, 'name'), 'Vega\n')
# 96 in real Vega
# Random comment for comparison purposes
caches = 0
# GPU links. Only thing that matters is we tell the runtime that GPU is
# connected through PCIe to CPU socket 0.
io_links = 1
io_dir = joinpath(node_dir, 'io_links/0')
remake_dir(io_dir)
io_prop = 'type 2\n' + \
'version_major 0\n' + \
'version_minor 0\n' + \
'node_from 1\n' + \
'node_to 0\n' + \
'weight 20\n' + \
'min_latency 0\n' + \
'max_latency 0\n' + \
'min_bandwidth 0\n' + \
'max_bandwidth 0\n' + \
'recommended_transfer_size 0\n' + \
'flags 1\n'
file_append((io_dir, 'properties'), io_prop)
# Populate GPU node properties
cu_scratch = options.simds_per_cu * options.wfs_per_simd
node_prop = 'cpu_cores_count 0\n' + \
'simd_count 256\n' + \
'mem_banks_count 1\n' + \
'caches_count %s\n' % caches + \
'io_links_count %s\n' % io_links + \
'cpu_core_id_base 0\n' + \
'simd_id_base 2147487744\n' + \
'max_waves_per_simd 10\n' + \
'lds_size_in_kb 64\n' + \
'gds_size_in_kb 0\n' + \
'wave_front_size 64\n' + \
'array_count 4\n' + \
'simd_arrays_per_engine 1\n' + \
'cu_per_simd_array 16\n' + \
'simd_per_cu 4\n' + \
'max_slots_scratch_cu %s\n' % cu_scratch + \
'vendor_id 4098\n' + \
'device_id 26720\n' + \
'location_id 1024\n' + \
'drm_render_minor 128\n' + \
'hive_id 0\n' + \
'num_sdma_engines 2\n' + \
'num_sdma_xgmi_engines 0\n' + \
'max_engine_clk_fcompute 1500\n' + \
'local_mem_size 17163091968\n' + \
'fw_version 421\n' + \
'capability 238208\n' + \
'debug_prop 32768\n' + \
'sdma_fw_version 430\n' + \
'max_engine_clk_ccompute 3400\n'
file_append((node_dir, 'properties'), node_prop)
# Fiji HBM reporting
# TODO: Extract size, clk, and width from sim paramters
mem_dir = joinpath(node_dir, 'mem_banks/0')
remake_dir(mem_dir)
mem_prop = 'heap_type %s\n' % heap_type.value + \
'size_in_bytes 17163091968\n' + \
'flags 0\n' + \
'width 2048\n' + \
'mem_clk_max 945\n'
file_append((mem_dir, 'properties'), mem_prop)
# This fakes out a dGPU setup so the runtime correctly operations. The spoofed
# system has a single dGPU and a single socket CPU. Note that more complex
# topologies (multi-GPU, multi-socket CPUs) need to have a different setup

7
src/gpu-compute/GPU.py
View File

@@ -50,6 +50,12 @@ class PrefetchType(Enum): vals = [
'PF_END',
]
class GfxVersion(ScopedEnum): vals = [
'gfx801',
'gfx803',
'gfx900',
]
class PoolManager(SimObject):
type = 'PoolManager'
abstract = True
@@ -237,6 +243,7 @@ class GPUComputeDriver(HSADriver):
type = 'GPUComputeDriver'
cxx_header = 'gpu-compute/gpu_compute_driver.hh'
isdGPU = Param.Bool(False, 'Driver is for a dGPU')
gfxVersion = Param.GfxVersion('gfx801', 'ISA of gpu to model')
dGPUPoolID = Param.Int(False, 'Pool ID for dGPU.')
# Default Mtype for caches
#-- 1 1 1 C_RW_S (Cached-ReadWrite-Shared)

59
src/gpu-compute/gpu_compute_driver.cc
View File

@@ -47,7 +47,8 @@
#include "sim/syscall_emul_buf.hh"
GPUComputeDriver::GPUComputeDriver(const Params &p)
: HSADriver(p), isdGPU(p.isdGPU)
: HSADriver(p), isdGPU(p.isdGPU), gfxVersion(p.gfxVersion),
dGPUPoolID(p.dGPUPoolID)
{
device->attachDriver(this);
DPRINTF(GPUDriver, "Constructing KFD: device\n");
@@ -175,10 +176,34 @@ GPUComputeDriver::ioctl(ThreadContext *tc, unsigned req, Addr ioc_buf)
gpuVmApeLimit(args->process_apertures[i].gpuvm_base);
// NOTE: Must match ID populated by hsaTopology.py
if (isdGPU)
args->process_apertures[i].gpu_id = 50156;
else
args->process_apertures[i].gpu_id = 2765;
//
// https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver/
// blob/6a986c0943e9acd8c4c0cf2a9d510ff42167b43f/include/uapi/
// linux/kfd_ioctl.h#L564
//
// The gpu_id is a device identifier used by the driver for
// ioctls that allocate arguments. Each device has an unique
// id composed out of a non-zero base and an offset.
if (isdGPU) {
switch (gfxVersion) {
case GfxVersion::gfx803:
args->process_apertures[i].gpu_id = 50156;
break;
case GfxVersion::gfx900:
args->process_apertures[i].gpu_id = 22124;
break;
default:
fatal("Invalid gfx version for dGPU\n");
}
} else {
switch (gfxVersion) {
case GfxVersion::gfx801:
args->process_apertures[i].gpu_id = 2765;
break;
default:
fatal("Invalid gfx version for APU\n");
}
}
DPRINTF(GPUDriver, "GPUVM base for node[%i] = %#x\n", i,
args->process_apertures[i].gpuvm_base);
@@ -610,10 +635,26 @@ GPUComputeDriver::ioctl(ThreadContext *tc, unsigned req, Addr ioc_buf)
ape_args->gpuvm_limit = gpuVmApeLimit(ape_args->gpuvm_base);
// NOTE: Must match ID populated by hsaTopology.py
if (isdGPU)
ape_args->gpu_id = 50156;
else
ape_args->gpu_id = 2765;
if (isdGPU) {
switch (gfxVersion) {
case GfxVersion::gfx803:
ape_args->gpu_id = 50156;
break;
case GfxVersion::gfx900:
ape_args->gpu_id = 22124;
break;
default:
fatal("Invalid gfx version for dGPU\n");
}
} else {
switch (gfxVersion) {
case GfxVersion::gfx801:
ape_args->gpu_id = 2765;
break;
default:
fatal("Invalid gfx version for APU\n");
}
}
assert(bits<Addr>(ape_args->scratch_base, 63, 47) != 0x1ffff);
assert(bits<Addr>(ape_args->scratch_base, 63, 47) != 0);

2
src/gpu-compute/gpu_compute_driver.hh
View File

@@ -44,6 +44,7 @@
#include "base/addr_range_map.hh"
#include "dev/hsa/hsa_driver.hh"
#include "enums/GfxVersion.hh"
#include "mem/request.hh"
struct GPUComputeDriverParams;
@@ -68,6 +69,7 @@ class GPUComputeDriver final : public HSADriver
private:
bool isdGPU;
GfxVersion gfxVersion;
int dGPUPoolID;
/**