From 8b7e55339a0d1b550bc02f7b8aa49717f148556d Mon Sep 17 00:00:00 2001 From: Vishnu Ramadas Date: Mon, 27 Mar 2023 08:37:05 -0500 Subject: [PATCH] dev-amdgpu: Add GART translations to GPUFS checkpoint Earlier, the GART entries were not being checkpointed. Therefore, during checkpoint restore, certain SDMA instances were initialized with incorrect addresses that led to incorrect behavior. This commit checkpoints the GART entries and restores them. Change-Id: I5464a39ed431e482ff7519b89bd5b664fd992ccf Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/69299 Reviewed-by: Matt Sinclair Reviewed-by: Matthew Poremba Maintainer: Matt Sinclair Tested-by: kokoro --- src/dev/amdgpu/amdgpu_device.cc | 2 ++ src/dev/amdgpu/amdgpu_vm.cc | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/dev/amdgpu/amdgpu_device.cc b/src/dev/amdgpu/amdgpu_device.cc index 2b58b200ea..cb180b6dc5 100644 --- a/src/dev/amdgpu/amdgpu_device.cc +++ b/src/dev/amdgpu/amdgpu_device.cc @@ -536,6 +536,7 @@ AMDGPUDevice::serialize(CheckpointOut &cp) const // Serialize the device memory deviceMem.serializeSection(cp, "deviceMem"); + gpuvm.serializeSection(cp, "GPUVM"); } void @@ -597,6 +598,7 @@ AMDGPUDevice::unserialize(CheckpointIn &cp) // Unserialize the device memory deviceMem.unserializeSection(cp, "deviceMem"); + gpuvm.unserializeSection(cp, "GPUVM"); } uint16_t diff --git a/src/dev/amdgpu/amdgpu_vm.cc b/src/dev/amdgpu/amdgpu_vm.cc index 7a30917b21..5a13ac9ba0 100644 --- a/src/dev/amdgpu/amdgpu_vm.cc +++ b/src/dev/amdgpu/amdgpu_vm.cc @@ -186,6 +186,7 @@ AMDGPUVM::serialize(CheckpointOut &cp) const Addr vm0PTBase = vmContext0.ptBase; Addr vm0PTStart = vmContext0.ptStart; Addr vm0PTEnd = vmContext0.ptEnd; + uint64_t gartTableSize; SERIALIZE_SCALAR(vm0PTBase); SERIALIZE_SCALAR(vm0PTStart); SERIALIZE_SCALAR(vm0PTEnd); @@ -213,6 +214,21 @@ AMDGPUVM::serialize(CheckpointOut &cp) const SERIALIZE_ARRAY(ptBase, AMDGPU_VM_COUNT); SERIALIZE_ARRAY(ptStart, AMDGPU_VM_COUNT); SERIALIZE_ARRAY(ptEnd, AMDGPU_VM_COUNT); + + gartTableSize = gartTable.size(); + uint64_t* gartTableKey = new uint64_t[gartTableSize]; + uint64_t* gartTableValue = new uint64_t[gartTableSize]; + SERIALIZE_SCALAR(gartTableSize); + int i = 0; + for (auto it = gartTable.begin(); it != gartTable.end(); ++it) { + gartTableKey[i] = it->first; + gartTableValue[i] = it->second; + i++; + } + SERIALIZE_ARRAY(gartTableKey, gartTableSize); + SERIALIZE_ARRAY(gartTableValue, gartTableSize); + delete[] gartTableKey; + delete[] gartTableValue; } void @@ -222,6 +238,7 @@ AMDGPUVM::unserialize(CheckpointIn &cp) Addr vm0PTBase; Addr vm0PTStart; Addr vm0PTEnd; + uint64_t gartTableSize, *gartTableKey, *gartTableValue; UNSERIALIZE_SCALAR(vm0PTBase); UNSERIALIZE_SCALAR(vm0PTStart); UNSERIALIZE_SCALAR(vm0PTEnd); @@ -252,6 +269,16 @@ AMDGPUVM::unserialize(CheckpointIn &cp) vmContexts[i].ptStart = ptStart[i]; vmContexts[i].ptEnd = ptEnd[i]; } + UNSERIALIZE_SCALAR(gartTableSize); + gartTableKey = new uint64_t[gartTableSize]; + gartTableValue = new uint64_t[gartTableSize]; + UNSERIALIZE_ARRAY(gartTableKey, gartTableSize); + UNSERIALIZE_ARRAY(gartTableValue, gartTableSize); + for (uint64_t i = 0; i < gartTableSize; i++) { + gartTable[gartTableKey[i]] = gartTableValue[i]; + } + delete[] gartTableKey; + delete[] gartTableValue; } void