gem5-dev@gem5.org

The gem5 Developer List

View all threads

[S] Change in gem5/gem5[develop]: dev-amdgpu: Add GART translations to GPUFS checkpoint

VR
VISHNU RAMADAS (Gerrit)
Mon, Mar 27, 2023 5:38 PM

VISHNU RAMADAS has uploaded this change for review. (
https://gem5-review.googlesource.com/c/public/gem5/+/69299?usp=email )

Change subject: dev-amdgpu: Add GART translations to GPUFS checkpoint
......................................................................

dev-amdgpu: Add GART translations to GPUFS checkpoint

Earlier, the GART entries were not being checkpointed. Therefore, during
checkpoint restore, certain SDMA instances were initialized with
incorrect addresses that led to incorrect behavior. This commit
checkpoints the GART entries and restores them.

Change-Id: I5464a39ed431e482ff7519b89bd5b664fd992ccf

M src/dev/amdgpu/amdgpu_device.cc
M src/dev/amdgpu/amdgpu_vm.cc
2 files changed, 29 insertions(+), 0 deletions(-)

diff --git a/src/dev/amdgpu/amdgpu_device.cc
b/src/dev/amdgpu/amdgpu_device.cc
index 2b58b20..cb180b6 100644
--- a/src/dev/amdgpu/amdgpu_device.cc
+++ b/src/dev/amdgpu/amdgpu_device.cc
@@ -536,6 +536,7 @@

  // Serialize the device memory
  deviceMem.serializeSection(cp, "deviceMem");
  • gpuvm.serializeSection(cp, "GPUVM");
    }

void
@@ -597,6 +598,7 @@

  // Unserialize the device memory
  deviceMem.unserializeSection(cp, "deviceMem");
  • gpuvm.unserializeSection(cp, "GPUVM");
    }

uint16_t
diff --git a/src/dev/amdgpu/amdgpu_vm.cc b/src/dev/amdgpu/amdgpu_vm.cc
index 7a30917..5a13ac9 100644
--- a/src/dev/amdgpu/amdgpu_vm.cc
+++ b/src/dev/amdgpu/amdgpu_vm.cc
@@ -186,6 +186,7 @@
Addr vm0PTBase = vmContext0.ptBase;
Addr vm0PTStart = vmContext0.ptStart;
Addr vm0PTEnd = vmContext0.ptEnd;

  • uint64_t gartTableSize;
    SERIALIZE_SCALAR(vm0PTBase);
    SERIALIZE_SCALAR(vm0PTStart);
    SERIALIZE_SCALAR(vm0PTEnd);
    @@ -213,6 +214,21 @@
    SERIALIZE_ARRAY(ptBase, AMDGPU_VM_COUNT);
    SERIALIZE_ARRAY(ptStart, AMDGPU_VM_COUNT);
    SERIALIZE_ARRAY(ptEnd, AMDGPU_VM_COUNT);
  • gartTableSize = gartTable.size();
  • uint64_t* gartTableKey = new uint64_t[gartTableSize];
  • uint64_t* gartTableValue = new uint64_t[gartTableSize];
  • SERIALIZE_SCALAR(gartTableSize);
  • int i = 0;
  • for (auto it = gartTable.begin(); it != gartTable.end(); ++it) {
  •    gartTableKey[i] = it->first;
    
  •    gartTableValue[i] = it->second;
    
  •    i++;
    
  • }
  • SERIALIZE_ARRAY(gartTableKey, gartTableSize);
  • SERIALIZE_ARRAY(gartTableValue, gartTableSize);
  • delete[] gartTableKey;
  • delete[] gartTableValue;
    }

void
@@ -222,6 +238,7 @@
Addr vm0PTBase;
Addr vm0PTStart;
Addr vm0PTEnd;

  • uint64_t gartTableSize, *gartTableKey, *gartTableValue;
    UNSERIALIZE_SCALAR(vm0PTBase);
    UNSERIALIZE_SCALAR(vm0PTStart);
    UNSERIALIZE_SCALAR(vm0PTEnd);
    @@ -252,6 +269,16 @@
    vmContexts[i].ptStart = ptStart[i];
    vmContexts[i].ptEnd = ptEnd[i];
    }
  • UNSERIALIZE_SCALAR(gartTableSize);
  • gartTableKey = new uint64_t[gartTableSize];
  • gartTableValue = new uint64_t[gartTableSize];
  • UNSERIALIZE_ARRAY(gartTableKey, gartTableSize);
  • UNSERIALIZE_ARRAY(gartTableValue, gartTableSize);
  • for (uint64_t i = 0; i < gartTableSize; i++) {
  •    gartTable[gartTableKey[i]] = gartTableValue[i];
    
  • }
  • delete[] gartTableKey;
  • delete[] gartTableValue;
    }

void

--
To view, visit
https://gem5-review.googlesource.com/c/public/gem5/+/69299?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gem5-review.googlesource.com/settings

Gerrit-Project: public/gem5
Gerrit-Branch: develop
Gerrit-Change-Id: I5464a39ed431e482ff7519b89bd5b664fd992ccf
Gerrit-Change-Number: 69299
Gerrit-PatchSet: 1
Gerrit-Owner: VISHNU RAMADAS vramadas@wisc.edu
Gerrit-MessageType: newchange

VISHNU RAMADAS has uploaded this change for review. ( https://gem5-review.googlesource.com/c/public/gem5/+/69299?usp=email ) Change subject: dev-amdgpu: Add GART translations to GPUFS checkpoint ...................................................................... dev-amdgpu: Add GART translations to GPUFS checkpoint Earlier, the GART entries were not being checkpointed. Therefore, during checkpoint restore, certain SDMA instances were initialized with incorrect addresses that led to incorrect behavior. This commit checkpoints the GART entries and restores them. Change-Id: I5464a39ed431e482ff7519b89bd5b664fd992ccf --- M src/dev/amdgpu/amdgpu_device.cc M src/dev/amdgpu/amdgpu_vm.cc 2 files changed, 29 insertions(+), 0 deletions(-) diff --git a/src/dev/amdgpu/amdgpu_device.cc b/src/dev/amdgpu/amdgpu_device.cc index 2b58b20..cb180b6 100644 --- a/src/dev/amdgpu/amdgpu_device.cc +++ b/src/dev/amdgpu/amdgpu_device.cc @@ -536,6 +536,7 @@ // Serialize the device memory deviceMem.serializeSection(cp, "deviceMem"); + gpuvm.serializeSection(cp, "GPUVM"); } void @@ -597,6 +598,7 @@ // Unserialize the device memory deviceMem.unserializeSection(cp, "deviceMem"); + gpuvm.unserializeSection(cp, "GPUVM"); } uint16_t diff --git a/src/dev/amdgpu/amdgpu_vm.cc b/src/dev/amdgpu/amdgpu_vm.cc index 7a30917..5a13ac9 100644 --- a/src/dev/amdgpu/amdgpu_vm.cc +++ b/src/dev/amdgpu/amdgpu_vm.cc @@ -186,6 +186,7 @@ Addr vm0PTBase = vmContext0.ptBase; Addr vm0PTStart = vmContext0.ptStart; Addr vm0PTEnd = vmContext0.ptEnd; + uint64_t gartTableSize; SERIALIZE_SCALAR(vm0PTBase); SERIALIZE_SCALAR(vm0PTStart); SERIALIZE_SCALAR(vm0PTEnd); @@ -213,6 +214,21 @@ SERIALIZE_ARRAY(ptBase, AMDGPU_VM_COUNT); SERIALIZE_ARRAY(ptStart, AMDGPU_VM_COUNT); SERIALIZE_ARRAY(ptEnd, AMDGPU_VM_COUNT); + + gartTableSize = gartTable.size(); + uint64_t* gartTableKey = new uint64_t[gartTableSize]; + uint64_t* gartTableValue = new uint64_t[gartTableSize]; + SERIALIZE_SCALAR(gartTableSize); + int i = 0; + for (auto it = gartTable.begin(); it != gartTable.end(); ++it) { + gartTableKey[i] = it->first; + gartTableValue[i] = it->second; + i++; + } + SERIALIZE_ARRAY(gartTableKey, gartTableSize); + SERIALIZE_ARRAY(gartTableValue, gartTableSize); + delete[] gartTableKey; + delete[] gartTableValue; } void @@ -222,6 +238,7 @@ Addr vm0PTBase; Addr vm0PTStart; Addr vm0PTEnd; + uint64_t gartTableSize, *gartTableKey, *gartTableValue; UNSERIALIZE_SCALAR(vm0PTBase); UNSERIALIZE_SCALAR(vm0PTStart); UNSERIALIZE_SCALAR(vm0PTEnd); @@ -252,6 +269,16 @@ vmContexts[i].ptStart = ptStart[i]; vmContexts[i].ptEnd = ptEnd[i]; } + UNSERIALIZE_SCALAR(gartTableSize); + gartTableKey = new uint64_t[gartTableSize]; + gartTableValue = new uint64_t[gartTableSize]; + UNSERIALIZE_ARRAY(gartTableKey, gartTableSize); + UNSERIALIZE_ARRAY(gartTableValue, gartTableSize); + for (uint64_t i = 0; i < gartTableSize; i++) { + gartTable[gartTableKey[i]] = gartTableValue[i]; + } + delete[] gartTableKey; + delete[] gartTableValue; } void -- To view, visit https://gem5-review.googlesource.com/c/public/gem5/+/69299?usp=email To unsubscribe, or for help writing mail filters, visit https://gem5-review.googlesource.com/settings Gerrit-Project: public/gem5 Gerrit-Branch: develop Gerrit-Change-Id: I5464a39ed431e482ff7519b89bd5b664fd992ccf Gerrit-Change-Number: 69299 Gerrit-PatchSet: 1 Gerrit-Owner: VISHNU RAMADAS <vramadas@wisc.edu> Gerrit-MessageType: newchange