derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tests: Refactor downloading of pannotia tests (#1653)

Browse Source 
With this patch the pannotia tests now:

1. Download the resources to 'gpu-pannotia' in the
'tests/gem5/resources' directory. This is where other test resources are
store.
2. Download thr USA-road-d.NY.gr dataset from Google cloud bucket in a
decompressed state.
2. Avoid re-download the resources if they are already present on the
host machine.
This commit is contained in:
Bobby R. Bruce
2024-10-10 10:17:32 -07:00
committed by GitHub
parent 6195b33960
commit 65ba2dcae5
1 changed files with 9 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tests/gem5/gpu/test_gpu_pannotia.py
View File

@@ -27,11 +27,14 @@
import gzip import gzip
import os.path import os.path
import shutil import shutil
from pathlib import Path
from urllib.request import urlretrieve from urllib.request import urlretrieve
from testlib import * from testlib import *
resource_path = joinpath(absdirpath(__file__), "..", "gpu-pannotia-resources") resource_path = joinpath(
absdirpath(__file__), "..", "resources", "gpu-pannotia"
)
binary_path = joinpath(resource_path, "pannotia-bins") binary_path = joinpath(resource_path, "pannotia-bins")
dataset_path = joinpath(resource_path, "pannotia-datasets") dataset_path = joinpath(resource_path, "pannotia-datasets")
@@ -57,7 +60,7 @@ dataset_links = {
"G3_circuit.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/G3_circuit.graph", "G3_circuit.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/G3_circuit.graph",
"ecology1.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/ecology1.graph", "ecology1.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/ecology1.graph",
"coAuthorsDBLP.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/pagerank/coAuthorsDBLP.graph", "coAuthorsDBLP.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/pagerank/coAuthorsDBLP.graph",
"USA-road-d.NY.gr.gz": "http://www.diag.uniroma1.it/challenge9/data/USA-road-d/USA-road-d.NY.gr.gz", "USA-road-d.NY.gr": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/USA-road-d/USA-road-d.NY.gr",
} }
@@ -66,15 +69,14 @@ if not os.path.isdir(resource_path):
os.makedirs(dataset_path) os.makedirs(dataset_path)
for name in binary_links.keys(): for name in binary_links.keys():
if Path(f"{binary_path}/{name}").exists():
continue
urlretrieve(binary_links[name], f"{binary_path}/{name}") urlretrieve(binary_links[name], f"{binary_path}/{name}")
for name in dataset_links.keys(): for name in dataset_links.keys():
if Path(f"{dataset_path}/{name}").exists():
continue
urlretrieve(dataset_links[name], f"{dataset_path}/{name}") urlretrieve(dataset_links[name], f"{dataset_path}/{name}")
with gzip.open(f"{dataset_path}/USA-road-d.NY.gr.gz", "rb") as f_in:
with open(f"{dataset_path}/USA-road-d.NY.gr", "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(f"{dataset_path}/USA-road-d.NY.gr.gz")
if len(os.listdir(binary_path)) < len(binary_links): if len(os.listdir(binary_path)) < len(binary_links):
testlib.log.test_log.warn( testlib.log.test_log.warn(
"One or more binaries for the Pannotia GPU tests are missing! Try deleting gpu-pannotia-resources and rerunning." "One or more binaries for the Pannotia GPU tests are missing! Try deleting gpu-pannotia-resources and rerunning."