tests: Refactor downloading of pannotia tests (#1653)

With this patch the pannotia tests now:

1. Download the resources to 'gpu-pannotia' in the
'tests/gem5/resources' directory. This is where other test resources are
store.
2. Download thr USA-road-d.NY.gr dataset from Google cloud bucket in a
decompressed state.
2. Avoid re-download the resources if they are already present on the
host machine.
This commit is contained in:
Bobby R. Bruce
2024-10-10 10:17:32 -07:00
committed by GitHub
parent 6195b33960
commit 65ba2dcae5

View File

@@ -27,11 +27,14 @@
import gzip
import os.path
import shutil
from pathlib import Path
from urllib.request import urlretrieve
from testlib import *
resource_path = joinpath(absdirpath(__file__), "..", "gpu-pannotia-resources")
resource_path = joinpath(
absdirpath(__file__), "..", "resources", "gpu-pannotia"
)
binary_path = joinpath(resource_path, "pannotia-bins")
dataset_path = joinpath(resource_path, "pannotia-datasets")
@@ -57,7 +60,7 @@ dataset_links = {
"G3_circuit.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/G3_circuit.graph",
"ecology1.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/ecology1.graph",
"coAuthorsDBLP.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/pagerank/coAuthorsDBLP.graph",
"USA-road-d.NY.gr.gz": "http://www.diag.uniroma1.it/challenge9/data/USA-road-d/USA-road-d.NY.gr.gz",
"USA-road-d.NY.gr": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/USA-road-d/USA-road-d.NY.gr",
}
@@ -66,15 +69,14 @@ if not os.path.isdir(resource_path):
os.makedirs(dataset_path)
for name in binary_links.keys():
if Path(f"{binary_path}/{name}").exists():
continue
urlretrieve(binary_links[name], f"{binary_path}/{name}")
for name in dataset_links.keys():
if Path(f"{dataset_path}/{name}").exists():
continue
urlretrieve(dataset_links[name], f"{dataset_path}/{name}")
with gzip.open(f"{dataset_path}/USA-road-d.NY.gr.gz", "rb") as f_in:
with open(f"{dataset_path}/USA-road-d.NY.gr", "wb") as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(f"{dataset_path}/USA-road-d.NY.gr.gz")
if len(os.listdir(binary_path)) < len(binary_links):
testlib.log.test_log.warn(
"One or more binaries for the Pannotia GPU tests are missing! Try deleting gpu-pannotia-resources and rerunning."