tests: Refactor downloading of pannotia tests (#1653)
With this patch the pannotia tests now: 1. Download the resources to 'gpu-pannotia' in the 'tests/gem5/resources' directory. This is where other test resources are store. 2. Download thr USA-road-d.NY.gr dataset from Google cloud bucket in a decompressed state. 2. Avoid re-download the resources if they are already present on the host machine.
This commit is contained in:
@@ -27,11 +27,14 @@
|
||||
import gzip
|
||||
import os.path
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from urllib.request import urlretrieve
|
||||
|
||||
from testlib import *
|
||||
|
||||
resource_path = joinpath(absdirpath(__file__), "..", "gpu-pannotia-resources")
|
||||
resource_path = joinpath(
|
||||
absdirpath(__file__), "..", "resources", "gpu-pannotia"
|
||||
)
|
||||
binary_path = joinpath(resource_path, "pannotia-bins")
|
||||
dataset_path = joinpath(resource_path, "pannotia-datasets")
|
||||
|
||||
@@ -57,7 +60,7 @@ dataset_links = {
|
||||
"G3_circuit.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/G3_circuit.graph",
|
||||
"ecology1.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/mis/ecology1.graph",
|
||||
"coAuthorsDBLP.graph": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/pagerank/coAuthorsDBLP.graph",
|
||||
"USA-road-d.NY.gr.gz": "http://www.diag.uniroma1.it/challenge9/data/USA-road-d/USA-road-d.NY.gr.gz",
|
||||
"USA-road-d.NY.gr": "https://storage.googleapis.com/dist.gem5.org/dist/develop/datasets/pannotia/USA-road-d/USA-road-d.NY.gr",
|
||||
}
|
||||
|
||||
|
||||
@@ -66,15 +69,14 @@ if not os.path.isdir(resource_path):
|
||||
os.makedirs(dataset_path)
|
||||
|
||||
for name in binary_links.keys():
|
||||
if Path(f"{binary_path}/{name}").exists():
|
||||
continue
|
||||
urlretrieve(binary_links[name], f"{binary_path}/{name}")
|
||||
for name in dataset_links.keys():
|
||||
if Path(f"{dataset_path}/{name}").exists():
|
||||
continue
|
||||
urlretrieve(dataset_links[name], f"{dataset_path}/{name}")
|
||||
|
||||
with gzip.open(f"{dataset_path}/USA-road-d.NY.gr.gz", "rb") as f_in:
|
||||
with open(f"{dataset_path}/USA-road-d.NY.gr", "wb") as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
os.remove(f"{dataset_path}/USA-road-d.NY.gr.gz")
|
||||
|
||||
if len(os.listdir(binary_path)) < len(binary_links):
|
||||
testlib.log.test_log.warn(
|
||||
"One or more binaries for the Pannotia GPU tests are missing! Try deleting gpu-pannotia-resources and rerunning."
|
||||
|
||||
Reference in New Issue
Block a user