derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

util,resources,stdlib: Add 'obtain-resource.py' utility to easily obtain resources from the CLI (#317)

Browse Source 
This allows users to obtain resources via the CLI instead of having to
write a python script to do so. It is essentially a nice CLI wrapper for
"gem5.resources.resource.obtain_resource"

## Usage

```sh
> scons build/ALL/gem5.opt -j `nproc`
> ./build/ALL/gem5.opt util/obtain-resource.py --help

usage: obtain-resource.py [-h] [-p PATH] [-q] id

positional arguments:
  id                    The resource id to download.

options:
  -h, --help            show this help message and exit
  -p PATH, --path PATH  The path the resource is to be downloaded to. If not specified, the resource will be downloaded to the default
                        location in the gem5 local cache of resources
  -q, --quiet           Suppress output.
```

E.g.:

```sh
./build/ALL/gem5.opt util/obtain-resource.py arm-hello64-static -p arm-hello
```

Will download the resource with ID `arm-hello64-static` to `arm-hello`
in the CWD.
This commit is contained in:
Bobby R. Bruce
2023-09-14 21:04:30 -07:00
committed by GitHub
parent 46be2d2339 600ea81031
commit 23442727f7
3 changed files with 163 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/python/gem5/resources/downloader.py
View File

@@ -204,6 +204,7 @@ def get_resource(
resource_version: Optional[str] = None,
clients: Optional[List] = None,
gem5_version: Optional[str] = core.gem5Version,
quiet: bool = False,
) -> None:
"""
Obtains a gem5 resource and stored it to a specified location. If the
@@ -236,6 +237,9 @@ def get_resource(
By default, the version of gem5 being used is used. This is used primarily
for testing purposes.
:param quiet: If true, no output will be printed to the console (baring
exceptions). False by default.
:raises Exception: An exception is thrown if a file is already present at
`to_path` but it does not have the correct md5 sum. An exception will also
be thrown is a directory is present at `to_path`
@@ -326,37 +330,41 @@ def get_resource(
)
shutil.copy(file_uri_path, download_dest)
else:
# TODO: Might be nice to have some kind of download status bar here.
# TODO: There might be a case where this should be silenced.
print(
"Resource '{}' was not found locally. Downloading to '{}'...".format(
resource_name, download_dest
# TODO: Might be nice to have some kind of download status bar here..
if not quiet:
print(
f"Resource '{resource_name}' was not found locally. "
f"Downloading to '{download_dest}'..."
)
)
# Get the URL.
url = resource_json["url"]
_download(url=url, download_to=download_dest)
print(f"Finished downloading resource '{resource_name}'.")
if not quiet:
print(f"Finished downloading resource '{resource_name}'.")
if run_unzip:
print(
f"Decompressing resource '{resource_name}' ('{download_dest}')..."
)
if not quiet:
print(
f"Decompressing resource '{resource_name}' "
f"('{download_dest}')..."
)
unzip_to = download_dest[: -len(zip_extension)]
with gzip.open(download_dest, "rb") as f:
with open(unzip_to, "wb") as o:
shutil.copyfileobj(f, o)
os.remove(download_dest)
download_dest = unzip_to
print(f"Finished decompressing resource '{resource_name}'.")
if not quiet:
print(f"Finished decompressing resource '{resource_name}'.")
if run_tar_extract:
print(
f"Unpacking the the resource '{resource_name}' "
f"('{download_dest}')"
)
if not quiet:
print(
f"Unpacking the the resource '{resource_name}' "
f"('{download_dest}')"
)
unpack_to = download_dest[: -len(tar_extension)]
with tarfile.open(download_dest) as f:

81
src/python/gem5/resources/resource.py
View File

@@ -621,6 +621,8 @@ def obtain_resource(
resource_version: Optional[str] = None,
clients: Optional[List] = None,
gem5_version=core.gem5Version,
to_path: Optional[str] = None,
quiet: bool = False,
) -> AbstractResource:
"""
This function primarily serves as a factory for resources. It will return
@@ -633,6 +635,7 @@ def obtain_resource(
resource is to be stored. If this parameter is not set, it will set to
the environment variable `GEM5_RESOURCE_DIR`. If the environment is not
set it will default to `~/.cache/gem5` if available, otherwise the CWD.
**Note**: This argument is ignored if the `to_path` parameter is specified.
:param download_md5_mismatch: If the resource is present, but does not
have the correct md5 value, the resoruce will be deleted and
re-downloaded if this value is True. Otherwise an exception will be
@@ -644,6 +647,11 @@ def obtain_resource(
:param gem5_version: The gem5 version to use to filter incompatible
resource versions. By default set to the current gem5 version. If None,
this filtering is not performed.
:param to_path: The path to which the resource is to be downloaded. If
None, the resource will be downloaded to the resource directory with
the file/directory name equal to the ID of the resource. **Note**: Usage
of this parameter will override the `resource_directory` parameter.
:param quiet: If True, suppress output. False by default.
"""
# Obtain the resource object entry for this resource
@@ -654,47 +662,64 @@ def obtain_resource(
gem5_version=gem5_version,
)
to_path = None
# If the "url" field is specified, the resoruce must be downloaded.
if "url" in resource_json and resource_json["url"]:
# If the `resource_directory` parameter is not set via this function, we
# check the "GEM5_RESOURCE_DIR" environment variable. If this too is not
# set we call `_get_default_resource_dir()` to determine where the
# resource directory is, or should be, located.
if resource_directory == None:
resource_directory = os.getenv(
"GEM5_RESOURCE_DIR", _get_default_resource_dir()
)
# Small checks here to ensure the resource directory is valid.
if os.path.exists(resource_directory):
if not os.path.isdir(resource_directory):
raise Exception(
"gem5 resource directory, "
"'{}', exists but is not a directory".format(
resource_directory
)
# If the `to_path` parameter is set, we use that as the path to which
# the resource is to be downloaded. Otherwise, default to the
# `resource_directory` parameter plus the resource ID.
if not to_path:
# If the `resource_directory` parameter is not set via this
# function, we heck the "GEM5_RESOURCE_DIR" environment variable.
# If this too is not set we call `_get_default_resource_dir()` to
# determine where the resource directory is, or should be, located.
if resource_directory == None:
resource_directory = os.getenv(
"GEM5_RESOURCE_DIR", _get_default_resource_dir()
)
else:
# `exist_ok=True` here as, occasionally, if multiple instance of
# gem5 are started simultaneously, a race condition can exist to
# create the resource directory. Without `exit_ok=True`, threads
# which lose this race will thrown a `FileExistsError` exception.
# `exit_ok=True` ensures no exception is thrown.
os.makedirs(resource_directory, exist_ok=True)
# This is the path to which the resource is to be stored.
to_path = os.path.join(resource_directory, resource_id)
# Small checks here to ensure the resource directory is valid.
if os.path.exists(resource_directory):
if not os.path.isdir(resource_directory):
raise Exception(
"gem5 resource directory, "
"'{}', exists but is not a directory".format(
resource_directory
)
)
# This is the path to which the resource is to be stored.
to_path = os.path.join(resource_directory, resource_id)
assert to_path
# Here we ensure the directory in which the resource is to be stored
# is created.
#
# `exist_ok=True` here as, occasionally, if multiple instance of gem5
# are started simultaneously, a race condition can exist to create the
# resource directory. Without `exit_ok=True`, threads which lose this
# race will thrown a `FileExistsError` exception. `exit_ok=True`
# ensures no exception is thrown.
try:
Path(to_path).parent.mkdir(parents=True, exist_ok=True)
except Exception as e:
fatal(
f"Recursive creation of the directory "
f"'{Path(to_path).parent.absolute}' failed. \n"
f"Perhaps the path specified, '{to_path}', is incorrect?\n"
f"Failed with Exception:\n{e}"
)
# Download the resource if it does not already exist.
get_resource(
resource_name=resource_id,
to_path=os.path.join(resource_directory, resource_id),
to_path=to_path,
download_md5_mismatch=download_md5_mismatch,
resource_version=resource_version,
clients=clients,
gem5_version=gem5_version,
quiet=quiet,
)
# Obtain the type from the JSON. From this we will determine what subclass

87
util/obtain-resource.py Normal file
View File

@@ -0,0 +1,87 @@
# Copyright (c) 2023 The Regents of the University of California
# All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
Obtain a resource from gem5 resource.
Usage
-----
```sh
scons build/ALL/gem5.opt -j$(nproc)
build/ALL/gem5.opt util/obtain-resource.py <resource_id> [-p <path>] [-q]
# Example:
# `build/ALL/gem5.opt util/obtain-resource.py arm-hello64-static -p arm-hello`
# This will download the resource with id `arm-hello64-static` to the
# "arm-hello" in the CWD.
```
"""
if __name__ == "__m5_main__":
from gem5.resources.resource import obtain_resource
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"id",
type=str,
help="The resource id to download.",
)
parser.add_argument(
"-p",
"--path",
type=str,
required=False,
help="The path the resource is to be downloaded to. If not specified, "
"the resource will be downloaded to the default location in the "
"gem5 local cache of resources",
)
parser.add_argument(
"-q",
"--quiet",
action="store_true",
default=False,
help="Suppress output.",
)
args = parser.parse_args()
resource = obtain_resource(
resource_id=args.id,
quiet=args.quiet,
to_path=args.path,
)
if not args.quiet:
print(f"Resource at: '" + str(resource.get_local_path()) + "'")
exit(0)
print("Error: This script is meant to be run with the gem5 binary")
exit(1)