diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index 5bca67f677..3260c97a4b 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -204,6 +204,7 @@ def get_resource( resource_version: Optional[str] = None, clients: Optional[List] = None, gem5_version: Optional[str] = core.gem5Version, + quiet: bool = False, ) -> None: """ Obtains a gem5 resource and stored it to a specified location. If the @@ -236,6 +237,9 @@ def get_resource( By default, the version of gem5 being used is used. This is used primarily for testing purposes. + :param quiet: If true, no output will be printed to the console (baring + exceptions). False by default. + :raises Exception: An exception is thrown if a file is already present at `to_path` but it does not have the correct md5 sum. An exception will also be thrown is a directory is present at `to_path` @@ -326,37 +330,41 @@ def get_resource( ) shutil.copy(file_uri_path, download_dest) else: - # TODO: Might be nice to have some kind of download status bar here. - # TODO: There might be a case where this should be silenced. - print( - "Resource '{}' was not found locally. Downloading to '{}'...".format( - resource_name, download_dest + # TODO: Might be nice to have some kind of download status bar here.. + if not quiet: + print( + f"Resource '{resource_name}' was not found locally. " + f"Downloading to '{download_dest}'..." ) - ) # Get the URL. url = resource_json["url"] _download(url=url, download_to=download_dest) - print(f"Finished downloading resource '{resource_name}'.") + if not quiet: + print(f"Finished downloading resource '{resource_name}'.") if run_unzip: - print( - f"Decompressing resource '{resource_name}' ('{download_dest}')..." - ) + if not quiet: + print( + f"Decompressing resource '{resource_name}' " + f"('{download_dest}')..." + ) unzip_to = download_dest[: -len(zip_extension)] with gzip.open(download_dest, "rb") as f: with open(unzip_to, "wb") as o: shutil.copyfileobj(f, o) os.remove(download_dest) download_dest = unzip_to - print(f"Finished decompressing resource '{resource_name}'.") + if not quiet: + print(f"Finished decompressing resource '{resource_name}'.") if run_tar_extract: - print( - f"Unpacking the the resource '{resource_name}' " - f"('{download_dest}')" - ) + if not quiet: + print( + f"Unpacking the the resource '{resource_name}' " + f"('{download_dest}')" + ) unpack_to = download_dest[: -len(tar_extension)] with tarfile.open(download_dest) as f: diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 9e6c79ab01..aafb3b70b0 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -621,6 +621,8 @@ def obtain_resource( resource_version: Optional[str] = None, clients: Optional[List] = None, gem5_version=core.gem5Version, + to_path: Optional[str] = None, + quiet: bool = False, ) -> AbstractResource: """ This function primarily serves as a factory for resources. It will return @@ -633,6 +635,7 @@ def obtain_resource( resource is to be stored. If this parameter is not set, it will set to the environment variable `GEM5_RESOURCE_DIR`. If the environment is not set it will default to `~/.cache/gem5` if available, otherwise the CWD. + **Note**: This argument is ignored if the `to_path` parameter is specified. :param download_md5_mismatch: If the resource is present, but does not have the correct md5 value, the resoruce will be deleted and re-downloaded if this value is True. Otherwise an exception will be @@ -644,6 +647,11 @@ def obtain_resource( :param gem5_version: The gem5 version to use to filter incompatible resource versions. By default set to the current gem5 version. If None, this filtering is not performed. + :param to_path: The path to which the resource is to be downloaded. If + None, the resource will be downloaded to the resource directory with + the file/directory name equal to the ID of the resource. **Note**: Usage + of this parameter will override the `resource_directory` parameter. + :param quiet: If True, suppress output. False by default. """ # Obtain the resource object entry for this resource @@ -654,47 +662,64 @@ def obtain_resource( gem5_version=gem5_version, ) - to_path = None # If the "url" field is specified, the resoruce must be downloaded. if "url" in resource_json and resource_json["url"]: - # If the `resource_directory` parameter is not set via this function, we - # check the "GEM5_RESOURCE_DIR" environment variable. If this too is not - # set we call `_get_default_resource_dir()` to determine where the - # resource directory is, or should be, located. - if resource_directory == None: - resource_directory = os.getenv( - "GEM5_RESOURCE_DIR", _get_default_resource_dir() - ) - - # Small checks here to ensure the resource directory is valid. - if os.path.exists(resource_directory): - if not os.path.isdir(resource_directory): - raise Exception( - "gem5 resource directory, " - "'{}', exists but is not a directory".format( - resource_directory - ) + # If the `to_path` parameter is set, we use that as the path to which + # the resource is to be downloaded. Otherwise, default to the + # `resource_directory` parameter plus the resource ID. + if not to_path: + # If the `resource_directory` parameter is not set via this + # function, we heck the "GEM5_RESOURCE_DIR" environment variable. + # If this too is not set we call `_get_default_resource_dir()` to + # determine where the resource directory is, or should be, located. + if resource_directory == None: + resource_directory = os.getenv( + "GEM5_RESOURCE_DIR", _get_default_resource_dir() ) - else: - # `exist_ok=True` here as, occasionally, if multiple instance of - # gem5 are started simultaneously, a race condition can exist to - # create the resource directory. Without `exit_ok=True`, threads - # which lose this race will thrown a `FileExistsError` exception. - # `exit_ok=True` ensures no exception is thrown. - os.makedirs(resource_directory, exist_ok=True) - # This is the path to which the resource is to be stored. - to_path = os.path.join(resource_directory, resource_id) + # Small checks here to ensure the resource directory is valid. + if os.path.exists(resource_directory): + if not os.path.isdir(resource_directory): + raise Exception( + "gem5 resource directory, " + "'{}', exists but is not a directory".format( + resource_directory + ) + ) + + # This is the path to which the resource is to be stored. + to_path = os.path.join(resource_directory, resource_id) + + assert to_path + + # Here we ensure the directory in which the resource is to be stored + # is created. + # + # `exist_ok=True` here as, occasionally, if multiple instance of gem5 + # are started simultaneously, a race condition can exist to create the + # resource directory. Without `exit_ok=True`, threads which lose this + # race will thrown a `FileExistsError` exception. `exit_ok=True` + # ensures no exception is thrown. + try: + Path(to_path).parent.mkdir(parents=True, exist_ok=True) + except Exception as e: + fatal( + f"Recursive creation of the directory " + f"'{Path(to_path).parent.absolute}' failed. \n" + f"Perhaps the path specified, '{to_path}', is incorrect?\n" + f"Failed with Exception:\n{e}" + ) # Download the resource if it does not already exist. get_resource( resource_name=resource_id, - to_path=os.path.join(resource_directory, resource_id), + to_path=to_path, download_md5_mismatch=download_md5_mismatch, resource_version=resource_version, clients=clients, gem5_version=gem5_version, + quiet=quiet, ) # Obtain the type from the JSON. From this we will determine what subclass diff --git a/util/obtain-resource.py b/util/obtain-resource.py new file mode 100644 index 0000000000..de6a7b90e2 --- /dev/null +++ b/util/obtain-resource.py @@ -0,0 +1,87 @@ +# Copyright (c) 2023 The Regents of the University of California +# All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Obtain a resource from gem5 resource. + +Usage +----- + +```sh +scons build/ALL/gem5.opt -j$(nproc) +build/ALL/gem5.opt util/obtain-resource.py [-p ] [-q] +# Example: +# `build/ALL/gem5.opt util/obtain-resource.py arm-hello64-static -p arm-hello` +# This will download the resource with id `arm-hello64-static` to the +# "arm-hello" in the CWD. +``` +""" + +if __name__ == "__m5_main__": + from gem5.resources.resource import obtain_resource + import argparse + + parser = argparse.ArgumentParser() + + parser.add_argument( + "id", + type=str, + help="The resource id to download.", + ) + + parser.add_argument( + "-p", + "--path", + type=str, + required=False, + help="The path the resource is to be downloaded to. If not specified, " + "the resource will be downloaded to the default location in the " + "gem5 local cache of resources", + ) + + parser.add_argument( + "-q", + "--quiet", + action="store_true", + default=False, + help="Suppress output.", + ) + + args = parser.parse_args() + + resource = obtain_resource( + resource_id=args.id, + quiet=args.quiet, + to_path=args.path, + ) + + if not args.quiet: + print(f"Resource at: '" + str(resource.get_local_path()) + "'") + + exit(0) + +print("Error: This script is meant to be run with the gem5 binary") +exit(1)