diff --git a/src/python/SConscript b/src/python/SConscript index e261bfa7f2..f98b5700c7 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -264,12 +264,24 @@ PySource('gem5.prebuilt.riscvmatched', PySource('gem5.prebuilt.riscvmatched', 'gem5/prebuilt/riscvmatched/riscvmatched_core.py') PySource('gem5.resources', 'gem5/resources/__init__.py') +PySource('gem5.resources', 'gem5/resources/client.py') PySource('gem5.resources', 'gem5/resources/downloader.py') PySource('gem5.resources', 'gem5/resources/md5_utils.py') PySource('gem5.resources', 'gem5/resources/resource.py') PySource('gem5.resources', 'gem5/resources/workload.py') PySource('gem5.resources', 'gem5/resources/looppoint.py') PySource('gem5.resources', 'gem5/resources/elfie.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/__init__.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/jsonclient.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/atlasclient.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/client_wrapper.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/abstract_client.py') +PySource('gem5', 'gem5_default_config.py') PySource('gem5.utils', 'gem5/utils/__init__.py') PySource('gem5.utils', 'gem5/utils/filelock.py') PySource('gem5.utils', 'gem5/utils/override.py') diff --git a/src/python/gem5/resources/client.py b/src/python/gem5/resources/client.py new file mode 100644 index 0000000000..bd473eb038 --- /dev/null +++ b/src/python/gem5/resources/client.py @@ -0,0 +1,84 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pathlib import Path +import os +from typing import Optional, Dict, List +from .client_api.client_wrapper import ClientWrapper +from gem5.gem5_default_config import config +from m5.util import inform + + +def getFileContent(file_path: Path) -> Dict: + """ + Get the content of the file at the given path + :param file_path: The path of the file + :return: The content of the file + """ + if file_path.exists(): + with open(file_path, "r") as file: + return json.load(file) + else: + raise Exception(f"File not found at {file_path}") + + +clientwrapper = None + + +def get_resource_json_obj( + resource_id, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, +) -> Dict: + """ + Get the resource json object from the clients wrapper + :param resource_id: The resource id + :param resource_version: The resource version + :param clients: The list of clients to query + """ + global clientwrapper + if clientwrapper is None: + # First check if the config file path is provided in the environment variable + if "GEM5_CONFIG" in os.environ: + config_file_path = Path(os.environ["GEM5_CONFIG"]) + gem5_config = getFileContent(config_file_path) + inform("Using config file specified by $GEM5_CONFIG") + inform(f"Using config file at {os.environ['GEM5_CONFIG']}") + # If not, check if the config file is present in the current directory + elif (Path().cwd().resolve() / "gem5-config.json").exists(): + config_file_path = Path().resolve() / "gem5-config.json" + gem5_config = getFileContent(config_file_path) + inform(f"Using config file at {config_file_path}") + # If not, use the default config in the build directory + else: + gem5_config = config + inform("Using default config") + clientwrapper = ClientWrapper(gem5_config) + + return clientwrapper.get_resource_json_obj_from_client( + resource_id, resource_version, clients + ) diff --git a/src/python/gem5/resources/client_api/__init__.py b/src/python/gem5/resources/client_api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/python/gem5/resources/client_api/abstract_client.py b/src/python/gem5/resources/client_api/abstract_client.py new file mode 100644 index 0000000000..74a513fc56 --- /dev/null +++ b/src/python/gem5/resources/client_api/abstract_client.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from abc import ABC, abstractmethod +from typing import Any, Dict, List +import urllib.parse + + +class AbstractClient(ABC): + def verify_status_code(self, status_code: int) -> None: + """ + Verifies that the status code is 200. + :param status_code: The status code to verify. + """ + if status_code == 200: + return + if status_code == 429: + raise Exception("Panic: Too many requests") + if status_code == 401: + raise Exception("Panic: Unauthorized") + if status_code == 404: + raise Exception("Panic: Not found") + if status_code == 400: + raise Exception("Panic: Bad request") + if status_code == 500: + raise Exception("Panic: Internal server error") + + raise Exception(f"Panic: Unknown status code {status_code}") + + def _url_validator(self, url: str) -> bool: + """ + Validates the provided URL. + :param url: The URL to be validated. + :return: True if the URL is valid, False otherwise. + """ + try: + result = urllib.parse.urlparse(url) + return all([result.scheme, result.netloc, result.path]) + except: + return False + + @abstractmethod + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. + :return: A list of all the Resources with the given ID. + """ + raise NotImplementedError diff --git a/src/python/gem5/resources/client_api/atlasclient.py b/src/python/gem5/resources/client_api/atlasclient.py new file mode 100644 index 0000000000..4a6e5cf691 --- /dev/null +++ b/src/python/gem5/resources/client_api/atlasclient.py @@ -0,0 +1,91 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from urllib import request, parse +from urllib.error import HTTPError, URLError +from typing import Optional, Dict, Union, Type, Tuple, List, Any +import json +from .abstract_client import AbstractClient + + +class AtlasClient(AbstractClient): + def __init__(self, config: Dict[str, str]): + """ + Initializes a connection to a MongoDB Atlas database. + :param uri: The URI for connecting to the MongoDB server. + :param db: The name of the database to connect to. + :param collection: The name of the collection within the database. + """ + self.apiKey = config["apiKey"] + self.url = config["url"] + self.collection = config["collection"] + self.database = config["database"] + self.dataSource = config["dataSource"] + self.authUrl = config["authUrl"] + + def get_token(self): + data = {"key": self.apiKey} + data = json.dumps(data).encode("utf-8") + + req = request.Request( + self.authUrl, + data=data, + headers={"Content-Type": "application/json"}, + ) + try: + response = request.urlopen(req) + except HTTPError as e: + self.verify_status_code(e.status) + return None + result = json.loads(response.read().decode("utf-8")) + token = result["access_token"] + return token + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + url = f"{self.url}/action/find" + data = { + "dataSource": self.dataSource, + "collection": self.collection, + "database": self.database, + "filter": {"id": resource_id}, + } + data = json.dumps(data).encode("utf-8") + + headers = { + "Authorization": f"Bearer {self.get_token()}", + "Content-Type": "application/json", + } + + req = request.Request(url, data=data, headers=headers) + try: + response = request.urlopen(req) + except HTTPError as e: + self.verify_status_code(e.status) + return None + result = json.loads(response.read().decode("utf-8")) + resources = result["documents"] + + return resources diff --git a/src/python/gem5/resources/client_api/client_wrapper.py b/src/python/gem5/resources/client_api/client_wrapper.py new file mode 100644 index 0000000000..74ee831c1f --- /dev/null +++ b/src/python/gem5/resources/client_api/client_wrapper.py @@ -0,0 +1,228 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .jsonclient import JSONClient +from .atlasclient import AtlasClient +from _m5 import core +from typing import Optional, Dict, List +from distutils.version import StrictVersion +import itertools +from m5.util import warn + + +class ClientWrapper: + def __init__(self, config): + self.clients = self.create_clients(config) + + def create_clients( + self, + config: Dict, + ) -> Dict: + """ + This function creates respective client object for each source in the + config file according to the type of source. + Params: config: config file containing the source information + Returns: clients: dictionary of clients for each source + """ + clients = {} + for client in config["sources"]: + client_source = config["sources"][client] + try: + if client_source["isMongo"]: + clients[client] = AtlasClient(client_source) + else: + clients[client] = JSONClient(client_source["url"]) + except Exception as e: + warn(f"Error creating client {client}: {str(e)}") + return clients + + def get_all_resources_by_id( + self, + resource_id: str, + clients: Optional[List[str]] = None, + ) -> List[Dict]: + """ + This function returns all the resources with the given id from all the + sources. + :param resource_id: The id of the resource to search for. + :param clients: A list of clients to search through. If None, all + clients are searched. + :return: A list of resources as Python dictionaries. + """ + resources = [] + if not clients: + clients = list(self.clients.keys()) + for client in clients: + if client not in self.clients: + raise Exception(f"Client: {client} does not exist") + try: + resources.extend( + self.clients[client].get_resources_by_id(resource_id) + ) + except Exception as e: + warn(f"Error getting resources from client {client}: {str(e)}") + # check if no 2 resources have the same id and version + for res1, res2 in itertools.combinations(resources, 2): + if res1["resource_version"] == res2["resource_version"]: + raise Exception( + f"Resource {resource_id} has multiple resources with " + f"the same version: {res1['resource_version']}" + ) + return resources + + def get_resource_json_obj_from_client( + self, + resource_id: str, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, + ) -> Dict: + """ + This function returns the resource object from the client with the + given id and version. + :param resource_id: The id of the resource to search for. + :param resource_version: The version of the resource to search for. + :param clients: A list of clients to search through. If None, all + clients are searched. + :return: The resource object as a Python dictionary if found. + If not found, exception is thrown. + """ + # getting all the resources with the given id from the dictionary + resources = self.get_all_resources_by_id(resource_id, clients) + # if no resource with the given id is found, return None + if len(resources) == 0: + raise Exception(f"Resource with ID '{resource_id}' not found.") + + resource_to_return = None + + if resource_version: + resource_to_return = self._search_version_in_resources( + resources, resource_id, resource_version + ) + + else: + compatible_resources = ( + self._get_resources_compatible_with_gem5_version(resources) + ) + if len(compatible_resources) == 0: + resource_to_return = self._sort_resources(resources)[0] + else: + resource_to_return = self._sort_resources( + compatible_resources + )[0] + + self._check_resource_version_compatibility(resource_to_return) + + return resource_to_return + + def _search_version_in_resources( + self, resources: List, resource_id: str, resource_version: str + ) -> Dict: + """ + Searches for the resource with the given version. If the resource is + not found, an exception is thrown. + :param resources: A list of resources to search through. + :param resource_version: The version of the resource to search for. + :return: The resource object as a Python dictionary if found. + If not found, None is returned. + """ + return_resource = next( + iter( + [ + resource + for resource in resources + if resource["resource_version"] == resource_version + ] + ), + None, + ) + if not return_resource: + raise Exception( + f"Resource {resource_id} with version '{resource_version}'" + " not found.\nResource versions can be found at: " + "https://resources.gem5.org/" + f"resources/{resource_id}/versions" + ) + return return_resource + + def _get_resources_compatible_with_gem5_version( + self, resources: List, gem5_version: str = core.gem5Version + ) -> List: + """ + Returns a list of compatible resources with the current gem5 version. + :param resources: A list of resources to filter. + :return: A list of compatible resources as Python dictionaries. + If no compatible resources are found, the original list of resources + is returned. + """ + compatible_resources = [ + resource + for resource in resources + if gem5_version in resource["gem5_versions"] + ] + return compatible_resources + + def _sort_resources(self, resources: List) -> List: + """ + Sorts the resources by ID. + If the IDs are the same, the resources are sorted by version. + :param resources: A list of resources to sort. + :return: A list of sorted resources. + """ + return sorted( + resources, + key=lambda resource: ( + resource["id"].lower(), + StrictVersion(resource["resource_version"]), + ), + reverse=True, + ) + + def _check_resource_version_compatibility( + self, resource: dict, gem5_version: Optional[str] = core.gem5Version + ) -> bool: + """ + Checks if the resource is compatible with the gem5 version. + Prints a warning if the resource is not compatible. + :param resource: The resource to check. + :optional param gem5_version: The gem5 version to check + compatibility with. + :return: True if the resource is compatible, False otherwise. + """ + if not resource: + return False + if gem5_version not in resource["gem5_versions"]: + warn( + f"Resource {resource['id']} with version " + f"{resource['resource_version']} is not known to be compatible" + f" with gem5 version {gem5_version}. " + "This may cause problems with your simulation. " + "This resource's compatibility " + "with different gem5 versions can be found here: " + "https://resources.gem5.org" + f"/resources/{resource['id']}/versions" + ) + return False + return True diff --git a/src/python/gem5/resources/client_api/jsonclient.py b/src/python/gem5/resources/client_api/jsonclient.py new file mode 100644 index 0000000000..225126e2a8 --- /dev/null +++ b/src/python/gem5/resources/client_api/jsonclient.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pathlib import Path +from urllib import request +from typing import Optional, Dict, Union, Type, Tuple, List, Any +from .abstract_client import AbstractClient +from urllib.error import URLError +from m5.util import warn + + +class JSONClient(AbstractClient): + def __init__(self, path: str): + """ + Initializes a JSON client. + :param path: The path to the Resource, either URL or local. + """ + self.path = path + self.resources = [] + + if Path(self.path).is_file(): + self.resources = json.load(open(self.path)) + elif not self._url_validator(self.path): + raise Exception( + f"Resources location '{self.path}' is not a valid path or URL." + ) + else: + req = request.Request(self.path) + try: + response = request.urlopen(req) + except URLError as e: + raise Exception( + f"Unable to open Resources location '{self.path}': {e}" + ) + self.resources = json.loads(response.read().decode("utf-8")) + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. + :return: A list of all the Resources with the given ID. + """ + return [ + resource + for resource in self.resources + if resource["id"] == resource_id + ] diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index 16b0147b74..0781d9b15a 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -41,6 +41,7 @@ from tempfile import gettempdir from urllib.error import HTTPError from typing import List, Dict, Set, Optional +from .client import get_resource_json_obj from .md5_utils import md5_file, md5_dir from ..utils.progress_bar import tqdm, progress_hook @@ -398,6 +399,8 @@ def get_resource( unzip: bool = True, untar: bool = True, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> None: """ Obtains a gem5 resource and stored it to a specified location. If the @@ -419,6 +422,13 @@ def get_resource( will delete this local resource and re-download it if this parameter is True. True by default. + :param resource_version: The version of the resource to be obtained. If + None, the latest version of the resource compatible with the working + directory's gem5 version will be obtained. None by default. + + :param clients: A list of clients to use when obtaining the resource. If + None, all clients will be used. None by default. + :raises Exception: An exception is thrown if a file is already present at `to_path` but it does not have the correct md5 sum. An exception will also be thrown is a directory is present at `to_path` @@ -430,11 +440,13 @@ def get_resource( # minutes.Most resources should be downloaded and decompressed in this # timeframe, even on the most constrained of systems. with FileLock(f"{to_path}.lock", timeout=900): - - resource_json = get_resources_json_obj(resource_name) + resource_json = get_resource_json_obj( + resource_name, + resource_version=resource_version, + clients=clients, + ) if os.path.exists(to_path): - if os.path.isfile(to_path): md5 = md5_file(Path(to_path)) else: @@ -495,9 +507,8 @@ def get_resource( ) ) - # Get the URL. The URL may contain '{url_base}' which needs replaced - # with the correct value. - url = resource_json["url"].format(url_base=_get_url_base()) + # Get the URL. + url = resource_json["url"] _download(url=url, download_to=download_dest) print(f"Finished downloading resource '{resource_name}'.") diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 9cba9cf88f..22adf15670 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -29,13 +29,15 @@ import os from pathlib import Path from m5.util import warn, fatal -from .downloader import get_resource, get_resources_json_obj +from .downloader import get_resource from .looppoint import LooppointCsvLoader, LooppointJsonLoader from ..isas import ISA, get_isa_from_str from typing import Optional, Dict, Union, Type, Tuple, List +from .client import get_resource_json_obj + """ Resources are items needed to run a simulation, such as a disk image, kernel, or binary. The gem5 project provides pre-built resources, with sources, at @@ -67,18 +69,20 @@ class AbstractResource: def __init__( self, + resource_version: Optional[str] = None, local_path: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, ): """ :param local_path: The path on the host system where this resource is located. - :param documentation: Documentation describing this resource. Not a + :param description: Description describing this resource. Not a required parameter. By default is None. :param source: The source (as in "source code") for this resource. This string should navigate users to where the source for this resource may be found. Not a required parameter. By default is None. + :param resource_version: Version of the resource itself. """ if local_path and not os.path.exists(local_path): @@ -88,16 +92,21 @@ class AbstractResource: ) self._local_path = local_path - self._documentation = documentation + self._description = description self._source = source + self._version = resource_version + + def get_resource_version(self) -> str: + """Returns the version of the resource.""" + return self._version def get_local_path(self) -> Optional[str]: """Returns the local path of the resource.""" return self._local_path - def get_documentation(self) -> Optional[str]: - """Returns documentation associated with this resource.""" - return self._documentation + def get_description(self) -> Optional[str]: + """Returns description associated with this resource.""" + return self._description def get_source(self) -> Optional[str]: """Returns information as to where the source for this resource may be @@ -112,7 +121,8 @@ class FileResource(AbstractResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): @@ -123,8 +133,9 @@ class FileResource(AbstractResource): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -134,11 +145,11 @@ class DirectoryResource(AbstractResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): - if not os.path.isdir(local_path): raise Exception( f"DirectoryResource path specified, {local_path}, is not a " @@ -147,8 +158,9 @@ class DirectoryResource(AbstractResource): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -158,15 +170,17 @@ class DiskImageResource(FileResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, root_partition: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._root_partition = root_partition @@ -181,15 +195,17 @@ class BinaryResource(FileResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._architecture = None @@ -210,16 +226,18 @@ class BootloaderResource(BinaryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, architecture=architecture, source=source, + resource_version=resource_version, ) @@ -229,14 +247,16 @@ class GitResource(DirectoryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -246,16 +266,18 @@ class KernelResource(BinaryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, architecture=architecture, + resource_version=resource_version, ) @@ -270,14 +292,16 @@ class CheckpointResource(DirectoryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -290,12 +314,13 @@ class SimpointResource(AbstractResource): def __init__( self, + resource_version: Optional[str] = None, simpoint_interval: int = None, simpoint_list: List[int] = None, weight_list: List[float] = None, warmup_interval: int = 0, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, local_path: Optional[str] = None, **kwargs, @@ -314,8 +339,9 @@ class SimpointResource(AbstractResource): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._weight_list = weight_list @@ -402,15 +428,17 @@ class LooppointCsvResource(FileResource, LooppointCsvLoader): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointCsvLoader.__init__(self, pinpoints_file=Path(local_path)) @@ -419,16 +447,18 @@ class LooppointJsonResource(FileResource, LooppointJsonLoader): def __init__( self, local_path: str, + resource_version: Optional[str] = None, region_id: Optional[Union[str, int]] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointJsonLoader.__init__( self, looppoint_file=local_path, region_id=region_id @@ -446,8 +476,9 @@ class SimpointDirectoryResource(SimpointResource): weight_file: str, simpoint_interval: int, warmup_interval: int, + resource_version: Optional[str] = None, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): @@ -478,8 +509,9 @@ class SimpointDirectoryResource(SimpointResource): warmup_interval=warmup_interval, workload_name=workload_name, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) def get_simpoint_file(self) -> Path: @@ -522,9 +554,11 @@ class SimpointDirectoryResource(SimpointResource): def obtain_resource( - resource_name: str, + resource_id: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> AbstractResource: """ This function primarily serves as a factory for resources. It will return @@ -544,10 +578,16 @@ def obtain_resource( have the correct md5 value, the resoruce will be deleted and re-downloaded if this value is True. Otherwise an exception will be thrown. True by default. + :param resource_version: Version of the resource itself. + Not a required parameter. None by default. + :param clients: A list of clients to search for the resource. If this + parameter is not set, it will default search all clients. """ - # Obtain the JSON resource entry for this resource - resource_json = get_resources_json_obj(resource_name) + # Obtain the resource object entry for this resource + resource_json = get_resource_json_obj( + resource_id, resource_version=resource_version, clients=clients + ) to_path = None # If the "url" field is specified, the resoruce must be downloaded. @@ -580,38 +620,37 @@ def obtain_resource( os.makedirs(resource_directory, exist_ok=True) # This is the path to which the resource is to be stored. - to_path = os.path.join(resource_directory, resource_name) + to_path = os.path.join(resource_directory, resource_id) # Download the resource if it does not already exist. get_resource( - resource_name=resource_name, - to_path=os.path.join(resource_directory, resource_name), + resource_name=resource_id, + to_path=os.path.join(resource_directory, resource_id), download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + clients=clients, ) # Obtain the type from the JSON. From this we will determine what subclass # of `AbstractResource` we are to create and return. - resources_type = resource_json["type"] + resources_category = resource_json["category"] - if resources_type == "resource": + if resources_category == "resource": # This is a stop-gap measure to ensure to work with older versions of # the "resource.json" file. These should be replaced with their # respective specializations ASAP and this case removed. - if ( - "additional_metadata" in resource_json - and "root_partition" in resource_json["additional_metadata"] - ): + if "root_partition" in resource_json: # In this case we should return a DiskImageResource. - root_partition = resource_json["additional_metadata"][ - "root_partition" - ] + root_partition = resource_json["root_partition"] return DiskImageResource( - local_path=to_path, root_partition=root_partition + local_path=to_path, + root_partition=root_partition, + **resource_json, ) return CustomResource(local_path=to_path) - assert resources_type in _get_resource_json_type_map - resource_class = _get_resource_json_type_map[resources_type] + assert resources_category in _get_resource_json_type_map + resource_class = _get_resource_json_type_map[resources_category] # Once we know what AbstractResource subclass we are using, we create it. # The fields in the JSON object are assumed to map like-for-like to the @@ -694,6 +733,7 @@ class CustomDiskImageResource(DiskImageResource): def __init__( self, local_path: str, + resource_version: Optional[str] = None, root_partition: Optional[str] = None, metadata: Dict = {}, ): @@ -702,6 +742,7 @@ class CustomDiskImageResource(DiskImageResource): :param root_partition: The root disk partition to use. :param metadata: Metadata for the resource. **Warning:** As of " "v22.1.1, this parameter is not used. + :param resource_version: Version of the resource itself. """ warn( "The `CustomDiskImageResource` class is deprecated. Please use " @@ -713,13 +754,19 @@ class CustomDiskImageResource(DiskImageResource): "`CustomDiskImageResource` constructor. This parameter is not " "used." ) - super().__init__(local_path=local_path, root_partition=root_partition) + super().__init__( + local_path=local_path, + root_partition=root_partition, + resource_version=resource_version, + ) def Resource( - resource_name: str, + resource_id: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, ) -> AbstractResource: """ This function was created to maintain backwards compability for v21.1.0 @@ -737,9 +784,11 @@ def Resource( ) return obtain_resource( - resource_name=resource_name, + resource_id=resource_id, resource_directory=resource_directory, download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + clients=clients, ) diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index e0a19d0792..148ab3f35a 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -24,10 +24,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .downloader import get_workload_json_obj from .resource import obtain_resource +from .client import get_resource_json_obj -from typing import Dict, Any, Optional +from typing import Dict, Any, List, Optional class AbstractWorkload: @@ -155,7 +155,11 @@ class Workload(AbstractWorkload): """ def __init__( - self, workload_name: str, resource_directory: Optional[str] = None + self, + workload_name: str, + resource_directory: Optional[str] = None, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> None: """ This constructor will load the workload details from the workload with @@ -167,13 +171,13 @@ class Workload(AbstractWorkload): ```json { - "type" : "workload", - "name" : "x86-ubuntu-18.04-echo-hello", - "documentation" : "Description of workload here", + "category" : "workload", + "id" : "x86-ubuntu-18.04-echo-hello", + "description" : "Description of workload here", "function" : "set_kernel_disk_workload", "resources" : { "kernel" : "x86-linux-kernel-5.4.49", - "disk_image" : "x86-ubuntu-18.04-img" + "disk-image" : "x86-ubuntu-18.04-img" }, "additional_params" : { "readfile_contents" : "m5_exit; echo 'hello'; m5_exit" @@ -187,7 +191,7 @@ class Workload(AbstractWorkload): ```python board.set_kernel_disk_workload( kernel = Resource("x86-linux-kernel-5.4.49"), - disk_image = Resource("x86-ubuntu-18.04-img"), + disk-image = Resource("x86-ubuntu-18.04-img"), readfile_contents = "m5_exit; echo 'hello'; m5_exit", ) ``` @@ -198,7 +202,12 @@ class Workload(AbstractWorkload): any resources should be download and accessed from. If None, a default location will be used. None by default. """ - workload_json = get_workload_json_obj(workload_name=workload_name) + + workload_json = get_resource_json_obj( + workload_name, + resource_version=resource_version, + clients=clients, + ) func = workload_json["function"] assert isinstance(func, str) diff --git a/src/python/gem5/utils/simpoint.py b/src/python/gem5/utils/simpoint.py index eab92e2291..0d1af4b1cf 100644 --- a/src/python/gem5/utils/simpoint.py +++ b/src/python/gem5/utils/simpoint.py @@ -83,15 +83,11 @@ class SimPoint: simpoint_file_path = simpoint_directory.get_simpoint_file() weight_file_path = simpoint_resource.get_weight_file() - simpoint_interval = ( - simpoint_resource.get_metadata() - .get("additional_metadata") - .get("simpoint_interval") + simpoint_interval = simpoint_resource.get_metadata().get( + "simpoint_interval" ) - warmup_interval = ( - simpoint_resource.get_metadata() - .get("additional_metadata") - .get("warmup_interval") + warmup_interval = simpoint_resource.get_metadata().get( + "warmup_interval" ) self._simpoint_interval = simpoint_interval diff --git a/src/python/gem5_default_config.py b/src/python/gem5_default_config.py new file mode 100644 index 0000000000..980c883109 --- /dev/null +++ b/src/python/gem5_default_config.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +config = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + } +} diff --git a/tests/gem5/verifier.py b/tests/gem5/verifier.py index 93d47c8d10..c725fc68b9 100644 --- a/tests/gem5/verifier.py +++ b/tests/gem5/verifier.py @@ -172,6 +172,7 @@ class MatchStdout(DerivedGoldStandard): re.compile("^Using GPU kernel code file\(s\) "), re.compile("^.* not found locally\. Downloading"), re.compile("^Finished downloading"), + re.compile("^info: Using default config"), ] diff --git a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py new file mode 100644 index 0000000000..96aadf6879 --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py @@ -0,0 +1,511 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from gem5.isas import ISA +from gem5.resources.client import get_resource_json_obj +import gem5.resources.client +from gem5.resources.client_api.client_wrapper import ClientWrapper +from typing import Dict +from unittest.mock import patch +from unittest import mock +import json +from urllib.error import HTTPError +import io +import contextlib +from pathlib import Path + +mock_json_path = Path(__file__).parent / "refs/resources.json" +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + +mock_config_mongo = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, +} + +mock_config_combined = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + }, + "baba": { + "url": mock_json_path, + "isMongo": False, + }, + }, +} + +mock_json = {} + +with open(Path(__file__).parent / "refs/mongo_mock.json", "r") as f: + mock_json = json.load(f) + +duplicate_mock_json = {} + +with open(Path(__file__).parent / "refs/mongo_dup_mock.json", "r") as f: + duplicate_mock_json = json.load(f) + + +def mocked_requests_post(*args): + # mokcing urllib.request.urlopen + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status = status_code + + def read(self): + return json.dumps(self.json_data).encode("utf-8") + + data = json.loads(args[0].data) + if "/api-key/login" in args[0].full_url: + return MockResponse({"access_token": "test-token"}, 200) + if "/endpoint/data/v1/action/find" in args[0].full_url: + if data: + if data["filter"]["id"] == "x86-ubuntu-18.04-img": + return MockResponse( + { + "documents": mock_json, + }, + 200, + ) + if data["filter"]["id"] == "test-duplicate": + return MockResponse( + { + "documents": duplicate_mock_json, + }, + 200, + ) + if data["filter"]["id"] == "test-too-many": + error_file = io.BytesIO() + error_file.status = 429 + raise HTTPError( + args[0].full_url, 429, "Too Many Requests", {}, error_file + ) + return MockResponse( + { + "documents": [], + }, + 200, + ) + error_file = io.BytesIO() + error_file.status = 404 + raise HTTPError(args[0].full_url, 404, "Not Found", {}, error_file) + + +class ClientWrapperTestSuite(unittest.TestCase): + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj(self): + # Test that the resource object is correctly returned + resource = "this-is-a-test-resource" + resource = get_resource_json_obj(resource) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "2.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual( + resource["description"], "This is a test resource but double newer" + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_invalid_client(self): + # Test that an exception is raised when an invalid client is passed + resource_id = "test-id" + client = "invalid" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, clients=[client]) + self.assertTrue( + f"Client: {client} does not exist" in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_with_version(self): + # Test that the resource object is correctly returned + resource_id = "this-is-a-test-resource" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, resource_version=resource_version + ) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_1(self, mock_get): + resource = "x86-ubuntu-18.04-img" + resource = get_resource_json_obj(resource) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "1.1.0") + self.assertEqual(resource["category"], "disk-image") + self.assertEqual( + resource["description"], + "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_mongodb(self, mock_get): + # Test that the resource object is correctly returned + resource_id = "x86-ubuntu-18.04-img" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, + resource_version=resource_version, + clients=["gem5-resources"], + ) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "disk-image") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_id_invalid_mongodb(self, mock_get): + resource_id = "invalid-id" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, clients=["gem5-resources"]) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_invalid_mongodb( + self, mock_get + ): + resource_id = "x86-ubuntu-18.04-img" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + clients=["gem5-resources"], + ) + self.assertTrue( + f"Resource x86-ubuntu-18.04-img with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/x86-ubuntu-18.04-img/versions" + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_with_version_invalid_json(self): + resource_id = "this-is-a-test-resource" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + ) + self.assertTrue( + f"Resource this-is-a-test-resource with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/this-is-a-test-resource/versions" + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_combine(self, mock_get): + resource_id_mongo = "x86-ubuntu-18.04-img" + resource_version_mongo = "1.0.0" + resource_id_json = "this-is-a-test-resource" + resource_version_json = "1.0.0" + resource_mongo = get_resource_json_obj( + resource_id_mongo, + resource_version=resource_version_mongo, + clients=["gem5-resources"], + ) + resource_json = get_resource_json_obj( + resource_id_json, + resource_version=resource_version_json, + clients=["baba"], + ) + self.assertEqual(resource_mongo["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_mongo["resource_version"], "1.0.0") + self.assertEqual(resource_mongo["category"], "disk-image") + self.assertEqual( + resource_mongo["description"], "This is a test resource" + ) + self.assertEqual( + resource_mongo["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource_mongo["architecture"], "X86") + + self.assertEqual(resource_json["id"], "this-is-a-test-resource") + self.assertEqual(resource_json["resource_version"], "1.0.0") + self.assertEqual(resource_json["category"], "binary") + self.assertEqual( + resource_json["description"], "This is a test resource" + ) + self.assertEqual( + resource_json["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource_json["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_multi_database_second_only(self, mock_get): + resource_id = "simpoint-resource" + resource = get_resource_json_obj( + resource_id, + ) + self.assertEqual(resource["id"], resource_id) + self.assertEqual(resource["resource_version"], "0.2.0") + self.assertEqual(resource["category"], "file") + self.assertEqual( + resource["description"], + ( + "Simpoints for running the 'x86-print-this' resource with" + ' the parameters `"print this" 15000`. This is encapsulated' + " in the 'x86-print-this-15000-with-simpoints' workload." + ), + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_same_resource_different_versions( + self, mock_get + ): + resource_id = "x86-ubuntu-18.04-img" + resource_json = get_resource_json_obj( + resource_id, + ) + + self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_json["resource_version"], "2.0.0") + self.assertEqual(resource_json["category"], "disk-image") + + resource_json = get_resource_json_obj( + resource_id, + resource_version="1.0.0", + ) + + self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_json["resource_version"], "1.0.0") + self.assertEqual(resource_json["category"], "disk-image") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_same_resource_same_version(self, mock_get): + resource_id = "test-duplicate" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + f"Resource {resource_id} has multiple resources with" + f" the same version: 0.2.0" in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/logi", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_auth_url(self, mock_get): + resource_id = "test-resource" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Not found" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-resource' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_url(self, mock_get): + resource_id = "test-resource" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Not found" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-resource' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_url(self, mock_get): + resource_id = "test-too-many" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Too many requests" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-too-many' not found." + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py index 08736bbc32..19169e480e 100644 --- a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 The Regents of the University of California +# Copyright (c) 2023 The Regents of the University of California # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,6 +28,7 @@ import unittest import tempfile import os from typing import Dict +import json from gem5.resources.downloader import ( _get_resources_json_at_path, @@ -42,48 +43,102 @@ class ResourceDownloaderTestSuite(unittest.TestCase): @classmethod def setUpClass(cls) -> str: """ - This creates a simple resource.json temp file for testing purposes. + This creates a simple resources collection for testing """ - - file_contents = ( - "{" - + f'"version" : "{_resources_json_version_required()}",' - + """ - "url_base" : "http://dist.gem5.org/dist/v21-2", - "previous-versions" : {}, - "resources": [ - { - "type": "resource", - "name" : "riscv-disk-img", - "documentation" : "A simple RISCV disk image based on busybox.", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "d6126db9f6bed7774518ae25aa35f153", - "url": "{url_base}/images/riscv/busybox/riscv-disk.img.gz", - "source" : "src/riscv-fs", - "additional_metadata" : { - "root_partition": null - } - }, - { - "type": "resource", - "name" : "riscv-lupio-busybox-img", - "documentation" : "A RISCV disk image, based on busybox, to ...", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "e5bee8a31f45f4803f87c0d781553ccc", - "url": "{url_base}/images/riscv/busybox/riscv-lupio-busybox.img", - "source" : "src/lupv", - "additional_metadata" : { - "root_partition": "1" - } - } - ] -} - """ - ) + file_contents = [ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest", "testing", "riscv", "testing"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": ["23.1"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + ] file = tempfile.NamedTemporaryFile(mode="w", delete=False) - file.write(file_contents) + file.write(json.dumps(file_contents)) file.close() cls.file_path = file.name @@ -100,14 +155,16 @@ class ResourceDownloaderTestSuite(unittest.TestCase): "create_temp_resources_json" has been loaded correctly into a Python dictionary. """ - self.assertTrue("resources" in json) - self.assertEquals(2, len(json["resources"])) - self.assertTrue("name" in json["resources"][0]) - self.assertEquals("riscv-disk-img", json["resources"][0]["name"]) - self.assertTrue("name" in json["resources"][1]) - self.assertEquals( - "riscv-lupio-busybox-img", json["resources"][1]["name"] - ) + self.assertEquals(4, len(json)) + self.assertTrue("id" in json[0]) + self.assertEquals("this-is-a-test-resource", json[0]["id"]) + self.assertEquals("binary", json[0]["category"]) + self.assertTrue("id" in json[1]) + self.assertEquals("this-is-a-test-resource", json[1]["id"]) + self.assertTrue("id" in json[2]) + self.assertEquals("test-version", json[2]["id"]) + self.assertTrue("id" in json[3]) + self.assertEquals("test-version", json[3]["id"]) def test_get_resources_json_at_path(self) -> None: # Tests the gem5.resources.downloader._get_resources_json_at_path() diff --git a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py new file mode 100644 index 0000000000..791d96c1f1 --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py @@ -0,0 +1,192 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import os +import io +import contextlib +from pathlib import Path + +from gem5.resources.resource import * + +from gem5.resources.looppoint import ( + LooppointCsvLoader, + LooppointJsonLoader, +) + +from gem5.isas import ISA + +from _m5 import core + +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch + +mock_json_path = Path(__file__).parent / "refs/obtain-resource.json" + +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + + +@patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), +) +class TestObtainResourcesCheck(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Prior to running the suite we set the resource directory to + "ref/resource-specialization.json" + """ + os.environ["GEM5_RESOURCE_JSON"] = os.path.join( + os.path.realpath(os.path.dirname(__file__)), + "refs", + "obtain-resource.json", + ) + + @classmethod + def tearDownClass(cls) -> None: + """After running the suite we unset the gem5-resource JSON file, as to + not interfere with others tests. + """ + del os.environ["GEM5_RESOURCE_JSON"] + + def get_resource_dir(cls) -> str: + """To ensure the resources are cached to the same directory as all + other tests, this function returns the location of the testing + directories "resources" directory. + """ + return os.path.join( + os.path.realpath(os.path.dirname(__file__)), + os.pardir, + os.pardir, + os.pardir, + "gem5", + "resources", + ) + + def test_obtain_resources_no_version(self): + """Test that the resource loader returns latest version compatible with that version of gem5 when no version is specified.""" + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + ) + self.assertEquals("2.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals("test description", resource.get_description()) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_compatible(self): + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertEquals("1.7.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals( + "test description v1.7.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_incompatible(self): + resource = None + f = io.StringIO() + with contextlib.redirect_stderr(f): + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + self.assertTrue( + f"warn: Resource test-binary-resource with version 1.5.0 is not known to be compatible with gem5 version {core.gem5Version}. " + "This may cause problems with your simulation. This resource's compatibility with different gem5 versions can be found here: " + f"https://resources.gem5.org/resources/test-binary-resource/versions" + in f.getvalue() + ) + + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + self.assertEquals("1.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + self.assertEquals( + "test description for 1.5.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_no_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + ) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_version(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="3.0.0", + ) + print("context.exception: ", context.exception) + print(str(context.exception)) + self.assertTrue( + f"Resource test-binary-resource with version '3.0.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/test-binary-resource/versions" + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py new file mode 100644 index 0000000000..8f6674ff0d --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import tempfile +import os +from typing import Dict + +from gem5.resources.downloader import ( + get_resources_json_obj, +) + + +class ResourceDownloadTestSuite(unittest.TestCase): + """Test cases for gem5.resources.downloader""" + + @classmethod + def setUpClass(cls) -> str: + pass + + def get_resource_json_by_id(self) -> None: + """Get a resource by its id""" + resources = get_resources_json_obj("test-version") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "2.0.0") + + def get_resource_json_invalid_id(self) -> None: + """Should throw an exception when trying to get a resource that doesn't exist""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("this-resource-doesnt-exist") + self.assertTrue( + f"Error: Resource with name 'this-resource-doesnt-exist' does not exist" + in str(context.exception) + ) + + def get_resource_json_by_id_and_version(self) -> None: + """Get a resource by its id and version""" + resources = get_resources_json_obj("test-version", "1.0.0") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "1.0.0") + + def get_resource_json_by_id_and_invalid_version(self) -> None: + """Get a resource by its id and an invalid version (does not exist)""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("test-version", "3.0.0") + self.assertTrue( + f"Specified Version 3.0.0 does not exist for the resource 'test-version'." + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index 660bf5f38f..5c22a7341e 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -37,7 +37,25 @@ from gem5.resources.looppoint import ( from gem5.isas import ISA +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch +mock_json_path = Path(__file__).parent / "refs/resource-specialization.json" + +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + + +@patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), +) class ResourceSpecializationSuite(unittest.TestCase): """This suite tests that `gem5.resource.resource` casts to the correct `AbstractResource` specialization when using the `obtain_resource` @@ -79,14 +97,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_binary_resource(self) -> None: """Tests the loading of of a BinaryResource""" resource = obtain_resource( - resource_name="binary-example", + resource_id="binary-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BinaryResource) self.assertEquals( - "binary-example documentation.", resource.get_documentation() + "binary-example documentation.", resource.get_description() ) self.assertEquals("src/simple", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) @@ -94,14 +112,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_kernel_resource(self) -> None: """Tests the loading of a KernelResource.""" resource = obtain_resource( - resource_name="kernel-example", + resource_id="kernel-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, KernelResource) self.assertEquals( - "kernel-example documentation.", resource.get_documentation() + "kernel-example documentation.", resource.get_description() ) self.assertEquals("src/linux-kernel", resource.get_source()) self.assertEquals(ISA.RISCV, resource.get_architecture()) @@ -109,14 +127,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_bootloader_resource(self) -> None: """Tests the loading of a BootloaderResource.""" resource = obtain_resource( - resource_name="bootloader-example", + resource_id="bootloader-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BootloaderResource) self.assertEquals( - "bootloader documentation.", resource.get_documentation() + "bootloader documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_architecture()) @@ -124,14 +142,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_disk_image_resource(self) -> None: """Tests the loading of a DiskImageResource.""" resource = obtain_resource( - resource_name="disk-image-example", + resource_id="disk-image-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DiskImageResource) self.assertEquals( - "disk-image documentation.", resource.get_documentation() + "disk-image documentation.", resource.get_description() ) self.assertEquals("src/x86-ubuntu", resource.get_source()) self.assertEquals("1", resource.get_root_partition()) @@ -139,40 +157,40 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_checkpoint_resource(self) -> None: """Tests the loading of a CheckpointResource.""" resource = obtain_resource( - resource_name="checkpoint-example", + resource_id="checkpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, CheckpointResource) self.assertEquals( - "checkpoint-example documentation.", resource.get_documentation() + "checkpoint-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) def test_git_resource(self) -> None: """Tests the loading of a GitResource.""" resource = obtain_resource( - resource_name="git-example", + resource_id="git-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, GitResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_simpoint_directory_resource(self) -> None: """Tests the loading of a Simpoint directory resource.""" resource = obtain_resource( - resource_name="simpoint-directory-example", + resource_id="simpoint-directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointDirectoryResource) self.assertEquals( - "simpoint directory documentation.", resource.get_documentation() + "simpoint directory documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -199,14 +217,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_simpoint_resource(self) -> None: """Tests the loading of a Simpoint resource.""" resource = obtain_resource( - resource_name="simpoint-example", + resource_id="simpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointResource) self.assertEquals( - "simpoint documentation.", resource.get_documentation() + "simpoint documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_local_path()) @@ -219,26 +237,27 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_file_resource(self) -> None: """Tests the loading of a FileResource.""" resource = obtain_resource( - resource_name="file-example", + resource_id="file-example", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, FileResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_directory_resource(self) -> None: """Tests the loading of a DirectoryResource.""" resource = obtain_resource( - resource_name="directory-example", + resource_id="directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DirectoryResource) self.assertEquals( - "directory-example documentation.", resource.get_documentation() + "directory-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -247,7 +266,7 @@ class ResourceSpecializationSuite(unittest.TestCase): pinpoints csv file.""" resource = obtain_resource( - resource_name="looppoint-pinpoint-csv-resource", + resource_id="looppoint-pinpoint-csv-resource", resource_directory=self.get_resource_dir(), ) @@ -258,7 +277,7 @@ class ResourceSpecializationSuite(unittest.TestCase): self.assertIsInstance(resource, LooppointCsvLoader) self.assertEquals( - "A looppoint pinpoints csv file.", resource.get_documentation() + "A looppoint pinpoints csv file.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -267,8 +286,9 @@ class ResourceSpecializationSuite(unittest.TestCase): Looppoint JSON file.""" resource = obtain_resource( - resource_name="looppoint-json-restore-resource-region-1", + resource_id="looppoint-json-restore-resource-region-1", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, LooppointJsonResource) @@ -278,6 +298,6 @@ class ResourceSpecializationSuite(unittest.TestCase): self.assertTrue("1" in resource.get_regions()) self.assertEquals( - "A looppoint json file resource.", resource.get_documentation() + "A looppoint json file resource.", resource.get_description() ) self.assertIsNone(resource.get_source()) diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index 2bc31f5a3f..b898faeb79 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 The Regents of the University of California +# Copyright (c) 2023 The Regents of the University of California # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -36,6 +36,29 @@ from gem5.resources.resource import ( from typing import Dict +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch +from pathlib import Path + +mock_config_json1 = { + "sources": { + "baba": { + "url": Path(__file__).parent + / "refs/workload-checks-custom-workload.json", + "isMongo": False, + } + }, +} + +mock_config_json2 = { + "sources": { + "baba": { + "url": Path(__file__).parent / "refs/workload-checks.json", + "isMongo": False, + } + }, +} + class CustomWorkloadTestSuite(unittest.TestCase): """ @@ -43,8 +66,11 @@ class CustomWorkloadTestSuite(unittest.TestCase): """ @classmethod + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json1), + ) def setUpClass(cls) -> None: - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", @@ -114,8 +140,7 @@ class CustomWorkloadTestSuite(unittest.TestCase): "test", self.custom_workload.get_parameters()["binary"] ) - # We set the overridden parameter back to it's old value. - self.custom_workload.set_parameter("binary", old_value) + # We set the overridden parameter back to it's old valu self.custom_workload.set_parameter("binary", old_value) class WorkloadTestSuite(unittest.TestCase): @@ -124,8 +149,11 @@ class WorkloadTestSuite(unittest.TestCase): """ @classmethod + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json2), + ) def setUpClass(cls): - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", @@ -157,9 +185,9 @@ class WorkloadTestSuite(unittest.TestCase): self.assertTrue("kernel" in parameters) self.assertTrue(isinstance(parameters["kernel"], BinaryResource)) - self.assertTrue("disk_image" in parameters) + self.assertTrue("disk-image" in parameters) self.assertTrue( - isinstance(parameters["disk_image"], DiskImageResource) + isinstance(parameters["disk-image"], DiskImageResource) ) self.assertTrue("readfile_contents" in parameters) diff --git a/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json b/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json new file mode 100644 index 0000000000..fe87cc8c28 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json @@ -0,0 +1,30 @@ +[ + { + "category": "file", + "id": "test-duplicate", + "description": "test", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/mongo_mock.json b/tests/pyunit/stdlib/resources/refs/mongo_mock.json new file mode 100644 index 0000000000..b6376cc5e4 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/mongo_mock.json @@ -0,0 +1,56 @@ +[ + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "1.1.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] diff --git a/tests/pyunit/stdlib/resources/refs/obtain-resource.json b/tests/pyunit/stdlib/resources/refs/obtain-resource.json new file mode 100644 index 0000000000..fac95e106a --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/obtain-resource.json @@ -0,0 +1,59 @@ +[ + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.5.0", + "gem5_versions": [ + "25.0" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description v1.7.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.7.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description for 1.5.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.5.0", + "gem5_versions": [ + "21.1", + "22.1" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index c4d5eb4714..1129f1bd05 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -1,132 +1,181 @@ - -{ - "version" : "develop", - "url_base" : "http://dist.gem5.org/dist/v22-1", - "previous-versions" : { - "develop" : "https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/develop/resources.json?format=TEXT", - "21.2" : "http://resources.gem5.org/prev-resources-json/resources-21-2.json" +[ + { + "category": "kernel", + "id": "kernel-example", + "description": "kernel-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "60a53c7d47d7057436bf4b9df707a841", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.4.49", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] }, - "resources": [ - { - "type" : "kernel", - "name" : "kernel-example", - "documentation" : "kernel-example documentation.", - "architecture" : "RISCV", - "is_zipped" : false, - "md5sum" : "60a53c7d47d7057436bf4b9df707a841", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.4.49", - "source" : "src/linux-kernel" - }, - { - "type" : "disk-image", - "name" : "disk-image-example", - "documentation" : "disk-image documentation.", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" - }, - { - "type" : "binary", - "name" : "binary-example", - "documentation" : "binary-example documentation.", - "architecture" : "ARM", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static", - "source" : "src/simple" - }, - { - "type" : "bootloader", - "name" : "bootloader-example", - "documentation" : "bootloader documentation.", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static" - }, - { - "type" : "checkpoint", - "name" : "checkpoint-example", - "documentation" : "checkpoint-example documentation.", - "architecture": "RISCV", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "git", - "name" : "git-example", - "documentation" : null, - "is_zipped" : false, - "is_tar_archive" : true, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "file", - "name" : "file-example", - "documentation" : null, - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar", - "source" : null - }, - { - "type" : "directory", - "name" : "directory-example", - "documentation" : "directory-example documentation.", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type": "simpoint-directory", - "name": "simpoint-directory-example", - "documentation": "simpoint directory documentation.", - "is_zipped" : false, - "md5sum" : "3fcffe3956c8a95e3fb82e232e2b41fb", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/simpoints/x86-print-this-15000-simpoints-20221013.tar", - "simpoint_interval": 1000000, - "warmup_interval": 1000000, - "simpoint_file": "simpoint.simpt", - "weight_file": "simpoint.weight", - "workload_name": "Example Workload" - }, - { - "type": "simpoint", - "name": "simpoint-example", - "documentation": "simpoint documentation.", - "simpoint_interval": 1000000, - "warmup_interval": 23445, - "simpoint_list" : [2,3,4,15], - "weight_list" : [0.1, 0.2, 0.4, 0.3] - }, - { - "type": "looppoint-pinpoint-csv", - "name": "looppoint-pinpoint-csv-resource", - "documentation" : "A looppoint pinpoints csv file.", - "is_zipped" : false, - "md5sum" : "199ab22dd463dc70ee2d034bfe045082", - "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", - "source" : null - }, - { - "type": "looppoint-json", - "name": "looppoint-json-restore-resource-region-1", - "documentation" : "A looppoint json file resource.", - "is_zipped" : false, - "region_id" : "1", - "md5sum" : "a71ed64908b082ea619b26b940a643c1", - "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", - "source" : null - } - ] -} + { + "category": "disk-image", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "binary", + "id": "binary-example", + "description": "binary-example documentation.", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "bootloader", + "id": "bootloader-example", + "description": "bootloader documentation.", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "checkpoint", + "id": "checkpoint-example", + "description": "checkpoint-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "git", + "id": "git-example", + "description": null, + "is_zipped": false, + "is_tar_archive": true, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "file", + "id": "file-example", + "description": null, + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "directory", + "id": "directory-example", + "description": "directory-example documentation.", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint-directory", + "id": "simpoint-directory-example", + "description": "simpoint directory documentation.", + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "simpoint_file": "simpoint.simpt", + "weight_file": "simpoint.weight", + "workload_name": "Example Workload", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint", + "id": "simpoint-example", + "description": "simpoint documentation.", + "simpoint_interval": 1000000, + "warmup_interval": 23445, + "simpoint_list": [ + 2, + 3, + 4, + 15 + ], + "weight_list": [ + 0.1, + 0.2, + 0.4, + 0.3 + ], + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-pinpoint-csv", + "id": "looppoint-pinpoint-csv-resource", + "description": "A looppoint pinpoints csv file.", + "is_zipped": false, + "md5sum": "199ab22dd463dc70ee2d034bfe045082", + "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-json", + "id": "looppoint-json-restore-resource-region-1", + "description": "A looppoint json file resource.", + "is_zipped": false, + "region_id": "1", + "md5sum": "a71ed64908b082ea619b26b940a643c1", + "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/resources.json b/tests/pyunit/stdlib/resources/refs/resources.json new file mode 100644 index 0000000000..812caeff43 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/resources.json @@ -0,0 +1,329 @@ +[ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing", + "new" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.1.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.1" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "test-duplicate", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "aa", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "asfsaf", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "simpoint-resource", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "bat43f34fman", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "adadadas", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json index a9dd2aaa46..8349b92658 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json @@ -1,17 +1,16 @@ -{ - "version" : null, - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "binary", - "name" : "x86-hello64-static", - "documentation" : "A 'Hello World!' binary.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "dbf120338b37153e3334603970cebd8c", - "url" : "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", - "source" : "src/simple" - } - ] -} +[ + { + "category": "binary", + "id": "x86-hello64-static", + "description": "A 'Hello World!' binary.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index 4f7e76bfb5..d19396bf8c 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -1,40 +1,48 @@ -{ - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "kernel", - "name" : "x86-linux-kernel-5.2.3", - "documentation" : "The linux kernel (v5.2.3), compiled to X86.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "4838c99b77d33c8307b939c16624e4ac", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.2.3", - "source" : "src/linux-kernel" +[ + { + "category": "kernel", + "id": "x86-linux-kernel-5.2.3", + "description": "The linux kernel (v5.2.3), compiled to X86.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "4838c99b77d33c8307b939c16624e4ac", + "url": "{url_base}/kernels/x86/static/vmlinux-5.2.3", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86..", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "workload", + "id": "simple-boot", + "description": "Description of workload here", + "function": "set_kernel_disk_workload", + "resources": { + "kernel": "x86-linux-kernel-5.2.3", + "disk-image": "x86-ubuntu-18.04-img" }, - { - "type" : "disk-image", - "name" : "x86-ubuntu-18.04-img", - "documentation" : "A disk image containing Ubuntu 18.04 for x86..", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" + "additional_params": { + "readfile_contents": "echo 'Boot successful'; m5 exit" }, - { - "type" : "workload", - "name" : "simple-boot", - "documentation" : "Description of workload here", - "function" : "set_kernel_disk_workload", - "resources" : { - "kernel" : "x86-linux-kernel-5.2.3", - "disk_image" : "x86-ubuntu-18.04-img" - }, - "additional_params" : { - "readfile_contents" : "echo 'Boot successful'; m5 exit" - } - } - ] -} + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +]