diff --git a/src/python/SConscript b/src/python/SConscript index 1016fee5c6..fc2241fa09 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -296,10 +296,10 @@ PySource('gem5.resources.client_api', 'gem5/resources/client_api/jsonclient.py') PySource('gem5.resources.client_api', 'gem5/resources/client_api/atlasclient.py') -PySource('gem5.resources.client_api', - 'gem5/resources/client_api/client_wrapper.py') PySource('gem5.resources.client_api', 'gem5/resources/client_api/abstract_client.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/client_query.py') PySource('gem5', 'gem5_default_config.py') PySource('gem5.utils', 'gem5/utils/__init__.py') PySource('gem5.utils', 'gem5/utils/filelock.py') diff --git a/src/python/gem5/resources/client.py b/src/python/gem5/resources/client.py index 1fecbb9c08..d45d73c499 100644 --- a/src/python/gem5/resources/client.py +++ b/src/python/gem5/resources/client.py @@ -24,13 +24,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import itertools import json import os +import sys from pathlib import Path from typing import ( + Any, Dict, List, Optional, + Tuple, ) from m5.util import ( @@ -42,7 +46,9 @@ from _m5 import core from gem5.gem5_default_config import config -from .client_api.client_wrapper import ClientWrapper +from .client_api.atlasclient import AtlasClient +from .client_api.client_query import ClientQuery +from .client_api.jsonclient import JSONClient def getFileContent(file_path: Path) -> Dict: @@ -125,7 +131,7 @@ def _get_clientwrapper(): f"Appending resources from {os.environ['GEM5_RESOURCE_JSON_APPEND']}" ) - clientwrapper = ClientWrapper(gem5_config) + clientwrapper = _create_clients(gem5_config) return clientwrapper @@ -143,7 +149,8 @@ def list_resources( :return: A Python Dict where the key is the resource id and the value is a list of all the supported resource versions. """ - return _get_clientwrapper().list_resources(clients, gem5_version) + _get_clientwrapper() + return _list_all_resources(clients, gem5_version) def get_resource_json_obj( @@ -163,7 +170,214 @@ def get_resource_json_obj( current build. If ``None``, filtering based on compatibility is not performed. """ + _get_clientwrapper() + if resource_version: + client_queries = [ + ClientQuery(resource_id, resource_version, gem5_version) + ] + else: + client_queries = [ClientQuery(resource_id, gem5_version=gem5_version)] - return _get_clientwrapper().get_resource_json_obj_from_client( - resource_id, resource_version, clients, gem5_version + # We will return a list when we refactor ontain_resources to handle multiple + # resources + return _get_resource_json_obj_from_client(client_queries, clients)[0] + + +def get_multiple_resource_json_obj( + client_queries: List[ClientQuery], + clients: Optional[List[str]] = None, +) -> List[Dict]: + """ + Get the resource json object from the clients wrapper. + + :param client_queries: This is a list of ClientQuery objects that contain + information about the resources to fetch from datasources. + :param clients: The list of clients to query. + """ + _get_clientwrapper() + return _get_resource_json_obj_from_client(client_queries, clients) + + +def _create_clients( + config: Dict, +) -> Dict: + """ + This function creates respective client object for each source in the + config file according to the type of source. + + :param config: config file containing the source information + + :returns: clients: dictionary of clients for each source + """ + clients = {} + for client in config["sources"]: + client_source = config["sources"][client] + try: + if client_source["isMongo"]: + clients[client] = AtlasClient(client_source) + else: + clients[client] = JSONClient(client_source["url"]) + except Exception as e: + warn(f"Error creating client {client}: {str(e)}") + return clients + + +def _list_all_resources( + clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, +) -> Dict[str, List[str]]: + global clientwrapper + clients_to_search = ( + list(clientwrapper.keys()) if clients is None else clients + ) + # There's some duplications of functionality here (similar code in + # `get_all_resources_by_id`. This code could be refactored to avoid + # this). + resources = [] + for client in clients_to_search: + if client not in clientwrapper: + raise Exception(f"Client: {client} does not exist") + try: + resources.extend( + clientwrapper[client].get_resources(gem5_version=gem5_version) + ) + except Exception as e: + warn(f"Error getting resources from client {client}: {str(e)}") + + to_return = {} + for resource in resources: + if resource["id"] not in to_return: + to_return[resource["id"]] = [] + to_return[resource["id"]].append(resource["resource_version"]) + return to_return + + +def _get_resource_json_obj_from_client( + client_queries: List[ClientQuery], + clients: Optional[List[str]] = None, +) -> Dict: + """ + This function returns the resource object from the client with the + given id and version. + + :param client_queries: This is a list of ClientQuery objects that contain + information about the resources to fetch from datasources. + :param resource_version: The version of the resource to search for. + :param clients: A list of clients to search through. If ``None``, all + clients are searched. + :param gem5_version: The gem5 version to check compatibility with. If + ``None``, no compatibility check is performed. By + default, is the current version of gem5. + :return: The resource object as a Python dictionary if found. + If not found, exception is thrown. + """ + # getting all the resources with the given id from the dictionary + resources_list = _get_all_resources_by_id(client_queries, clients) + + for id, resources in resources_list.items(): + # if no resource with the given id is found, return None + if len(resources) == 0: + raise Exception(f"Resource with ID '{id}' not found.") + + resource_to_return = [] + + # if there are multiple resources with the same id, return the one with + # the highest version + for id, resources in resources_list.items(): + resources_list[id] = _sort_resources(resources) + resource_to_return.append(resources_list[id][0]) + + return resource_to_return + + +def _get_all_resources_by_id( + client_queries: List[ClientQuery], + clients: Optional[List[str]] = None, +) -> Dict[str, Any]: + """ + This function returns all the resources with the given id from all the + sources. + + :param client_queries: This is a list of ClientQuery objects that contain + information about the resources to fetch from datasources. + :param clients: A list of clients to search through. If ``None``, all + clients are searched. + :return: A list of resources as Python dictionaries. + """ + global clientwrapper + + # creating a dictionary with the resource id as the key and an empty + # list as the value, the list will be populated with different versions + # of the resource + resources = {} + for client_query in client_queries: + id = client_query.get_resource_id() + resources[id] = [] + + if not clients: + clients = list(clientwrapper.keys()) + for client in clients: + if client not in clientwrapper: + raise Exception(f"Client: {client} does not exist") + try: + filtered_resources = clientwrapper[client].get_resources_by_id( + client_queries + ) + for k in resources.keys(): + if k in filtered_resources.keys(): + resources[k].append(filtered_resources[k]) + + except Exception as e: + print( + f"Exception thrown while getting resources '{client_queries}' " + f"from client '{client}'\n", + file=sys.stderr, + ) + raise e + # check if no 2 resources have the same id and version + for resource_id, different_version_of_resource in resources.items(): + for res1, res2 in itertools.combinations( + different_version_of_resource, 2 + ): + if res1["resource_version"] == res2["resource_version"]: + raise Exception( + f"Resource {resource_id} has multiple resources with " + f"the same version: {res1['resource_version']}" + ) + + return resources + + +def _sort_resources(resources: List) -> List: + """ + Sorts the resources by ID. + + If the IDs are the same, the resources are sorted by version. + + :param resources: A list of resources to sort. + + :return: A list of sorted resources. + """ + + def sort_tuple(resource: Dict) -> Tuple: + """This is used for sorting resources by ID and version. First + the ID is sorted, then the version. In cases where the version + contains periods, it's assumed this is to separate a + ``major.minor.hotfix`` style versioning system. In which case, the + value separated in the most-significant position is sorted before + those less significant. If the value is a digit it is cast as an + int, otherwise, it is cast as a string, to lower-case. + """ + to_return = (resource["id"].lower(),) + for val in resource["resource_version"].split("."): + if val.isdigit(): + to_return += (int(val),) + else: + to_return += (str(val).lower(),) + return to_return + + return sorted( + resources, + key=lambda resource: sort_tuple(resource), + reverse=True, ) diff --git a/src/python/gem5/resources/client_api/abstract_client.py b/src/python/gem5/resources/client_api/abstract_client.py index 2ec4a3cdb8..0dd84aba02 100644 --- a/src/python/gem5/resources/client_api/abstract_client.py +++ b/src/python/gem5/resources/client_api/abstract_client.py @@ -34,8 +34,11 @@ from typing import ( Dict, List, Optional, + Tuple, ) +from .client_query import ClientQuery + class AbstractClient(ABC): def _url_validator(self, url: str) -> bool: @@ -55,13 +58,15 @@ class AbstractClient(ABC): @abstractmethod def get_resources( self, - resource_id: Optional[str] = None, - resource_version: Optional[str] = None, - gem5_version: Optional[str] = None, + client_queries: List[ClientQuery], ) -> List[Dict[str, Any]]: """ - :param resource_id: The ID of the Resource. Optional, if not set, all - resources will be returned. + :param client_queries: A list of client queries containing the + information to query the resources. Each + ClientQuery object can contain the following information: + - resource_id: The ID of the Resource. + - resource_version: The version of the `Resource`. + - gem5_version: The version of gem5. :param resource_version: The version of the `Resource`. Optional, if not set, all resource versions will be returned. Note: If ``resource_id`` is not set, this @@ -72,6 +77,40 @@ class AbstractClient(ABC): """ raise NotImplementedError + def sort_resources(self, resources: List) -> List: + """ + Sorts the resources by ID. + + If the IDs are the same, the resources are sorted by version. + + :param resources: A list of resources to sort. + + :return: A list of sorted resources. + """ + + def sort_tuple(resource: Dict) -> Tuple: + """This is used for sorting resources by ID and version. First + the ID is sorted, then the version. In cases where the version + contains periods, it's assumed this is to separate a + ``major.minor.hotfix`` style versioning system. In which case, the + value separated in the most-significant position is sorted before + those less significant. If the value is a digit it is cast as an + int, otherwise, it is cast as a string, to lower-case. + """ + to_return = (resource["id"].lower(),) + for val in resource["resource_version"].split("."): + if val.isdigit(): + to_return += (int(val),) + else: + to_return += (str(val).lower(),) + return to_return + + return sorted( + resources, + key=lambda resource: sort_tuple(resource), + reverse=True, + ) + def filter_incompatible_resources( self, resources_to_filter: List[Dict[str, Any]], @@ -111,10 +150,17 @@ class AbstractClient(ABC): filtered_resources.append(resource) return filtered_resources - def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + def get_resources_by_id( + self, client_queries: List[ClientQuery] + ) -> List[Dict[str, Any]]: """ - :param resource_id: The ID of the Resource. + :param client_queries: A list of ClientQuery objects containing the + information to query the resources. Each + ClientQuery object can contain the following information: + - resource_id: The ID of the Resource. + - resource_version: The version of the `Resource`. + - gem5_version: The version of gem5. :return: A list of all the Resources with the given ID. """ - return self.get_resources(resource_id=resource_id) + return self.get_resources(client_queries=client_queries) diff --git a/src/python/gem5/resources/client_api/atlasclient.py b/src/python/gem5/resources/client_api/atlasclient.py index 83933f5497..7dd5cd92b8 100644 --- a/src/python/gem5/resources/client_api/atlasclient.py +++ b/src/python/gem5/resources/client_api/atlasclient.py @@ -45,6 +45,7 @@ from m5.util import warn from ...utils.socks_ssl_context import get_proxy_context from .abstract_client import AbstractClient +from .client_query import ClientQuery class AtlasClientHttpJsonRequestError(Exception): @@ -156,24 +157,39 @@ class AtlasClient(AbstractClient): def get_resources( self, - resource_id: Optional[str] = None, - resource_version: Optional[str] = None, - gem5_version: Optional[str] = None, - ) -> List[Dict[str, Any]]: + client_queries: List[ClientQuery], + ) -> Dict[str, Any]: url = f"{self.url}/action/find" data = { "dataSource": self.dataSource, "collection": self.collection, "database": self.database, } - filter = {} - if resource_id: - filter["id"] = resource_id - if resource_version is not None: - filter["resource_version"] = resource_version - if filter: - data["filter"] = filter + search_conditions = [] + for resource in client_queries: + condition = { + "id": resource.get_resource_id(), + } + + if not resource.get_gem5_version().startswith("DEVELOP"): + # This is a regex search that matches the beginning of the + # string. So if the resource version is '20.1', it will + # match '20.1.1'. + condition["gem5_versions"] = { + "$regex": f"^{resource.get_gem5_version()}", + "$options": "i", + } + + # If the resource has a resource_version, add it to the search + # conditions. + if resource.get_resource_version(): + condition["resource_version"] = resource.get_resource_version() + + search_conditions.append(condition) + + filter = {"$or": search_conditions} + data["filter"] = filter headers = { "Authorization": f"Bearer {self.get_token()}", @@ -187,8 +203,15 @@ class AtlasClient(AbstractClient): purpose_of_request="Get Resources", )["documents"] - # I do this as a lazy post-processing step because I can't figure out - # how to do this via an Atlas query, which may be more efficient. - return self.filter_incompatible_resources( - resources_to_filter=resources, gem5_version=gem5_version - ) + resources_by_id = {} + for resource in resources: + if resource["id"] in resources_by_id.keys(): + resources_by_id[resource["id"]].append(resource) + else: + resources_by_id[resource["id"]] = [resource] + + # Sort the resources by version and return the latest version. + for id, resource_list in resources_by_id.items(): + resources_by_id[id] = self.sort_resources(resource_list)[0] + + return resources_by_id diff --git a/src/python/gem5/resources/client_api/client_query.py b/src/python/gem5/resources/client_api/client_query.py new file mode 100644 index 0000000000..b41ef6b2f8 --- /dev/null +++ b/src/python/gem5/resources/client_api/client_query.py @@ -0,0 +1,57 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import Optional + +from _m5 import core + +""" +This class is a data class that represents a query to the client. +It encapsulates the fields required to query resources from the client. +Right now, it only contains the resource_id, resource_version, and gem5_version +fields, but it can be expanded to include more fields in the future, if needed. +""" + + +class ClientQuery: + def __init__( + self, + resource_id: str, + resource_version: Optional[str] = None, + gem5_version: Optional[str] = core.gem5Version, + ): + self.resource_id = resource_id + self.resource_version = resource_version + self.gem5_version = gem5_version + + def get_resource_id(self) -> str: + return self.resource_id + + def get_resource_version(self) -> Optional[str]: + return self.resource_version + + def get_gem5_version(self) -> Optional[str]: + return self.gem5_version diff --git a/src/python/gem5/resources/client_api/client_wrapper.py b/src/python/gem5/resources/client_api/client_wrapper.py deleted file mode 100644 index 4e93fc46b1..0000000000 --- a/src/python/gem5/resources/client_api/client_wrapper.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright (c) 2023 The Regents of the University of California -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import itertools -import sys -from typing import ( - Dict, - List, - Optional, - Tuple, -) - -from m5.util import warn - -from _m5 import core - -from .atlasclient import AtlasClient -from .jsonclient import JSONClient - - -class ClientWrapper: - def __init__(self, config): - self.clients = self.create_clients(config) - - def create_clients( - self, - config: Dict, - ) -> Dict: - """ - This function creates respective client object for each source in the - config file according to the type of source. - - :param config: config file containing the source information - - :returns: clients: dictionary of clients for each source - """ - clients = {} - for client in config["sources"]: - client_source = config["sources"][client] - try: - if client_source["isMongo"]: - clients[client] = AtlasClient(client_source) - else: - clients[client] = JSONClient(client_source["url"]) - except Exception as e: - warn(f"Error creating client {client}: {str(e)}") - return clients - - def list_resources( - self, - clients: Optional[List[str]] = None, - gem5_version: Optional[str] = core.gem5Version, - ) -> Dict[str, List[str]]: - clients_to_search = ( - list(self.clients.keys()) if clients is None else clients - ) - # There's some duplications of functionality here (similar code in - # `get_all_resources_by_id`. This code could be refactored to avoid - # this). - resources = [] - for client in clients_to_search: - if client not in self.clients: - raise Exception(f"Client: {client} does not exist") - try: - resources.extend( - self.clients[client].get_resources( - gem5_version=gem5_version - ) - ) - except Exception as e: - warn(f"Error getting resources from client {client}: {str(e)}") - - to_return = {} - for resource in resources: - if resource["id"] not in to_return: - to_return[resource["id"]] = [] - to_return[resource["id"]].append(resource["resource_version"]) - return to_return - - def get_all_resources_by_id( - self, - resource_id: str, - clients: Optional[List[str]] = None, - ) -> List[Dict]: - """ - This function returns all the resources with the given id from all the - sources. - - :param resource_id: The id of the resource to search for. - :param clients: A list of clients to search through. If ``None``, all - clients are searched. - :return: A list of resources as Python dictionaries. - """ - resources = [] - if not clients: - clients = list(self.clients.keys()) - for client in clients: - if client not in self.clients: - raise Exception(f"Client: {client} does not exist") - try: - resources.extend( - self.clients[client].get_resources_by_id(resource_id) - ) - except Exception as e: - print( - f"Exception thrown while getting resource '{resource_id}' " - f"from client '{client}'\n", - file=sys.stderr, - ) - raise e - # check if no 2 resources have the same id and version - for res1, res2 in itertools.combinations(resources, 2): - if res1["resource_version"] == res2["resource_version"]: - raise Exception( - f"Resource {resource_id} has multiple resources with " - f"the same version: {res1['resource_version']}" - ) - return resources - - def get_resource_json_obj_from_client( - self, - resource_id: str, - resource_version: Optional[str] = None, - clients: Optional[List[str]] = None, - gem5_version: Optional[str] = core.gem5Version, - ) -> Dict: - """ - This function returns the resource object from the client with the - given id and version. - - :param resource_id: The id of the resource to search for. - :param resource_version: The version of the resource to search for. - :param clients: A list of clients to search through. If ``None``, all - clients are searched. - :param gem5_version: The gem5 version to check compatibility with. If - ``None``, no compatibility check is performed. By - default, is the current version of gem5. - :return: The resource object as a Python dictionary if found. - If not found, exception is thrown. - """ - # getting all the resources with the given id from the dictionary - resources = self.get_all_resources_by_id(resource_id, clients) - # if no resource with the given id is found, return None - if len(resources) == 0: - raise Exception(f"Resource with ID '{resource_id}' not found.") - - resource_to_return = None - - if resource_version: - resource_to_return = self._search_version_in_resources( - resources, resource_id, resource_version - ) - - else: - compatible_resources = ( - self._get_resources_compatible_with_gem5_version( - resources, gem5_version=gem5_version - ) - ) - if len(compatible_resources) == 0: - resource_to_return = self._sort_resources(resources)[0] - else: - resource_to_return = self._sort_resources( - compatible_resources - )[0] - - if gem5_version: - self._check_resource_version_compatibility( - resource_to_return, gem5_version=gem5_version - ) - - return resource_to_return - - def _search_version_in_resources( - self, resources: List, resource_id: str, resource_version: str - ) -> Dict: - """ - Searches for the resource with the given version. If the resource is - not found, an exception is thrown. - - :param resources: A list of resources to search through. - :param resource_version: The version of the resource to search for. - - :return: The resource object as a Python dictionary if found. - If not found, ``None`` is returned. - """ - return_resource = next( - iter( - [ - resource - for resource in resources - if resource["resource_version"] == resource_version - ] - ), - None, - ) - if not return_resource: - raise Exception( - f"Resource {resource_id} with version '{resource_version}'" - " not found.\nResource versions can be found at: " - "https://resources.gem5.org/" - f"resources/{resource_id}/versions" - ) - return return_resource - - def _get_resources_compatible_with_gem5_version( - self, resources: List, gem5_version: str = core.gem5Version - ) -> List: - """ - Returns a list of compatible resources with the current gem5 version. - - .. note:: - - This function assumes if the minor component of a resource's - gem5_version is not specified, it that the resource is compatible - all minor versions of the same major version. - - Likewise, if no hot-fix component is specified, it is assumed that - the resource is compatible with all hot-fix versions of the same - minor version. - - * '20.1' would be compatible with gem5 '20.1.1.0' and '20.1.2.0'. - * '21.5.2' would be compatible with gem5 '21.5.2.0' and '21.5.2.0'. - * '22.3.2.4' would only be compatible with gem5 '22.3.2.4'. - - :param resources: A list of resources to filter. - - :return: A list of compatible resources as Python dictionaries. - - .. note:: - - This is a big duplication of code. This functionality already - exists in the `AbstractClient` class. This code should be refactored - to avoid this duplication. - """ - - compatible_resources = [] - for resource in resources: - for version in resource["gem5_versions"]: - if gem5_version.startswith(version): - compatible_resources.append(resource) - return compatible_resources - - def _sort_resources(self, resources: List) -> List: - """ - Sorts the resources by ID. - - If the IDs are the same, the resources are sorted by version. - - :param resources: A list of resources to sort. - - :return: A list of sorted resources. - """ - - def sort_tuple(resource: Dict) -> Tuple: - """This is used for sorting resources by ID and version. First - the ID is sorted, then the version. In cases where the version - contains periods, it's assumed this is to separate a - ``major.minor.hotfix`` style versioning system. In which case, the - value separated in the most-significant position is sorted before - those less significant. If the value is a digit it is cast as an - int, otherwise, it is cast as a string, to lower-case. - """ - to_return = (resource["id"].lower(),) - for val in resource["resource_version"].split("."): - if val.isdigit(): - to_return += (int(val),) - else: - to_return += (str(val).lower(),) - return to_return - - return sorted( - resources, - key=lambda resource: sort_tuple(resource), - reverse=True, - ) - - def _check_resource_version_compatibility( - self, resource: dict, gem5_version: Optional[str] = core.gem5Version - ) -> bool: - """ - Checks if the resource is compatible with the gem5 version. - - Prints a warning if the resource is not compatible. - - :param resource: The resource to check. - :optional param gem5_version: The gem5 version to check - compatibility with. - :return: ``True`` if the resource is compatible, ``False`` otherwise. - """ - if not resource: - return False - if ( - gem5_version - and not gem5_version.upper().startswith("DEVELOP") - and not self._get_resources_compatible_with_gem5_version( - [resource], gem5_version=gem5_version - ) - ): - if not gem5_version.upper().startswith("DEVELOP"): - warn( - f"Resource {resource['id']} with version " - f"{resource['resource_version']} is not known to be compatible" - f" with gem5 version {gem5_version}. " - "This may cause problems with your simulation. " - "This resource's compatibility " - "with different gem5 versions can be found here: " - "https://resources.gem5.org" - f"/resources/{resource['id']}/versions" - ) - return False - return True diff --git a/src/python/gem5/resources/client_api/jsonclient.py b/src/python/gem5/resources/client_api/jsonclient.py index f242a788a9..81f4ae174e 100644 --- a/src/python/gem5/resources/client_api/jsonclient.py +++ b/src/python/gem5/resources/client_api/jsonclient.py @@ -41,6 +41,7 @@ from urllib.error import URLError from m5.util import warn from .abstract_client import AbstractClient +from .client_query import ClientQuery class JSONClient(AbstractClient): @@ -75,25 +76,49 @@ class JSONClient(AbstractClient): def get_resources( self, - resource_id: Optional[str] = None, - resource_version: Optional[str] = None, - gem5_version: Optional[str] = None, - ) -> List[Dict[str, Any]]: - filter = self.resources # Unfiltered. - if resource_id: - filter = [ # Filter by resource_id. - resource - for resource in filter - if resource["id"] == resource_id - ] - if resource_version: - filter = [ # Filter by resource_version. - resource - for resource in filter - if resource["resource_version"] == resource_version - ] + client_queries: List[ClientQuery], + ) -> Dict[str, Any]: + def filter_resource(resource, client_queries): + for resource_query in client_queries: + gem5_version_match = False + resource_version_match = False - # Filter by gem5_version. - return self.filter_incompatible_resources( - resources_to_filter=filter, gem5_version=gem5_version + if ( + resource_query.get_gem5_version() is not None + and not resource_query.get_gem5_version().startswith( + "DEVELOP" + ) + ): + gem5_version_match = ( + resource_query.get_gem5_version() + in resource["gem5_versions"] + ) + + if resource_query.get_resource_version() is not None: + resource_version_match = ( + resource["resource_version"] + == resource_query.get_resource_version() + ) + + if gem5_version_match and resource_version_match: + return True + + return False + + filtered_resources = filter( + lambda resource: filter_resource(resource, client_queries), + self.resources, ) + + resources_by_id = {} + for resource in filtered_resources: + if resource["resource_id"] in resources_by_id.keys(): + resources_by_id[resource["resource_id"]].append(resource) + else: + resources_by_id[resource["resource_id"]] = [resource] + + # Sort the resoruces by resoruce version and get the latest version. + for id, resource_list in resources_by_id.items(): + resources_by_id[id] = self.sort_resources(resource_list)[0] + + return resources_by_id diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 410dd3b018..b3432ffe07 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -24,7 +24,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json import os from abc import ABCMeta from functools import partial @@ -52,7 +51,11 @@ from ..isas import ( ISA, get_isa_from_str, ) -from .client import get_resource_json_obj +from .client import ( + get_multiple_resource_json_obj, + get_resource_json_obj, +) +from .client_api.client_query import ClientQuery from .downloader import get_resource from .looppoint import ( LooppointCsvLoader, @@ -754,8 +757,10 @@ class SuiteResource(AbstractResource): **kwargs, ) -> None: """ - :param workloads: A list of ``WorkloadResource`` objects - created from the ``_workloads`` parameter. + :param workloads: A Dict of Tuples containing the WorkloadResource + object as the key and a set of input groups as the + value. This Dict is created from the ``_workloads`` + parameter. :param local_path: The path on the host system where this resource is located. :param description: Description describing this resource. Not a @@ -834,46 +839,6 @@ class SuiteResource(AbstractResource): } -class ShadowResource(AbstractResource): - """A special resource class which delays the `obtain_resource` call. It is, - in a sense, half constructed. Only when a function or attribute is called - which is is neither `get_id` or `get_resource_version` does this class - fully construct itself by calling the `obtain_resource_call` partial - function. - - **Note:** This class is a hack. The ideal solution to this would be to - enable the bundled obtaining of resources in the gem5 Standard Library. - Use of the class is discouraged and should not be depended on. Issue - https://github.com/gem5/gem5/issues/644 is tracking the implementation of - an alternative. - """ - - def __init__( - self, - id: str, - resource_version: str, - obtain_resource_call: partial, - ): - super().__init__( - id=id, - resource_version=resource_version, - ) - self._workload: Optional[AbstractResource] = None - self._obtain_resource_call = obtain_resource_call - - def __getattr__(self, attr): - """if getting the id or resource version, we keep the object in the - "shdow state" where the `obtain_resource` function has not been called. - When more information is needed by calling another attribute, we call - the `obtain_resource` function and store the result in the `_workload`. - """ - if attr in {"get_id", "get_resource_version"}: - return getattr(super(), attr) - if not self._workload: - self._workload = self._obtain_resource_call() - return getattr(self._workload, attr) - - class WorkloadResource(AbstractResource): """A workload resource. This resource is used to specify a workload to run on a board. It contains the function to call and the parameters to pass to @@ -996,6 +961,234 @@ def obtain_resource( gem5_version=gem5_version, ) + to_path, downloader = _get_to_path_and_downloader_partial( + resource_json=resource_json, + to_path=to_path, + resource_directory=resource_directory, + download_md5_mismatch=download_md5_mismatch, + clients=clients, + gem5_version=gem5_version, + quiet=quiet, + ) + + # Obtain the type from the JSON. From this we will determine what subclass + # of `AbstractResource` we are to create and return. + resources_category = resource_json["category"] + + if resources_category == "resource": + # This is a stop-gap measure to ensure to work with older versions of + # the "resource.json" file. These should be replaced with their + # respective specializations ASAP and this case removed. + if "root_partition" in resource_json: + # In this case we should return a DiskImageResource. + root_partition = resource_json["root_partition"] + return DiskImageResource( + local_path=to_path, + root_partition=root_partition, + downloader=downloader, + **resource_json, + ) + return CustomResource(local_path=to_path, downloader=downloader) + + assert resources_category in _get_resource_json_type_map + resource_class = _get_resource_json_type_map[resources_category] + + if resources_category == "suite": + return _get_suite( + resource_json, + to_path, + resource_directory, + download_md5_mismatch, + clients, + gem5_version, + quiet, + ) + if resources_category == "workload": + # This parses the "resources" and "additional_params" fields of the + # workload resource into a dictionary of AbstractResource objects and + # strings respectively. + return _get_workload( + resource_json, + to_path, + resource_directory, + download_md5_mismatch, + clients, + gem5_version, + quiet, + ) + # Once we know what AbstractResource subclass we are using, we create it. + # The fields in the JSON object are assumed to map like-for-like to the + # subclass contructor, so we can pass the resource_json map directly. + return resource_class( + local_path=to_path, downloader=downloader, **resource_json + ) + + +def _get_suite( + suite: Dict[str, Any], + local_path: str, + resource_directory: str, + download_md5_mismatch: bool, + clients: List[str], + gem5_version: str, + quiet: bool, +) -> SuiteResource: + """ + :param suite: The suite JSON object. + :param local_path: The local path of the suite. + :param resource_directory: The resource directory. + :param download_md5_mismatch: If the resource is present, but does not have + the correct md5 value, the resource will be + deleted and re-downloaded if this value is ``True``. + Otherwise an exception will be thrown. + :param clients: A list of clients to search for the resource. If this + parameter is not set, it will default search all clients. + :param gem5_version: The gem5 version to use to filter incompatible + resource versions. By default set to the current gem5 + version. + :param quiet: If ``True``, suppress output. ``False`` by default. + """ + # Mapping input groups to workload IDs + id_input_group_dict = {} + for workload in suite["workloads"]: + id_input_group_dict[workload["id"]] = workload["input_group"] + + # Fetching the workload resources as a list of dicts + db_query = [ + ClientQuery( + resource_id=resource_info["id"], + resource_version=resource_info["resource_version"], + gem5_version=gem5_version, + ) + for resource_info in suite["workloads"] + ] + workload_json = get_multiple_resource_json_obj(db_query, clients) + + # Creating the workload resource objects for each workload + # and setting the input group for each workload + workload_input_group_dict = {} + for workload in workload_json: + workload_input_group_dict[ + _get_workload( + workload, + local_path, + resource_directory, + download_md5_mismatch, + clients, + gem5_version, + quiet, + ) + ] = id_input_group_dict[workload["id"]] + + suite["workloads"] = workload_input_group_dict + return SuiteResource( + local_path=local_path, + downloader=None, + **suite, + ) + + +def _get_workload( + workload: Dict[str, Any], + local_path: str, + resource_directory: str, + download_md5_mismatch: bool, + clients: List[str], + gem5_version: str, + quiet: bool, +) -> WorkloadResource: + """ + :param workload: The workload JSON object. + :param local_path: The local path of the workload. + :param resource_directory: The resource directory. + :param download_md5_mismatch: If the resource is present, but does not have + the correct md5 value, the resource will be + deleted and re-downloaded if this value is ``True``. + Otherwise an exception will be thrown. + :param clients: A list of clients to search for the resource. If this + parameter is not set, it will default search all clients. + :param gem5_version: The gem5 version to use to filter incompatible + resource versions. By default set to the current gem5 + version. + :param quiet: If ``True``, suppress output. ``False`` by default. + """ + params = {} + + db_query = [] + for resource in workload["resources"].values(): + db_query.append( + ClientQuery( + resource_id=resource["id"], + resource_version=resource["resource_version"], + gem5_version=gem5_version, + ) + ) + # Fetching resources as a list of dicts + resource_details_list = get_multiple_resource_json_obj(db_query, clients) + + # Creating the resource objects for each resource + for param_name, param_resource in workload["resources"].items(): + resource_match = None + for resource in resource_details_list: + if ( + param_resource["id"] == resource["id"] + and param_resource["resource_version"] + == resource["resource_version"] + ): + resource_match = resource + break + + if resource_match is None: + raise Exception( + f"Resource {param_resource['id']} with version {param_resource['resource_version']} not found" + ) + assert isinstance(param_name, str) + to_path, downloader = _get_to_path_and_downloader_partial( + resource_json=resource_match, + to_path=local_path, + resource_directory=resource_directory, + download_md5_mismatch=download_md5_mismatch, + clients=clients, + gem5_version=gem5_version, + quiet=quiet, + ) + + resource_class = _get_resource_json_type_map[ + resource_match["category"] + ] + + params[param_name] = resource_class( + local_path=to_path, + downloader=downloader, + **resource, + ) + + # Adding the additional parameters to the workload parameters + if workload["additional_params"]: + for key in workload["additional_params"].keys(): + assert isinstance(key, str) + value = workload["additional_params"][key] + params[key] = value + + return WorkloadResource( + local_path=local_path, + downloader=None, + parameters=params, + **workload, + ) + + +def _get_to_path_and_downloader_partial( + resource_json: Dict[str, str], + to_path: str, + resource_directory: str, + download_md5_mismatch: bool, + clients: List[str], + gem5_version: str, + quiet: bool, +) -> Tuple[str, Optional[partial]]: + resource_id = resource_json["id"] + resource_version = resource_json["resource_version"] # This is is used to store the partial function which is used to download # the resource when the `get_local_path` function is called. downloader: Optional[partial] = None @@ -1059,79 +1252,7 @@ def obtain_resource( gem5_version=gem5_version, quiet=quiet, ) - - # Obtain the type from the JSON. From this we will determine what subclass - # of `AbstractResource` we are to create and return. - resources_category = resource_json["category"] - - if resources_category == "resource": - # This is a stop-gap measure to ensure to work with older versions of - # the "resource.json" file. These should be replaced with their - # respective specializations ASAP and this case removed. - if "root_partition" in resource_json: - # In this case we should return a DiskImageResource. - root_partition = resource_json["root_partition"] - return DiskImageResource( - local_path=to_path, - root_partition=root_partition, - downloader=downloader, - **resource_json, - ) - return CustomResource(local_path=to_path, downloader=downloader) - - assert resources_category in _get_resource_json_type_map - resource_class = _get_resource_json_type_map[resources_category] - - if resources_category == "suite": - workloads = resource_json["workloads"] - workloads_obj = {} - for workload in workloads: - workloads_obj[ - ShadowResource( - id=workload["id"], - resource_version=workload["resource_version"], - obtain_resource_call=partial( - obtain_resource, - workload["id"], - resource_version=workload["resource_version"], - resource_directory=resource_directory, - clients=clients, - gem5_version=gem5_version, - ), - ) - ] = set(workload["input_group"]) - resource_json["workloads"] = workloads_obj - - if resources_category == "workload": - # This parses the "resources" and "additional_params" fields of the - # workload resource into a dictionary of AbstractResource objects and - # strings respectively. - params = {} - if "resources" in resource_json: - for key in resource_json["resources"].keys(): - assert isinstance(key, str) - value = resource_json["resources"][key] - - assert isinstance(value, dict) - params[key] = obtain_resource( - value["id"], - resource_version=value["resource_version"], - resource_directory=resource_directory, - clients=clients, - gem5_version=gem5_version, - ) - if "additional_params" in resource_json: - for key in resource_json["additional_params"].keys(): - assert isinstance(key, str) - value = resource_json["additional_params"][key] - params[key] = value - resource_json["parameters"] = params - # Once we know what AbstractResource subclass we are using, we create it. - # The fields in the JSON object are assumed to map like-for-like to the - # subclass contructor, so we can pass the resource_json map directly. - return resource_class( - local_path=to_path, downloader=downloader, **resource_json - ) + return to_path, downloader def _get_default_resource_dir() -> str: