derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stdlib: Enable bundled resource requests from the databases (#779)

Browse Source
This commit is contained in:
Harshil Patel
2024-04-22 11:53:23 -07:00
committed by GitHub
parent 40fdf368d8
commit 97a0530452
8 changed files with 654 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/python/SConscript
View File

@@ -296,10 +296,10 @@ PySource('gem5.resources.client_api',
'gem5/resources/client_api/jsonclient.py')
PySource('gem5.resources.client_api',
'gem5/resources/client_api/atlasclient.py')
PySource('gem5.resources.client_api',
'gem5/resources/client_api/client_wrapper.py')
PySource('gem5.resources.client_api',
'gem5/resources/client_api/abstract_client.py')
PySource('gem5.resources.client_api',
'gem5/resources/client_api/client_query.py')
PySource('gem5', 'gem5_default_config.py')
PySource('gem5.utils', 'gem5/utils/__init__.py')
PySource('gem5.utils', 'gem5/utils/filelock.py')

224
src/python/gem5/resources/client.py
View File

@@ -24,13 +24,17 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import itertools
import json
import os
import sys
from pathlib import Path
from typing import (
Any,
Dict,
List,
Optional,
Tuple,
)
from m5.util import (
@@ -42,7 +46,9 @@ from _m5 import core
from gem5.gem5_default_config import config
from .client_api.client_wrapper import ClientWrapper
from .client_api.atlasclient import AtlasClient
from .client_api.client_query import ClientQuery
from .client_api.jsonclient import JSONClient
def getFileContent(file_path: Path) -> Dict:
@@ -125,7 +131,7 @@ def _get_clientwrapper():
f"Appending resources from {os.environ['GEM5_RESOURCE_JSON_APPEND']}"
)
clientwrapper = ClientWrapper(gem5_config)
clientwrapper = _create_clients(gem5_config)
return clientwrapper
@@ -143,7 +149,8 @@ def list_resources(
:return: A Python Dict where the key is the resource id and the value is
a list of all the supported resource versions.
"""
return _get_clientwrapper().list_resources(clients, gem5_version)
_get_clientwrapper()
return _list_all_resources(clients, gem5_version)
def get_resource_json_obj(
@@ -163,7 +170,214 @@ def get_resource_json_obj(
current build. If ``None``, filtering based on compatibility
is not performed.
"""
_get_clientwrapper()
if resource_version:
client_queries = [
ClientQuery(resource_id, resource_version, gem5_version)
]
else:
client_queries = [ClientQuery(resource_id, gem5_version=gem5_version)]
return _get_clientwrapper().get_resource_json_obj_from_client(
resource_id, resource_version, clients, gem5_version
# We will return a list when we refactor ontain_resources to handle multiple
# resources
return _get_resource_json_obj_from_client(client_queries, clients)[0]
def get_multiple_resource_json_obj(
client_queries: List[ClientQuery],
clients: Optional[List[str]] = None,
) -> List[Dict]:
"""
Get the resource json object from the clients wrapper.
:param client_queries: This is a list of ClientQuery objects that contain
information about the resources to fetch from datasources.
:param clients: The list of clients to query.
"""
_get_clientwrapper()
return _get_resource_json_obj_from_client(client_queries, clients)
def _create_clients(
config: Dict,
) -> Dict:
"""
This function creates respective client object for each source in the
config file according to the type of source.
:param config: config file containing the source information
:returns: clients: dictionary of clients for each source
"""
clients = {}
for client in config["sources"]:
client_source = config["sources"][client]
try:
if client_source["isMongo"]:
clients[client] = AtlasClient(client_source)
else:
clients[client] = JSONClient(client_source["url"])
except Exception as e:
warn(f"Error creating client {client}: {str(e)}")
return clients
def _list_all_resources(
clients: Optional[List[str]] = None,
gem5_version: Optional[str] = core.gem5Version,
) -> Dict[str, List[str]]:
global clientwrapper
clients_to_search = (
list(clientwrapper.keys()) if clients is None else clients
)
# There's some duplications of functionality here (similar code in
# `get_all_resources_by_id`. This code could be refactored to avoid
# this).
resources = []
for client in clients_to_search:
if client not in clientwrapper:
raise Exception(f"Client: {client} does not exist")
try:
resources.extend(
clientwrapper[client].get_resources(gem5_version=gem5_version)
)
except Exception as e:
warn(f"Error getting resources from client {client}: {str(e)}")
to_return = {}
for resource in resources:
if resource["id"] not in to_return:
to_return[resource["id"]] = []
to_return[resource["id"]].append(resource["resource_version"])
return to_return
def _get_resource_json_obj_from_client(
client_queries: List[ClientQuery],
clients: Optional[List[str]] = None,
) -> Dict:
"""
This function returns the resource object from the client with the
given id and version.
:param client_queries: This is a list of ClientQuery objects that contain
information about the resources to fetch from datasources.
:param resource_version: The version of the resource to search for.
:param clients: A list of clients to search through. If ``None``, all
clients are searched.
:param gem5_version: The gem5 version to check compatibility with. If
``None``, no compatibility check is performed. By
default, is the current version of gem5.
:return: The resource object as a Python dictionary if found.
If not found, exception is thrown.
"""
# getting all the resources with the given id from the dictionary
resources_list = _get_all_resources_by_id(client_queries, clients)
for id, resources in resources_list.items():
# if no resource with the given id is found, return None
if len(resources) == 0:
raise Exception(f"Resource with ID '{id}' not found.")
resource_to_return = []
# if there are multiple resources with the same id, return the one with
# the highest version
for id, resources in resources_list.items():
resources_list[id] = _sort_resources(resources)
resource_to_return.append(resources_list[id][0])
return resource_to_return
def _get_all_resources_by_id(
client_queries: List[ClientQuery],
clients: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""
This function returns all the resources with the given id from all the
sources.
:param client_queries: This is a list of ClientQuery objects that contain
information about the resources to fetch from datasources.
:param clients: A list of clients to search through. If ``None``, all
clients are searched.
:return: A list of resources as Python dictionaries.
"""
global clientwrapper
# creating a dictionary with the resource id as the key and an empty
# list as the value, the list will be populated with different versions
# of the resource
resources = {}
for client_query in client_queries:
id = client_query.get_resource_id()
resources[id] = []
if not clients:
clients = list(clientwrapper.keys())
for client in clients:
if client not in clientwrapper:
raise Exception(f"Client: {client} does not exist")
try:
filtered_resources = clientwrapper[client].get_resources_by_id(
client_queries
)
for k in resources.keys():
if k in filtered_resources.keys():
resources[k].append(filtered_resources[k])
except Exception as e:
print(
f"Exception thrown while getting resources '{client_queries}' "
f"from client '{client}'\n",
file=sys.stderr,
)
raise e
# check if no 2 resources have the same id and version
for resource_id, different_version_of_resource in resources.items():
for res1, res2 in itertools.combinations(
different_version_of_resource, 2
):
if res1["resource_version"] == res2["resource_version"]:
raise Exception(
f"Resource {resource_id} has multiple resources with "
f"the same version: {res1['resource_version']}"
)
return resources
def _sort_resources(resources: List) -> List:
"""
Sorts the resources by ID.
If the IDs are the same, the resources are sorted by version.
:param resources: A list of resources to sort.
:return: A list of sorted resources.
"""
def sort_tuple(resource: Dict) -> Tuple:
"""This is used for sorting resources by ID and version. First
the ID is sorted, then the version. In cases where the version
contains periods, it's assumed this is to separate a
``major.minor.hotfix`` style versioning system. In which case, the
value separated in the most-significant position is sorted before
those less significant. If the value is a digit it is cast as an
int, otherwise, it is cast as a string, to lower-case.
"""
to_return = (resource["id"].lower(),)
for val in resource["resource_version"].split("."):
if val.isdigit():
to_return += (int(val),)
else:
to_return += (str(val).lower(),)
return to_return
return sorted(
resources,
key=lambda resource: sort_tuple(resource),
reverse=True,
)

62
src/python/gem5/resources/client_api/abstract_client.py
View File

@@ -34,8 +34,11 @@ from typing import (
Dict,
List,
Optional,
Tuple,
)
from .client_query import ClientQuery
class AbstractClient(ABC):
def _url_validator(self, url: str) -> bool:
@@ -55,13 +58,15 @@ class AbstractClient(ABC):
@abstractmethod
def get_resources(
self,
resource_id: Optional[str] = None,
resource_version: Optional[str] = None,
gem5_version: Optional[str] = None,
client_queries: List[ClientQuery],
) -> List[Dict[str, Any]]:
"""
:param resource_id: The ID of the Resource. Optional, if not set, all
resources will be returned.
:param client_queries: A list of client queries containing the
information to query the resources. Each
ClientQuery object can contain the following information:
- resource_id: The ID of the Resource.
- resource_version: The version of the `Resource`.
- gem5_version: The version of gem5.
:param resource_version: The version of the `Resource`. Optional, if
not set, all resource versions will be returned.
Note: If ``resource_id`` is not set, this
@@ -72,6 +77,40 @@ class AbstractClient(ABC):
"""
raise NotImplementedError
def sort_resources(self, resources: List) -> List:
"""
Sorts the resources by ID.
If the IDs are the same, the resources are sorted by version.
:param resources: A list of resources to sort.
:return: A list of sorted resources.
"""
def sort_tuple(resource: Dict) -> Tuple:
"""This is used for sorting resources by ID and version. First
the ID is sorted, then the version. In cases where the version
contains periods, it's assumed this is to separate a
``major.minor.hotfix`` style versioning system. In which case, the
value separated in the most-significant position is sorted before
those less significant. If the value is a digit it is cast as an
int, otherwise, it is cast as a string, to lower-case.
"""
to_return = (resource["id"].lower(),)
for val in resource["resource_version"].split("."):
if val.isdigit():
to_return += (int(val),)
else:
to_return += (str(val).lower(),)
return to_return
return sorted(
resources,
key=lambda resource: sort_tuple(resource),
reverse=True,
)
def filter_incompatible_resources(
self,
resources_to_filter: List[Dict[str, Any]],
@@ -111,10 +150,17 @@ class AbstractClient(ABC):
filtered_resources.append(resource)
return filtered_resources
def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]:
def get_resources_by_id(
self, client_queries: List[ClientQuery]
) -> List[Dict[str, Any]]:
"""
:param resource_id: The ID of the Resource.
:param client_queries: A list of ClientQuery objects containing the
information to query the resources. Each
ClientQuery object can contain the following information:
- resource_id: The ID of the Resource.
- resource_version: The version of the `Resource`.
- gem5_version: The version of gem5.
:return: A list of all the Resources with the given ID.
"""
return self.get_resources(resource_id=resource_id)
return self.get_resources(client_queries=client_queries)

55
src/python/gem5/resources/client_api/atlasclient.py
View File

@@ -45,6 +45,7 @@ from m5.util import warn
from ...utils.socks_ssl_context import get_proxy_context
from .abstract_client import AbstractClient
from .client_query import ClientQuery
class AtlasClientHttpJsonRequestError(Exception):
@@ -156,24 +157,39 @@ class AtlasClient(AbstractClient):
def get_resources(
self,
resource_id: Optional[str] = None,
resource_version: Optional[str] = None,
gem5_version: Optional[str] = None,
) -> List[Dict[str, Any]]:
client_queries: List[ClientQuery],
) -> Dict[str, Any]:
url = f"{self.url}/action/find"
data = {
"dataSource": self.dataSource,
"collection": self.collection,
"database": self.database,
}
filter = {}
if resource_id:
filter["id"] = resource_id
if resource_version is not None:
filter["resource_version"] = resource_version
if filter:
data["filter"] = filter
search_conditions = []
for resource in client_queries:
condition = {
"id": resource.get_resource_id(),
}
if not resource.get_gem5_version().startswith("DEVELOP"):
# This is a regex search that matches the beginning of the
# string. So if the resource version is '20.1', it will
# match '20.1.1'.
condition["gem5_versions"] = {
"$regex": f"^{resource.get_gem5_version()}",
"$options": "i",
}
# If the resource has a resource_version, add it to the search
# conditions.
if resource.get_resource_version():
condition["resource_version"] = resource.get_resource_version()
search_conditions.append(condition)
filter = {"$or": search_conditions}
data["filter"] = filter
headers = {
"Authorization": f"Bearer {self.get_token()}",
@@ -187,8 +203,15 @@ class AtlasClient(AbstractClient):
purpose_of_request="Get Resources",
)["documents"]
# I do this as a lazy post-processing step because I can't figure out
# how to do this via an Atlas query, which may be more efficient.
return self.filter_incompatible_resources(
resources_to_filter=resources, gem5_version=gem5_version
)
resources_by_id = {}
for resource in resources:
if resource["id"] in resources_by_id.keys():
resources_by_id[resource["id"]].append(resource)
else:
resources_by_id[resource["id"]] = [resource]
# Sort the resources by version and return the latest version.
for id, resource_list in resources_by_id.items():
resources_by_id[id] = self.sort_resources(resource_list)[0]
return resources_by_id

57
src/python/gem5/resources/client_api/client_query.py Normal file
View File

@@ -0,0 +1,57 @@
# Copyright (c) 2024 The Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from typing import Optional
from _m5 import core
"""
This class is a data class that represents a query to the client.
It encapsulates the fields required to query resources from the client.
Right now, it only contains the resource_id, resource_version, and gem5_version
fields, but it can be expanded to include more fields in the future, if needed.
"""
class ClientQuery:
def __init__(
self,
resource_id: str,
resource_version: Optional[str] = None,
gem5_version: Optional[str] = core.gem5Version,
):
self.resource_id = resource_id
self.resource_version = resource_version
self.gem5_version = gem5_version
def get_resource_id(self) -> str:
return self.resource_id
def get_resource_version(self) -> Optional[str]:
return self.resource_version
def get_gem5_version(self) -> Optional[str]:
return self.gem5_version

335
src/python/gem5/resources/client_api/client_wrapper.py
View File

@@ -1,335 +0,0 @@
# Copyright (c) 2023 The Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import itertools
import sys
from typing import (
Dict,
List,
Optional,
Tuple,
)
from m5.util import warn
from _m5 import core
from .atlasclient import AtlasClient
from .jsonclient import JSONClient
class ClientWrapper:
def __init__(self, config):
self.clients = self.create_clients(config)
def create_clients(
self,
config: Dict,
) -> Dict:
"""
This function creates respective client object for each source in the
config file according to the type of source.
:param config: config file containing the source information
:returns: clients: dictionary of clients for each source
"""
clients = {}
for client in config["sources"]:
client_source = config["sources"][client]
try:
if client_source["isMongo"]:
clients[client] = AtlasClient(client_source)
else:
clients[client] = JSONClient(client_source["url"])
except Exception as e:
warn(f"Error creating client {client}: {str(e)}")
return clients
def list_resources(
self,
clients: Optional[List[str]] = None,
gem5_version: Optional[str] = core.gem5Version,
) -> Dict[str, List[str]]:
clients_to_search = (
list(self.clients.keys()) if clients is None else clients
)
# There's some duplications of functionality here (similar code in
# `get_all_resources_by_id`. This code could be refactored to avoid
# this).
resources = []
for client in clients_to_search:
if client not in self.clients:
raise Exception(f"Client: {client} does not exist")
try:
resources.extend(
self.clients[client].get_resources(
gem5_version=gem5_version
)
)
except Exception as e:
warn(f"Error getting resources from client {client}: {str(e)}")
to_return = {}
for resource in resources:
if resource["id"] not in to_return:
to_return[resource["id"]] = []
to_return[resource["id"]].append(resource["resource_version"])
return to_return
def get_all_resources_by_id(
self,
resource_id: str,
clients: Optional[List[str]] = None,
) -> List[Dict]:
"""
This function returns all the resources with the given id from all the
sources.
:param resource_id: The id of the resource to search for.
:param clients: A list of clients to search through. If ``None``, all
clients are searched.
:return: A list of resources as Python dictionaries.
"""
resources = []
if not clients:
clients = list(self.clients.keys())
for client in clients:
if client not in self.clients:
raise Exception(f"Client: {client} does not exist")
try:
resources.extend(
self.clients[client].get_resources_by_id(resource_id)
)
except Exception as e:
print(
f"Exception thrown while getting resource '{resource_id}' "
f"from client '{client}'\n",
file=sys.stderr,
)
raise e
# check if no 2 resources have the same id and version
for res1, res2 in itertools.combinations(resources, 2):
if res1["resource_version"] == res2["resource_version"]:
raise Exception(
f"Resource {resource_id} has multiple resources with "
f"the same version: {res1['resource_version']}"
)
return resources
def get_resource_json_obj_from_client(
self,
resource_id: str,
resource_version: Optional[str] = None,
clients: Optional[List[str]] = None,
gem5_version: Optional[str] = core.gem5Version,
) -> Dict:
"""
This function returns the resource object from the client with the
given id and version.
:param resource_id: The id of the resource to search for.
:param resource_version: The version of the resource to search for.
:param clients: A list of clients to search through. If ``None``, all
clients are searched.
:param gem5_version: The gem5 version to check compatibility with. If
``None``, no compatibility check is performed. By
default, is the current version of gem5.
:return: The resource object as a Python dictionary if found.
If not found, exception is thrown.
"""
# getting all the resources with the given id from the dictionary
resources = self.get_all_resources_by_id(resource_id, clients)
# if no resource with the given id is found, return None
if len(resources) == 0:
raise Exception(f"Resource with ID '{resource_id}' not found.")
resource_to_return = None
if resource_version:
resource_to_return = self._search_version_in_resources(
resources, resource_id, resource_version
)
else:
compatible_resources = (
self._get_resources_compatible_with_gem5_version(
resources, gem5_version=gem5_version
)
)
if len(compatible_resources) == 0:
resource_to_return = self._sort_resources(resources)[0]
else:
resource_to_return = self._sort_resources(
compatible_resources
)[0]
if gem5_version:
self._check_resource_version_compatibility(
resource_to_return, gem5_version=gem5_version
)
return resource_to_return
def _search_version_in_resources(
self, resources: List, resource_id: str, resource_version: str
) -> Dict:
"""
Searches for the resource with the given version. If the resource is
not found, an exception is thrown.
:param resources: A list of resources to search through.
:param resource_version: The version of the resource to search for.
:return: The resource object as a Python dictionary if found.
If not found, ``None`` is returned.
"""
return_resource = next(
iter(
[
resource
for resource in resources
if resource["resource_version"] == resource_version
]
),
None,
)
if not return_resource:
raise Exception(
f"Resource {resource_id} with version '{resource_version}'"
" not found.\nResource versions can be found at: "
"https://resources.gem5.org/"
f"resources/{resource_id}/versions"
)
return return_resource
def _get_resources_compatible_with_gem5_version(
self, resources: List, gem5_version: str = core.gem5Version
) -> List:
"""
Returns a list of compatible resources with the current gem5 version.
.. note::
This function assumes if the minor component of a resource's
gem5_version is not specified, it that the resource is compatible
all minor versions of the same major version.
Likewise, if no hot-fix component is specified, it is assumed that
the resource is compatible with all hot-fix versions of the same
minor version.
* '20.1' would be compatible with gem5 '20.1.1.0' and '20.1.2.0'.
* '21.5.2' would be compatible with gem5 '21.5.2.0' and '21.5.2.0'.
* '22.3.2.4' would only be compatible with gem5 '22.3.2.4'.
:param resources: A list of resources to filter.
:return: A list of compatible resources as Python dictionaries.
.. note::
This is a big duplication of code. This functionality already
exists in the `AbstractClient` class. This code should be refactored
to avoid this duplication.
"""
compatible_resources = []
for resource in resources:
for version in resource["gem5_versions"]:
if gem5_version.startswith(version):
compatible_resources.append(resource)
return compatible_resources
def _sort_resources(self, resources: List) -> List:
"""
Sorts the resources by ID.
If the IDs are the same, the resources are sorted by version.
:param resources: A list of resources to sort.
:return: A list of sorted resources.
"""
def sort_tuple(resource: Dict) -> Tuple:
"""This is used for sorting resources by ID and version. First
the ID is sorted, then the version. In cases where the version
contains periods, it's assumed this is to separate a
``major.minor.hotfix`` style versioning system. In which case, the
value separated in the most-significant position is sorted before
those less significant. If the value is a digit it is cast as an
int, otherwise, it is cast as a string, to lower-case.
"""
to_return = (resource["id"].lower(),)
for val in resource["resource_version"].split("."):
if val.isdigit():
to_return += (int(val),)
else:
to_return += (str(val).lower(),)
return to_return
return sorted(
resources,
key=lambda resource: sort_tuple(resource),
reverse=True,
)
def _check_resource_version_compatibility(
self, resource: dict, gem5_version: Optional[str] = core.gem5Version
) -> bool:
"""
Checks if the resource is compatible with the gem5 version.
Prints a warning if the resource is not compatible.
:param resource: The resource to check.
:optional param gem5_version: The gem5 version to check
compatibility with.
:return: ``True`` if the resource is compatible, ``False`` otherwise.
"""
if not resource:
return False
if (
gem5_version
and not gem5_version.upper().startswith("DEVELOP")
and not self._get_resources_compatible_with_gem5_version(
[resource], gem5_version=gem5_version
)
):
if not gem5_version.upper().startswith("DEVELOP"):
warn(
f"Resource {resource['id']} with version "
f"{resource['resource_version']} is not known to be compatible"
f" with gem5 version {gem5_version}. "
"This may cause problems with your simulation. "
"This resource's compatibility "
"with different gem5 versions can be found here: "
"https://resources.gem5.org"
f"/resources/{resource['id']}/versions"
)
return False
return True

65
src/python/gem5/resources/client_api/jsonclient.py
View File

@@ -41,6 +41,7 @@ from urllib.error import URLError
from m5.util import warn
from .abstract_client import AbstractClient
from .client_query import ClientQuery
class JSONClient(AbstractClient):
@@ -75,25 +76,49 @@ class JSONClient(AbstractClient):
def get_resources(
self,
resource_id: Optional[str] = None,
resource_version: Optional[str] = None,
gem5_version: Optional[str] = None,
) -> List[Dict[str, Any]]:
filter = self.resources # Unfiltered.
if resource_id:
filter = [ # Filter by resource_id.
resource
for resource in filter
if resource["id"] == resource_id
]
if resource_version:
filter = [ # Filter by resource_version.
resource
for resource in filter
if resource["resource_version"] == resource_version
]
client_queries: List[ClientQuery],
) -> Dict[str, Any]:
def filter_resource(resource, client_queries):
for resource_query in client_queries:
gem5_version_match = False
resource_version_match = False
# Filter by gem5_version.
return self.filter_incompatible_resources(
resources_to_filter=filter, gem5_version=gem5_version
if (
resource_query.get_gem5_version() is not None
and not resource_query.get_gem5_version().startswith(
"DEVELOP"
)
):
gem5_version_match = (
resource_query.get_gem5_version()
in resource["gem5_versions"]
)
if resource_query.get_resource_version() is not None:
resource_version_match = (
resource["resource_version"]
== resource_query.get_resource_version()
)
if gem5_version_match and resource_version_match:
return True
return False
filtered_resources = filter(
lambda resource: filter_resource(resource, client_queries),
self.resources,
)
resources_by_id = {}
for resource in filtered_resources:
if resource["resource_id"] in resources_by_id.keys():
resources_by_id[resource["resource_id"]].append(resource)
else:
resources_by_id[resource["resource_id"]] = [resource]
# Sort the resoruces by resoruce version and get the latest version.
for id, resource_list in resources_by_id.items():
resources_by_id[id] = self.sort_resources(resource_list)[0]
return resources_by_id

355
src/python/gem5/resources/resource.py
View File

@@ -24,7 +24,6 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import json
import os
from abc import ABCMeta
from functools import partial
@@ -52,7 +51,11 @@ from ..isas import (
ISA,
get_isa_from_str,
)
from .client import get_resource_json_obj
from .client import (
get_multiple_resource_json_obj,
get_resource_json_obj,
)
from .client_api.client_query import ClientQuery
from .downloader import get_resource
from .looppoint import (
LooppointCsvLoader,
@@ -754,8 +757,10 @@ class SuiteResource(AbstractResource):
**kwargs,
) -> None:
"""
:param workloads: A list of ``WorkloadResource`` objects
created from the ``_workloads`` parameter.
:param workloads: A Dict of Tuples containing the WorkloadResource
object as the key and a set of input groups as the
value. This Dict is created from the ``_workloads``
parameter.
:param local_path: The path on the host system where this resource is
located.
:param description: Description describing this resource. Not a
@@ -834,46 +839,6 @@ class SuiteResource(AbstractResource):
}
class ShadowResource(AbstractResource):
"""A special resource class which delays the `obtain_resource` call. It is,
in a sense, half constructed. Only when a function or attribute is called
which is is neither `get_id` or `get_resource_version` does this class
fully construct itself by calling the `obtain_resource_call` partial
function.
**Note:** This class is a hack. The ideal solution to this would be to
enable the bundled obtaining of resources in the gem5 Standard Library.
Use of the class is discouraged and should not be depended on. Issue
https://github.com/gem5/gem5/issues/644 is tracking the implementation of
an alternative.
"""
def __init__(
self,
id: str,
resource_version: str,
obtain_resource_call: partial,
):
super().__init__(
id=id,
resource_version=resource_version,
)
self._workload: Optional[AbstractResource] = None
self._obtain_resource_call = obtain_resource_call
def __getattr__(self, attr):
"""if getting the id or resource version, we keep the object in the
"shdow state" where the `obtain_resource` function has not been called.
When more information is needed by calling another attribute, we call
the `obtain_resource` function and store the result in the `_workload`.
"""
if attr in {"get_id", "get_resource_version"}:
return getattr(super(), attr)
if not self._workload:
self._workload = self._obtain_resource_call()
return getattr(self._workload, attr)
class WorkloadResource(AbstractResource):
"""A workload resource. This resource is used to specify a workload to run
on a board. It contains the function to call and the parameters to pass to
@@ -996,6 +961,234 @@ def obtain_resource(
gem5_version=gem5_version,
)
to_path, downloader = _get_to_path_and_downloader_partial(
resource_json=resource_json,
to_path=to_path,
resource_directory=resource_directory,
download_md5_mismatch=download_md5_mismatch,
clients=clients,
gem5_version=gem5_version,
quiet=quiet,
)
# Obtain the type from the JSON. From this we will determine what subclass
# of `AbstractResource` we are to create and return.
resources_category = resource_json["category"]
if resources_category == "resource":
# This is a stop-gap measure to ensure to work with older versions of
# the "resource.json" file. These should be replaced with their
# respective specializations ASAP and this case removed.
if "root_partition" in resource_json:
# In this case we should return a DiskImageResource.
root_partition = resource_json["root_partition"]
return DiskImageResource(
local_path=to_path,
root_partition=root_partition,
downloader=downloader,
**resource_json,
)
return CustomResource(local_path=to_path, downloader=downloader)
assert resources_category in _get_resource_json_type_map
resource_class = _get_resource_json_type_map[resources_category]
if resources_category == "suite":
return _get_suite(
resource_json,
to_path,
resource_directory,
download_md5_mismatch,
clients,
gem5_version,
quiet,
)
if resources_category == "workload":
# This parses the "resources" and "additional_params" fields of the
# workload resource into a dictionary of AbstractResource objects and
# strings respectively.
return _get_workload(
resource_json,
to_path,
resource_directory,
download_md5_mismatch,
clients,
gem5_version,
quiet,
)
# Once we know what AbstractResource subclass we are using, we create it.
# The fields in the JSON object are assumed to map like-for-like to the
# subclass contructor, so we can pass the resource_json map directly.
return resource_class(
local_path=to_path, downloader=downloader, **resource_json
)
def _get_suite(
suite: Dict[str, Any],
local_path: str,
resource_directory: str,
download_md5_mismatch: bool,
clients: List[str],
gem5_version: str,
quiet: bool,
) -> SuiteResource:
"""
:param suite: The suite JSON object.
:param local_path: The local path of the suite.
:param resource_directory: The resource directory.
:param download_md5_mismatch: If the resource is present, but does not have
the correct md5 value, the resource will be
deleted and re-downloaded if this value is ``True``.
Otherwise an exception will be thrown.
:param clients: A list of clients to search for the resource. If this
parameter is not set, it will default search all clients.
:param gem5_version: The gem5 version to use to filter incompatible
resource versions. By default set to the current gem5
version.
:param quiet: If ``True``, suppress output. ``False`` by default.
"""
# Mapping input groups to workload IDs
id_input_group_dict = {}
for workload in suite["workloads"]:
id_input_group_dict[workload["id"]] = workload["input_group"]
# Fetching the workload resources as a list of dicts
db_query = [
ClientQuery(
resource_id=resource_info["id"],
resource_version=resource_info["resource_version"],
gem5_version=gem5_version,
)
for resource_info in suite["workloads"]
]
workload_json = get_multiple_resource_json_obj(db_query, clients)
# Creating the workload resource objects for each workload
# and setting the input group for each workload
workload_input_group_dict = {}
for workload in workload_json:
workload_input_group_dict[
_get_workload(
workload,
local_path,
resource_directory,
download_md5_mismatch,
clients,
gem5_version,
quiet,
)
] = id_input_group_dict[workload["id"]]
suite["workloads"] = workload_input_group_dict
return SuiteResource(
local_path=local_path,
downloader=None,
**suite,
)
def _get_workload(
workload: Dict[str, Any],
local_path: str,
resource_directory: str,
download_md5_mismatch: bool,
clients: List[str],
gem5_version: str,
quiet: bool,
) -> WorkloadResource:
"""
:param workload: The workload JSON object.
:param local_path: The local path of the workload.
:param resource_directory: The resource directory.
:param download_md5_mismatch: If the resource is present, but does not have
the correct md5 value, the resource will be
deleted and re-downloaded if this value is ``True``.
Otherwise an exception will be thrown.
:param clients: A list of clients to search for the resource. If this
parameter is not set, it will default search all clients.
:param gem5_version: The gem5 version to use to filter incompatible
resource versions. By default set to the current gem5
version.
:param quiet: If ``True``, suppress output. ``False`` by default.
"""
params = {}
db_query = []
for resource in workload["resources"].values():
db_query.append(
ClientQuery(
resource_id=resource["id"],
resource_version=resource["resource_version"],
gem5_version=gem5_version,
)
)
# Fetching resources as a list of dicts
resource_details_list = get_multiple_resource_json_obj(db_query, clients)
# Creating the resource objects for each resource
for param_name, param_resource in workload["resources"].items():
resource_match = None
for resource in resource_details_list:
if (
param_resource["id"] == resource["id"]
and param_resource["resource_version"]
== resource["resource_version"]
):
resource_match = resource
break
if resource_match is None:
raise Exception(
f"Resource {param_resource['id']} with version {param_resource['resource_version']} not found"
)
assert isinstance(param_name, str)
to_path, downloader = _get_to_path_and_downloader_partial(
resource_json=resource_match,
to_path=local_path,
resource_directory=resource_directory,
download_md5_mismatch=download_md5_mismatch,
clients=clients,
gem5_version=gem5_version,
quiet=quiet,
)
resource_class = _get_resource_json_type_map[
resource_match["category"]
]
params[param_name] = resource_class(
local_path=to_path,
downloader=downloader,
**resource,
)
# Adding the additional parameters to the workload parameters
if workload["additional_params"]:
for key in workload["additional_params"].keys():
assert isinstance(key, str)
value = workload["additional_params"][key]
params[key] = value
return WorkloadResource(
local_path=local_path,
downloader=None,
parameters=params,
**workload,
)
def _get_to_path_and_downloader_partial(
resource_json: Dict[str, str],
to_path: str,
resource_directory: str,
download_md5_mismatch: bool,
clients: List[str],
gem5_version: str,
quiet: bool,
) -> Tuple[str, Optional[partial]]:
resource_id = resource_json["id"]
resource_version = resource_json["resource_version"]
# This is is used to store the partial function which is used to download
# the resource when the `get_local_path` function is called.
downloader: Optional[partial] = None
@@ -1059,79 +1252,7 @@ def obtain_resource(
gem5_version=gem5_version,
quiet=quiet,
)
# Obtain the type from the JSON. From this we will determine what subclass
# of `AbstractResource` we are to create and return.
resources_category = resource_json["category"]
if resources_category == "resource":
# This is a stop-gap measure to ensure to work with older versions of
# the "resource.json" file. These should be replaced with their
# respective specializations ASAP and this case removed.
if "root_partition" in resource_json:
# In this case we should return a DiskImageResource.
root_partition = resource_json["root_partition"]
return DiskImageResource(
local_path=to_path,
root_partition=root_partition,
downloader=downloader,
**resource_json,
)
return CustomResource(local_path=to_path, downloader=downloader)
assert resources_category in _get_resource_json_type_map
resource_class = _get_resource_json_type_map[resources_category]
if resources_category == "suite":
workloads = resource_json["workloads"]
workloads_obj = {}
for workload in workloads:
workloads_obj[
ShadowResource(
id=workload["id"],
resource_version=workload["resource_version"],
obtain_resource_call=partial(
obtain_resource,
workload["id"],
resource_version=workload["resource_version"],
resource_directory=resource_directory,
clients=clients,
gem5_version=gem5_version,
),
)
] = set(workload["input_group"])
resource_json["workloads"] = workloads_obj
if resources_category == "workload":
# This parses the "resources" and "additional_params" fields of the
# workload resource into a dictionary of AbstractResource objects and
# strings respectively.
params = {}
if "resources" in resource_json:
for key in resource_json["resources"].keys():
assert isinstance(key, str)
value = resource_json["resources"][key]
assert isinstance(value, dict)
params[key] = obtain_resource(
value["id"],
resource_version=value["resource_version"],
resource_directory=resource_directory,
clients=clients,
gem5_version=gem5_version,
)
if "additional_params" in resource_json:
for key in resource_json["additional_params"].keys():
assert isinstance(key, str)
value = resource_json["additional_params"][key]
params[key] = value
resource_json["parameters"] = params
# Once we know what AbstractResource subclass we are using, we create it.
# The fields in the JSON object are assumed to map like-for-like to the
# subclass contructor, so we can pass the resource_json map directly.
return resource_class(
local_path=to_path, downloader=downloader, **resource_json
)
return to_path, downloader
def _get_default_resource_dir() -> str: