aff1e7b64c
This patch includes several changes to the gem5 tools interface to the
gem5-resources infrastructure. These are:
* The old download and JSON query functions have been removed from the
downloader module. These functions were used for directly downloading
and inspecting the resource JSON file, hosted at
https://resources.gem5.org/resources. This information is now obtained
via `gem5.client`. If a resources JSON file is specified as a client,
it should conform to the new schema:
https//resources.gem5.org/gem5-resources-schema.json. The old schema
(pre-v23.0) is no longer valid. Tests have been updated to reflect
this change. Those which tested these old functions have been removed.
* Unused imports have been removed.
* For the resource query functions, and those tasked with obtaining the
resources, the parameter `gem5_version` has been added. In all cases
it does the same thing:
* It will filter results based on compatibility to the
`gem5_version` specified. If no resources are compatible the
latest version of that resource is chosen (though a warning is
thrown).
* By default it is set to the current gem5 version.
* It is optional. If `None`, this filtering functionality is not
carried out.
* Tests have been updated to fix the version to “develop” so the
they do not break between versions.
* The `gem5_version` parameters will filter using a logic which will
base compatibility on the specificity of the gem5-version specified in
a resource’s data. If a resource has a compatible gem5-version of
“v18.4” it will be compatible with any minor/hotfix version within the
v18.4 release (this can be seen as matching on “v18.4.*.*”.) Likewise,
if a resource has a compatible gem5-version of “v18.4.1” then it’s
only compatible with the v18.4.1 release but any of it’s hot fix
releases (“v18.4.1.*”).
* The ‘list_resources’ function has been updated to use the
“gem5.client” APIs to get resource information from the clients
(MongoDB or a JSON file). This has been designed to remain backwards
compatible to as much as is possible, though, due to schema changes,
the function does search across all versions of gem5.
* `get_resources` function was added to the `AbstractClient`. This is a
more general function than `get_resource_by_id`. It was
primarily created to handle the `list_resources` update but is a
useful update to the API. The `get_resource_by_id` function has been
altered to function as a wrapped to the `get_resources` function.
* Removed “GEM5_RESOURCE_JSON” code has been removed. This is no longer
used.
* Tests have been cleaned up a little bit to be easier to read.
* Some docstrings have been updated.
Things that are left TODO with this code:
* The client_wrapper/client/abstract_client abstractions are rather
pointless. In particular the client_wrapper and client classes could
be merged.
* The downloader module no longer does much and should have its
functions merged into other modules.
* With the addition of the `get_resources` function, much of the code in
the `AbstractClient` could be simplified.
Change-Id: I0ce48e88b93a2b9db53d4749861fa0b5f9472053
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71506
Reviewed-by: Kunal Pai <kunpai@ucdavis.edu>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>
(cherry picked from commit 82587ce71b)
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71739
Reviewed-by: Bobby Bruce <bbruce@ucdavis.edu>
Maintainer: Bobby Bruce <bbruce@ucdavis.edu>
182 lines
8.0 KiB
Python
182 lines
8.0 KiB
Python
# Copyright (c) 2023 The Regents of the University of California
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met: redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer;
|
|
# redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution;
|
|
# neither the name of the copyright holders nor the names of its
|
|
# contributors may be used to endorse or promote products derived from
|
|
# this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
import unittest
|
|
import tempfile
|
|
import os
|
|
from typing import Dict
|
|
import json
|
|
|
|
from gem5.resources.client_api.jsonclient import JSONClient
|
|
|
|
|
|
class JSONClientTestSuite(unittest.TestCase):
|
|
"""Test cases for gem5.resources.client_api.jsonclient"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls) -> str:
|
|
"""
|
|
This creates a simple resources collection for testing
|
|
"""
|
|
file_contents = [
|
|
{
|
|
"category": "binary",
|
|
"id": "this-is-a-test-resource",
|
|
"description": "This is a test resource",
|
|
"architecture": "X86",
|
|
"size": 13816,
|
|
"tags": ["asmtest", "testing", "riscv", "testing"],
|
|
"is_zipped": False,
|
|
"md5sum": "4e70a98b6976969deffff91eed17fba1",
|
|
"source": "src/asmtest",
|
|
"url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak",
|
|
"code_examples": [],
|
|
"license": " BSD-3-Clause",
|
|
"author": [],
|
|
"source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest",
|
|
"resource_version": "1.0.0",
|
|
"gem5_versions": ["23.0"],
|
|
"example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")',
|
|
},
|
|
{
|
|
"category": "binary",
|
|
"id": "this-is-a-test-resource",
|
|
"description": "This is a test resource but double newer",
|
|
"architecture": "X86",
|
|
"size": 13816,
|
|
"tags": ["asmtest"],
|
|
"is_zipped": False,
|
|
"md5sum": "4e70a98b6976969deffff91eed17fba1",
|
|
"source": "src/asmtest",
|
|
"url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak",
|
|
"code_examples": [],
|
|
"license": " BSD-3-Clause",
|
|
"author": [],
|
|
"source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest",
|
|
"resource_version": "2.0.0",
|
|
"gem5_versions": ["23.1"],
|
|
"example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")',
|
|
},
|
|
{
|
|
"category": "simpoint",
|
|
"id": "test-version",
|
|
"description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.",
|
|
"architecture": "X86",
|
|
"size": 10240,
|
|
"tags": [],
|
|
"is_zipped": False,
|
|
"md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb",
|
|
"is_tar_archive": True,
|
|
"url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar",
|
|
"simpoint_interval": 1000000,
|
|
"warmup_interval": 1000000,
|
|
"code_examples": [],
|
|
"license": "",
|
|
"author": [],
|
|
"source_url": "",
|
|
"resource_version": "1.0.0",
|
|
"gem5_versions": ["23.0"],
|
|
"workload_name": "x86-print-this-15000-with-simpoints",
|
|
"example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")',
|
|
"workloads": [
|
|
"x86-print-this-15000-with-simpoints",
|
|
"x86-print-this-15000-with-simpoints-and-checkpoint",
|
|
],
|
|
},
|
|
{
|
|
"category": "file",
|
|
"id": "test-version",
|
|
"description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.",
|
|
"architecture": "X86",
|
|
"size": 10240,
|
|
"tags": [],
|
|
"is_zipped": False,
|
|
"md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb",
|
|
"is_tar_archive": True,
|
|
"url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar",
|
|
"simpoint_interval": 1000000,
|
|
"warmup_interval": 1000000,
|
|
"code_examples": [],
|
|
"license": "",
|
|
"author": [],
|
|
"source_url": "",
|
|
"resource_version": "0.2.0",
|
|
"gem5_versions": ["23.0"],
|
|
"workload_name": "x86-print-this-15000-with-simpoints",
|
|
"example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")',
|
|
"workloads": [
|
|
"x86-print-this-15000-with-simpoints",
|
|
"x86-print-this-15000-with-simpoints-and-checkpoint",
|
|
],
|
|
},
|
|
]
|
|
file = tempfile.NamedTemporaryFile(mode="w", delete=False)
|
|
file.write(json.dumps(file_contents))
|
|
file.close()
|
|
cls.file_path = file.name
|
|
|
|
@classmethod
|
|
def tearDownClass(cls) -> None:
|
|
os.remove(cls.file_path)
|
|
|
|
def verify_json(self, json: Dict) -> None:
|
|
"""
|
|
This verifies the JSON file created in created in
|
|
"create_temp_resources_json" has been loaded correctly into a Python
|
|
dictionary.
|
|
"""
|
|
self.assertEquals(4, len(json))
|
|
self.assertTrue("id" in json[0])
|
|
self.assertEquals("this-is-a-test-resource", json[0]["id"])
|
|
self.assertEquals("binary", json[0]["category"])
|
|
self.assertTrue("id" in json[1])
|
|
self.assertEquals("this-is-a-test-resource", json[1]["id"])
|
|
self.assertTrue("id" in json[2])
|
|
self.assertEquals("test-version", json[2]["id"])
|
|
self.assertTrue("id" in json[3])
|
|
self.assertEquals("test-version", json[3]["id"])
|
|
|
|
def test_get_resources_json_at_path(self) -> None:
|
|
# Tests JSONClient.get_resources_json()
|
|
|
|
client = JSONClient(path=self.file_path)
|
|
json = client.get_resources_json()
|
|
self.verify_json(json=json)
|
|
|
|
def test_get_resources_json_invalid_url(self) -> None:
|
|
# Tests the JSONClient.get_resources_json() function in case where an
|
|
# invalid url is passed as the URL/PATH of the resources JSON file.
|
|
|
|
path = "NotAURLorFilePath"
|
|
with self.assertRaises(Exception) as context:
|
|
client = JSONClient(path=path)
|
|
json = client.get_resources_json()
|
|
|
|
self.assertTrue(
|
|
f"Resources location '{path}' is not a valid path or URL."
|
|
in str(context.exception)
|
|
)
|