diff --git a/util/gem5-resources-manager/.gitignore b/util/gem5-resources-manager/.gitignore new file mode 100644 index 0000000000..ce625cd446 --- /dev/null +++ b/util/gem5-resources-manager/.gitignore @@ -0,0 +1,12 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ + +# Unit test / coverage reports +.coverage +database/* +instance +instance/* + +# Environments +.env +.venv diff --git a/util/gem5-resources-manager/README.md b/util/gem5-resources-manager/README.md new file mode 100644 index 0000000000..efbbf97b16 --- /dev/null +++ b/util/gem5-resources-manager/README.md @@ -0,0 +1,216 @@ +# gem5 Resources Manager + +This directory contains the code to convert the JSON file to a MongoDB database. This also contains tools to manage the database as well as the JSON file. + +# Table of Contents +- [gem5 Resources Manager](#gem5-resources-manager) +- [Table of Contents](#table-of-contents) +- [Resources Manager](#resources-manager) + - [Setup](#setup) + - [Selecting a Database](#selecting-a-database) + - [MongoDB](#mongodb) + - [JSON File](#json-file) + - [Adding a Resource](#adding-a-resource) + - [Updating a Resource](#updating-a-resource) + - [Deleting a Resource](#deleting-a-resource) + - [Adding a New Version](#adding-a-new-version) + - [Validation](#validation) +- [CLI tool](#cli-tool) + - [create\_resources\_json](#create_resources_json) + - [restore\_backup](#restore_backup) + - [backup\_mongodb](#backup_mongodb) + - [get\_resource](#get_resource) +- [Changes to Structure of JSON](#changes-to-structure-of-json) +- [Testing](#testing) + +# Resources Manager + +This is a tool to manage the resources JSON file and the MongoDB database. This tool is used to add, delete, update, view, and search for resources. + +## Setup + +First, install the requirements: + +```bash +pip3 install -r requirements.txt +``` + +Then run the flask server: + +```bash +python3 server.py +``` + +Then, you can access the server at `http://localhost:5000`. + +## Selecting a Database + +The Resource Manager currently supports 2 database options: MongoDB and JSON file. + +Select the database you want to use by clicking on the button on home page. + +### MongoDB + +The MongoDB database is hosted on MongoDB Atlas. To use this database, you need to have the MongoDB URI, collection name, and database name. Once you have the information, enter it into the form and click "login" or "save and login" to login to the database. + +Another way to use the MongoDB database is to switch to the Generate URI tab and enter the information there. This would generate a URI that you can use to login to the database. + +### JSON File + +There are currently 3 ways to use the JSON file: + +1. Adding a URL to the JSON file +2. Uploading a JSON file +3. Using an existing JSON file + +## Adding a Resource + +Once you are logged in, you can use the search bar to search for resources. If the ID doesn't exist, it would be prefilled with the required fields. You can then edit the fields and click "add" to add the resource to the database. + +## Updating a Resource + +If the ID exists, the form would be prefilled with the existing data. You can then edit the fields and click "update" to update the resource in the database. + +## Deleting a Resource + +If the ID exists, the form would be prefilled with the existing data. You can then click "delete" to delete the resource from the database. + +## Adding a New Version + +If the ID exists, the form would be prefilled with the existing data. Change the `resource_version` field to the new version and click "add" to add the new version to the database. You will only be able to add a new version if the `resource_version` field is different from any of the existing versions. + +## Validation + +The Resource Manager validates the data before adding it to the database. If the data is invalid, it would show an error message and not add the data to the database. The validation is done using the [schema](schema/schema.json) file. The Monaco editor automatically validates the data as you type and displays the errors in the editor. + +To view the schema, click on the "Show Schema" button on the left side of the page. + +# CLI tool + +```bash +usage: gem5_resource_cli.py [-h] [-u URI] [-d DATABASE] [-c COLLECTION] {get_resource,backup_mongodb,restore_backup,create_resources_json} ... + +CLI for gem5-resources. + +positional arguments: + {get_resource,backup_mongodb,restore_backup,create_resources_json} + The command to run. + get_resource Retrieves a resource from the collection based on the given ID. if a resource version is provided, it will retrieve the resource + with the given ID and version. + backup_mongodb Backs up the MongoDB collection to a JSON file. + restore_backup Restores a backup of the MongoDB collection from a JSON file. + create_resources_json + Creates a JSON file of all the resources in the collection. + +optional arguments: + -h, --help show this help message and exit + -u URI, --uri URI The URI of the MongoDB database. (default: None) + -d DATABASE, --database DATABASE + The MongoDB database to use. (default: gem5-vision) + -c COLLECTION, --collection COLLECTION + The MongoDB collection to use. (default: versions_test) +``` + +By default, the cli uses environment variables to get the URI. You can create a .env file with the `MONGO_URI` variable set to your URI. If you want to use a different URI, you can use the `-u` flag to specify the URI. + +## create_resources_json + +This command is used to create a new JSON file from the old JSON file. This is used to make the JSON file "parseable" by removing the nested JSON and adding the new fields. + +```bash +usage: gem5_resource_cli.py create_resources_json [-h] [-v VERSION] [-o OUTPUT] [-s SOURCE] + +optional arguments: + -h, --help show this help message and exit + -v VERSION, --version VERSION + The version of the resources to create the JSON file for. (default: dev) + -o OUTPUT, --output OUTPUT + The JSON file to create. (default: resources.json) + -s SOURCE, --source SOURCE + The path to the gem5 source code. (default: ) +``` + +A sample command to run this is: + +```bash +python3 gem5_resource_cli.py create_resources_json -o resources_new.json -s ./gem5 +``` + +## restore_backup + +This command is used to update the MongoDB database with the new JSON file. This is used to update the database with the new JSON file. + +```bash +usage: gem5_resource_cli.py restore_backup [-h] [-f FILE] + +optional arguments: + -h, --help show this help message and exit + +required arguments: + -f FILE, --file FILE The JSON file to restore the MongoDB collection from. +``` + +A sample command to run this is: + +```bash +python3 gem5_resource_cli.py restore_backup -f resources.json +``` + +## backup_mongodb + +This command is used to backup the MongoDB database to a JSON file. This is used to create a backup of the database. + +```bash +usage: gem5_resource_cli.py backup_mongodb [-h] -f FILE + +optional arguments: + -h, --help show this help message and exit + +required arguments: + -f FILE, --file FILE The JSON file to back up the MongoDB collection to. +``` + +A sample command to run this is: + +```bash +python3 gem5_resource_cli.py backup_mongodb -f resources.json +``` + +## get_resource + +This command is used to get a resource from the MongoDB database. This is used to get a resource from the database. + +```bash +usage: gem5_resource_cli.py get_resource [-h] -i ID [-v VERSION] + +optional arguments: + -h, --help show this help message and exit + -v VERSION, --version VERSION + The version of the resource to retrieve. + +required arguments: + -i ID, --id ID The ID of the resource to retrieve. +``` + +A sample command to run this is: + +```bash +python3 gem5_resource_cli.py get_resource -i x86-ubuntu-18.04-img -v 1.0.0 +``` +# Changes to Structure of JSON + +To view the new schema, see [schema.json](https://resources.gem5.org/gem5-resources-schema.json). + +# Testing + +To run the tests, run the following command: + +```bash +coverage run -m unittest discover -s test -p '*_test.py' +``` + +To view the coverage report, run the following command: + +```bash +coverage report +``` diff --git a/util/gem5-resources-manager/api/client.py b/util/gem5-resources-manager/api/client.py new file mode 100644 index 0000000000..20a91b50d2 --- /dev/null +++ b/util/gem5-resources-manager/api/client.py @@ -0,0 +1,134 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from abc import ABC, abstractmethod +from typing import Dict, List + + +class Client(ABC): + def __init__(self): + self.__undo_stack = [] + self.__redo_stack = [] + self.__undo_limit = 10 + + @abstractmethod + def find_resource(self, query: Dict) -> Dict: + raise NotImplementedError + + @abstractmethod + def get_versions(self, query: Dict) -> List[Dict]: + raise NotImplementedError + + @abstractmethod + def update_resource(self, query: Dict) -> Dict: + raise NotImplementedError + + @abstractmethod + def check_resource_exists(self, query: Dict) -> Dict: + raise NotImplementedError + + @abstractmethod + def insert_resource(self, query: Dict) -> Dict: + raise NotImplementedError + + @abstractmethod + def delete_resource(self, query: Dict) -> Dict: + raise NotImplementedError + + @abstractmethod + def save_session(self) -> Dict: + raise NotImplementedError + + def undo_operation(self) -> Dict: + """ + This function undoes the last operation performed on the database. + """ + if len(self.__undo_stack) == 0: + return {"status": "Nothing to undo"} + operation = self.__undo_stack.pop() + print(operation) + if operation["operation"] == "insert": + self.delete_resource(operation["resource"]) + elif operation["operation"] == "delete": + self.insert_resource(operation["resource"]) + elif operation["operation"] == "update": + self.update_resource(operation["resource"]) + temp = operation["resource"]["resource"] + operation["resource"]["resource"] = operation["resource"][ + "original_resource" + ] + operation["resource"]["original_resource"] = temp + else: + raise Exception("Invalid Operation") + self.__redo_stack.append(operation) + return {"status": "Undone"} + + def redo_operation(self) -> Dict: + """ + This function redoes the last operation performed on the database. + """ + if len(self.__redo_stack) == 0: + return {"status": "No operations to redo"} + operation = self.__redo_stack.pop() + print(operation) + if operation["operation"] == "insert": + self.insert_resource(operation["resource"]) + elif operation["operation"] == "delete": + self.delete_resource(operation["resource"]) + elif operation["operation"] == "update": + self.update_resource(operation["resource"]) + temp = operation["resource"]["resource"] + operation["resource"]["resource"] = operation["resource"][ + "original_resource" + ] + operation["resource"]["original_resource"] = temp + else: + raise Exception("Invalid Operation") + self.__undo_stack.append(operation) + return {"status": "Redone"} + + def _add_to_stack(self, operation: Dict) -> Dict: + if len(self.__undo_stack) == self.__undo_limit: + self.__undo_stack.pop(0) + self.__undo_stack.append(operation) + self.__redo_stack.clear() + return {"status": "Added to stack"} + + def get_revision_status(self) -> Dict: + """ + This function saves the status of revision operations to a dictionary. + + The revision operations whose statuses are saved are undo and redo. + + If the stack of a given revision operation is empty, the status of + that operation is set to 1 else the status is set to 0. + + :return: A dictionary containing the status of revision operations. + """ + return { + "undo": 1 if len(self.__undo_stack) == 0 else 0, + "redo": 1 if len(self.__redo_stack) == 0 else 0, + } diff --git a/util/gem5-resources-manager/api/create_resources_json.py b/util/gem5-resources-manager/api/create_resources_json.py new file mode 100644 index 0000000000..8d406a9ad5 --- /dev/null +++ b/util/gem5-resources-manager/api/create_resources_json.py @@ -0,0 +1,333 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import requests +import base64 +import os +from jsonschema import validate + + +class ResourceJsonCreator: + """ + This class generates the JSON which is pushed onto MongoDB. + On a high-level, it does the following: + - Adds certain fields to the JSON. + - Populates those fields. + - Makes sure the JSON follows the schema. + """ + + # Global Variables + base_url = "https://github.com/gem5/gem5/tree/develop" # gem5 GitHub URL + resource_url_map = { + "dev": ( + "https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/" + "develop/resources.json?format=TEXT" + ), + "22.1": ( + "https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/" + "stable/resources.json?format=TEXT" + ), + "22.0": ( + "http://resources.gem5.org/prev-resources-json/" + "resources-21-2.json" + ), + "21.2": ( + "http://resources.gem5.org/prev-resources-json/" + "resources-22-0.json" + ), + } + + def __init__(self): + self.schema = {} + with open("schema/schema.json", "r") as f: + self.schema = json.load(f) + + def _get_file_data(self, url): + json_data = None + try: + json_data = requests.get(url).text + json_data = base64.b64decode(json_data).decode("utf-8") + return json.loads(json_data) + except: + json_data = requests.get(url).json() + return json_data + + def _get_size(self, url): + """ + Helper function to return the size of a download through its URL. + Returns 0 if URL has an error. + + :param url: Download URL + """ + try: + response = requests.head(url) + size = int(response.headers.get("content-length", 0)) + return size + except Exception as e: + return 0 + + def _search_folder(self, folder_path, id): + """ + Helper function to find the instance of a string in a folder. + This is recursive, i.e., subfolders will also be searched. + + :param folder_path: Path to the folder to begin searching + :param id: Phrase to search in the folder + + :returns matching_files: List of file paths to the files containing id + """ + matching_files = [] + for filename in os.listdir(folder_path): + file_path = os.path.join(folder_path, filename) + if os.path.isfile(file_path): + with open( + file_path, "r", encoding="utf-8", errors="ignore" + ) as f: + contents = f.read() + if id in contents: + file_path = file_path.replace("\\", "/") + matching_files.append(file_path) + elif os.path.isdir(file_path): + matching_files.extend(self._search_folder(file_path, id)) + return matching_files + + def _change_type(self, resource): + if resource["type"] == "workload": + # get the architecture from the name and remove 64 from it + resource["architecture"] = ( + resource["name"].split("-")[0].replace("64", "").upper() + ) + return resource + if "kernel" in resource["name"]: + resource["type"] = "kernel" + elif "bootloader" in resource["name"]: + resource["type"] = "bootloader" + elif "benchmark" in resource["documentation"]: + resource["type"] = "disk-image" + # if tags not in resource: + if "tags" not in resource: + resource["tags"] = [] + resource["tags"].append("benchmark") + if ( + "additional_metadata" in resource + and "root_partition" in resource["additional_metadata"] + and resource["additional_metadata"]["root_partition"] + is not None + ): + resource["root_partition"] = resource["additional_metadata"][ + "root_partition" + ] + else: + resource["root_partition"] = "" + elif resource["url"] is not None and ".img.gz" in resource["url"]: + resource["type"] = "disk-image" + if ( + "additional_metadata" in resource + and "root_partition" in resource["additional_metadata"] + and resource["additional_metadata"]["root_partition"] + is not None + ): + resource["root_partition"] = resource["additional_metadata"][ + "root_partition" + ] + else: + resource["root_partition"] = "" + elif "binary" in resource["documentation"]: + resource["type"] = "binary" + elif "checkpoint" in resource["documentation"]: + resource["type"] = "checkpoint" + elif "simpoint" in resource["documentation"]: + resource["type"] = "simpoint" + return resource + + def _extract_code_examples(self, resource, source): + """ + This function goes by IDs present in the resources DataFrame. + It finds which files use those IDs in gem5/configs. + It adds the GitHub URL of those files under "example". + It finds whether those files are used in gem5/tests/gem5. + If yes, it marks "tested" as True. If not, it marks "tested" as False. + "example" and "tested" are made into a JSON for every code example. + This list of JSONs is assigned to the 'code_examples' field of the + DataFrame. + + :param resources: A DataFrame containing the current state of + resources. + :param source: Path to gem5 + + :returns resources: DataFrame with ['code-examples'] populated. + """ + id = resource["id"] + # search for files in the folder tree that contain the 'id' value + matching_files = self._search_folder( + source + "/configs", '"' + id + '"' + ) + filenames = [os.path.basename(path) for path in matching_files] + tested_files = [] + for file in filenames: + tested_files.append( + True + if len(self._search_folder(source + "/tests/gem5", file)) > 0 + else False + ) + + matching_files = [ + file.replace(source, self.base_url) for file in matching_files + ] + + code_examples = [] + + for i in range(len(matching_files)): + json_obj = { + "example": matching_files[i], + "tested": tested_files[i], + } + code_examples.append(json_obj) + return code_examples + + def unwrap_resources(self, ver): + data = self._get_file_data(self.resource_url_map[ver]) + resources = data["resources"] + new_resources = [] + for resource in resources: + if resource["type"] == "group": + for group in resource["contents"]: + new_resources.append(group) + else: + new_resources.append(resource) + return new_resources + + def _get_example_usage(self, resource): + if resource["category"] == "workload": + return f"Workload(\"{resource['id']}\")" + else: + return f"obtain_resource(resource_id=\"{resource['id']}\")" + + def _parse_readme(self, url): + metadata = { + "tags": [], + "author": [], + "license": "", + } + try: + request = requests.get(url) + content = request.text + content = content.split("---")[1] + content = content.split("---")[0] + if "tags:" in content: + tags = content.split("tags:\n")[1] + tags = tags.split(":")[0] + tags = tags.split("\n")[:-1] + tags = [tag.strip().replace("- ", "") for tag in tags] + if tags == [""] or tags == None: + tags = [] + metadata["tags"] = tags + if "author:" in content: + author = content.split("author:")[1] + author = author.split("\n")[0] + author = ( + author.replace("[", "").replace("]", "").replace('"', "") + ) + author = author.split(",") + author = [a.strip() for a in author] + metadata["author"] = author + if "license:" in content: + license = content.split("license:")[1].split("\n")[0] + metadata["license"] = license + except: + pass + return metadata + + def _add_fields(self, resources, source): + new_resources = [] + for resource in resources: + res = self._change_type(resource) + res["gem5_versions"] = ["23.0"] + res["resource_version"] = "1.0.0" + res["category"] = res["type"] + del res["type"] + res["id"] = res["name"] + del res["name"] + res["description"] = res["documentation"] + del res["documentation"] + if "additional_metadata" in res: + for k, v in res["additional_metadata"].items(): + res[k] = v + del res["additional_metadata"] + res["example_usage"] = self._get_example_usage(res) + if "source" in res: + url = ( + "https://raw.githubusercontent.com/gem5/" + "gem5-resources/develop/" + + str(res["source"]) + + "/README.md" + ) + res["source_url"] = ( + "https://github.com/gem5/gem5-resources/tree/develop/" + + str(res["source"]) + ) + else: + url = "" + res["source_url"] = "" + metadata = self._parse_readme(url) + if "tags" in res: + res["tags"].extend(metadata["tags"]) + else: + res["tags"] = metadata["tags"] + res["author"] = metadata["author"] + res["license"] = metadata["license"] + + res["code_examples"] = self._extract_code_examples(res, source) + + if "url" in resource: + download_url = res["url"].replace( + "{url_base}", "http://dist.gem5.org/dist/develop" + ) + res["url"] = download_url + res["size"] = self._get_size(download_url) + else: + res["size"] = 0 + + res = {k: v for k, v in res.items() if v is not None} + + new_resources.append(res) + return new_resources + + def _validate_schema(self, resources): + for resource in resources: + try: + validate(resource, schema=self.schema) + except Exception as e: + print(resource) + raise e + + def create_json(self, version, source, output): + resources = self.unwrap_resources(version) + resources = self._add_fields(resources, source) + self._validate_schema(resources) + with open(output, "w") as f: + json.dump(resources, f, indent=4) diff --git a/util/gem5-resources-manager/api/json_client.py b/util/gem5-resources-manager/api/json_client.py new file mode 100644 index 0000000000..24cfaee88c --- /dev/null +++ b/util/gem5-resources-manager/api/json_client.py @@ -0,0 +1,217 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from pathlib import Path +import json +from api.client import Client +from typing import Dict, List + + +class JSONClient(Client): + def __init__(self, file_path): + super().__init__() + self.file_path = Path("database/") / file_path + self.resources = self._get_resources(self.file_path) + + def _get_resources(self, path: Path) -> List[Dict]: + """ + Retrieves the resources from the JSON file. + :param path: The path to the JSON file. + :return: The resources as a JSON string. + """ + with open(path) as f: + return json.load(f) + + def find_resource(self, query: Dict) -> Dict: + """ + Finds a resource within a list of resources based on the + provided query. + :param query: The query object containing the search criteria. + :return: The resource that matches the query. + """ + found_resources = [] + for resource in self.resources: + if ( + "resource_version" not in query + or query["resource_version"] == "" + or query["resource_version"] == "Latest" + ): + if resource["id"] == query["id"]: + found_resources.append(resource) + else: + if ( + resource["id"] == query["id"] + and resource["resource_version"] + == query["resource_version"] + ): + return resource + if not found_resources: + return {"exists": False} + return max( + found_resources, + key=lambda resource: tuple( + map(int, resource["resource_version"].split(".")) + ), + ) + + def get_versions(self, query: Dict) -> List[Dict]: + """ + Retrieves all versions of a resource with the given ID from the + list of resources. + :param query: The query object containing the search criteria. + :return: A list of all versions of the resource. + """ + versions = [] + for resource in self.resources: + if resource["id"] == query["id"]: + versions.append( + {"resource_version": resource["resource_version"]} + ) + versions.sort( + key=lambda resource: tuple( + map(int, resource["resource_version"].split(".")) + ), + reverse=True, + ) + return versions + + def update_resource(self, query: Dict) -> Dict: + """ + Updates a resource within a list of resources based on the + provided query. + + The function iterates over the resources and checks if the "id" and + "resource_version" of a resource match the values in the query. + If there is a match, it removes the existing resource from the list + and appends the updated resource. + + After updating the resources, the function saves the updated list to + the specified file path. + + :param query: The query object containing the resource + identification criteria. + :return: A dictionary indicating that the resource was updated. + """ + original_resource = query["original_resource"] + modified_resource = query["resource"] + if ( + original_resource["id"] != modified_resource["id"] + and original_resource["resource_version"] + != modified_resource["resource_version"] + ): + return {"status": "Cannot change resource id"} + for resource in self.resources: + if ( + resource["id"] == original_resource["id"] + and resource["resource_version"] + == original_resource["resource_version"] + ): + self.resources.remove(resource) + self.resources.append(modified_resource) + + self.write_to_file() + return {"status": "Updated"} + + def check_resource_exists(self, query: Dict) -> Dict: + """ + Checks if a resource exists within a list of resources based on the + provided query. + + The function iterates over the resources and checks if the "id" and + "resource_version" of a resource match the values in the query. + If a matching resource is found, it returns a dictionary indicating + that the resource exists. + If no matching resource is found, it returns a dictionary indicating + that the resource does not exist. + + :param query: The query object containing the resource identification + criteria. + :return: A dictionary indicating whether the resource exists. + """ + for resource in self.resources: + if ( + resource["id"] == query["id"] + and resource["resource_version"] == query["resource_version"] + ): + return {"exists": True} + return {"exists": False} + + def insert_resource(self, query: Dict) -> Dict: + """ + Inserts a new resource into a list of resources. + + The function appends the query (new resource) to the resources list, + indicating the insertion. + It then writes the updated resources to the specified file path. + + :param query: The query object containing the resource identification + criteria. + :return: A dictionary indicating that the resource was inserted. + """ + if self.check_resource_exists(query)["exists"]: + return {"status": "Resource already exists"} + self.resources.append(query) + self.write_to_file() + return {"status": "Inserted"} + + def delete_resource(self, query: Dict) -> Dict: + """ + This function deletes a resource from the list of resources based on + the provided query. + + :param query: The query object containing the resource identification + criteria. + :return: A dictionary indicating that the resource was deleted. + """ + for resource in self.resources: + if ( + resource["id"] == query["id"] + and resource["resource_version"] == query["resource_version"] + ): + self.resources.remove(resource) + self.write_to_file() + return {"status": "Deleted"} + + def write_to_file(self) -> None: + """ + This function writes the list of resources to a file at the specified + file path. + + :return: None + """ + with Path(self.file_path).open("w") as outfile: + json.dump(self.resources, outfile, indent=4) + + def save_session(self) -> Dict: + """ + This function saves the client session to a dictionary. + :return: A dictionary containing the client session. + """ + session = { + "client": "json", + "filename": self.file_path.name, + } + return session diff --git a/util/gem5-resources-manager/api/mongo_client.py b/util/gem5-resources-manager/api/mongo_client.py new file mode 100644 index 0000000000..845524b886 --- /dev/null +++ b/util/gem5-resources-manager/api/mongo_client.py @@ -0,0 +1,237 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from bson import json_util +from api.client import Client +from pymongo.errors import ConnectionFailure, ConfigurationError +from pymongo import MongoClient +from typing import Dict, List +import pymongo + + +class DatabaseConnectionError(Exception): + "Raised for failure to connect to MongoDB client" + pass + + +class MongoDBClient(Client): + def __init__(self, mongo_uri, database_name, collection_name): + super().__init__() + self.mongo_uri = mongo_uri + self.collection_name = collection_name + self.database_name = database_name + self.collection = self._get_database( + mongo_uri, database_name, collection_name + ) + + def _get_database( + self, + mongo_uri: str, + database_name: str, + collection_name: str, + ) -> pymongo.collection.Collection: + """ + This function returns a MongoDB database object for the specified + collection. + It takes three arguments: 'mongo_uri', 'database_name', and + 'collection_name'. + + :param: mongo_uri: URI of the MongoDB instance + :param: database_name: Name of the database + :param: collection_name: Name of the collection + :return: database: MongoDB database object + """ + + try: + client = MongoClient(mongo_uri) + client.admin.command("ping") + except ConnectionFailure: + client.close() + raise DatabaseConnectionError( + "Could not connect to MongoClient with given URI!" + ) + except ConfigurationError as e: + raise DatabaseConnectionError(e) + + database = client[database_name] + if database.name not in client.list_database_names(): + raise DatabaseConnectionError("Database Does not Exist!") + + collection = database[collection_name] + if collection.name not in database.list_collection_names(): + raise DatabaseConnectionError("Collection Does not Exist!") + + return collection + + def find_resource(self, query: Dict) -> Dict: + """ + Find a resource in the database + + :param query: JSON object with id and resource_version + :return: json_resource: JSON object with request resource or + error message + """ + if "resource_version" not in query or query["resource_version"] == "": + resource = ( + self.collection.find({"id": query["id"]}, {"_id": 0}) + .sort("resource_version", -1) + .limit(1) + ) + else: + resource = ( + self.collection.find( + { + "id": query["id"], + "resource_version": query["resource_version"], + }, + {"_id": 0}, + ) + .sort("resource_version", -1) + .limit(1) + ) + json_resource = json_util.dumps(resource) + res = json.loads(json_resource) + if res == []: + return {"exists": False} + return res[0] + + def update_resource(self, query: Dict) -> Dict[str, str]: + """ + This function updates a resource in the database by first checking if + the resource version in the request matches the resource version + stored in the database. + If they match, the resource is updated in the database. If they do not + match, the update is rejected. + + :param: query: JSON object with original_resource and the + updated resource + :return: json_response: JSON object with status message + """ + original_resource = query["original_resource"] + modified_resource = query["resource"] + try: + self.collection.replace_one( + { + "id": original_resource["id"], + "resource_version": original_resource["resource_version"], + }, + modified_resource, + ) + except Exception as e: + print(e) + return {"status": "Resource does not exist"} + return {"status": "Updated"} + + def get_versions(self, query: Dict) -> List[Dict]: + """ + This function retrieves all versions of a resource with the given ID + from the database. + It takes two arguments, the database object and a JSON object + containing the 'id' key of the resource to be retrieved. + + :param: query: JSON object with id + :return: json_resource: JSON object with all resource versions + """ + versions = self.collection.find( + {"id": query["id"]}, {"resource_version": 1, "_id": 0} + ).sort("resource_version", -1) + # convert to json + res = json_util.dumps(versions) + return json_util.loads(res) + + def delete_resource(self, query: Dict) -> Dict[str, str]: + """ + This function deletes a resource from the database by first checking + if the resource version in the request matches the resource version + stored in the database. + If they match, the resource is deleted from the database. If they do + not match, the delete operation is rejected + + :param: query: JSON object with id and resource_version + :return: json_response: JSON object with status message + """ + self.collection.delete_one( + {"id": query["id"], "resource_version": query["resource_version"]} + ) + return {"status": "Deleted"} + + def insert_resource(self, query: Dict) -> Dict[str, str]: + """ + This function inserts a new resource into the database using the + 'insert_one' method of the MongoDB client. + The function takes two arguments, the database object and the JSON + object representing the new resource to be inserted. + + :param: json: JSON object representing the new resource to be inserted + :return: json_response: JSON object with status message + """ + try: + self.collection.insert_one(query) + except Exception as e: + return {"status": "Resource already exists"} + return {"status": "Inserted"} + + def check_resource_exists(self, query: Dict) -> Dict: + """ + This function checks if a resource exists in the database by searching + for a resource with a matching 'id' and 'resource_version' in + the database. + The function takes two arguments, the database object and a JSON object + containing the 'id' and 'resource_version' keys. + + :param: json: JSON object with id and resource_version + :return: json_response: JSON object with boolean 'exists' key + """ + resource = ( + self.collection.find( + { + "id": query["id"], + "resource_version": query["resource_version"], + }, + {"_id": 0}, + ) + .sort("resource_version", -1) + .limit(1) + ) + json_resource = json_util.dumps(resource) + res = json.loads(json_resource) + if res == []: + return {"exists": False} + return {"exists": True} + + def save_session(self) -> Dict: + """ + This function saves the client session to a dictionary. + :return: A dictionary containing the client session. + """ + session = { + "client": "mongodb", + "uri": self.mongo_uri, + "database": self.database_name, + "collection": self.collection_name, + } + return session diff --git a/util/gem5-resources-manager/gem5_resource_cli.py b/util/gem5-resources-manager/gem5_resource_cli.py new file mode 100644 index 0000000000..28528bec92 --- /dev/null +++ b/util/gem5-resources-manager/gem5_resource_cli.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pymongo import MongoClient +from api.create_resources_json import ResourceJsonCreator +import os +from dotenv import load_dotenv +import argparse +from itertools import cycle +from shutil import get_terminal_size +from threading import Thread +from time import sleep + +load_dotenv() + +# read MONGO_URI from environment variable +MONGO_URI = os.getenv("MONGO_URI") + + +class Loader: + def __init__(self, desc="Loading...", end="Done!", timeout=0.1): + """ + A loader-like context manager + + Args: + desc (str, optional): The loader's description. + Defaults to "Loading...". + end (str, optional): Final print. Defaults to "Done!". + timeout (float, optional): Sleep time between prints. + Defaults to 0.1. + """ + self.desc = desc + self.end = end + self.timeout = timeout + + self._thread = Thread(target=self._animate, daemon=True) + self.steps = ["⢿", "⣻", "⣽", "⣾", "⣷", "⣯", "⣟", "⡿"] + self.done = False + + def start(self): + self._thread.start() + return self + + def _animate(self): + for c in cycle(self.steps): + if self.done: + break + print(f"\r{self.desc} {c}", flush=True, end="") + sleep(self.timeout) + + def __enter__(self): + self.start() + + def stop(self): + self.done = True + cols = get_terminal_size((80, 20)).columns + print("\r" + " " * cols, end="", flush=True) + print(f"\r{self.end}", flush=True) + + def __exit__(self, exc_type, exc_value, tb): + # handle exceptions with those variables ^ + self.stop() + + +def get_database(collection="versions_test", uri=MONGO_URI, db="gem5-vision"): + """ + Retrieves the MongoDB database for gem5-vision. + """ + CONNECTION_STRING = uri + try: + client = MongoClient(CONNECTION_STRING) + client.server_info() + except: + print("\nCould not connect to MongoDB") + exit(1) + return client[db][collection] + + +collection = None + + +def cli(): + parser = argparse.ArgumentParser( + description="CLI for gem5-resources.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-u", + "--uri", + help="The URI of the MongoDB database.", + type=str, + default=MONGO_URI, + ) + parser.add_argument( + "-d", + "--database", + help="The MongoDB database to use.", + type=str, + default="gem5-vision", + ) + parser.add_argument( + "-c", + "--collection", + help="The MongoDB collection to use.", + type=str, + default="versions_test", + ) + + subparsers = parser.add_subparsers( + help="The command to run.", dest="command", required=True + ) + + parser_get_resource = subparsers.add_parser( + "get_resource", + help=( + "Retrieves a resource from the collection based on the given ID." + "\n if a resource version is provided, it will retrieve the " + "resource with the given ID and version." + ), + ) + req_group = parser_get_resource.add_argument_group( + title="required arguments" + ) + req_group.add_argument( + "-i", + "--id", + help="The ID of the resource to retrieve.", + type=str, + required=True, + ) + parser_get_resource.add_argument( + "-v", + "--version", + help="The version of the resource to retrieve.", + type=str, + required=False, + ) + parser_get_resource.set_defaults(func=get_resource) + + parser_backup_mongodb = subparsers.add_parser( + "backup_mongodb", + help="Backs up the MongoDB collection to a JSON file.", + ) + req_group = parser_backup_mongodb.add_argument_group( + title="required arguments" + ) + req_group.add_argument( + "-f", + "--file", + help="The JSON file to back up the MongoDB collection to.", + type=str, + required=True, + ) + parser_backup_mongodb.set_defaults(func=backup_mongodb) + + parser_update_mongodb = subparsers.add_parser( + "restore_backup", + help="Restores a backup of the MongoDB collection from a JSON file.", + ) + req_group = parser_update_mongodb.add_argument_group( + title="required arguments" + ) + req_group.add_argument( + "-f", + "--file", + help="The JSON file to restore the MongoDB collection from.", + type=str, + ) + parser_update_mongodb.set_defaults(func=restore_backup) + + parser_create_resources_json = subparsers.add_parser( + "create_resources_json", + help="Creates a JSON file of all the resources in the collection.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser_create_resources_json.add_argument( + "-v", + "--version", + help="The version of the resources to create the JSON file for.", + type=str, + default="dev", + ) + parser_create_resources_json.add_argument( + "-o", + "--output", + help="The JSON file to create.", + type=str, + default="resources.json", + ) + parser_create_resources_json.add_argument( + "-s", + "--source", + help="The path to the gem5 source code.", + type=str, + default="", + ) + parser_create_resources_json.set_defaults(func=create_resources_json) + + args = parser.parse_args() + if args.collection: + global collection + with Loader("Connecting to MongoDB...", end="Connected to MongoDB"): + collection = get_database(args.collection, args.uri, args.database) + args.func(args) + + +def get_resource(args): + # set the end after the loader is created + loader = Loader("Retrieving resource...").start() + resource = None + if args.version: + resource = collection.find_one( + {"id": args.id, "resource_version": args.version}, {"_id": 0} + ) + else: + resource = collection.find({"id": args.id}, {"_id": 0}) + resource = list(resource) + if resource: + loader.end = json.dumps(resource, indent=4) + else: + loader.end = "Resource not found" + + loader.stop() + + +def backup_mongodb(args): + """ + Backs up the MongoDB collection to a JSON file. + + :param file: The JSON file to back up the MongoDB collection to. + """ + with Loader( + "Backing up the database...", + end="Backed up the database to " + args.file, + ): + # get all the data from the collection + resources = collection.find({}, {"_id": 0}) + # write to resources.json + with open(args.file, "w") as f: + json.dump(list(resources), f, indent=4) + + +def restore_backup(args): + with Loader("Restoring backup...", end="Updated the database\n"): + with open(args.file) as f: + resources = json.load(f) + # clear the collection + collection.delete_many({}) + # push the new data + collection.insert_many(resources) + + +def create_resources_json(args): + with Loader("Creating resources JSON...", end="Created " + args.output): + creator = ResourceJsonCreator() + creator.create_json(args.version, args.source, args.output) + + +if __name__ == "__main__": + cli() diff --git a/util/gem5-resources-manager/requirements.txt b/util/gem5-resources-manager/requirements.txt new file mode 100644 index 0000000000..72771183ed --- /dev/null +++ b/util/gem5-resources-manager/requirements.txt @@ -0,0 +1,29 @@ +attrs==23.1.0 +blinker==1.6.2 +certifi==2023.5.7 +cffi==1.15.1 +charset-normalizer==3.1.0 +click==8.1.3 +colorama==0.4.6 +coverage==7.2.7 +cryptography==39.0.2 +dnspython==2.3.0 +Flask==2.3.2 +idna==3.4 +importlib-metadata==6.6.0 +itsdangerous==2.1.2 +Jinja2==3.1.2 +jsonschema==4.17.3 +Markdown==3.4.3 +MarkupSafe==2.1.3 +mongomock==4.1.2 +packaging==23.1 +pycparser==2.21 +pymongo==4.3.3 +pyrsistent==0.19.3 +requests==2.31.0 +sentinels==1.0.0 +urllib3==2.0.2 +Werkzeug==2.3.4 +zipp==3.15.0 +python-dotenv==1.0.0 diff --git a/util/gem5-resources-manager/server.py b/util/gem5-resources-manager/server.py new file mode 100644 index 0000000000..ec298d6c70 --- /dev/null +++ b/util/gem5-resources-manager/server.py @@ -0,0 +1,884 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from flask import ( + render_template, + Flask, + request, + redirect, + url_for, + make_response, +) +from bson import json_util +import json +import jsonschema +import requests +import markdown +import base64 +import secrets +from pathlib import Path +from werkzeug.utils import secure_filename +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives.kdf.scrypt import Scrypt +from cryptography.exceptions import InvalidSignature +from api.json_client import JSONClient +from api.mongo_client import MongoDBClient + +databases = {} + +response = requests.get( + "https://resources.gem5.org/gem5-resources-schema.json" +) +schema = json.loads(response.content) + + +UPLOAD_FOLDER = Path("database/") +TEMP_UPLOAD_FOLDER = Path("database/.tmp/") +CONFIG_FILE = Path("instance/config.py") +SESSIONS_COOKIE_KEY = "sessions" +ALLOWED_EXTENSIONS = {"json"} +CLIENT_TYPES = ["mongodb", "json"] + + +app = Flask(__name__, instance_relative_config=True) + + +if not CONFIG_FILE.exists(): + CONFIG_FILE.parent.mkdir() + with CONFIG_FILE.open("w+") as f: + f.write(f"SECRET_KEY = {secrets.token_bytes(32)}") + + +app.config.from_pyfile(CONFIG_FILE.name) + + +# Sorts keys in any serialized dict +# Default = True +# Set False to persevere JSON key order +app.json.sort_keys = False + + +def startup_config_validation(): + """ + Validates the startup configuration. + + Raises: + ValueError: If the 'SECRET_KEY' is not set or is not of type 'bytes'. + """ + if not app.secret_key: + raise ValueError("SECRET_KEY not set") + if not isinstance(app.secret_key, bytes): + raise ValueError("SECRET_KEY must be of type 'bytes'") + + +def startup_dir_file_validation(): + """ + Validates the startup directory and file configuration. + + Creates the required directories if they do not exist. + """ + for dir in [UPLOAD_FOLDER, TEMP_UPLOAD_FOLDER]: + if not dir.is_dir(): + dir.mkdir() + + +with app.app_context(): + startup_config_validation() + startup_dir_file_validation() + + +@app.route("/") +def index(): + """ + Renders the index HTML template. + + :return: The rendered index HTML template. + """ + return render_template("index.html") + + +@app.route("/login/mongodb") +def login_mongodb(): + """ + Renders the MongoDB login HTML template. + + :return: The rendered MongoDB login HTML template. + """ + return render_template("login/login_mongodb.html") + + +@app.route("/login/json") +def login_json(): + """ + Renders the JSON login HTML template. + + :return: The rendered JSON login HTML template. + """ + return render_template("login/login_json.html") + + +@app.route("/validateMongoDB", methods=["POST"]) +def validate_mongodb(): + """ + Validates the MongoDB connection parameters and redirects to the editor route if successful. + + This route expects a POST request with a JSON payload containing an alias for the session and the listed parameters in order to validate the MongoDB instance. + + This route expects the following JSON payload parameters: + - uri: The MongoDB connection URI. + - collection: The name of the collection in the MongoDB database. + - database: The name of the MongoDB database. + - alias: The value by which the session will be keyed in `databases`. + + If the 'uri' parameter is empty, a JSON response with an error message and status code 400 (Bad Request) is returned. + If the connection parameters are valid, the route redirects to the 'editor' route with the appropriate query parameters. + + :return: A redirect response to the 'editor' route or a JSON response with an error message and status code 400. + """ + global databases + try: + databases[request.json["alias"]] = MongoDBClient( + mongo_uri=request.json["uri"], + database_name=request.json["database"], + collection_name=request.json["collection"], + ) + except Exception as e: + return {"error": str(e)}, 400 + return redirect( + url_for("editor", alias=request.json["alias"]), + 302, + ) + + +@app.route("/validateJSON", methods=["GET"]) +def validate_json_get(): + """ + Validates the provided JSON URL and redirects to the editor route if successful. + + This route expects the following query parameters: + - q: The URL of the JSON file. + - filename: An optional filename for the uploaded JSON file. + + If the 'q' parameter is empty, a JSON response with an error message and status code 400 (Bad Request) is returned. + If the JSON URL is valid, the function retrieves the JSON content, saves it to a file, and redirects to the 'editor' + route with the appropriate query parameters. + + :return: A redirect response to the 'editor' route or a JSON response with an error message and status code 400. + """ + filename = request.args.get("filename") + url = request.args.get("q") + if not url: + return {"error": "empty"}, 400 + response = requests.get(url) + if response.status_code != 200: + return {"error": "invalid status"}, response.status_code + filename = secure_filename(request.args.get("filename")) + path = UPLOAD_FOLDER / filename + if (UPLOAD_FOLDER / filename).is_file(): + temp_path = TEMP_UPLOAD_FOLDER / filename + with temp_path.open("wb") as f: + f.write(response.content) + return {"conflict": "existing file in server"}, 409 + with path.open("wb") as f: + f.write(response.content) + global databases + if filename in databases: + return {"error": "alias already exists"}, 409 + try: + databases[filename] = JSONClient(filename) + except Exception as e: + return {"error": str(e)}, 400 + return redirect( + url_for("editor", alias=filename), + 302, + ) + + +@app.route("/validateJSON", methods=["POST"]) +def validate_json_post(): + """ + Validates and processes the uploaded JSON file. + + This route expects a file with the key 'file' in the request files. + If the file is not present, a JSON response with an error message + and status code 400 (Bad Request) is returned. + If the file already exists in the server, a JSON response with a + conflict error message and status code 409 (Conflict) is returned. + If the file's filename conflicts with an existing alias, a JSON + response with an error message and status code 409 (Conflict) is returned. + If there is an error while processing the JSON file, a JSON response + with the error message and status code 400 (Bad Request) is returned. + If the file is successfully processed, a redirect response to the + 'editor' route with the appropriate query parameters is returned. + + :return: A JSON response with an error message and + status code 400 or 409, or a redirect response to the 'editor' route. + """ + temp_path = None + if "file" not in request.files: + return {"error": "empty"}, 400 + file = request.files["file"] + filename = secure_filename(file.filename) + path = UPLOAD_FOLDER / filename + if path.is_file(): + temp_path = TEMP_UPLOAD_FOLDER / filename + file.save(temp_path) + return {"conflict": "existing file in server"}, 409 + file.save(path) + global databases + if filename in databases: + return {"error": "alias already exists"}, 409 + try: + databases[filename] = JSONClient(filename) + except Exception as e: + return {"error": str(e)}, 400 + return redirect( + url_for("editor", alias=filename), + 302, + ) + + +@app.route("/existingJSON", methods=["GET"]) +def existing_json(): + """ + Handles the request for an existing JSON file. + + This route expects a query parameter 'filename' + specifying the name of the JSON file. + If the file is not present in the 'databases', + it tries to create a 'JSONClient' instance for the file. + If there is an error while creating the 'JSONClient' + instance, a JSON response with the error message + and status code 400 (Bad Request) is returned. + If the file is present in the 'databases', a redirect + response to the 'editor' route with the appropriate + query parameters is returned. + + :return: A JSON response with an error message + and status code 400, or a redirect response to the 'editor' route. + """ + filename = request.args.get("filename") + global databases + if filename not in databases: + try: + databases[filename] = JSONClient(filename) + except Exception as e: + return {"error": str(e)}, 400 + return redirect( + url_for("editor", alias=filename), + 302, + ) + + +@app.route("/existingFiles", methods=["GET"]) +def get_existing_files(): + """ + Retrieves the list of existing files in the upload folder. + + This route returns a JSON response containing the names of the existing files in the upload folder configured in the + Flask application. + + :return: A JSON response with the list of existing files. + """ + files = [f.name for f in UPLOAD_FOLDER.iterdir() if f.is_file()] + return json.dumps(files) + + +@app.route("/resolveConflict", methods=["GET"]) +def resolve_conflict(): + """ + Resolves file conflict with JSON files. + + This route expects the following query parameters: + - filename: The name of the file that is conflicting or an updated name for it to resolve the name conflict + - resolution: A resolution option, defined as follows: + - clearInput: Deletes the conflicting file and does not proceed with login + - openExisting: Opens the existing file in `UPLOAD_FOLDER` + - overwrite: Overwrites the existing file with the conflicting file + - newFilename: Renames conflicting file, moving it to `UPLOAD_FOLDER` + + If the resolution parameter is not from the list given, an error is returned. + + The conflicting file in `TEMP_UPLOAD_FOLDER` is deleted. + + :return: A JSON response containing an error, or a success response, or a redirect to the editor. + """ + filename = secure_filename(request.args.get("filename")) + resolution = request.args.get("resolution") + resolution_options = [ + "clearInput", + "openExisting", + "overwrite", + "newFilename", + ] + temp_path = TEMP_UPLOAD_FOLDER / filename + if not resolution: + return {"error": "empty"}, 400 + if resolution not in resolution_options: + return {"error": "invalid resolution"}, 400 + if resolution == resolution_options[0]: + temp_path.unlink() + return {"success": "input cleared"}, 204 + if resolution in resolution_options[-2:]: + next(TEMP_UPLOAD_FOLDER.glob("*")).replace(UPLOAD_FOLDER / filename) + if temp_path.is_file(): + temp_path.unlink() + global databases + if filename in databases: + return {"error": "alias already exists"}, 409 + try: + databases[filename] = JSONClient(filename) + except Exception as e: + return {"error": str(e)}, 400 + return redirect( + url_for("editor", alias=filename), + 302, + ) + + +@app.route("/editor") +def editor(): + """ + Renders the editor page based on the specified database type. + + This route expects a GET request with specific query parameters: + - "alias": An optional alias for the MongoDB database. + + The function checks if the query parameters are present. If not, it returns a 404 error. + + The function determines the database type based on the instance of the client object stored in the databases['alias']. If the type is not in the + "CLIENT_TYPES" configuration, it returns a 404 error. + + :return: The rendered editor template based on the specified database type. + """ + global databases + if not request.args: + return render_template("404.html"), 404 + alias = request.args.get("alias") + if alias not in databases: + return render_template("404.html"), 404 + + client_type = "" + if isinstance(databases[alias], JSONClient): + client_type = CLIENT_TYPES[1] + elif isinstance(databases[alias], MongoDBClient): + client_type = CLIENT_TYPES[0] + else: + return render_template("404.html"), 404 + + response = make_response( + render_template("editor.html", client_type=client_type, alias=alias) + ) + + response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" + + return response + + +@app.route("/help") +def help(): + """ + Renders the help page. + + This route reads the contents of the "help.md" file located in the "static" folder and renders it as HTML using the + Markdown syntax. The rendered HTML is then passed to the "help.html" template for displaying the help page. + + :return: The rendered help page HTML. + """ + with Path("static/help.md").open("r") as f: + return render_template( + "help.html", rendered_html=markdown.markdown(f.read()) + ) + + +@app.route("/find", methods=["POST"]) +def find(): + """ + Finds a resource based on the provided search criteria. + + This route expects a POST request with a JSON payload containing the alias of the session which is to be searched for the + resource and the search criteria. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is returned. + + The Client API is used to find the resource by calling `find_resource()` on the session where the operation is + accomplished by the concrete client class. + + The result of the `find_resource` operation is returned as a JSON response. + + :return: A JSON response containing the result of the `find_resource` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + return database.find_resource(request.json) + + +@app.route("/update", methods=["POST"]) +def update(): + """ + Updates a resource with provided changes. + + This route expects a POST request with a JSON payload containing the alias of the session which contains the resource + that is to be updated and the data for updating the resource. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is returned. + + The Client API is used to update the resource by calling `update_resource()` on the session where the operation is + accomplished by the concrete client class. + + The `_add_to_stack` function of the session is called to insert the operation, update, and necessary data onto the revision + operations stack. + + The result of the `update_resource` operation is returned as a JSON response. It contains the original and the modified resources. + + :return: A JSON response containing the result of the `update_resource` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + original_resource = request.json["original_resource"] + modified_resource = request.json["resource"] + status = database.update_resource( + { + "original_resource": original_resource, + "resource": modified_resource, + } + ) + database._add_to_stack( + { + "operation": "update", + "resource": { + "original_resource": modified_resource, + "resource": original_resource, + }, + } + ) + return status + + +@app.route("/versions", methods=["POST"]) +def getVersions(): + """ + Retrieves the versions of a resource based on the provided search criteria. + + This route expects a POST request with a JSON payload containing the alias of the session which contains the resource + whose versions are to be retrieved and the search criteria. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is returned. + + The Client API is used to get the versions of a resource by calling `get_versions()` on the session where the operation is + accomplished by the concrete client class. + + The result of the `get_versions` operation is returned as a JSON response. + + :return: A JSON response containing the result of the `get_versions` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + return database.get_versions(request.json) + + +@app.route("/categories", methods=["GET"]) +def getCategories(): + """ + Retrieves the categories of the resources. + + This route returns a JSON response containing the categories of the resources. The categories are obtained from the + "enum" property of the "category" field in the schema. + + :return: A JSON response with the categories of the resources. + """ + return json.dumps(schema["properties"]["category"]["enum"]) + + +@app.route("/schema", methods=["GET"]) +def getSchema(): + """ + Retrieves the schema definition of the resources. + + This route returns a JSON response containing the schema definition of the resources. The schema is obtained from the + `schema` variable. + + :return: A JSON response with the schema definition of the resources. + """ + return json_util.dumps(schema) + + +@app.route("/keys", methods=["POST"]) +def getFields(): + """ + Retrieves the required fields for a specific category based on the provided data. + + This route expects a POST request with a JSON payload containing the data for retrieving the required fields. + The function constructs an empty object `empty_object` with the "category" and "id" values from the request payload. + + The function then uses the JSONSchema validator to validate the `empty_object` against the `schema`. It iterates + through the validation errors and handles two types of errors: + + 1. "is a required property" error: If a required property is missing in the `empty_object`, the function retrieves + the default value for that property from the schema and sets it in the `empty_object`. + + 2. "is not valid under any of the given schemas" error: If a property is not valid under the current schema, the + function evolves the validator to use the schema corresponding to the requested category. It then iterates + through the validation errors again and handles any missing required properties as described in the previous + step. + + Finally, the `empty_object` with the required fields populated (including default values if applicable) is returned + as a JSON response. + + :return: A JSON response containing the `empty_object` with the required fields for the specified category. + """ + empty_object = { + "category": request.json["category"], + "id": request.json["id"], + } + validator = jsonschema.Draft7Validator(schema) + errors = list(validator.iter_errors(empty_object)) + for error in errors: + if "is a required property" in error.message: + required = error.message.split("'")[1] + empty_object[required] = error.schema["properties"][required][ + "default" + ] + if "is not valid under any of the given schemas" in error.message: + validator = validator.evolve( + schema=error.schema["definitions"][request.json["category"]] + ) + for e in validator.iter_errors(empty_object): + if "is a required property" in e.message: + required = e.message.split("'")[1] + if "default" in e.schema["properties"][required]: + empty_object[required] = e.schema["properties"][ + required + ]["default"] + else: + empty_object[required] = "" + return json.dumps(empty_object) + + +@app.route("/delete", methods=["POST"]) +def delete(): + """ + Deletes a resource. + + This route expects a POST request with a JSON payload containing the alias of the session from which a resource is to be + deleted and the data for deleting the resource. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is returned. + + The Client API is used to delete the resource by calling `delete_resource()` on the session where the operation is + accomplished by the concrete client class. + + The `_add_to_stack` function of the session is called to insert the operation, delete, and necessary data onto the revision + operations stack. + + The result of the `delete` operation is returned as a JSON response. + + :return: A JSON response containing the result of the `delete` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + resource = request.json["resource"] + status = database.delete_resource(resource) + database._add_to_stack({"operation": "delete", "resource": resource}) + return status + + +@app.route("/insert", methods=["POST"]) +def insert(): + """ + Inserts a new resource. + + This route expects a POST request with a JSON payload containing the alias of the session to which the data + is to be inserted and the data for inserting the resource. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is returned. + + The Client API is used to insert the new resource by calling `insert_resource()` on the session where the operation is + accomplished by the concrete client class. + + The `_add_to_stack` function of the session is called to insert the operation, insert, and necessary data onto the revision + operations stack. + + The result of the `insert` operation is returned as a JSON response. + + :return: A JSON response containing the result of the `insert` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + resource = request.json["resource"] + status = database.insert_resource(resource) + database._add_to_stack({"operation": "insert", "resource": resource}) + return status + + +@app.route("/undo", methods=["POST"]) +def undo(): + """ + Undoes last operation performed on the session. + + This route expects a POST request with a JSON payload containing the alias of the session whose last operation + is to be undone. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is returned. + + The Client API is used to undo the last operation performed on the session by calling `undo_operation()` on the + session where the operation is accomplished by the concrete client class. + + The result of the `undo_operation` operation is returned as a JSON response. + + :return: A JSON response containing the result of the `undo_operation` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + return database.undo_operation() + + +@app.route("/redo", methods=["POST"]) +def redo(): + """ + Redoes last operation performed on the session. + + This route expects a POST request with a JSON payload containing the alias of the session whose last operation + is to be redone. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is returned. + + The Client API is used to redo the last operation performed on the session by calling `redo_operation()` on the + session where the operation is accomplished by the concrete client class. + + The result of the `redo_operation` operation is returned as a JSON response. + + :return: A JSON response containing the result of the `redo_operation` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + return database.redo_operation() + + +@app.route("/getRevisionStatus", methods=["POST"]) +def get_revision_status(): + """ + Gets the status of revision operations. + + This route expects a POST request with a JSON payload containing the alias of the session whose revision operations + statuses is being requested. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is + returned. + + The Client API is used to get the status of the revision operations by calling `get_revision_status()` on the + session where the operation is accomplished by the concrete client class. + + The result of the `get_revision_status` is returned as a JSON response. + + :return: A JSON response contain the result of the `get_revision_status` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + return database.get_revision_status() + + +def fernet_instance_generation(password): + """ + Generates Fernet instance for use in Saving and Loading Session. + + Utilizes Scrypt Key Derivation Function with `SECRET_KEY` as salt value and recommended + values for `length`, `n`, `r`, and `p` parameters. Derives key using `password`. Derived + key is then used to initialize Fernet instance. + + :param password: User provided password + :return: Fernet instance + """ + return Fernet( + base64.urlsafe_b64encode( + Scrypt(salt=app.secret_key, length=32, n=2**16, r=8, p=1).derive( + password.encode() + ) + ) + ) + + +@app.route("/saveSession", methods=["POST"]) +def save_session(): + """ + Generates ciphertext of session that is to be saved. + + This route expects a POST request with a JSON payload containing the alias of the session that is to be + saved and a password to be used in encrypting the session data. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is + returned. + + The `save_session()` method is called to get the necessary session data from the corresponding `Client` + as a dictionary. + + A Fernet instance, using the user provided password, is instantiated. The session data is encrypted using this + instance. If an Exception is raised, an error response is returned. + + The result of the save_session operation is returned as a JSON response. The ciphertext is returned or an error + message if an error occurred. + + :return: A JSON response containing the result of the save_session operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + session = databases[alias].save_session() + try: + fernet_instance = fernet_instance_generation(request.json["password"]) + ciphertext = fernet_instance.encrypt(json.dumps(session).encode()) + except (TypeError, ValueError): + return {"error": "Failed to Encrypt Session!"}, 400 + return {"ciphertext": ciphertext.decode()}, 200 + + +@app.route("/loadSession", methods=["POST"]) +def load_session(): + """ + Loads session from data specified in user request. + + This route expects a POST request with a JSON payload containing the encrypted ciphertext containing the session + data, the alias of the session that is to be restored, and the password associated with it. + + A Fernet instance, using the user provided password, is instantiated. The session data is decrypted using this + instance. If an Exception is raised, an error response is returned. + + The `Client` type is retrieved from the session data and a redirect to the appropriate login with the stored + parameters from the session data is applied. + + The result of the load_session operation is returned either as a JSON response containing the error message + or a redirect. + + :return: A JSON response containing the error of the load_session operation or a redirect. + """ + alias = request.json["alias"] + session = request.json["session"] + try: + fernet_instance = fernet_instance_generation(request.json["password"]) + session_data = json.loads(fernet_instance.decrypt(session)) + except (InvalidSignature, InvalidToken): + return {"error": "Incorrect Password! Please Try Again!"}, 400 + client_type = session_data["client"] + if client_type == CLIENT_TYPES[0]: + try: + databases[alias] = MongoDBClient( + mongo_uri=session_data["uri"], + database_name=session_data["database"], + collection_name=session_data["collection"], + ) + except Exception as e: + return {"error": str(e)}, 400 + + return redirect( + url_for("editor", type=CLIENT_TYPES[0], alias=alias), + 302, + ) + elif client_type == CLIENT_TYPES[1]: + return redirect( + url_for("existing_json", filename=session_data["filename"]), + 302, + ) + else: + return {"error": "Invalid Client Type!"}, 409 + + +@app.errorhandler(404) +def handle404(error): + """ + Error handler for 404 (Not Found) errors. + + This function is called when a 404 error occurs. It renders the "404.html" template and returns it as a response with + a status code of 404. + + :param error: The error object representing the 404 error. + :return: A response containing the rendered "404.html" template with a status code of 404. + """ + return render_template("404.html"), 404 + + +@app.route("/checkExists", methods=["POST"]) +def checkExists(): + """ + Checks if a resource exists based on the provided data. + + This route expects a POST request with a JSON payload containing the alias of the session in which it is to be + determined whether a given resource exists and the necessary data for checking the existence of the resource. + + The alias is used in retrieving the session from `databases`. If the session is not found, an error is + returned. + + The Client API is used to check the existence of the resource by calling `check_resource_exists()` on the + session where the operation is accomplished by the concrete client class. + + The result of the `check_resource_exists` is returned as a JSON response. + + :return: A JSON response contain the result of the `check_resource_exists` operation. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + database = databases[alias] + return database.check_resource_exists(request.json) + + +@app.route("/logout", methods=["POST"]) +def logout(): + """ + Logs the user out of the application. + + Deletes the alias from the `databases` dictionary. + + :param alias: The alias of the database to logout from. + + :return: A redirect to the index page. + """ + alias = request.json["alias"] + if alias not in databases: + return {"error": "database not found"}, 400 + databases.pop(alias) + return (redirect(url_for("index")), 302) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/util/gem5-resources-manager/static/help.md b/util/gem5-resources-manager/static/help.md new file mode 100644 index 0000000000..c79d26dea4 --- /dev/null +++ b/util/gem5-resources-manager/static/help.md @@ -0,0 +1,65 @@ +# Help + +## Load Previous Session +Retrieves list of saved sessions from browser localStorage. +If found, displays list, can select a session to restore, and if entered password is correct session is restored and redirects to editor. + +## MongoDB +Set up editor view for MongoDB Instance. + +### Login: Enter URI +Utilize if the MongoDB connection string is known. + +#### Fields: + - URI: [MongoDB](https://www.mongodb.com/docs/manual/reference/connection-string/) + +#### Additional Fields: + - Collection: Specify collection in MongoDB instance to retrieve + - Database: Specify database in MongoDB instance to retrieve + - Alias: Optional. Provide a display alias to show on editor view instead of URI + +### Login: Generate URI +Provides method to generate MongoDB URI connection string if it is not known or to supply with additional parameters. + +#### Fields: + + - Connection: Specify connection mode, Standard or DNS Seed List, as defined by [MongoDB](https://www.mongodb.com/docs/manual/reference/connection-string/) + - Username: Optional. + - Password: Optional. + - Host: Specify host/list of hosts for instance + - Retry Writes: Allow MongoDB to retry a write to database once if they fail the first time + - Write Concern: Determines level of acknowledgement required from database for write operations, specifies how many nodes must acknowledge the operation before it is considered successful. (Currently set to majority) + - Options: Optional. Additional parameters that can be set when connecting to the instance + +#### Additional Fields: + - Collection: Specify collection in MongoDB instance to retrieve + - Database: Specify database in MongoDB instance to retrieve + - Alias: Optional field to provide a display alias to show on editor view instead of URI + +## JSON +Set up editor view for JSON file. Can Specify a URL to a remote JSON file to be imported +or select a local JSON file. + + +## Editor +Page containing Monaco VSCode Diff Editor to allow editing of database entries. + +### Database Actions: +Actions that can be performed on database currently in use. + +- Search: Search for resource in database with exact Resource ID +- Version: Dropdown that allows for selection of a particular resource version of resource currently in view +- Category: Specify category of resource to viewed as defined by schema +- Undo: Undoes last edit to database +- Redo: Redoes last undone change to database +- Show Schema: Sets view for schema of current database (read only) +- Save Session: Save session in encrypted format to browser localStorage +- Logout: Removes sessions from list of active sessions + +### Editing Actions: +Actions that can be performed on resource currently in view. + +- Add New Resource: Add a new resource to database +- Add New Version: Insert a new version of current resource +- Delete: Permanently delete resource +- Update: Update resource with edits made diff --git a/util/gem5-resources-manager/static/images/favicon.png b/util/gem5-resources-manager/static/images/favicon.png new file mode 100644 index 0000000000..d0103efa4d Binary files /dev/null and b/util/gem5-resources-manager/static/images/favicon.png differ diff --git a/util/gem5-resources-manager/static/images/gem5ColorLong.gif b/util/gem5-resources-manager/static/images/gem5ColorLong.gif new file mode 100644 index 0000000000..552e4d12fc Binary files /dev/null and b/util/gem5-resources-manager/static/images/gem5ColorLong.gif differ diff --git a/util/gem5-resources-manager/static/images/gem5ResourcesManager.png b/util/gem5-resources-manager/static/images/gem5ResourcesManager.png new file mode 100644 index 0000000000..dac4cb595e Binary files /dev/null and b/util/gem5-resources-manager/static/images/gem5ResourcesManager.png differ diff --git a/util/gem5-resources-manager/static/js/app.js b/util/gem5-resources-manager/static/js/app.js new file mode 100644 index 0000000000..ed5025a2f2 --- /dev/null +++ b/util/gem5-resources-manager/static/js/app.js @@ -0,0 +1,135 @@ +const loadingContainer = document.getElementById("loading-container"); +const alertPlaceholder = document.getElementById('liveAlertPlaceholder'); +const interactiveElems = document.querySelectorAll('button, input, select'); + +const appendAlert = (errorHeader, id, message, type) => { + const alertDiv = document.createElement('div'); + alertDiv.classList.add("alert", `alert-${type}`, "alert-dismissible", "fade", "show", "d-flex", "flex-column", "shadow-sm"); + alertDiv.setAttribute("role", "alert"); + alertDiv.setAttribute("id", id); + alertDiv.style.maxWidth = "320px"; + + alertDiv.innerHTML = [ + `
`, + ` `, + ` `, + ` `, + ` ${errorHeader}`, + ` `, + `
`, + `
`, + `
${message}
`, + ].join(''); + + window.scrollTo(0, 0); + + alertPlaceholder.append(alertDiv); + + setTimeout(function () { + bootstrap.Alert.getOrCreateInstance(document.getElementById(`${id}`)).close(); + }, 5000); +} + +function toggleInteractables(isBlocking, excludedOnNotBlockingIds = [], otherBlockingUpdates = () => {}) { + if (isBlocking) { + loadingContainer.classList.add("d-flex"); + interactiveElems.forEach(elems => { + elems.disabled = true; + }); + window.scrollTo(0, 0); + otherBlockingUpdates(); + return; + } + + setTimeout(() => { + loadingContainer.classList.remove("d-flex"); + interactiveElems.forEach(elems => { + !excludedOnNotBlockingIds.includes(elems.id) ? elems.disabled = false : null; + }); + otherBlockingUpdates(); + }, 250); +} + +function showResetSavedSessionsModal() { + let sessions = localStorage.getItem("sessions"); + if (sessions === null) { + appendAlert('Error!', 'noSavedSessions', `No Saved Sessions Exist!`, 'danger'); + return; + } + sessions = JSON.parse(sessions); + + const resetSavedSessionsModal = new bootstrap.Modal(document.getElementById('resetSavedSessionsModal'), { + focus: true, keyboard: false + }); + + + let select = document.getElementById("delete-session-dropdown"); + select.innerHTML = ""; + Object.keys(sessions).forEach((alias) => { + let option = document.createElement("option"); + option.value = alias; + option.innerHTML = alias; + select.appendChild(option); + }); + + document.getElementById("selected-session").innerText = `"${document.getElementById("delete-session-dropdown").value}"`; + + resetSavedSessionsModal.show(); +} + +function resetSavedSessions() { + bootstrap.Modal.getInstance(document.getElementById("resetSavedSessionsModal")).hide(); + + const sessions = JSON.parse(localStorage.getItem("sessions")); + if (sessions === null) { + appendAlert('Error!', 'noSavedSessions', `No Saved Sessions Exist!`, 'danger'); + return; + } + + const activeTab = document.getElementById("reset-tabs").querySelector(".nav-link.active").getAttribute("id"); + if (activeTab === "delete-one-tab") { + const deleteOneConfirmation = document.getElementById("delete-one-confirmation").value; + if (deleteOneConfirmation !== document.getElementById("delete-session-dropdown").value) { + document.getElementById("resetSavedSessionsModal").querySelectorAll("form").forEach(form => { + form.reset(); + }) + appendAlert('Error!', 'noSavedSessions', `Invalid Confirmation Entry!`, 'danger'); + return; + } + + delete sessions[document.getElementById("delete-session-dropdown").value]; + Object.keys(sessions).length === 0 + ? localStorage.removeItem("sessions") + : localStorage.setItem("sessions", JSON.stringify(sessions)); + + } else { + const deleteAllConfirmation = document.getElementById("delete-all-confirmation").value; + if (deleteAllConfirmation !== "Delete All") { + document.getElementById("resetSavedSessionsModal").querySelectorAll("form").forEach(form => { + form.reset(); + }) + appendAlert('Error!', 'noSavedSessions', `Invalid Confirmation Entry!`, 'danger'); + return; + } + + localStorage.removeItem("sessions"); + } + + appendAlert('Success!', 'resetCookies', `Saved Session Reset Successful!`, 'success'); + setTimeout(() => { + location.reload(); + }, 750); +} + +document.getElementById("close-reset-modal").addEventListener("click", () => { + document.getElementById("resetSavedSessionsModal").querySelectorAll("form").forEach(form => { + form.reset(); + }) +}); + +document.getElementById("delete-session-dropdown").addEventListener("change", () => { + document.getElementById("selected-session").innerText = + `"${document.getElementById("delete-session-dropdown").value}"`; +}); diff --git a/util/gem5-resources-manager/static/js/editor.js b/util/gem5-resources-manager/static/js/editor.js new file mode 100644 index 0000000000..64786da0bd --- /dev/null +++ b/util/gem5-resources-manager/static/js/editor.js @@ -0,0 +1,589 @@ +const diffEditorContainer = document.getElementById("diff-editor"); +var diffEditor; +var originalModel; +var modifiedModel; + +const schemaEditorContainer = document.getElementById("schema-editor"); +var schemaEditor; +var schemaModel; + +const schemaButton = document.getElementById("schema-toggle"); +const editingActionsButtons = Array.from( + document.querySelectorAll("#editing-actions button") +); +var editingActionsState; + +const tooltipTriggerList = document.querySelectorAll('[data-bs-toggle="tooltip"]'); +tooltipTriggerList.forEach(tooltip => { + tooltip.setAttribute("data-bs-trigger", "hover"); +}); +const tooltipList = [...tooltipTriggerList].map(tooltipTriggerEl => new bootstrap.Tooltip(tooltipTriggerEl)); + +require.config({ + paths: { + vs: "https://cdnjs.cloudflare.com/ajax/libs/monaco-editor/0.26.1/min/vs", + }, +}); +require(["vs/editor/editor.main"], () => { + originalModel = monaco.editor.createModel(`{\n}`, "json"); + modifiedModel = monaco.editor.createModel(`{\n}`, "json"); + diffEditor = monaco.editor.createDiffEditor(diffEditorContainer, { + theme: "vs-dark", + language: "json", + automaticLayout: true, + }); + diffEditor.setModel({ + original: originalModel, + modified: modifiedModel, + }); + fetch("/schema") + .then((res) => res.json()) + .then((data) => { + monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ + trailingCommas: "error", + comments: "error", + validate: true, + schemas: [ + { + uri: "http://json-schema.org/draft-07/schema", + fileMatch: ["*"], + schema: data, + }, + ], + }); + + schemaEditor = monaco.editor.create(schemaEditorContainer, { + theme: "vs-dark", + language: "json", + automaticLayout: true, + readOnly: true, + }); + + schemaModel = monaco.editor.createModel(`{\n}`, "json"); + schemaEditor.setModel(schemaModel); + schemaModel.setValue(JSON.stringify(data, null, 4)); + + schemaEditorContainer.style.display = "none"; + }); +}); + +let clientType = document.getElementById('client-type'); +clientType.textContent = clientType.textContent === "mongodb" ? "MongoDB" : clientType.textContent.toUpperCase(); + +const revisionButtons = [document.getElementById("undo-operation"), document.getElementById("redo-operation")]; +revisionButtons.forEach(btn => { + btn.disabled = true; +}); + +const editorGroupIds = []; +document.querySelectorAll(".editorButtonGroup button, .revisionButtonGroup button") + .forEach(btn => { + editorGroupIds.push(btn.id); + }); + +function checkErrors() { + let errors = monaco.editor.getModelMarkers({ resource: modifiedModel.uri }); + if (errors.length > 0) { + console.log(errors); + let str = ""; + errors.forEach((error) => { + str += error.message + "\n"; + }); + appendAlert('Error!', 'schemaError', { str }, 'danger'); + return true; + } + return false; +} +let didChange = false; + +function update(e) { + e.preventDefault(); + if (checkErrors()) { + return; + } + let json = JSON.parse(modifiedModel.getValue()); + let original_json = JSON.parse(originalModel.getValue()); + + console.log(json); + fetch("/update", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + resource: json, + original_resource: original_json, + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then(async (data) => { + console.log(data); + await addVersions(); + //Select last option + document.getElementById("version-dropdown").value = + json["resource_version"]; + console.log(document.getElementById("version-dropdown").value); + find(e); + }); +} + +function addNewResource(e) { + e.preventDefault(); + if (checkErrors()) { + return; + } + let json = JSON.parse(modifiedModel.getValue()); + console.log(json); + fetch("/insert", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + resource: json, + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then(async (data) => { + console.log(data); + await addVersions(); + //Select last option + document.getElementById("version-dropdown").value = + json["resource_version"]; + console.log(document.getElementById("version-dropdown").value); + find(e); + }); +} + +function addVersion(e) { + e.preventDefault(); + console.log("add version"); + if (checkErrors()) { + return; + } + let json = JSON.parse(modifiedModel.getValue()); + console.log(json["resource_version"]); + fetch("/checkExists", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: json["id"], + resource_version: json["resource_version"], + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then((data) => { + console.log(data["exists"]); + if (data["exists"] == true) { + appendAlert("Error!", "existingResourceVersion", "Resource version already exists!", "danger"); + return; + } else { + fetch("/insert", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + resource: json, + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then(async (data) => { + console.log("added version"); + console.log(data); + await addVersions(); + //Select last option + document.getElementById("version-dropdown").value = + json["resource_version"]; + console.log(document.getElementById("version-dropdown").value); + find(e); + }); + } + }); +} + +function deleteRes(e) { + e.preventDefault(); + console.log("delete"); + let id = document.getElementById("id").value; + let resource_version = JSON.parse(originalModel.getValue())[ + "resource_version" + ]; + let json = JSON.parse(originalModel.getValue()); + console.log(resource_version); + fetch("/delete", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + resource: json, + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then(async (data) => { + console.log(data); + await addVersions(); + //Select first option + document.getElementById("version-dropdown").value = + document.getElementById("version-dropdown").options[0].value; + console.log(document.getElementById("version-dropdown").value); + find(e); + }); +} + +document.getElementById("id").onchange = function () { + console.log("id changed"); + didChange = true; +}; + +async function addVersions() { + let select = document.getElementById("version-dropdown"); + select.innerHTML = "Latest"; + await fetch("/versions", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: document.getElementById("id").value, + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then((data) => { + let select = document.getElementById("version-dropdown"); + if (data.length == 0) { + data = [{ resource_version: "Latest" }]; + } + data.forEach((version) => { + let option = document.createElement("option"); + option.value = version["resource_version"]; + option.innerText = version["resource_version"]; + select.appendChild(option); + }); + }); +} + +function find(e) { + e.preventDefault(); + if (didChange) { + addVersions(); + didChange = false; + } + + closeSchema(); + + toggleInteractables(true, editorGroupIds, () => { + diffEditor.updateOptions({ readOnly: true }); + updateRevisionBtnsDisabledAttr(); + }); + + fetch("/find", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: document.getElementById("id").value, + resource_version: document.getElementById("version-dropdown").value, + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then((data) => { + console.log(data); + toggleInteractables(false, editorGroupIds, () => { + diffEditor.updateOptions({ readOnly: false }); + updateRevisionBtnsDisabledAttr(); + }); + + if (data["exists"] == false) { + fetch("/keys", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + category: document.getElementById("category").value, + id: document.getElementById("id").value, + }), + }) + .then((res) => res.json()) + .then((data) => { + console.log(data) + data["id"] = document.getElementById("id").value; + data["category"] = document.getElementById("category").value; + originalModel.setValue(JSON.stringify(data, null, 4)); + modifiedModel.setValue(JSON.stringify(data, null, 4)); + + document.getElementById("add_new_resource").disabled = false; + document.getElementById("add_version").disabled = true; + document.getElementById("delete").disabled = true; + document.getElementById("update").disabled = true; + }); + } else { + console.log(data); + originalModel.setValue(JSON.stringify(data, null, 4)); + modifiedModel.setValue(JSON.stringify(data, null, 4)); + + document.getElementById("version-dropdown").value = + data.resource_version; + document.getElementById("category").value = data.category; + + document.getElementById("add_new_resource").disabled = true; + document.getElementById("add_version").disabled = false; + document.getElementById("delete").disabled = false; + document.getElementById("update").disabled = false; + } + }); +} + +window.onload = () => { + let ver_dropdown = document.getElementById("version-dropdown"); + let option = document.createElement("option"); + option.value = "Latest"; + option.innerHTML = "Latest"; + ver_dropdown.appendChild(option); + fetch("/categories") + .then((res) => res.json()) + .then((data) => { + console.log(data); + let select = document.getElementById("category"); + data.forEach((category) => { + let option = document.createElement("option"); + option.value = category; + option.innerHTML = category; + select.appendChild(option); + }); + fetch("/keys", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + category: document.getElementById("category").value, + id: "", + }), + }) + .then((res) => res.json()) + .then((data) => { + data["id"] = ""; + data["category"] = document.getElementById("category").value; + originalModel.setValue(JSON.stringify(data, null, 4)); + modifiedModel.setValue(JSON.stringify(data, null, 4)); + document.getElementById("add_new_resource").disabled = false; + }); + }); + + checkExistingSavedSession(); +}; + +const myModal = new bootstrap.Modal("#ConfirmModal", { + keyboard: false, +}); + +let confirmButton = document.getElementById("confirm"); + +function showModal(event, callback) { + event.preventDefault(); + myModal.show(); + confirmButton.onclick = () => { + callback(event); + myModal.hide(); + }; +} + +let editorTitle = document.getElementById("editor-title"); + +function showSchema() { + if (diffEditorContainer.style.display !== "none") { + diffEditorContainer.style.display = "none"; + schemaEditorContainer.classList.add("editor-sizing"); + schemaEditor.setPosition({ column: 1, lineNumber: 1 }); + schemaEditor.revealPosition({ column: 1, lineNumber: 1 }); + schemaEditorContainer.style.display = "block"; + + editingActionsState = editingActionsButtons.map( + (button) => button.disabled + ); + + editingActionsButtons.forEach((btn) => { + btn.disabled = true; + }); + + editorTitle.children[0].style.display = "none"; + editorTitle.children[1].textContent = "Schema (Read Only)"; + + schemaButton.textContent = "Close Schema"; + schemaButton.onclick = closeSchema; + } +} + +function closeSchema() { + if (schemaEditorContainer.style.display !== "none") { + schemaEditorContainer.style.display = "none"; + diffEditorContainer.style.display = "block"; + + editingActionsButtons.forEach((btn, i) => { + btn.disabled = editingActionsState[i]; + }); + + editorTitle.children[0].style.display = "unset"; + editorTitle.children[1].textContent = "Edited"; + + schemaButton.textContent = "Show Schema"; + schemaButton.onclick = showSchema; + } +} + +const saveSessionBtn = document.getElementById("saveSession"); +saveSessionBtn.disabled = true; + +let password = document.getElementById("session-password"); +password.addEventListener("input", () => { + saveSessionBtn.disabled = password.value === ""; +}); + +function showSaveSessionModal() { + const saveSessionModal = new bootstrap.Modal(document.getElementById('saveSessionModal'), { + focus: true, keyboard: false + }); + saveSessionModal.show(); +} + +function saveSession() { + alias = document.getElementById("alias").innerText; + + bootstrap.Modal.getInstance(document.getElementById("saveSessionModal")).hide(); + + let preserveDisabled = []; + document.querySelectorAll(".editorButtonGroup button, .revisionButtonGroup button") + .forEach(btn => { + btn.disabled === true ? preserveDisabled.push(btn.id) : null; + }); + + toggleInteractables(true); + + fetch("/saveSession", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + alias: alias, + password: document.getElementById("session-password").value + }), + }) + .then((res) => { + document.getElementById("saveSessionForm").reset(); + + toggleInteractables(false, preserveDisabled); + + res.json() + .then((data) => { + if (res.status === 400) { + appendAlert('Error!', 'saveSessionError', `${data["error"]}`, 'danger'); + return; + } + + let sessions = JSON.parse(localStorage.getItem("sessions")) || {}; + sessions[alias] = data["ciphertext"]; + localStorage.setItem("sessions", JSON.stringify(sessions)); + + document.getElementById("showSaveSessionModal").innerText = "Session Saved"; + checkExistingSavedSession(); + }) + }) +} + +function executeRevision(event, operation) { + if (!["undo", "redo"].includes(operation)) { + appendAlert("Error!", "invalidRevOp", "Fatal! Invalid Revision Operation!", "danger"); + return; + } + + toggleInteractables(true, editorGroupIds, () => { + diffEditor.updateOptions({ readOnly: true }); + updateRevisionBtnsDisabledAttr(); + }); + fetch(`/${operation}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + alias: document.getElementById("alias").innerText, + }), + }) + .then(() => { + toggleInteractables(false, editorGroupIds, () => { + diffEditor.updateOptions({ readOnly: false }); + updateRevisionBtnsDisabledAttr(); + }); + find(event); + }) +} + +function updateRevisionBtnsDisabledAttr() { + fetch("/getRevisionStatus", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => res.json()) + .then((data) => { + revisionButtons[0].disabled = data.undo; + revisionButtons[1].disabled = data.redo; + }) +} + +function logout() { + toggleInteractables(true); + + fetch("/logout", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + alias: document.getElementById("alias").innerText, + }), + }) + .then((res) => { + toggleInteractables(false); + + if (res.status !== 302) { + res.json() + .then((data) => { + appendAlert('Error!', 'logoutError', `${data["error"]}`, 'danger'); + return; + }) + } + + window.location = res.url; + }) +} + +function checkExistingSavedSession() { + document.getElementById("existing-session-warning").style.display = + document.getElementById("alias").innerText in JSON.parse(localStorage.getItem("sessions") || "{}") + ? "flex" + : "none"; +} + +document.getElementById("close-save-session-modal").addEventListener("click", () => { + document.getElementById("saveSessionModal").querySelector("form").reset(); + saveSessionBtn.disabled = password.value === ""; +}); diff --git a/util/gem5-resources-manager/static/js/index.js b/util/gem5-resources-manager/static/js/index.js new file mode 100644 index 0000000000..1509d2d893 --- /dev/null +++ b/util/gem5-resources-manager/static/js/index.js @@ -0,0 +1,75 @@ +window.onload = () => { + let select = document.getElementById("sessions-dropdown"); + const sessions = JSON.parse(localStorage.getItem("sessions")); + + if (sessions === null) { + document.getElementById("showSavedSessionModal").disabled = true; + return; + } + + Object.keys(sessions).forEach((alias) => { + let option = document.createElement("option"); + option.value = alias; + option.innerHTML = alias; + select.appendChild(option); + }); +} + +const loadSessionBtn = document.getElementById("loadSession"); +loadSessionBtn.disabled = true; + +let password = document.getElementById("session-password"); +password.addEventListener("input", () => { + loadSessionBtn.disabled = password.value === ""; +}); + +document.getElementById("close-load-session-modal").addEventListener("click", () => { + document.getElementById("savedSessionModal").querySelector("form").reset(); +}) + +function showSavedSessionModal() { + const savedSessionModal = new bootstrap.Modal(document.getElementById('savedSessionModal'), { focus: true, keyboard: false }); + savedSessionModal.show(); +} + +function loadSession() { + bootstrap.Modal.getInstance(document.getElementById("savedSessionModal")).hide(); + + const alias = document.getElementById("sessions-dropdown").value; + const session = JSON.parse(localStorage.getItem("sessions"))[alias]; + + if (session === null) { + appendAlert("Error!", "sessionNotFound", "Saved Session Not Found!", "danger"); + return; + } + + toggleInteractables(true); + + fetch("/loadSession", { + method: "POST", + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + password: document.getElementById("session-password").value, + alias: alias, + session: session + }) + }) + .then((res) => { + toggleInteractables(false); + + if (res.status !== 200) { + res.json() + .then((error) => { + document.getElementById("savedSessionModal").querySelector("form").reset(); + appendAlert("Error!", "invalidStatus", `${error["error"]}`, "danger"); + return; + }) + } + + if (res.redirected) { + window.location = res.url; + } + }) +} diff --git a/util/gem5-resources-manager/static/js/login.js b/util/gem5-resources-manager/static/js/login.js new file mode 100644 index 0000000000..b21ffeb458 --- /dev/null +++ b/util/gem5-resources-manager/static/js/login.js @@ -0,0 +1,330 @@ +function handleMongoDBLogin(event) { + event.preventDefault(); + const activeTab = document.getElementById("mongodb-login-tabs").querySelector(".nav-link.active").getAttribute("id"); + + activeTab === "enter-uri-tab" ? handleEnteredURI() : handleGenerateURI(); + + return; +} + +function handleEnteredURI() { + const uri = document.getElementById('uri').value; + const collection = document.getElementById('collection').value; + const database = document.getElementById('database').value; + const alias = document.getElementById('alias').value; + const emptyInputs = [{ type: "Alias", value: alias }, { type: "Collection", value: collection }, { type: "Database", value: database }, { type: "URI", value: uri }]; + let error = false; + + for (let i = 0; i < emptyInputs.length; i++) { + if (emptyInputs[i].value === "") { + appendAlert("Error", `${emptyInputs[i].type}`, `Cannot Proceed Without ${emptyInputs[i].type} Value!`, 'danger'); + error = true; + } + } + + if (error) { + return; + } + + handleMongoURLFetch(uri, collection, database, alias); +} + +function handleGenerateURI() { + const connection = document.getElementById('connection').checked; + const username = document.getElementById('username').value; + const password = document.getElementById('password').value; + const collection = document.getElementById('collectionGenerate').value; + const database = document.getElementById('databaseGenerate').value; + const host = document.getElementById('host').value; + const alias = document.getElementById('aliasGenerate').value; + const options = document.getElementById('options').value.split(","); + let generatedURI = ""; + const emptyInputs = [{ type: "Alias", value: alias }, { type: "Host", value: host }, { type: "Collection", value: collection }, { type: "Database", value: database }]; + let error = false; + + for (let i = 0; i < emptyInputs.length; i++) { + if (emptyInputs[i].value === "") { + appendAlert("Error", `${emptyInputs[i].type}`, `Cannot Proceed Without ${emptyInputs[i].type} Value!`, 'danger'); + error = true; + } + } + + if (error) { + return; + } + + generatedURI = connection ? "mongodb+srv://" : "mongodb://"; + if (username && password) { + generatedURI += `${encodeURIComponent(username)}:${encodeURIComponent(password)}@`; + } + + generatedURI += host; + + if (options.length) { + generatedURI += `/?${options.join("&")}`; + } + + handleMongoURLFetch(generatedURI, collection, database, alias); +} + +function handleMongoURLFetch(uri, collection, database, alias) { + toggleInteractables(true); + + fetch("/validateMongoDB", + { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + uri: uri, + collection: collection, + database: database, + alias: alias + }) + }) + .then((res) => { + toggleInteractables(false); + + if (!res.ok) { + res.json() + .then(error => { + appendAlert('Error!', 'mongodbValidationError', `${error.error}`, 'danger'); + }); + return; + } + + res.redirected ? window.location = res.url : appendAlert('Error!', 'invalidRes', 'Invalid Server Response!', 'danger'); + }) +} + +function handleJSONLogin(event) { + event.preventDefault(); + const activeTab = document.getElementById("json-login-tabs").querySelector(".nav-link.active").getAttribute("id"); + if (activeTab === "remote-tab") { + handleRemoteJSON(); + } else if (activeTab === "existing-tab") { + const filename = document.getElementById("existing-dropdown").value; + if (filename !== "No Existing Files") { + toggleInteractables(true); + + fetch(`/existingJSON?filename=${filename}`, + { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + }) + .then((res) => { + toggleInteractables(false); + + if (res.status !== 200) { + appendAlert('Error!', 'invalidURL', 'Invalid JSON File URL!', 'danger'); + } + if (res.redirected) { + window.location = res.url; + } + }) + } + } else { + handleUploadJSON(); + } + return; +} + +function handleRemoteJSON() { + const url = document.getElementById("jsonRemoteURL").value; + const filename = document.getElementById("remoteFilename").value; + const emptyInputs = [{ type: "URL", value: url }, { type: "Filename", value: filename }]; + let error = false; + + for (let i = 0; i < emptyInputs.length; i++) { + if (emptyInputs[i].value === "") { + appendAlert("Error", `${emptyInputs[i].type}`, `Cannot Proceed Without ${emptyInputs[i].type} Value!`, 'danger'); + error = true; + } + } + + if (error) { + return; + } + + const params = new URLSearchParams(); + params.append('filename', filename + ".json"); + params.append('q', url); + + const flask_url = `/validateJSON?${params.toString()}`; + + toggleInteractables(true); + + fetch(flask_url, { + method: 'GET', + }) + .then((res) => { + toggleInteractables(false); + + if (res.status === 400) { + appendAlert('Error!', 'invalidURL', 'Invalid JSON File URL!', 'danger'); + } + + if (res.status === 409) { + const myModal = new bootstrap.Modal(document.getElementById('conflictResolutionModal'), { focus: true, keyboard: false }); + document.getElementById("header-filename").textContent = `"${filename}"`; + myModal.show(); + } + + if (res.redirected) { + window.location = res.url; + } + }) +} + +var filename; + +function handleUploadJSON() { + const jsonFile = document.getElementById("jsonFile"); + const file = jsonFile.files[0]; + + if (jsonFile.value === "") { + appendAlert('Error!', 'emptyUpload', 'Cannot Proceed Without Uploading a File!', 'danger'); + return; + } + + filename = file.name; + + const form = new FormData(); + form.append("file", file); + + toggleInteractables(true); + + fetch("/validateJSON", { + method: 'POST', + body: form + }) + .then((res) => { + toggleInteractables(false); + + if (res.status === 400) { + appendAlert('Error!', 'invalidUpload', 'Invalid JSON File Upload!', 'danger'); + } + + if (res.status === 409) { + const myModal = new bootstrap.Modal(document.getElementById('conflictResolutionModal'), { focus: true, keyboard: false }); + document.getElementById("header-filename").textContent = `"${filename}"`; + myModal.show(); + } + + if (res.redirected) { + window.location = res.url; + } + }) +} + +function saveConflictResolution() { + const conflictResolutionModal = bootstrap.Modal.getInstance(document.getElementById("conflictResolutionModal")); + const selectedValue = document.querySelector('input[name="conflictRadio"]:checked').id; + const activeTab = document.getElementById("json-login-tabs").querySelector(".nav-link.active").getAttribute("id"); + + if (selectedValue === null) { + appendAlert('Error!', 'nullRadio', 'Fatal! Null Radio!', 'danger'); + return; + } + + if (selectedValue === "clearInput") { + if (activeTab === "upload-tab") { + document.getElementById("jsonFile").value = ''; + } + + if (activeTab === "remote-tab") { + document.getElementById('remoteFilename').value = ''; + document.getElementById('jsonRemoteURL').value = ''; + } + + conflictResolutionModal.hide(); + handleConflictResolution("clearInput", filename.split(".")[0]); + return; + } + + if (selectedValue === "openExisting") { + conflictResolutionModal.hide(); + handleConflictResolution("openExisting", filename.split(".")[0]); + return; + } + + if (selectedValue === "overwrite") { + conflictResolutionModal.hide(); + handleConflictResolution("overwrite", filename.split(".")[0]); + return; + } + + if (selectedValue === "newFilename") { + const updatedFilename = document.getElementById("updatedFilename").value; + if (updatedFilename === "") { + appendAlert('Error!', 'emptyFilename', 'Must Enter A New Name!', 'danger'); + return; + } + + if (`${updatedFilename}.json` === filename) { + appendAlert('Error!', 'sameFilenames', 'Cannot Have Same Name as Current!', 'danger'); + return; + } + + conflictResolutionModal.hide(); + handleConflictResolution("newFilename", updatedFilename); + return; + } +} + +function handleConflictResolution(resolution, filename) { + const params = new URLSearchParams(); + params.append('resolution', resolution); + params.append('filename', filename !== "" ? filename + ".json" : ""); + + const flask_url = `/resolveConflict?${params.toString()}`; + toggleInteractables(true); + + fetch(flask_url, { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + }) + .then((res) => { + toggleInteractables(false); + + if (res.status === 204) { + console.log("Input Cleared, Cached File Deleted, Resources Unset"); + return; + } + + if (res.status !== 200) { + appendAlert('Error!', 'didNotRedirect', 'Server Did Not Redirect!', 'danger'); + return; + } + + if (res.redirected) { + window.location = res.url; + } + }) +} + +window.onload = () => { + if (window.location.pathname === "/login/json") { + fetch('/existingFiles', { + method: 'GET', + }) + .then((res) => res.json()) + .then((data) => { + let select = document.getElementById("existing-dropdown"); + if (data.length === 0) { + data = ["No Existing Files"]; + } + data.forEach((files) => { + let option = document.createElement("option"); + option.value = files; + option.innerHTML = files; + select.appendChild(option); + }); + }); + } +} diff --git a/util/gem5-resources-manager/static/styles/global.css b/util/gem5-resources-manager/static/styles/global.css new file mode 100644 index 0000000000..caa446a60b --- /dev/null +++ b/util/gem5-resources-manager/static/styles/global.css @@ -0,0 +1,231 @@ +@import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;600;700&display=swap'); +@import url('https://fonts.googleapis.com/css2?family=Mulish:wght@700&display=swap'); + +html, +body { + min-height: 100vh; + margin: 0; +} + +.btn-outline-primary { + --bs-btn-color: #0095AF; + --bs-btn-bg: #FFFFFF; + --bs-btn-border-color: #0095AF; + --bs-btn-hover-color: #fff; + --bs-btn-hover-bg: #0095AF; + --bs-btn-hover-border-color: #0095AF; + --bs-btn-focus-shadow-rgb: 13, 110, 253; + --bs-btn-active-color: #fff; + --bs-btn-active-bg: #0095AF; + --bs-btn-active-border-color: #0095AF; + --bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125); + --bs-btn-disabled-color: white; + --bs-btn-disabled-bg: grey; + --bs-btn-disabled-border-color: grey; + --bs-gradient: none; +} + +.btn-box-shadow { + box-shadow: rgba(0, 0, 0, 0.24) 0px 3px 8px; +} + +.calc-main-height { + height: calc(100vh - 81px); +} + +.main-text-semi { + font-family: 'Open Sans', sans-serif; + font-weight: 600; + font-size: 1rem; +} + +.main-text-regular, +.buttonGroup>button, +#markdown-body-styling p, +#markdown-body-styling li { + font-family: 'Open Sans', sans-serif; + font-weight: 400; + font-size: 1rem; +} + +.secondary-text-semi { + font-family: 'Open Sans', sans-serif; + font-weight: 600; + font-size: 1.25rem; +} + +.secondary-text-bold { + font-family: 'Open Sans', sans-serif; + font-weight: 600; + font-size: 1.25rem; +} + +.main-text-bold { + font-family: 'Open Sans', sans-serif; + font-weight: 700; + font-size: 1rem; +} + +.page-title, +#markdown-body-styling h1 { + color: #425469; + font-family: 'Mulish', sans-serif; + font-weight: 700; + font-size: 2.5rem; +} + +.main-panel-container { + max-width: 530px; + padding-top: 5rem; + padding-bottom: 5rem; +} + +.input-shadow, +.form-input-shadow>input { + box-shadow: rgba(0, 0, 0, 0.35) 0px 5px 15px; +} + +.panel-container { + background: rgba(0, 149, 175, 0.50); + border-radius: 1rem; + box-shadow: rgba(0, 0, 0, 0.24) 0px 3px 8px, rgba(0, 0, 0, 0.35) 0px 5px 15px; + height: 555px; + width: 530px; +} + +.panel-text-styling, +#generate-uri-form>label { + text-shadow: 0px 0px 5px rgba(0, 0, 0, 0.50); + color: white; +} + +.editorContainer { + width: 80%; +} + +.monaco-editor { + position: absolute !important; +} + +.editor-sizing { + min-height: 650px; + height: 75%; + width: 100%; +} + +#liveAlertPlaceholder { + position: absolute; + margin-top: 1rem; + right: 2rem; + margin-left: 2rem; + z-index: 1040; +} + +.alert-dismissible { + padding-right: 1rem; +} + +.reset-nav, +.login-nav { + --bs-nav-link-color: #0095AF; + --bs-nav-link-hover-color: white; + --bs-nav-tabs-link-active-color: #0095AF; +} + +.login-nav-link { + color: white; + text-shadow: 0px 0px 5px rgba(0, 0, 0, 0.50); +} + +.login-nav-link.active { + text-shadow: none; +} + +.navbar-nav>.nav-link:hover { + text-decoration: underline; +} + +.reset-nav-link:hover, +.login-nav-link:hover { + background-color: #0095AF; +} + +.reset-nav-link { + color: black; +} + +.form-check-input:checked { + background-color: #6c6c6c; + border-color: #6c6c6c; +} + +#markdown-body-styling h1 { + color: #425469; +} + +code { + display: inline-table; + overflow-x: auto; + padding: 2px; + color: #333; + background: #f8f8f8; + border: 1px solid #ccc; + border-radius: 3px; +} + +.editor-tooltips { + --bs-tooltip-bg: #0095AF; + --bs-tooltip-opacity: 1; +} + +#loading-container { + display: none; + position: absolute; + right: 2rem; + margin-top: 1rem; +} + +.spinner { + --bs-spinner-width: 2.25rem; + --bs-spinner-height: 2.25rem; + --bs-spinner-border-width: 0.45em; + border-color: #0095AF; + border-right-color: transparent; +} + +#saved-confirmation { + opacity: 0; + transition: opacity 0.5s; +} + +@media (max-width: 991px) { + .editorContainer { + width: 95%; + } +} + +@media (max-width: 425px) { + + .main-text-regular, + .main-text-semi, + .main-text-bold, + .buttonGroup>button, + #markdown-body-styling p { + font-size: 0.875rem; + } + + .secondary-text-semi { + font-size: 1rem; + } + + .page-title, + #markdown-body-styling h1 { + font-size: 2.25rem; + } +} + +@media (min-width: 425px) { + #databaseActions { + max-width: 375px; + } +} diff --git a/util/gem5-resources-manager/templates/404.html b/util/gem5-resources-manager/templates/404.html new file mode 100644 index 0000000000..0a38326b2e --- /dev/null +++ b/util/gem5-resources-manager/templates/404.html @@ -0,0 +1,20 @@ +{% extends 'base.html' %} {% block head %} +Page Not Found +{% endblock %} {% block body %} +
+
+

404

+

+ The page you are looking for does not seem to exist. +

+ Home +
+
+{% endblock %} diff --git a/util/gem5-resources-manager/templates/base.html b/util/gem5-resources-manager/templates/base.html new file mode 100644 index 0000000000..3b89f8f4c1 --- /dev/null +++ b/util/gem5-resources-manager/templates/base.html @@ -0,0 +1,96 @@ + + + + + + + + + {% block head %}{% endblock %} + + + +
+
+ Processing... + +
+ + {% block body %}{% endblock %} + + + diff --git a/util/gem5-resources-manager/templates/editor.html b/util/gem5-resources-manager/templates/editor.html new file mode 100644 index 0000000000..813a4d1a4b --- /dev/null +++ b/util/gem5-resources-manager/templates/editor.html @@ -0,0 +1,355 @@ +{% extends 'base.html' %} {% block head %} +Editor + +{% endblock %} {% block body %} + + +
+
+
+
+
+ Database Actions +
+ +
+
+ +
+ + +
+ + + +
+
+
+ Revision Actions +
+
+ + + + + + +
+
+
+
+ Other Actions +
+ + + + + + + +
+
+
+ +
+

{{ client_type }}

+

+ {{ alias }} +

+
+
+

Original

+

Modified

+
+
+
+
+ + + + + + + + + + + + +
+
+
+
+ +{% endblock %} diff --git a/util/gem5-resources-manager/templates/help.html b/util/gem5-resources-manager/templates/help.html new file mode 100644 index 0000000000..957a87b59e --- /dev/null +++ b/util/gem5-resources-manager/templates/help.html @@ -0,0 +1,20 @@ +{% extends 'base.html' %} {% block head %} +Help + +{% endblock %} {% block body %} +
+
+ {{ rendered_html|safe }} +
+
+{% endblock %} diff --git a/util/gem5-resources-manager/templates/index.html b/util/gem5-resources-manager/templates/index.html new file mode 100644 index 0000000000..6321a9e5b6 --- /dev/null +++ b/util/gem5-resources-manager/templates/index.html @@ -0,0 +1,116 @@ +{% extends 'base.html' %} {% block head %} +Resources Manager +{% endblock %} {% block body %} + +
+
+
+
+
+ gem5 +
+
+
+ + + + + + + +
+
+
+
+ +{% endblock %} diff --git a/util/gem5-resources-manager/templates/login/login_json.html b/util/gem5-resources-manager/templates/login/login_json.html new file mode 100644 index 0000000000..98663a304b --- /dev/null +++ b/util/gem5-resources-manager/templates/login/login_json.html @@ -0,0 +1,242 @@ +{% extends 'base.html' %} {% block head %} +JSON Login +{% endblock %} {% block body %} + +
+
+
+
+

JSON

+
+ +
+
+
+
+ +
+ + .json +
+
+ + +
+
+
+
+ +
+
+
+
+ + +
+
+
+
+
+
+ +
+
+
+
+ +{% endblock %} diff --git a/util/gem5-resources-manager/templates/login/login_mongodb.html b/util/gem5-resources-manager/templates/login/login_mongodb.html new file mode 100644 index 0000000000..83361b578e --- /dev/null +++ b/util/gem5-resources-manager/templates/login/login_mongodb.html @@ -0,0 +1,189 @@ +{% extends 'base.html' %} {% block head %} +MongoDB Login +{% endblock %} {% block body %} +
+
+
+
+

MongoDB

+
+ +
+
+
+ + + + + + + + +
+
+
+
+
+ Standard +
+ +
+ DNS Seed List +
+ + + + + + + + + + + + + + +
+
+
+
+
+
+ +
+
+
+
+ +{% endblock %} diff --git a/util/gem5-resources-manager/test/__init__.py b/util/gem5-resources-manager/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/util/gem5-resources-manager/test/api_test.py b/util/gem5-resources-manager/test/api_test.py new file mode 100644 index 0000000000..0ff439cd2e --- /dev/null +++ b/util/gem5-resources-manager/test/api_test.py @@ -0,0 +1,722 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import flask +import contextlib +import unittest +from server import app +import server +import json +from bson import json_util +from unittest.mock import patch +import mongomock +from api.mongo_client import MongoDBClient +import requests + + +@contextlib.contextmanager +def captured_templates(app): + """ + This is a context manager that allows you to capture the templates + that are rendered during a test. + """ + recorded = [] + + def record(sender, template, context, **extra): + recorded.append((template, context)) + + flask.template_rendered.connect(record, app) + try: + yield recorded + finally: + flask.template_rendered.disconnect(record, app) + + +class TestAPI(unittest.TestCase): + @patch.object( + MongoDBClient, + "_get_database", + return_value=mongomock.MongoClient().db.collection, + ) + def setUp(self, mock_get_database): + """This method sets up the test environment.""" + self.ctx = app.app_context() + self.ctx.push() + self.app = app + self.test_client = app.test_client() + self.alias = "test" + objects = [] + with open("./test/refs/resources.json", "rb") as f: + objects = json.loads(f.read(), object_hook=json_util.object_hook) + self.collection = mock_get_database() + for obj in objects: + self.collection.insert_one(obj) + + self.test_client.post( + "/validateMongoDB", + json={ + "uri": "mongodb://localhost:27017", + "database": "test", + "collection": "test", + "alias": self.alias, + }, + ) + + def tearDown(self): + """ + This method tears down the test environment. + """ + self.collection.drop() + self.ctx.pop() + + def test_get_helppage(self): + """ + This method tests the call to the help page. + It checks if the call is GET, status code is 200 and if the template + rendered is help.html. + """ + with captured_templates(self.app) as templates: + response = self.test_client.get("/help") + self.assertEqual(response.status_code, 200) + self.assertTrue(templates[0][0].name == "help.html") + + def test_get_mongodb_loginpage(self): + """ + This method tests the call to the MongoDB login page. + It checks if the call is GET, status code is 200 and if the template + rendered is mongoDBLogin.html. + """ + with captured_templates(self.app) as templates: + response = self.test_client.get("/login/mongodb") + self.assertEqual(response.status_code, 200) + self.assertTrue(templates[0][0].name == "login/login_mongodb.html") + + def test_get_json_loginpage(self): + """ + This method tests the call to the JSON login page. + It checks if the call is GET, status code is 200 and if the template + rendered is jsonLogin.html. + """ + with captured_templates(self.app) as templates: + response = self.test_client.get("/login/json") + self.assertEqual(response.status_code, 200) + self.assertTrue(templates[0][0].name == "login/login_json.html") + + def test_get_editorpage(self): + """This method tests the call to the editor page. + It checks if the call is GET, status code is 200 and if the template + rendered is editor.html. + """ + with captured_templates(self.app) as templates: + response = self.test_client.get("/editor?alias=test") + self.assertEqual(response.status_code, 200) + self.assertTrue(templates[0][0].name == "editor.html") + + def test_get_editorpage_invalid(self): + """This method tests the call to the editor page without required + query parameters. + It checks if the call is GET, status code is 404 and if the template + rendered is 404.html. + """ + with captured_templates(self.app) as templates: + response = self.test_client.get("/editor") + self.assertEqual(response.status_code, 404) + self.assertTrue(templates[0][0].name == "404.html") + response = self.test_client.get("/editor?alias=invalid") + self.assertEqual(response.status_code, 404) + self.assertTrue(templates[0][0].name == "404.html") + + def test_default_call(self): + """This method tests the default call to the API.""" + with captured_templates(self.app) as templates: + response = self.test_client.get("/") + self.assertEqual(response.status_code, 200) + self.assertTrue(templates[0][0].name == "index.html") + + def test_default_call_is_not_post(self): + """This method tests that the default call is not a POST.""" + + response = self.test_client.post("/") + self.assertEqual(response.status_code, 405) + + def test_get_categories(self): + """ + The methods tests if the category call returns the same categories as + the schema. + """ + + response = self.test_client.get("/categories") + post_response = self.test_client.post("/categories") + categories = [ + "workload", + "disk-image", + "binary", + "kernel", + "checkpoint", + "git", + "bootloader", + "file", + "directory", + "simpoint", + "simpoint-directory", + "resource", + "looppoint-pinpoint-csv", + "looppoint-json", + ] + self.assertEqual(post_response.status_code, 405) + self.assertEqual(response.status_code, 200) + returnedData = json.loads(response.data) + self.assertTrue(returnedData == categories) + + def test_get_schema(self): + """ + The methods tests if the schema call returns the same schema as the + schema file. + """ + + response = self.test_client.get("/schema") + post_response = self.test_client.post("/schema") + self.assertEqual(post_response.status_code, 405) + self.assertEqual(response.status_code, 200) + returnedData = json.loads(response.data) + schema = {} + schema = requests.get( + "https://resources.gem5.org/gem5-resources-schema.json" + ).json() + self.assertTrue(returnedData == schema) + + def test_insert(self): + """This method tests the insert method of the API.""" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + response = self.test_client.post( + "/insert", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + resource = self.collection.find({"id": "test-resource"}, {"_id": 0}) + + json_resource = json.loads(json_util.dumps(resource[0])) + self.assertTrue(json_resource == test_resource) + + def test_find_no_version(self): + """This method tests the find method of the API.""" + test_id = "test-resource" + test_resource_version = "1.0.0" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + response = self.test_client.post( + "/find", + json={"id": test_id, "resource_version": "", "alias": self.alias}, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + def test_find_not_exist(self): + """This method tests the find method of the API.""" + test_id = "test-resource" + response = self.test_client.post( + "/find", + json={"id": test_id, "resource_version": "", "alias": self.alias}, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == {"exists": False}) + + def test_find_with_version(self): + """This method tests the find method of the API.""" + test_id = "test-resource" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + test_resource["resource_version"] = "1.0.1" + test_resource["description"] = "test-description2" + self.collection.insert_one(test_resource.copy()) + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": "1.0.1", + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + return_json = response.json + self.assertTrue(return_json["description"] == "test-description2") + self.assertTrue(return_json["resource_version"] == "1.0.1") + self.assertTrue(return_json == test_resource) + + def test_delete(self): + """This method tests the delete method of the API.""" + test_id = "test-resource" + test_version = "1.0.0" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + response = self.test_client.post( + "/delete", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Deleted"}) + resource = self.collection.find({"id": "test-resource"}, {"_id": 0}) + json_resource = json.loads(json_util.dumps(resource)) + self.assertTrue(json_resource == []) + + def test_if_resource_exists_true(self): + """This method tests the checkExists method of the API.""" + test_id = "test-resource" + test_version = "1.0.0" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + response = self.test_client.post( + "/checkExists", + json={ + "id": test_id, + "resource_version": test_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"exists": True}) + + def test_if_resource_exists_false(self): + """This method tests the checkExists method of the API.""" + test_id = "test-resource" + test_version = "1.0.0" + response = self.test_client.post( + "/checkExists", + json={ + "id": test_id, + "resource_version": test_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"exists": False}) + + def test_get_resource_versions(self): + """This method tests the getResourceVersions method of the API.""" + test_id = "test-resource" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + test_resource["resource_version"] = "1.0.1" + test_resource["description"] = "test-description2" + self.collection.insert_one(test_resource.copy()) + response = self.test_client.post( + "/versions", json={"id": test_id, "alias": self.alias} + ) + return_json = json.loads(response.data) + self.assertEqual(response.status_code, 200) + self.assertEqual( + return_json, + [{"resource_version": "1.0.1"}, {"resource_version": "1.0.0"}], + ) + + def test_update_resource(self): + """This method tests the updateResource method of the API.""" + test_id = "test-resource" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + original_resource = test_resource.copy() + self.collection.insert_one(test_resource.copy()) + test_resource["description"] = "test-description2" + test_resource["example_usage"] = "test-usage2" + response = self.test_client.post( + "/update", + json={ + "original_resource": original_resource, + "resource": test_resource, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Updated"}) + resource = self.collection.find({"id": test_id}, {"_id": 0}) + json_resource = json.loads(json_util.dumps(resource)) + self.assertTrue(json_resource == [test_resource]) + + def test_keys_1(self): + """This method tests the keys method of the API.""" + response = self.test_client.post( + "/keys", json={"category": "simpoint", "id": "test-resource"} + ) + test_response = { + "category": "simpoint", + "id": "test-resource", + "author": [], + "description": "", + "license": "", + "source_url": "", + "tags": [], + "example_usage": "", + "gem5_versions": [], + "resource_version": "1.0.0", + "simpoint_interval": 0, + "warmup_interval": 0, + } + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.data), test_response) + + def test_keys_2(self): + """This method tests the keys method of the API.""" + response = self.test_client.post( + "/keys", json={"category": "disk-image", "id": "test-resource"} + ) + test_response = { + "category": "disk-image", + "id": "test-resource", + "author": [], + "description": "", + "license": "", + "source_url": "", + "tags": [], + "example_usage": "", + "gem5_versions": [], + "resource_version": "1.0.0", + } + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.data), test_response) + + def test_undo(self): + """This method tests the undo method of the API.""" + test_id = "test-resource" + test_resource = { + "category": "disk-image", + "id": "test-resource", + "author": [], + "description": "", + "license": "", + "source_url": "", + "tags": [], + "example_usage": "", + "gem5_versions": [], + "resource_version": "1.0.0", + } + original_resource = test_resource.copy() + # insert resource + response = self.test_client.post( + "/insert", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + # update resource + test_resource["description"] = "test-description2" + test_resource["example_usage"] = "test-usage2" + response = self.test_client.post( + "/update", + json={ + "original_resource": original_resource, + "resource": test_resource, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Updated"}) + # check if resource is updated + resource = self.collection.find({"id": test_id}, {"_id": 0}) + json_resource = json.loads(json_util.dumps(resource)) + self.assertTrue(json_resource == [test_resource]) + # undo update + response = self.test_client.post("/undo", json={"alias": self.alias}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Undone"}) + # check if resource is back to original + resource = self.collection.find({"id": test_id}, {"_id": 0}) + json_resource = json.loads(json_util.dumps(resource)) + self.assertTrue(json_resource == [original_resource]) + + def test_redo(self): + """This method tests the undo method of the API.""" + test_id = "test-resource" + test_resource = { + "category": "disk-image", + "id": "test-resource", + "author": [], + "description": "", + "license": "", + "source_url": "", + "tags": [], + "example_usage": "", + "gem5_versions": [], + "resource_version": "1.0.0", + } + original_resource = test_resource.copy() + # insert resource + response = self.test_client.post( + "/insert", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + # update resource + test_resource["description"] = "test-description2" + test_resource["example_usage"] = "test-usage2" + response = self.test_client.post( + "/update", + json={ + "original_resource": original_resource, + "resource": test_resource, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Updated"}) + # check if resource is updated + resource = self.collection.find({"id": test_id}, {"_id": 0}) + json_resource = json.loads(json_util.dumps(resource)) + self.assertTrue(json_resource == [test_resource]) + # undo update + response = self.test_client.post("/undo", json={"alias": self.alias}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Undone"}) + # check if resource is back to original + resource = self.collection.find({"id": test_id}, {"_id": 0}) + json_resource = json.loads(json_util.dumps(resource)) + self.assertTrue(json_resource == [original_resource]) + # redo update + response = self.test_client.post("/redo", json={"alias": self.alias}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Redone"}) + # check if resource is updated again + resource = self.collection.find({"id": test_id}, {"_id": 0}) + json_resource = json.loads(json_util.dumps(resource)) + self.assertTrue(json_resource == [test_resource]) + + def test_invalid_alias(self): + test_id = "test-resource" + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + alias = "invalid" + response = self.test_client.post( + "/insert", json={"resource": test_resource, "alias": alias} + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post( + "/find", + json={"id": test_id, "resource_version": "", "alias": alias}, + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post( + "/delete", json={"resource": test_resource, "alias": alias} + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post( + "/checkExists", + json={"id": test_id, "resource_version": "", "alias": alias}, + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post( + "/versions", json={"id": test_id, "alias": alias} + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post( + "/update", + json={ + "original_resource": test_resource, + "resource": test_resource, + "alias": alias, + }, + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post("/undo", json={"alias": alias}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post("/redo", json={"alias": alias}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post( + "/getRevisionStatus", json={"alias": alias} + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + response = self.test_client.post("/saveSession", json={"alias": alias}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json, {"error": "database not found"}) + + def test_get_revision_status_valid(self): + response = self.test_client.post( + "/getRevisionStatus", json={"alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"undo": 1, "redo": 1}) + + @patch.object( + MongoDBClient, + "_get_database", + return_value=mongomock.MongoClient().db.collection, + ) + def test_save_session_load_session(self, mock_get_database): + password = "test" + expected_session = server.databases["test"].save_session() + response = self.test_client.post( + "/saveSession", json={"alias": self.alias, "password": password} + ) + self.assertEqual(response.status_code, 200) + + response = self.test_client.post( + "/loadSession", + json={ + "alias": self.alias, + "session": response.json["ciphertext"], + "password": password, + }, + ) + self.assertEqual(response.status_code, 302) + self.assertEqual( + expected_session, server.databases[self.alias].save_session() + ) + + def test_logout(self): + response = self.test_client.post("/logout", json={"alias": self.alias}) + self.assertEqual(response.status_code, 302) + self.assertNotIn(self.alias, server.databases) diff --git a/util/gem5-resources-manager/test/comprehensive_test.py b/util/gem5-resources-manager/test/comprehensive_test.py new file mode 100644 index 0000000000..4c32087324 --- /dev/null +++ b/util/gem5-resources-manager/test/comprehensive_test.py @@ -0,0 +1,407 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from server import app +import json +from bson import json_util +import copy +import mongomock +from unittest.mock import patch +from api.mongo_client import MongoDBClient + + +class TestComprehensive(unittest.TestCase): + @patch.object( + MongoDBClient, + "_get_database", + return_value=mongomock.MongoClient().db.collection, + ) + def setUp(self, mock_get_database): + """This method sets up the test environment.""" + self.ctx = app.app_context() + self.ctx.push() + self.app = app + self.test_client = app.test_client() + self.alias = "test" + objects = [] + with open("./test/refs/resources.json", "rb") as f: + objects = json.loads(f.read(), object_hook=json_util.object_hook) + self.collection = mock_get_database() + for obj in objects: + self.collection.insert_one(obj) + + self.test_client.post( + "/validateMongoDB", + json={ + "uri": "mongodb://localhost:27017", + "database": "test", + "collection": "test", + "alias": self.alias, + }, + ) + + def tearDown(self): + """This method tears down the test environment.""" + self.collection.drop() + self.ctx.pop() + + def test_insert_find_update_find(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + original_resource = test_resource.copy() + test_id = test_resource["id"] + test_resource_version = test_resource["resource_version"] + # insert resource + response = self.test_client.post( + "/insert", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": test_resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + # update resource + test_resource["description"] = "test-description-2" + test_resource["author"].append("test-author-2") + response = self.test_client.post( + "/update", + json={ + "original_resource": original_resource, + "resource": test_resource, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Updated"}) + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": test_resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + def test_find_new_insert(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + test_id = test_resource["id"] + test_resource_version = test_resource["resource_version"] + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": test_resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"exists": False}) + # insert resource + response = self.test_client.post( + "/insert", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": test_resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + def test_insert_find_new_version_find_older(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + test_id = test_resource["id"] + test_resource_version = test_resource["resource_version"] + # insert resource + response = self.test_client.post( + "/insert", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": test_resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + # add new version + test_resource_new_version = copy.deepcopy(test_resource) + test_resource_new_version["description"] = "test-description-2" + test_resource_new_version["author"].append("test-author-2") + test_resource_new_version["resource_version"] = "1.0.1" + + response = self.test_client.post( + "/insert", + json={"resource": test_resource_new_version, "alias": self.alias}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + + # get resource versions + response = self.test_client.post( + "/versions", json={"id": test_id, "alias": self.alias} + ) + return_json = json.loads(response.data) + self.assertEqual(response.status_code, 200) + self.assertEqual( + return_json, + [{"resource_version": "1.0.1"}, {"resource_version": "1.0.0"}], + ) + + resource_version = return_json[1]["resource_version"] + # find older version + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + def test_find_add_new_version_delete_older(self): + test_resource = { + "category": "binary", + "id": "binary-example", + "description": "binary-example documentation.", + "architecture": "ARM", + "is_zipped": False, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": ( + "http://dist.gem5.org/dist/develop/" + "test-progs/hello/bin/arm/linux/hello64-static" + ), + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + test_id = test_resource["id"] + test_resource_version = test_resource["resource_version"] + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": test_resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + # add new version + test_resource_new_version = copy.deepcopy(test_resource) + test_resource_new_version["description"] = "test-description-2" + test_resource_new_version["resource_version"] = "1.0.1" + + response = self.test_client.post( + "/insert", + json={"resource": test_resource_new_version, "alias": self.alias}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + + # get resource versions + response = self.test_client.post( + "/versions", json={"id": test_id, "alias": self.alias} + ) + return_json = json.loads(response.data) + self.assertEqual(response.status_code, 200) + self.assertEqual( + return_json, + [{"resource_version": "1.0.1"}, {"resource_version": "1.0.0"}], + ) + # delete older version + response = self.test_client.post( + "/delete", json={"resource": test_resource, "alias": self.alias} + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Deleted"}) + + # get resource versions + response = self.test_client.post( + "/versions", json={"id": test_id, "alias": self.alias} + ) + return_json = json.loads(response.data) + self.assertEqual(response.status_code, 200) + self.assertEqual(return_json, [{"resource_version": "1.0.1"}]) + + def test_find_add_new_version_update_older(self): + test_resource = { + "category": "binary", + "id": "binary-example", + "description": "binary-example documentation.", + "architecture": "ARM", + "is_zipped": False, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": ( + "http://dist.gem5.org/dist/develop/" + "test-progs/hello/bin/arm/linux/hello64-static" + ), + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + original_resource = test_resource.copy() + test_id = test_resource["id"] + test_resource_version = test_resource["resource_version"] + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": test_resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) + + # add new version + test_resource_new_version = copy.deepcopy(test_resource) + test_resource_new_version["description"] = "test-description-2" + test_resource_new_version["resource_version"] = "1.0.1" + + response = self.test_client.post( + "/insert", + json={"resource": test_resource_new_version, "alias": self.alias}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Inserted"}) + + # get resource versions + response = self.test_client.post( + "/versions", json={"id": test_id, "alias": self.alias} + ) + return_json = json.loads(response.data) + self.assertEqual(response.status_code, 200) + self.assertEqual( + return_json, + [{"resource_version": "1.0.1"}, {"resource_version": "1.0.0"}], + ) + + resource_version = return_json[1]["resource_version"] + + # update older version + test_resource["description"] = "test-description-3" + + response = self.test_client.post( + "/update", + json={ + "original_resource": original_resource, + "resource": test_resource, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {"status": "Updated"}) + + # find resource + response = self.test_client.post( + "/find", + json={ + "id": test_id, + "resource_version": resource_version, + "alias": self.alias, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertTrue(response.json == test_resource) diff --git a/util/gem5-resources-manager/test/json_client_test.py b/util/gem5-resources-manager/test/json_client_test.py new file mode 100644 index 0000000000..e08eb18452 --- /dev/null +++ b/util/gem5-resources-manager/test/json_client_test.py @@ -0,0 +1,262 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from api.json_client import JSONClient +from server import app +import json +from bson import json_util +from unittest.mock import patch +from pathlib import Path +from api.json_client import JSONClient + + +def get_json(): + with open("test/refs/test_json.json", "r") as f: + jsonFile = f.read() + return json.loads(jsonFile) + + +def mockinit(self, file_path): + self.file_path = Path("test/refs/") / file_path + with open(self.file_path, "r") as f: + self.resources = json.load(f) + + +class TestJson(unittest.TestCase): + @classmethod + def setUpClass(cls): + with open("./test/refs/resources.json", "rb") as f: + jsonFile = f.read() + with open("./test/refs/test_json.json", "wb") as f: + f.write(jsonFile) + + @classmethod + def tearDownClass(cls): + Path("./test/refs/test_json.json").unlink() + + @patch.object(JSONClient, "__init__", mockinit) + def setUp(self): + """This method sets up the test environment.""" + with open("./test/refs/test_json.json", "rb") as f: + jsonFile = f.read() + self.original_json = json.loads(jsonFile) + self.json_client = JSONClient("test_json.json") + + def tearDown(self): + """This method tears down the test environment.""" + with open("./test/refs/test_json.json", "w") as f: + json.dump(self.original_json, f, indent=4) + + def test_insertResource(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + response = self.json_client.insert_resource(test_resource) + self.assertEqual(response, {"status": "Inserted"}) + json_data = get_json() + self.assertNotEqual(json_data, self.original_json) + self.assertIn(test_resource, json_data) + + def test_insertResource_duplicate(self): + test_resource = { + "category": "diskimage", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": True, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": ( + "http://dist.gem5.org/dist/develop/images" + "/x86/ubuntu-18-04/x86-ubuntu.img.gz" + ), + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + response = self.json_client.insert_resource(test_resource) + self.assertEqual(response, {"status": "Resource already exists"}) + + def test_find_no_version(self): + expected_response = { + "category": "diskimage", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": True, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": ( + "http://dist.gem5.org/dist/develop/images" + "/x86/ubuntu-18-04/x86-ubuntu.img.gz" + ), + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + response = self.json_client.find_resource( + {"id": expected_response["id"]} + ) + self.assertEqual(response, expected_response) + + def test_find_with_version(self): + expected_response = { + "category": "kernel", + "id": "kernel-example", + "description": "kernel-example documentation.", + "architecture": "RISCV", + "is_zipped": False, + "md5sum": "60a53c7d47d7057436bf4b9df707a841", + "url": ( + "http://dist.gem5.org/dist/develop" + "/kernels/x86/static/vmlinux-5.4.49" + ), + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + response = self.json_client.find_resource( + { + "id": expected_response["id"], + "resource_version": expected_response["resource_version"], + } + ) + self.assertEqual(response, expected_response) + + def test_find_not_found(self): + response = self.json_client.find_resource({"id": "not-found"}) + self.assertEqual(response, {"exists": False}) + + def test_deleteResource(self): + deleted_resource = { + "category": "diskimage", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": True, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": ( + "http://dist.gem5.org/dist/develop/" + "images/x86/ubuntu-18-04/x86-ubuntu.img.gz" + ), + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + response = self.json_client.delete_resource( + { + "id": deleted_resource["id"], + "resource_version": deleted_resource["resource_version"], + } + ) + self.assertEqual(response, {"status": "Deleted"}) + json_data = get_json() + self.assertNotEqual(json_data, self.original_json) + self.assertNotIn(deleted_resource, json_data) + + def test_updateResource(self): + updated_resource = { + "category": "diskimage", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": True, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": ( + "http://dist.gem5.org/dist/develop/images" + "/x86/ubuntu-18-04/x86-ubuntu.img.gz" + ), + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + original_resource = { + "category": "diskimage", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": True, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": ( + "http://dist.gem5.org/dist/develop/" + "images/x86/ubuntu-18-04/x86-ubuntu.img.gz" + ), + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + } + response = self.json_client.update_resource( + { + "original_resource": original_resource, + "resource": updated_resource, + } + ) + self.assertEqual(response, {"status": "Updated"}) + json_data = get_json() + self.assertNotEqual(json_data, self.original_json) + self.assertIn(updated_resource, json_data) + + def test_getVersions(self): + resource_id = "kernel-example" + response = self.json_client.get_versions({"id": resource_id}) + self.assertEqual( + response, + [{"resource_version": "2.0.0"}, {"resource_version": "1.0.0"}], + ) + + def test_checkResourceExists_True(self): + resource_id = "kernel-example" + resource_version = "1.0.0" + response = self.json_client.check_resource_exists( + {"id": resource_id, "resource_version": resource_version} + ) + self.assertEqual(response, {"exists": True}) + + def test_checkResourceExists_False(self): + resource_id = "kernel-example" + resource_version = "3.0.0" + response = self.json_client.check_resource_exists( + {"id": resource_id, "resource_version": resource_version} + ) + self.assertEqual(response, {"exists": False}) diff --git a/util/gem5-resources-manager/test/mongo_client_test.py b/util/gem5-resources-manager/test/mongo_client_test.py new file mode 100644 index 0000000000..761475ead8 --- /dev/null +++ b/util/gem5-resources-manager/test/mongo_client_test.py @@ -0,0 +1,281 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from server import app, databases +import json +from bson import json_util +import mongomock +from unittest.mock import patch +from api.mongo_client import MongoDBClient + + +class TestApi(unittest.TestCase): + """This is a test class that tests the API.""" + + API_URL = "http://127.0.0.1:5000" + + @patch.object( + MongoDBClient, + "_get_database", + return_value=mongomock.MongoClient().db.collection, + ) + def setUp(self, mock_get_database): + """This method sets up the test environment.""" + objects = [] + with open("./test/refs/resources.json", "rb") as f: + objects = json.loads(f.read(), object_hook=json_util.object_hook) + self.collection = mock_get_database() + for obj in objects: + self.collection.insert_one(obj) + self.mongo_client = MongoDBClient( + "mongodb://localhost:27017", "test", "test" + ) + + def tearDown(self): + """This method tears down the test environment.""" + self.collection.drop() + + def test_insertResource(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + ret_value = self.mongo_client.insert_resource(test_resource) + self.assertEqual(ret_value, {"status": "Inserted"}) + self.assertEqual( + self.collection.find({"id": "test-resource"})[0], test_resource + ) + self.collection.delete_one({"id": "test-resource"}) + + def test_insertResource_duplicate(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources/" + "tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource) + ret_value = self.mongo_client.insert_resource(test_resource) + self.assertEqual(ret_value, {"status": "Resource already exists"}) + + def test_findResource_no_version(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources" + "/tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + ret_value = self.mongo_client.find_resource({"id": "test-resource"}) + self.assertEqual(ret_value, test_resource) + self.collection.delete_one({"id": "test-resource"}) + + def test_findResource_with_version(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources" + "/tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + test_resource["resource_version"] = "2.0.0" + test_resource["description"] = "test-description2" + self.collection.insert_one(test_resource.copy()) + ret_value = self.mongo_client.find_resource( + {"id": "test-resource", "resource_version": "2.0.0"} + ) + self.assertEqual(ret_value, test_resource) + + def test_findResource_not_found(self): + ret_value = self.mongo_client.find_resource({"id": "test-resource"}) + self.assertEqual(ret_value, {"exists": False}) + + def test_deleteResource(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources" + "/tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + ret_value = self.mongo_client.delete_resource( + {"id": "test-resource", "resource_version": "1.0.0"} + ) + self.assertEqual(ret_value, {"status": "Deleted"}) + + self.assertEqual( + json.loads( + json_util.dumps(self.collection.find({"id": "test-resource"})) + ), + [], + ) + + def test_updateResource(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources" + "/tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + original_resource = test_resource.copy() + self.collection.insert_one(test_resource.copy()) + test_resource["author"].append("test-author2") + test_resource["description"] = "test-description2" + ret_value = self.mongo_client.update_resource( + {"original_resource": original_resource, "resource": test_resource} + ) + self.assertEqual(ret_value, {"status": "Updated"}) + self.assertEqual( + self.collection.find({"id": "test-resource"}, {"_id": 0})[0], + test_resource, + ) + + def test_checkResourceExists(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources" + "/tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + ret_value = self.mongo_client.check_resource_exists( + {"id": "test-resource", "resource_version": "1.0.0"} + ) + self.assertEqual(ret_value, {"exists": True}) + + def test_checkResourceExists_not_found(self): + ret_value = self.mongo_client.check_resource_exists( + {"id": "test-resource", "resource_version": "1.0.0"} + ) + self.assertEqual(ret_value, {"exists": False}) + + def test_getVersion(self): + test_resource = { + "category": "diskimage", + "id": "test-resource", + "author": ["test-author"], + "description": "test-description", + "license": "test-license", + "source_url": ( + "https://github.com/gem5/gem5-resources" + "/tree/develop/src/x86-ubuntu" + ), + "tags": ["test-tag", "test-tag2"], + "example_usage": " test-usage", + "gem5_versions": [ + "22.1", + ], + "resource_version": "1.0.0", + } + self.collection.insert_one(test_resource.copy()) + test_resource["resource_version"] = "2.0.0" + test_resource["description"] = "test-description2" + self.collection.insert_one(test_resource.copy()) + ret_value = self.mongo_client.get_versions({"id": "test-resource"}) + self.assertEqual( + ret_value, + [{"resource_version": "2.0.0"}, {"resource_version": "1.0.0"}], + ) diff --git a/util/gem5-resources-manager/test/refs/resources.json b/util/gem5-resources-manager/test/refs/resources.json new file mode 100644 index 0000000000..614f8dc764 --- /dev/null +++ b/util/gem5-resources-manager/test/refs/resources.json @@ -0,0 +1,196 @@ +[ + { + "category": "kernel", + "id": "kernel-example", + "description": "kernel-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "60a53c7d47d7057436bf4b9df707a841", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.4.49", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "kernel", + "id": "kernel-example", + "description": "kernel-example documentation 2.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "60a53c7d47d7057436bf4b9df707a841", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.4.49", + "source": "src/linux-kernel", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "diskimage", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "binary", + "id": "binary-example", + "description": "binary-example documentation.", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + + }, + { + "category": "bootloader", + "id": "bootloader-example", + "description": "bootloader documentation.", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "checkpoint", + "id": "checkpoint-example", + "description": "checkpoint-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "git", + "id": "git-example", + "description": null, + "is_zipped": false, + "is_tar_archive": true, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "file", + "id": "file-example", + "description": null, + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "directory", + "id": "directory-example", + "description": "directory-example documentation.", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint-directory", + "id": "simpoint-directory-example", + "description": "simpoint directory documentation.", + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "simpoint_file": "simpoint.simpt", + "weight_file": "simpoint.weight", + "workload_name": "Example Workload", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint", + "id": "simpoint-example", + "description": "simpoint documentation.", + "simpoint_interval": 1000000, + "warmup_interval": 23445, + "simpoint_list": [ + 2, + 3, + 4, + 15 + ], + "weight_list": [ + 0.1, + 0.2, + 0.4, + 0.3 + ], + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-pinpoint-csv", + "id": "looppoint-pinpoint-csv-resource", + "description": "A looppoint pinpoints csv file.", + "is_zipped": false, + "md5sum": "199ab22dd463dc70ee2d034bfe045082", + "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-json", + "id": "looppoint-json-restore-resource-region-1", + "description": "A looppoint json file resource.", + "is_zipped": false, + "region_id": "1", + "md5sum": "a71ed64908b082ea619b26b940a643c1", + "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +]