resources, stdlib: Add support for local files in obtain_resource (#204)

This patch allows a local JSON file to specify a local path in the JSON
object of a Resource, through the "url" field.

Local paths can be entered with the prefix "file:" in the "url" field.

If the local path exists, then the Resource from there is copied into
the resource directory defined in the
function earlier.

This behavior is the same as using specific Resource classes (ex.
BinaryResource) and passing a local_path into the function.

But, the above class does not allow simultaneous creation of local
Resources and Workloads of those local Resources.

With this patch, someone can use a local JSON, specify the location of
local Resources and create a Workload from those Resources and test both
together.

src/python/gem5/resources/downloader.py

@@ -34,6 +34,7 @@ import random
 from pathlib import Path
 import tarfile
 from urllib.error import HTTPError
+from urllib.parse import urlparse
 from typing import List, Optional, Dict
 
 from _m5 import core
@@ -310,19 +311,34 @@ def get_resource(
         if run_unzip:
             download_dest += zip_extension
 
-        # TODO: Might be nice to have some kind of download status bar here.
-        # TODO: There might be a case where this should be silenced.
-        print(
-            "Resource '{}' was not found locally. Downloading to '{}'...".format(
-                resource_name, download_dest
+        file_uri_path = _file_uri_to_path(resource_json["url"])
+        if file_uri_path:
+            if not file_uri_path.exists():
+                raise Exception(
+                    f"Could not find file at path '{file_uri_path}'"
+                )
+            print(
+                "Resource '{}' is being copied from '{}' to '{}'...".format(
+                    resource_name,
+                    urlparse(resource_json["url"]).path,
+                    download_dest,
+                )
+            )
+            shutil.copy(file_uri_path, download_dest)
+        else:
+            # TODO: Might be nice to have some kind of download status bar here.
+            # TODO: There might be a case where this should be silenced.
+            print(
+                "Resource '{}' was not found locally. Downloading to '{}'...".format(
+                    resource_name, download_dest
+                )
             )
-        )
 
-        # Get the URL.
-        url = resource_json["url"]
+            # Get the URL.
+            url = resource_json["url"]
 
-        _download(url=url, download_to=download_dest)
-        print(f"Finished downloading resource '{resource_name}'.")
+            _download(url=url, download_to=download_dest)
+            print(f"Finished downloading resource '{resource_name}'.")
 
         if run_unzip:
             print(
@@ -368,3 +384,27 @@ def get_resource(
 
                 safe_extract(f, unpack_to)
             os.remove(download_dest)
+
+
+def _file_uri_to_path(uri: str) -> Optional[Path]:
+    """
+    If the URI uses the File scheme (e.g, `file://host/path`) then
+    a Path object for the local path is returned, otherwise None.
+
+    **Note:** Only files from localhost are permitted. An exception
+    is thrown otherwise.
+
+    :param uri: The file URI to convert.
+
+    :returns: The path to the file.
+    """
+
+    if urlparse(uri).scheme == "file":
+        if urlparse(uri).netloc == "" or urlparse(uri).netloc == "localhost":
+            local_path = urlparse(uri).path
+            return Path(local_path)
+        raise Exception(
+            f"File URI '{uri}' specifies host '{urlparse(uri).netloc}'. "
+            "Only localhost is permitted."
+        )
+    return None

tests/pyunit/stdlib/resources/pyunit_local_file_path_check.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2023 The Regents of the University of California
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from gem5.resources.downloader import _file_uri_to_path
+from pathlib import Path
+import unittest
+
+
+class LocalPathTestSuite(unittest.TestCase):
+    def test_local_path_exists_single_slash(self):
+        # Test that a local path is returned as-is
+        path = "file:/test/test/file"
+        expected_path = Path("/test/test/file")
+        self.assertEqual(_file_uri_to_path(path), expected_path)
+
+    def test_non_localhost_exception(self):
+        # Test that a local path with different netloc throws an exception
+        path = "file://test/test/file"
+        # should raise Exception because netloc is not '' or 'localhost'
+        with self.assertRaises(Exception) as exception:
+            _file_uri_to_path(path)
+        self.assertEqual(
+            str(exception.exception),
+            f"File URI '{path}' specifies host 'test'. "
+            "Only localhost is permitted.",
+        )
+
+    def test_localhost_accepted(self):
+        path = "file://localhost/test/test/file"
+        # should work as expected because netloc is 'localhost'
+        expected_path = Path("/test/test/file")
+        self.assertEqual(_file_uri_to_path(path), expected_path)
+
+    def test_local_path_exists_triple_slash(self):
+        # Test that a local path is returned as-is
+        path = "file:///test/test/file"
+        expected_path = Path("/test/test/file")
+        self.assertEqual(_file_uri_to_path(path), expected_path)
+
+    def test_local_path_exists_quadruple_slash(self):
+        # Test that a local path is returned as-is
+        path = "file:////test/test/file"
+        expected_path = Path("//test/test/file")
+        self.assertEqual(_file_uri_to_path(path), expected_path)
+
+    def test_uri_not_file(self):
+        # Test that a URL returns None
+        path = "http://test/test/file"
+        self.assertIsNone(_file_uri_to_path(path))