derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scons: Update the special module importer API.

Browse Source 
In the SConscript, there is a special importer which enables importing
embedded code using various m5.* paths. This was implemented using an
API which has been deprecated and replaced in more recent versions of
python.

Change-Id: I5900f269af48befbcedcb9d25353f04f6297ce9d
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/48363
Tested-by: kokoro <noreply+kokoro@google.com>
Maintainer: Gabe Black <gabe.black@gmail.com>
Reviewed-by: Bobby R. Bruce <bbruce@ucdavis.edu>
This commit is contained in:
Gabe Black
2021-07-17 00:18:40 -07:00
parent 1853d57dc3
commit 9db4c91510
3 changed files with 82 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
src/SConscript
View File

@@ -42,6 +42,10 @@ import bisect
import distutils.spawn
import functools
import imp
import importlib
import importlib.abc
import importlib.machinery
import importlib.util
import os
import os.path
import re
@@ -735,10 +739,20 @@ env.Command('config/the_gpu_isa.hh', [],
#
SimObject.fixed = True
class DictImporter(object):
'''This importer takes a dictionary of arbitrary module names that
map to arbitrary filenames.'''
class SimpleModuleLoader(importlib.abc.Loader):
'''A simple wrapper which delegates setting up a module to a function.'''
def __init__(self, executor):
super(SimpleModuleLoader, self).__init__()
self.executor = executor
def create_module(self, spec):
return None
def exec_module(self, module):
self.executor(module)
class M5MetaPathFinder(importlib.abc.MetaPathFinder):
def __init__(self, modules):
super(M5MetaPathFinder, self).__init__()
self.modules = modules
self.installed = set()
@@ -748,42 +762,46 @@ class DictImporter(object):
del sys.modules[module]
self.installed = set()
def find_module(self, fullname, path):
if fullname == 'm5.defines':
return self
def find_spec(self, fullname, path, target=None):
spec = None
if fullname == 'm5.objects':
return self
# If this isn't even in the m5 package, ignore it.
if fullname.startswith('m5.'):
if fullname.startswith('m5.objects'):
# When imported in this context, return a spec for a dummy
# package which just serves to house the modules within it.
# This is subtley different from "import * from m5.objects"
# which relies on the __init__.py in m5.objects. That in turn
# indirectly relies on the c++ based _m5 package which doesn't
# exist yet.
if fullname == 'm5.objects':
dummy_loader = SimpleModuleLoader(lambda x: None)
spec = importlib.machinery.ModuleSpec(
name=fullname, loader=dummy_loader,
is_package=True)
spec.loader_state = self.modules.keys()
source = self.modules.get(fullname, None)
if source is not None and fullname.startswith('m5.objects'):
return self
# If this is a module within the m5.objects package, return a
# spec that maps to its source file.
elif fullname in self.modules:
source = self.modules[fullname]
spec = importlib.util.spec_from_file_location(
name=fullname, location=source.abspath)
return None
# The artificial m5.defines subpackage.
elif fullname == 'm5.defines':
def build_m5_defines(module):
module.__dict__['buildEnv'] = dict(build_env)
def load_module(self, fullname):
mod = imp.new_module(fullname)
sys.modules[fullname] = mod
self.installed.add(fullname)
spec = importlib.util.spec_from_loader(name=fullname,
loader=SimpleModuleLoader(build_m5_defines))
mod.__loader__ = self
if fullname == 'm5.objects':
mod.__path__ = fullname.split('.')
return mod
# If we're handling this module, write it down so we can unload it
# later.
if spec is not None:
self.installed.add(fullname)
if fullname == 'm5.defines':
mod.__dict__['buildEnv'] = dict(build_env)
return mod
source = self.modules[fullname]
if source.modname == '__init__':
mod.__path__ = source.modpath
mod.__file__ = source.abspath
compiled = compile(open(source.abspath).read(), source.abspath, 'exec')
exec(compiled, mod.__dict__)
return mod
return spec
import m5.SimObject
import m5.params
@@ -794,7 +812,7 @@ m5.params.clear()
# install the python importer so we can grab stuff from the source
# tree itself. We can't have SimObjects added after this point or
# else we won't know about them for the rest of the stuff.
importer = DictImporter(PySource.modules)
importer = M5MetaPathFinder(PySource.modules)
sys.meta_path[0:0] = [ importer ]
# import all sim objects so we can populate the all_objects list

67
src/python/importer.py
View File

@@ -24,12 +24,27 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import importlib
import importlib.abc
import importlib.util
import os
class ByteCodeLoader(importlib.abc.Loader):
def __init__(self, code):
super(ByteCodeLoader, self).__init__()
self.code = code
def exec_module(self, module):
exec(self.code, module.__dict__)
# Simple importer that allows python to import data from a dict of
# code objects. The keys are the module path, and the items are the
# filename and bytecode of the file.
class CodeImporter(object):
def __init__(self):
self.modules = {}
override_var = os.environ.get('M5_OVERRIDE_PY_SOURCE', 'false')
self.override = (override_var.lower() in ('true', 'yes'))
def add_module(self, filename, abspath, modpath, code):
if modpath in self.modules:
@@ -37,50 +52,24 @@ class CodeImporter(object):
self.modules[modpath] = (filename, abspath, code)
def find_module(self, fullname, path):
if fullname in self.modules:
return self
def find_spec(self, fullname, path, target=None):
if fullname not in self.modules:
return None
return None
srcfile, abspath, code = self.modules[fullname]
def load_module(self, fullname):
# Because the importer is created and initialized in its own
# little sandbox (in init.cc), the globals that were available
# when the importer module was loaded and CodeImporter was
# defined are not available when load_module is actually
# called. Soooo, the imports must live here.
import imp
import os
import sys
if self.override and os.path.exists(abspath):
src = open(abspath, 'r').read()
code = compile(src, abspath, 'exec')
try:
mod = sys.modules[fullname]
except KeyError:
mod = imp.new_module(fullname)
sys.modules[fullname] = mod
is_package = (os.path.basename(srcfile) == '__init__.py')
spec = importlib.util.spec_from_loader(
name=fullname, loader=ByteCodeLoader(code),
is_package=is_package)
try:
mod.__loader__ = self
srcfile,abspath,code = self.modules[fullname]
spec.loader_state = self.modules.keys()
override = os.environ.get('M5_OVERRIDE_PY_SOURCE', 'false').lower()
if override in ('true', 'yes') and os.path.exists(abspath):
src = open(abspath, 'r').read()
code = compile(src, abspath, 'exec')
if os.path.basename(srcfile) == '__init__.py':
mod.__path__ = fullname.split('.')
mod.__package__ = fullname
else:
mod.__package__ = fullname.rpartition('.')[0]
mod.__file__ = srcfile
exec(code, mod.__dict__)
except Exception:
del sys.modules[fullname]
raise
return mod
return spec
# Create an importer and add it to the meta_path so future imports can
# use it. There's currently nothing in the importer, but calls to

4
src/python/m5/objects/__init__.py
View File

@@ -28,10 +28,10 @@ from m5.internal import params
from m5.SimObject import *
try:
modules = __loader__.modules
modules = __spec__.loader_state
except NameError:
modules = { }
for module in modules.keys():
for module in modules:
if module.startswith('m5.objects.'):
exec("from %s import *" % module)