derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

python: Use PyBind11 instead of SWIG for Python wrappers

Browse Source 
Use the PyBind11 wrapping infrastructure instead of SWIG to generate
wrappers for functionality that needs to be exported to Python. This
has several benefits:

  * PyBind11 can be redistributed with gem5, which means that we have
    full control of the version used. This avoid a large number of
    hard-to-debug SWIG issues we have seen in the past.

  * PyBind11 doesn't rely on a custom C++ parser, instead it relies on
    wrappers being explicitly declared in C++. The leads to slightly
    more boiler-plate code in manually created wrappers, but doesn't
    doesn't increase the overall code size. A big benefit is that this
    avoids strange compilation errors when SWIG doesn't understand
    modern language features.

  * Unlike SWIG, there is no risk that the wrapper code incorporates
    incorrect type casts (this has happened on numerous occasions in
    the past) since these will result in compile-time errors.

As a part of this change, the mechanism to define exported methods has
been redesigned slightly. New methods can be exported either by
declaring them in the SimObject declaration and decorating them with
the cxxMethod decorator or by adding an instance of
PyBindMethod/PyBindProperty to the cxx_exports class variable. The
decorator has the added benefit of making it possible to add a
docstring and naming the method's parameters.

The new wrappers have the following known issues:

  * Global events can't be memory managed correctly. This was the
    case in SWIG as well.

Change-Id: I88c5a95b6cf6c32fa9e1ad31dfc08b2e8199a763
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Andreas Hansson <andreas.hansson@arm.com>
Reviewed-by: Andrew Bardsley <andrew.bardsley@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/2231
Reviewed-by: Tony Gutierrez <anthony.gutierrez@amd.com>
Reviewed-by: Pierre-Yves Péneau <pierre-yves.peneau@lirmm.fr>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
This commit is contained in:
Andreas Sandberg
2017-02-27 13:17:51 +00:00
parent ba42457254
commit 60e6e785f9
44 changed files with 1294 additions and 1453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
src/python/m5/SimObject.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2012 ARM Limited
# Copyright (c) 2017 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
@@ -41,12 +41,16 @@
# Authors: Steve Reinhardt
# Nathan Binkert
# Andreas Hansson
# Andreas Sandberg
import sys
from types import FunctionType, MethodType, ModuleType
from functools import wraps
import inspect
import m5
from m5.util import *
from m5.util.pybind import *
# Have to import params up top since Param is referenced on initial
# load (when SimObject class references Param to create a class
@@ -268,7 +272,6 @@ def createCxxConfigDirectoryEntryFile(code, name, simobj, is_header):
code('{')
code.indent()
code('this->name = name_;')
code('this->pyobj = NULL;')
code.dedent()
code('}')
@@ -393,12 +396,16 @@ def createCxxConfigDirectoryEntryFile(code, name, simobj, is_header):
# class are instantiated, and provides inherited instance behavior).
class MetaSimObject(type):
# Attributes that can be set only at initialization time
init_keywords = { 'abstract' : bool,
'cxx_class' : str,
'cxx_type' : str,
'cxx_header' : str,
'type' : str,
'cxx_bases' : list }
init_keywords = {
'abstract' : bool,
'cxx_class' : str,
'cxx_type' : str,
'cxx_header' : str,
'type' : str,
'cxx_bases' : list,
'cxx_exports' : list,
'cxx_param_exports' : list,
}
# Attributes that can be set any time
keywords = { 'check' : FunctionType }
@@ -415,7 +422,13 @@ class MetaSimObject(type):
# filtered in __init__.
cls_dict = {}
value_dict = {}
cxx_exports = []
for key,val in dict.items():
try:
cxx_exports.append(getattr(val, "__pybind"))
except AttributeError:
pass
if public_value(key, val):
cls_dict[key] = val
else:
@@ -425,6 +438,12 @@ class MetaSimObject(type):
value_dict['abstract'] = False
if 'cxx_bases' not in value_dict:
value_dict['cxx_bases'] = []
if 'cxx_exports' not in value_dict:
value_dict['cxx_exports'] = cxx_exports
else:
value_dict['cxx_exports'] += cxx_exports
if 'cxx_param_exports' not in value_dict:
value_dict['cxx_param_exports'] = []
cls_dict['_value_dict'] = value_dict
cls = super(MetaSimObject, mcls).__new__(mcls, name, bases, cls_dict)
if 'type' in value_dict:
@@ -654,6 +673,9 @@ class MetaSimObject(type):
def cxx_predecls(cls, code):
code('#include "params/$cls.hh"')
def pybind_predecls(cls, code):
code('#include "${{cls.cxx_header}}"')
# See ParamValue.swig_predecls for description.
def swig_predecls(cls, code):
code('%import "python/_m5/param_$cls.i"')
@@ -755,6 +777,85 @@ using std::ptrdiff_t;
code()
code('%include "params/$cls.hh"')
def pybind_decl(cls, code):
class_path = cls.cxx_class.split('::')
namespaces, classname = class_path[:-1], class_path[-1]
py_class_name = '_COLONS_'.join(class_path) if namespaces else \
classname;
# The 'local' attribute restricts us to the params declared in
# the object itself, not including inherited params (which
# will also be inherited from the base class's param struct
# here). Sort the params based on their key
params = map(lambda (k, v): v, sorted(cls._params.local.items()))
ports = cls._ports.local
code('''#include "pybind11/pybind11.h"
#include "pybind11/stl.h"
#include "sim/sim_object.hh"
#include "params/$cls.hh"
#include "sim/init.hh"
#include "${{cls.cxx_header}}"
''')
for param in params:
param.pybind_predecls(code)
code('''namespace py = pybind11;
static void
module_init(py::module &m_internal)
{
py::module m = m_internal.def_submodule("param_${cls}");
''')
code.indent()
if cls._base:
code('py::class_<${cls}Params, ${{cls._base.type}}Params>(m, ' \
'"${cls}Params")')
else:
code('py::class_<${cls}Params>(m, "${cls}Params")')
code.indent()
if not hasattr(cls, 'abstract') or not cls.abstract:
code('.def(py::init<>())')
code('.def("create", &${cls}Params::create)')
param_exports = cls.cxx_param_exports + [
PyBindProperty(k)
for k, v in sorted(cls._params.local.items())
] + [
PyBindProperty("port_%s_connection_count" % port.name)
for port in ports.itervalues()
]
for exp in param_exports:
exp.export(code, "%sParams" % cls)
code(';')
code()
code.dedent()
bases = [ cls._base.cxx_class ] + cls.cxx_bases if cls._base else \
cls.cxx_bases
if bases:
base_str = ", ".join(bases)
code('py::class_<${{cls.cxx_class}}, ${base_str}>(m, ' \
'"${py_class_name}")')
else:
code('py::class_<${{cls.cxx_class}}>(m, "${py_class_name}")')
code.indent()
for exp in cls.cxx_exports:
exp.export(code, cls.cxx_class)
code(';')
code.dedent()
code()
code.dedent()
code('}')
code()
code('static EmbeddedPyBind embed_obj("${0}", module_init, "${1}");',
cls, cls._base.type if cls._base else "")
# Generate the C++ declaration (.hh file) for this SimObject's
# param struct. Called from src/SConscript.
@@ -780,6 +881,14 @@ using std::ptrdiff_t;
''')
# The base SimObject has a couple of params that get
# automatically set from Python without being declared through
# the normal Param mechanism; we slip them in here (needed
# predecls now, actual declarations below)
if cls == SimObject:
code('''#include <string>''')
# A forward class declaration is sufficient since we are just
# declaring a pointer.
for ns in class_path[:-1]:
@@ -789,18 +898,6 @@ using std::ptrdiff_t;
code('} // namespace $ns')
code()
# The base SimObject has a couple of params that get
# automatically set from Python without being declared through
# the normal Param mechanism; we slip them in here (needed
# predecls now, actual declarations below)
if cls == SimObject:
code('''
#ifndef PY_VERSION
struct PyObject;
#endif
#include <string>
''')
for param in params:
param.cxx_predecls(code)
for port in ports.itervalues():
@@ -832,8 +929,8 @@ struct PyObject;
virtual ~SimObjectParams() {}
std::string name;
PyObject *pyobj;
''')
for param in params:
param.cxx_decl(code)
for port in ports.itervalues():
@@ -861,6 +958,47 @@ struct PyObject;
def isSimObjectOrVector(value):
return False
def cxxMethod(*args, **kwargs):
"""Decorator to export C++ functions to Python"""
def decorate(func):
name = func.func_name
override = kwargs.get("override", False)
cxx_name = kwargs.get("cxx_name", name)
args, varargs, keywords, defaults = inspect.getargspec(func)
if varargs or keywords:
raise ValueError("Wrapped methods must not contain variable " \
"arguments")
# Create tuples of (argument, default)
if defaults:
args = args[:-len(defaults)] + zip(args[-len(defaults):], defaults)
# Don't include self in the argument list to PyBind
args = args[1:]
@wraps(func)
def cxx_call(self, *args, **kwargs):
ccobj = self.getCCObject()
return getattr(ccobj, name)(*args, **kwargs)
@wraps(func)
def py_call(self, *args, **kwargs):
return self.func(*args, **kwargs)
f = py_call if override else cxx_call
f.__pybind = PyBindMethod(name, cxx_name=cxx_name, args=args)
return f
if len(args) == 0:
return decorate
elif len(args) == 1 and len(kwargs) == 0:
return decorate(*args)
else:
raise TypeError("One argument and no kwargs, or only kwargs expected")
# This class holds information about each simobject parameter
# that should be displayed on the command line for use in the
# configuration system.
@@ -921,6 +1059,27 @@ class SimObject(object):
void startup();
''')
cxx_exports = [
PyBindMethod("init"),
PyBindMethod("initState"),
PyBindMethod("memInvalidate"),
PyBindMethod("memWriteback"),
PyBindMethod("regStats"),
PyBindMethod("resetStats"),
PyBindMethod("regProbePoints"),
PyBindMethod("regProbeListeners"),
PyBindMethod("startup"),
]
cxx_param_exports = [
PyBindProperty("name"),
]
@cxxMethod
def loadState(self, cp):
"""Load SimObject state from a checkpoint"""
pass
# Returns a dict of all the option strings that can be
# generated as command line options for this simobject instance
# by tracing all reachable params in the top level instance and
@@ -1379,7 +1538,6 @@ class SimObject(object):
cc_params_struct = getattr(m5.internal.params, '%sParams' % self.type)
cc_params = cc_params_struct()
cc_params.pyobj = self
cc_params.name = str(self)
param_names = self._params.keys()
@@ -1395,8 +1553,7 @@ class SimObject(object):
assert isinstance(value, list)
vec = getattr(cc_params, param)
assert not len(vec)
for v in value:
vec.append(v)
setattr(cc_params, param, list(value))
else:
setattr(cc_params, param, value)
@@ -1517,4 +1674,9 @@ def clear():
# __all__ defines the list of symbols that get exported when
# 'from config import *' is invoked. Try to keep this reasonably
# short to avoid polluting other namespaces.
__all__ = [ 'SimObject' ]
__all__ = [
'SimObject',
'cxxMethod',
'PyBindMethod',
'PyBindProperty',
]

37
src/python/m5/event.py
View File

@@ -1,3 +1,15 @@
# Copyright (c) 2017 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder. You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Copyright (c) 2006 The Regents of The University of Michigan
# Copyright (c) 2013 Advanced Micro Devices, Inc.
# Copyright (c) 2013 Mark D. Hill and David A. Wood
@@ -31,24 +43,11 @@
import m5
import _m5.event
from _m5.event import PythonEvent, GlobalSimLoopExitEvent as SimExit
from _m5.event import GlobalSimLoopExitEvent as SimExit
from _m5.event import Event, getEventQueue, setEventQueue
mainq = None
def create(obj, priority=None):
if priority is None:
priority = Event.Default_Pri
return PythonEvent(obj, priority)
# As a reminder, priorities found in sim/eventq.hh are stuck into the
# Event class by swig
class Event(PythonEvent):
def __init__(self, priority=None):
if priority is None:
priority = Event.Default_Pri
super(Event, self).__init__(self, priority)
class ProgressEvent(Event):
def __init__(self, eventq, period):
super(ProgressEvent, self).__init__()
@@ -60,10 +59,4 @@ class ProgressEvent(Event):
print "Progress! Time now %fs" % (m5.curTick()/1e12)
self.eventq.schedule(self, m5.curTick() + self.period)
def getEventQueue(index):
return _m5.event.getEventQueue(index)
def setEventQueue(eventq):
_m5.event.curEventQueue(eventq)
__all__ = [ 'create', 'Event', 'ProgressEvent', 'SimExit', 'mainq' ]
__all__ = [ 'Event', 'ProgressEvent', 'SimExit', 'mainq' ]

25
src/python/m5/internal/params.py
View File

@@ -1,3 +1,15 @@
# Copyright (c) 2017 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder. You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Copyright (c) 2010 The Hewlett-Packard Development Company
# All rights reserved.
#
@@ -26,12 +38,9 @@
#
# Authors: Nathan Binkert
try:
modules = __loader__.modules
except NameError:
modules = { }
import inspect
import _m5
for module in modules.iterkeys():
if module.startswith('_m5.param_') or \
module.startswith('_m5.enum_'):
exec "from %s import *" % module
for name, module in inspect.getmembers(_m5):
if name.startswith('param_') or name.startswith('enum_'):
exec "from _m5.%s import *" % name

93
src/python/m5/params.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2012-2014 ARM Limited
# Copyright (c) 2012-2014, 2017 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
@@ -102,6 +102,10 @@ class ParamValue(object):
def cxx_predecls(cls, code):
pass
@classmethod
def pybind_predecls(cls, code):
cls.cxx_predecls(code)
# Generate the code needed as a prerequisite for including a
# reference to a C++ object of this type in a SWIG .i file.
# Typically generates one or more %import or %include statements.
@@ -222,6 +226,9 @@ class ParamDesc(object):
code('#include <cstddef>')
self.ptype.cxx_predecls(code)
def pybind_predecls(self, code):
self.ptype.pybind_predecls(code)
def swig_predecls(self, code):
self.ptype.swig_predecls(code)
@@ -408,6 +415,10 @@ class VectorParamDesc(ParamDesc):
code('#include <vector>')
self.ptype.cxx_predecls(code)
def pybind_predecls(self, code):
code('#include <vector>')
self.ptype.pybind_predecls(code)
def cxx_decl(self, code):
code('std::vector< ${{self.ptype.cxx_type}} > ${{self.name}};')
@@ -791,6 +802,11 @@ class AddrRange(ParamValue):
Addr.cxx_predecls(code)
code('#include "base/addr_range.hh"')
@classmethod
def pybind_predecls(cls, code):
Addr.pybind_predecls(code)
code('#include "base/addr_range.hh"')
@classmethod
def swig_predecls(cls, code):
Addr.swig_predecls(code)
@@ -941,7 +957,7 @@ class EthernetAddr(ParamValue):
return self
def getValue(self):
from m5.internal.params import EthAddr
from _m5.net import EthAddr
return EthAddr(self.value)
def __str__(self):
@@ -1007,7 +1023,7 @@ class IpAddress(ParamValue):
raise TypeError, "invalid ip address %#08x" % self.ip
def getValue(self):
from m5.internal.params import IpAddress
from _m5.net import IpAddress
return IpAddress(self.ip)
# When initializing an IpNetmask, pass in an existing IpNetmask, a string of
@@ -1086,7 +1102,7 @@ class IpNetmask(IpAddress):
raise TypeError, "invalid netmask %d" % netmask
def getValue(self):
from m5.internal.params import IpNetmask
from _m5.net import IpNetmask
return IpNetmask(self.ip, self.netmask)
# When initializing an IpWithPort, pass in an existing IpWithPort, a string of
@@ -1164,7 +1180,7 @@ class IpWithPort(IpAddress):
raise TypeError, "invalid port %d" % self.port
def getValue(self):
from m5.internal.params import IpWithPort
from _m5.net import IpWithPort
return IpWithPort(self.ip, self.port)
time_formats = [ "%a %b %d %H:%M:%S %Z %Y",
@@ -1224,30 +1240,10 @@ class Time(ParamValue):
return value
def getValue(self):
from m5.internal.params import tm
from _m5.core import tm
import calendar
c_time = tm()
py_time = self.value
# UNIX is years since 1900
c_time.tm_year = py_time.tm_year - 1900;
# Python starts at 1, UNIX starts at 0
c_time.tm_mon = py_time.tm_mon - 1;
c_time.tm_mday = py_time.tm_mday;
c_time.tm_hour = py_time.tm_hour;
c_time.tm_min = py_time.tm_min;
c_time.tm_sec = py_time.tm_sec;
# Python has 0 as Monday, UNIX is 0 as sunday
c_time.tm_wday = py_time.tm_wday + 1
if c_time.tm_wday > 6:
c_time.tm_wday -= 7;
# Python starts at 1, Unix starts at 0
c_time.tm_yday = py_time.tm_yday - 1;
return c_time
return tm.gmtime(calendar.timegm(self.value))
def __str__(self):
return time.asctime(self.value)
@@ -1375,6 +1371,40 @@ $wrapper $wrapper_name {
code('} // namespace $wrapper_name')
code.dedent(1)
def pybind_def(cls, code):
name = cls.__name__
wrapper_name = cls.wrapper_name
enum_name = cls.__name__ if cls.enum_name is None else cls.enum_name
code('''#include "pybind11/pybind11.h"
#include "pybind11/stl.h"
#include <sim/init.hh>
namespace py = pybind11;
static void
module_init(py::module &m_internal)
{
py::module m = m_internal.def_submodule("enum_${name}");
py::enum_<${wrapper_name}::${enum_name}>(m, "enum_${name}")
''')
code.indent()
code.indent()
for val in cls.vals:
code('.value("${val}", ${wrapper_name}::${val})')
code('.value("Num_${name}", ${wrapper_name}::Num_${enum_name})')
code('.export_values()')
code(';')
code.dedent()
code('}')
code.dedent()
code()
code('static EmbeddedPyBind embed_enum("enum_${name}", module_init);')
def swig_decl(cls, code):
name = cls.__name__
code('''\
@@ -1435,7 +1465,9 @@ class Enum(ParamValue):
code('}')
def getValue(self):
return int(self.map[self.value])
import m5.internal.params
e = getattr(m5.internal.params, "enum_%s" % self.__class__.__name__)
return e(self.map[self.value])
def __str__(self):
return self.value
@@ -2040,6 +2072,9 @@ class Port(object):
def cxx_predecls(self, code):
pass
def pybind_predecls(self, code):
cls.cxx_predecls(self, code)
# Declare an unsigned int with the same name as the port, that
# will eventually hold the number of connected ports (and thus the
# number of elements for a VectorPort).

85
src/python/m5/util/pybind.py Normal file
View File

@@ -0,0 +1,85 @@
# Copyright (c) 2017 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder. You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Andreas Sandberg
from abc import *
class PyBindExport(object):
__metaclass__ = ABCMeta
@abstractmethod
def export(self, code, cname):
pass
class PyBindProperty(PyBindExport):
def __init__(self, name, cxx_name=None, writable=True):
self.name = name
self.cxx_name = cxx_name if cxx_name else name
self.writable = writable
def export(self, code, cname):
export = "def_readwrite" if self.writable else "def_readonly"
code('.${export}("${{self.name}}", &${cname}::${{self.cxx_name}})')
class PyBindMethod(PyBindExport):
def __init__(self, name, cxx_name=None, args=None):
self.name = name
self.cxx_name = cxx_name if cxx_name else name
self.args = args
def _conv_arg(self, value):
if isinstance(value, bool):
return "true" if value else "false"
elif isinstance(value, float, int):
return repr(value)
else:
raise TypeError("Unsupported PyBind default value type")
def export(self, code, cname):
if self.args:
def get_arg_decl(arg):
if isinstance(arg, tuple):
name, default = arg
return 'py::arg("%s") = %s' % (
name, self._conv_arg(default))
else:
return 'py::arg("%s")' % arg
code('.def("${{self.name}}", &${cname}::${{self.name}}, ')
code(' ' + \
', '.join([ get_arg_decl(a) for a in self.args ]) + ')')
else:
code('.def("${{self.name}}", &${cname}::${{self.cxx_name}})')