python: Use PyBind11 instead of SWIG for Python wrappers

Use the PyBind11 wrapping infrastructure instead of SWIG to generate
wrappers for functionality that needs to be exported to Python. This
has several benefits:

  * PyBind11 can be redistributed with gem5, which means that we have
    full control of the version used. This avoid a large number of
    hard-to-debug SWIG issues we have seen in the past.

  * PyBind11 doesn't rely on a custom C++ parser, instead it relies on
    wrappers being explicitly declared in C++. The leads to slightly
    more boiler-plate code in manually created wrappers, but doesn't
    doesn't increase the overall code size. A big benefit is that this
    avoids strange compilation errors when SWIG doesn't understand
    modern language features.

  * Unlike SWIG, there is no risk that the wrapper code incorporates
    incorrect type casts (this has happened on numerous occasions in
    the past) since these will result in compile-time errors.

As a part of this change, the mechanism to define exported methods has
been redesigned slightly. New methods can be exported either by
declaring them in the SimObject declaration and decorating them with
the cxxMethod decorator or by adding an instance of
PyBindMethod/PyBindProperty to the cxx_exports class variable. The
decorator has the added benefit of making it possible to add a
docstring and naming the method's parameters.

The new wrappers have the following known issues:

  * Global events can't be memory managed correctly. This was the
    case in SWIG as well.

Change-Id: I88c5a95b6cf6c32fa9e1ad31dfc08b2e8199a763
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Andreas Hansson <andreas.hansson@arm.com>
Reviewed-by: Andrew Bardsley <andrew.bardsley@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/2231
Reviewed-by: Tony Gutierrez <anthony.gutierrez@amd.com>
Reviewed-by: Pierre-Yves Péneau <pierre-yves.peneau@lirmm.fr>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>

src/sim/Process.py

@@ -26,13 +26,18 @@
 #
 # Authors: Nathan Binkert
 
-from m5.SimObject import SimObject
+from m5.SimObject import *
 from m5.params import *
 from m5.proxy import *
 
 class Process(SimObject):
     type = 'Process'
     cxx_header = "sim/process.hh"
+
+    @cxxMethod
+    def map(self, vaddr, paddr, size, cacheable=False):
+        pass
+
     input = Param.String('cin', "filename for stdin")
     output = Param.String('cout', 'filename for stdout')
     errout = Param.String('cerr', 'filename for stderr')

src/sim/System.py

@@ -28,7 +28,7 @@
 # Authors: Nathan Binkert
 #          Rick Strong
 
-from m5.SimObject import SimObject
+from m5.SimObject import *
 from m5.defines import buildEnv
 from m5.params import *
 from m5.proxy import *
@@ -44,12 +44,10 @@ class System(MemObject):
     cxx_header = "sim/system.hh"
     system_port = MasterPort("System port")
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      Enums::MemoryMode getMemoryMode() const;
-      void setMemoryMode(Enums::MemoryMode mode);
-''')
+    cxx_exports = [
+        PyBindMethod("getMemoryMode"),
+        PyBindMethod("setMemoryMode"),
+    ]
 
     memories = VectorParam.AbstractMemory(Self.all,
                                           "All memories in the system")

src/sim/init.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012 ARM Limited
+ * Copyright (c) 2012, 2017 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -56,6 +56,7 @@
 #include "base/misc.hh"
 #include "base/types.hh"
 #include "config/have_protobuf.hh"
+#include "python/pybind11/pybind.hh"
 #include "sim/async.hh"
 #include "sim/core.hh"
 
@@ -65,6 +66,7 @@
 #endif
 
 using namespace std;
+namespace py = pybind11;
 
 // The python library is totally messed up with respect to constness,
 // so make a simple macro to make life a little easier
@@ -184,10 +186,87 @@ EmbeddedSwig::initAll()
     _Py_PackageContext = old_context;
 }
 
+EmbeddedPyBind::EmbeddedPyBind(const char *_name,
+                               void (*init_func)(py::module &),
+                               const char *_base)
+    : initFunc(init_func), registered(false), name(_name), base(_base)
+{
+    getMap()[_name] = this;
+}
+
+EmbeddedPyBind::EmbeddedPyBind(const char *_name,
+                               void (*init_func)(py::module &))
+    : initFunc(init_func), registered(false), name(_name), base("")
+{
+    getMap()[_name] = this;
+}
+
+void
+EmbeddedPyBind::init(py::module &m)
+{
+    if (!registered) {
+        initFunc(m);
+        registered = true;
+    } else {
+        cprintf("Warning: %s already registered.\n", name);
+    }
+}
+
+bool
+EmbeddedPyBind::depsReady() const
+{
+    return base.empty() || getMap()[base]->registered;
+}
+
+std::map<std::string, EmbeddedPyBind *> &
+EmbeddedPyBind::getMap()
+{
+    static std::map<std::string, EmbeddedPyBind *> objs;
+    return objs;
+}
+
+void
+EmbeddedPyBind::initAll()
+{
+    std::list<EmbeddedPyBind *> pending;
+
+    py::module m_m5 = py::module("_m5");
+    m_m5.attr("__package__") = py::cast("_m5");
+
+    pybind_init_core(m_m5);
+    pybind_init_debug(m_m5);
+
+    pybind_init_event(m_m5);
+    pybind_init_pyobject(m_m5);
+    pybind_init_stats(m_m5);
+
+    for (auto &kv : getMap()) {
+        auto &obj = kv.second;
+        if (obj->base.empty()) {
+            obj->init(m_m5);
+        } else {
+            pending.push_back(obj);
+        }
+    }
+
+    while (!pending.empty()) {
+        for (auto it = pending.begin(); it != pending.end(); ) {
+            EmbeddedPyBind &obj = **it;
+            if (obj.depsReady()) {
+                obj.init(m_m5);
+                it = pending.erase(it);
+            } else {
+                ++it;
+            }
+        }
+    }
+}
+
 int
 initM5Python()
 {
     EmbeddedSwig::initAll();
+    EmbeddedPyBind::initAll();
     return EmbeddedPython::initAll();
 }
 

src/sim/init.hh

@@ -1,4 +1,16 @@
 /*
+ * Copyright (c) 2017 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
  * Copyright (c) 2008 The Hewlett-Packard Development Company
  * All rights reserved.
  *
@@ -31,9 +43,10 @@
 #ifndef __SIM_INIT_HH__
 #define __SIM_INIT_HH__
 
-#include <Python.h>
+#include "pybind11/pybind11.h"
 
 #include <list>
+#include <map>
 #include <string>
 
 #include <inttypes.h>
@@ -79,6 +92,31 @@ struct EmbeddedSwig
     static void initAll();
 };
 
+class EmbeddedPyBind
+{
+  public:
+    EmbeddedPyBind(const char *_name,
+                   void (*init_func)(pybind11::module &),
+                   const char *_base);
+
+    EmbeddedPyBind(const char *_name,
+                   void (*init_func)(pybind11::module &));
+
+    static void initAll();
+
+  private:
+    void (*initFunc)(pybind11::module &);
+
+    bool depsReady() const;
+    void init(pybind11::module &m);
+
+    bool registered;
+    const std::string name;
+    const std::string base;
+
+    static std::map<std::string, EmbeddedPyBind *> &getMap();
+};
+
 int initM5Python();
 int m5Main(int argc, char **argv);
 PyMODINIT_FUNC initm5(void);

src/sim/power/PowerModel.py

@@ -35,7 +35,7 @@
 #
 # Authors: David Guillen Fandos
 
-from m5.SimObject import SimObject
+from m5.SimObject import *
 from m5.params import *
 from m5.proxy import Parent
 
@@ -46,12 +46,10 @@ class PowerModel(SimObject):
     type = 'PowerModel'
     cxx_header = "sim/power/power_model.hh"
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      double getDynamicPower() const;
-      double getStaticPower() const;
-''')
+    cxx_exports = [
+        PyBindMethod("getDynamicPower"),
+        PyBindMethod("getStaticPower"),
+    ]
 
     # Keep a list of every model for every power state
     pm = VectorParam.PowerModelState([], "List of per-state power models.")

src/sim/power/PowerModelState.py

@@ -35,7 +35,7 @@
 #
 # Authors: David Guillen Fandos
 
-from m5.SimObject import SimObject
+from m5.SimObject import *
 from m5.params import *
 
 # Represents a power model for a simobj
@@ -45,11 +45,10 @@ class PowerModelState(SimObject):
     abstract = True
     cxx_class = 'PowerModelState'
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      double getDynamicPower() const;
-      double getStaticPower() const;
-''')
+    cxx_exports = [
+        PyBindMethod("getDynamicPower"),
+        PyBindMethod("getStaticPower"),
+    ]
+
 
 

src/sim/power/ThermalDomain.py

@@ -35,7 +35,7 @@
 #
 # Authors: David Guillen Fandos
 
-from m5.SimObject import SimObject
+from m5.SimObject import *
 from m5.params import *
 
 # Represents a group of simobj which produce heat
@@ -43,11 +43,9 @@ class ThermalDomain(SimObject):
     type = 'ThermalDomain'
     cxx_header = "sim/power/thermal_domain.hh"
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      void setNode(ThermalNode * node);
-''')
+    cxx_exports = [
+        PyBindMethod("setNode"),
+    ]
 
     # Static temperature which may change over time
     initial_temperature = Param.Float(25.0, "Initial temperature")

src/sim/power/ThermalModel.py

@@ -35,7 +35,7 @@
 #
 # Authors: David Guillen Fandos
 
-from m5.SimObject import SimObject
+from m5.SimObject import *
 from ClockedObject import ClockedObject
 
 from m5.params import *
@@ -52,11 +52,9 @@ class ThermalResistor(SimObject):
     type = 'ThermalResistor'
     cxx_header = "sim/power/thermal_model.hh"
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      void setNodes(ThermalNode * node1, ThermalNode * node2);
-''')
+    cxx_exports = [
+        PyBindMethod("setNodes"),
+    ]
 
     resistance = Param.Float(1.0, "Thermal resistance, expressed in Kelvin per Watt")
 
@@ -65,11 +63,9 @@ class ThermalCapacitor(SimObject):
     type = 'ThermalCapacitor'
     cxx_header = "sim/power/thermal_model.hh"
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      void setNodes(ThermalNode * node1, ThermalNode * node2);
-''')
+    cxx_exports = [
+        PyBindMethod("setNodes"),
+    ]
 
     capacitance = Param.Float(1.0, "Thermal capacitance, expressed in Joules per Kelvin")
 
@@ -78,11 +74,9 @@ class ThermalReference(SimObject, object):
     type = 'ThermalReference'
     cxx_header = "sim/power/thermal_model.hh"
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      void setNode(ThermalNode * node);
-''')
+    cxx_exports = [
+        PyBindMethod("setNode"),
+    ]
 
     # Static temperature which may change over time
     temperature = Param.Float(25.0, "Operational temperature in Celsius")
@@ -93,16 +87,14 @@ class ThermalModel(ClockedObject):
     type = 'ThermalModel'
     cxx_header = "sim/power/thermal_model.hh"
 
-    @classmethod
-    def export_methods(cls, code):
-        code('''
-      void addCapacitor(ThermalCapacitor *obj);
-      void addResistor(ThermalResistor *obj);
-      void addReference(ThermalReference *obj);
-      void addDomain(ThermalDomain *obj);
-      void addNode(ThermalNode *obj);
-      void doStep();
-''')
+    cxx_exports = [
+        PyBindMethod("addCapacitor"),
+        PyBindMethod("addResistor"),
+        PyBindMethod("addReference"),
+        PyBindMethod("addDomain"),
+        PyBindMethod("addNode"),
+        PyBindMethod("doStep"),
+    ]
 
     step = Param.Float(0.01, "Simulation step (in seconds) for thermal simulation")