python,sim: Change how the m5.* importer code is integrated.

Previously, the importer module was built into gem5 as a compressed
bytecode blob like all the other code, and it had to be singled out and
installed manually so that it could help bring in all the other modules.
That adds some amount of complexity since it has to be identified and
treated as a special case.

Instead, this change builds it into gem5 using pybind11's
PYBIND11_EMBEDDED_MODULE macro, and a string that gets evaluated into
the new module's __dict__. This means the importer module is
automatically available just by building in that .cc, and it can just be
imported to start using it.

Theoretically all the embedded python could be handled this way, but
that would mean building it into gem5 as raw strings which wouldn't even
be compiled into byte code until run time. That would take more space in
the binary, and also delay catching simple errors.

Change-Id: Ic600bf6bce41a53289a2833484a655dd5a226e03
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/49410
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

src/python/SConscript

@@ -253,6 +253,10 @@ PySource('m5.ext.pystats', 'm5/ext/pystats/timeconversion.py')
 PySource('m5.ext.pystats', 'm5/ext/pystats/jsonloader.py')
 PySource('m5.stats', 'm5/stats/gem5stats.py')
 
+Source('importer.cc', add_tags='python')
+cc, hh = env.Blob('m5ImporterCode', 'importer.py')
+Source(cc, add_tags='python')
+
 Source('pybind11/core.cc', add_tags='python')
 Source('pybind11/debug.cc', add_tags='python')
 Source('pybind11/event.cc', add_tags='python')

src/python/importer.cc

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2021 Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met: redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer;
+ * redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution;
+ * neither the name of the copyright holders nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "pybind11/embed.h"
+#include "pybind11/pybind11.h"
+#include "python/m5ImporterCode.hh"
+
+namespace py = pybind11;
+
+PYBIND11_EMBEDDED_MODULE(importer, m)
+{
+    py::str importer_code(
+            reinterpret_cast<const char *>(gem5::Blobs::m5ImporterCode),
+            gem5::Blobs::m5ImporterCode_len);
+    py::exec(std::move(importer_code), m.attr("__dict__"));
+}

src/sim/init.cc

@@ -73,18 +73,11 @@ namespace gem5
 // so make a simple macro to make life a little easier
 #define PyCC(x) (const_cast<char *>(x))
 
-EmbeddedPython *EmbeddedPython::importer = NULL;
-PyObject *EmbeddedPython::importerModule = NULL;
 EmbeddedPython::EmbeddedPython(const char *abspath, const char *modpath,
         const unsigned char *code, int zlen, int len)
     : abspath(abspath), modpath(modpath), code(code), zlen(zlen), len(len)
 {
-    // if we've added the importer keep track of it because we need it
-    // to bootstrap.
-    if (std::string(modpath) == std::string("importer"))
-        importer = this;
-    else
-        getList().push_back(this);
+    getList().push_back(this);
 }
 
 std::list<EmbeddedPython *> &
@@ -114,15 +107,11 @@ EmbeddedPython::getCode() const
 bool
 EmbeddedPython::addModule() const
 {
-    PyObject *code = getCode();
-    PyObject *result = PyObject_CallMethod(importerModule, PyCC("add_module"),
-        PyCC("ssO"), abspath, modpath, code);
-    if (!result) {
-        PyErr_Print();
-        return false;
-    }
-
-    Py_DECREF(result);
+    auto code = py::reinterpret_borrow<py::object>(getCode());
+    // Ensure that "code" is not garbage collected.
+    code.inc_ref();
+    auto importer = py::module_::import("importer");
+    importer.attr("add_module")(abspath, modpath, code);
     return true;
 }
 
@@ -132,16 +121,7 @@ EmbeddedPython::addModule() const
 int
 EmbeddedPython::initAll()
 {
-    // Load the importer module
-    PyObject *code = importer->getCode();
-    importerModule = PyImport_ExecCodeModule(PyCC("importer"), code);
-    if (!importerModule) {
-        PyErr_Print();
-        return 1;
-    }
-
-    // Load the rest of the embedded python files into the embedded
-    // python importer
+    // Load the embedded python files into the embedded python importer.
     for (auto *embedded: getList()) {
         if (!embedded->addModule())
             return 1;

src/sim/init.hh

@@ -74,8 +74,6 @@ struct EmbeddedPython
     PyObject *getCode() const;
     bool addModule() const;
 
-    static EmbeddedPython *importer;
-    static PyObject *importerModule;
     static std::list<EmbeddedPython *> &getList();
     static int initAll();
 };