From 15157710f84ab7e079449f03607678ef5f2eca30 Mon Sep 17 00:00:00 2001 From: Gabe Black Date: Tue, 17 Aug 2021 02:12:45 -0700 Subject: [PATCH] python,sim: Change how the m5.* importer code is integrated. Previously, the importer module was built into gem5 as a compressed bytecode blob like all the other code, and it had to be singled out and installed manually so that it could help bring in all the other modules. That adds some amount of complexity since it has to be identified and treated as a special case. Instead, this change builds it into gem5 using pybind11's PYBIND11_EMBEDDED_MODULE macro, and a string that gets evaluated into the new module's __dict__. This means the importer module is automatically available just by building in that .cc, and it can just be imported to start using it. Theoretically all the embedded python could be handled this way, but that would mean building it into gem5 as raw strings which wouldn't even be compiled into byte code until run time. That would take more space in the binary, and also delay catching simple errors. Change-Id: Ic600bf6bce41a53289a2833484a655dd5a226e03 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/49410 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Tested-by: kokoro --- src/python/SConscript | 4 ++++ src/python/importer.cc | 40 ++++++++++++++++++++++++++++++++++++++++ src/sim/init.cc | 34 +++++++--------------------------- src/sim/init.hh | 2 -- 4 files changed, 51 insertions(+), 29 deletions(-) create mode 100644 src/python/importer.cc diff --git a/src/python/SConscript b/src/python/SConscript index 9f1cddbedd..1fab974180 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -253,6 +253,10 @@ PySource('m5.ext.pystats', 'm5/ext/pystats/timeconversion.py') PySource('m5.ext.pystats', 'm5/ext/pystats/jsonloader.py') PySource('m5.stats', 'm5/stats/gem5stats.py') +Source('importer.cc', add_tags='python') +cc, hh = env.Blob('m5ImporterCode', 'importer.py') +Source(cc, add_tags='python') + Source('pybind11/core.cc', add_tags='python') Source('pybind11/debug.cc', add_tags='python') Source('pybind11/event.cc', add_tags='python') diff --git a/src/python/importer.cc b/src/python/importer.cc new file mode 100644 index 0000000000..c7beb6243e --- /dev/null +++ b/src/python/importer.cc @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Google, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "pybind11/embed.h" +#include "pybind11/pybind11.h" +#include "python/m5ImporterCode.hh" + +namespace py = pybind11; + +PYBIND11_EMBEDDED_MODULE(importer, m) +{ + py::str importer_code( + reinterpret_cast(gem5::Blobs::m5ImporterCode), + gem5::Blobs::m5ImporterCode_len); + py::exec(std::move(importer_code), m.attr("__dict__")); +} diff --git a/src/sim/init.cc b/src/sim/init.cc index ed54742855..ce72b04e3f 100644 --- a/src/sim/init.cc +++ b/src/sim/init.cc @@ -73,18 +73,11 @@ namespace gem5 // so make a simple macro to make life a little easier #define PyCC(x) (const_cast(x)) -EmbeddedPython *EmbeddedPython::importer = NULL; -PyObject *EmbeddedPython::importerModule = NULL; EmbeddedPython::EmbeddedPython(const char *abspath, const char *modpath, const unsigned char *code, int zlen, int len) : abspath(abspath), modpath(modpath), code(code), zlen(zlen), len(len) { - // if we've added the importer keep track of it because we need it - // to bootstrap. - if (std::string(modpath) == std::string("importer")) - importer = this; - else - getList().push_back(this); + getList().push_back(this); } std::list & @@ -114,15 +107,11 @@ EmbeddedPython::getCode() const bool EmbeddedPython::addModule() const { - PyObject *code = getCode(); - PyObject *result = PyObject_CallMethod(importerModule, PyCC("add_module"), - PyCC("ssO"), abspath, modpath, code); - if (!result) { - PyErr_Print(); - return false; - } - - Py_DECREF(result); + auto code = py::reinterpret_borrow(getCode()); + // Ensure that "code" is not garbage collected. + code.inc_ref(); + auto importer = py::module_::import("importer"); + importer.attr("add_module")(abspath, modpath, code); return true; } @@ -132,16 +121,7 @@ EmbeddedPython::addModule() const int EmbeddedPython::initAll() { - // Load the importer module - PyObject *code = importer->getCode(); - importerModule = PyImport_ExecCodeModule(PyCC("importer"), code); - if (!importerModule) { - PyErr_Print(); - return 1; - } - - // Load the rest of the embedded python files into the embedded - // python importer + // Load the embedded python files into the embedded python importer. for (auto *embedded: getList()) { if (!embedded->addModule()) return 1; diff --git a/src/sim/init.hh b/src/sim/init.hh index c2f4cf2f58..9d73015ffd 100644 --- a/src/sim/init.hh +++ b/src/sim/init.hh @@ -74,8 +74,6 @@ struct EmbeddedPython PyObject *getCode() const; bool addModule() const; - static EmbeddedPython *importer; - static PyObject *importerModule; static std::list &getList(); static int initAll(); };