derek/DRAMSys
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to pybind11

Browse Source 
With the switch to pybind11, the complexity of the Python integration
in the TraceAnalyzer can be greatly reduced. The new code is much
easier to understand and fixes a number of bugs regarding the Python
integration.
This commit is contained in:
Derek Christ
2023-05-11 12:28:21 +02:00
parent 50e87b7a63
commit edd52e0fe1
13 changed files with 130 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
extensions/apps/traceAnalyzer/CMakeLists.txt
View File

@@ -44,7 +44,14 @@ file(GLOB_RECURSE SOURCE_FILES CONFIGURE_DEPENDS *.cpp)
file(GLOB_RECURSE HEADER_FILES CONFIGURE_DEPENDS *.h;*.hpp)
# Add Python3 Dependency:
find_package(Python3 COMPONENTS Development)
find_package(Python3 COMPONENTS Development Interpreter)
FetchContent_Declare(
pybind11
URL https://github.com/pybind/pybind11/archive/refs/tags/v2.10.4.zip
)
FetchContent_MakeAvailable(pybind11)
# Add QWT Dependency:
find_library(QWT_LIBRARY NAMES "qwt" "qwt-qt5" PATHS
@@ -74,11 +81,10 @@ add_executable(TraceAnalyzer ${SOURCE_FILES} ${HEADER_FILES})
target_include_directories(TraceAnalyzer
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
PRIVATE ${QWT_INCLUDE_DIRS}
PRIVATE ${Python3_INCLUDE_DIRS}
)
target_link_libraries(TraceAnalyzer
PRIVATE ${Python3_LIBRARIES}
PRIVATE pybind11::embed
PRIVATE ${QWT_LIBRARY}
PRIVATE Qt5::Widgets
PRIVATE Qt5::Sql
@@ -86,5 +92,12 @@ target_link_libraries(TraceAnalyzer
PRIVATE DRAMSys::config
)
set(DRAMSYS_TRACEANALYZER_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
target_compile_definitions(${PROJECT_NAME}
PUBLIC
DRAMSYS_TRACEANALYZER_DIR="${DRAMSYS_TRACEANALYZER_DIR}"
)
build_source_group()
diagnostics_print(${PROJECT_NAME})

15
extensions/apps/traceAnalyzer/README.md
View File

@@ -1,2 +1,13 @@
dram.vp.scheduler
=================
## TraceAnalyzer
### Python Dependencies
The used Python dependencies of this project include:
- matplotlib
- numpy
- pyvcd
- tqdm
To install all required packages, a `requirements.txt` file is provided, which can be found in `scripts/requirements.txt`.
Install the packages using this command:
```
python3 -m pip install -r scripts/requirements.txt
```

19
extensions/apps/traceAnalyzer/businessObjects/calculatedMetric.h
View File

@@ -33,30 +33,17 @@
* Janik Schlemminger
* Robert Gernhardt
* Matthias Jung
* Derek Christ
*/
#ifndef METRIC_H
#define METRIC_H
#include <QString>
class CalculatedMetric
struct CalculatedMetric
{
public:
CalculatedMetric(QString name, double value): name(name), value(value) {}
QString getName()
{
return name;
}
double getValue()
{
return value;
}
private:
QString name;
std::string name;
double value;
};
#endif // METRIC_H

235
extensions/apps/traceAnalyzer/businessObjects/pythoncaller.cpp
View File

@@ -41,205 +41,78 @@
#include <exception>
#include <string>
#include <iostream>
#include <QFileInfo>
#include <QDebug>
#include <QApplication>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
PythonCaller::PythonCaller() :
metricModuleName("metrics"),
metricFunctionName("calculateMetrics"),
getMetricFunctionName("getMetrics"),
pathToScripts(QApplication::applicationDirPath().toStdString() +
"/../../extensions/apps/traceAnalyzer/scripts/"),
plotsModuleName("plots"),
plotsFunctionName("generatePlots"),
checkDependenciesModuleName("checkDependencies"),
vcdExportModuleName("vcdExport"),
vcdExportFunctionName("dumpVcd"),
vcdExportDependenciesFunctionName("checkVcdExport")
std::string PythonCaller::generatePlots(std::string_view pathToTrace)
{
Py_Initialize();
PyObject *sysPath = PySys_GetObject((char *)"path");
PyObject *path = PyUnicode_FromString(this->pathToScripts.c_str());
PyList_Insert(sysPath, 0, path);
Py_DECREF(path);
qDebug() << "Metric:" << metricModuleName.c_str() << metricFunctionName.c_str();
qDebug() << "Plot:" << plotsModuleName.c_str() << plotsFunctionName.c_str();
qDebug() << "VcdExport:" << vcdExportModuleName.c_str() << vcdExportFunctionName.c_str();
qDebug() << "Script: " << pathToScripts.c_str();
pCalculateMetricsFunction = loadFunctionFromModule(metricModuleName,
metricFunctionName);
pGenPlotsFunction = loadFunctionFromModule(plotsModuleName, plotsFunctionName);
pGetMetricsFunction = loadFunctionFromModule(metricModuleName,
getMetricFunctionName);
pVcdExportDependenciesFunction = loadFunctionFromModule(checkDependenciesModuleName, vcdExportDependenciesFunctionName);
if (vcdExportDependenciesAvailable())
pVcdExportFunction = loadFunctionFromModule(vcdExportModuleName, vcdExportFunctionName);
else
std::cerr << "Warning: Python module pyvcd or tqdm not installed! Exporting as VCD not possible." << std::endl;
}
//returns new reference to the function (see: http://edcjones.tripod.com/refcount.html for the difference between "new reference" and "borrowed reference")
PyObject *PythonCaller::loadFunctionFromModule(std::string moduleName, std::string functionName)
{
PyObject *pModuleName = PyUnicode_FromString(moduleName.c_str());
PyObject *pModule = PyImport_Import(pModuleName);
if (!pModule) {
throw std::runtime_error(std::string("Could not load module " + moduleName));
try
{
pybind11::module_ metricsModule = pybind11::module_::import("plots");
auto result = metricsModule.attr("generatePlots")(pathToTrace).cast<std::string>();
return result;
}
catch (std::exception const &err)
{
std::cout << err.what() << std::endl;
}
PyObject *pFunction = PyObject_GetAttrString(pModule, functionName.c_str());
return {};
}
if (!pFunction || !PyCallable_Check(pFunction)) {
throw std::runtime_error(
std::string("Could not load test function " + functionName + "in module " + moduleName));
std::vector<std::string> PythonCaller::availableMetrics(std::string_view pathToTrace)
{
try
{
pybind11::module_ metricsModule = pybind11::module_::import("metrics");
pybind11::list result = metricsModule.attr("getMetrics")(pathToTrace);
return result.cast<std::vector<std::string>>();
}
catch (std::exception const &err)
{
std::cout << err.what() << std::endl;
}
Py_DECREF(pModuleName);
Py_DECREF(pModule);
return pFunction;
return {};
}
PythonCaller::~PythonCaller()
TraceCalculatedMetrics PythonCaller::evaluateMetrics(std::string_view pathToTrace, std::vector<long> selectedMetrics)
{
Py_DECREF(pCalculateMetricsFunction);
Py_DECREF(pGenPlotsFunction);
Py_DECREF(pGetMetricsFunction);
TraceCalculatedMetrics metrics(pathToTrace.data());
if (pVcdExportFunction)
Py_DECREF(pVcdExportFunction);
try
{
pybind11::module_ metricsModule = pybind11::module_::import("metrics");
pybind11::list result = metricsModule.attr("calculateMetrics")(pathToTrace, selectedMetrics);
auto metricList = result.cast<std::vector<pybind11::tuple>>();
Py_DECREF(pVcdExportDependenciesFunction);
Py_Finalize();
}
PyObject *PythonCaller::callMetricsFunction(PyObject *function, QString argument, std::vector<long> list)
{
assert(PyCallable_Check(function));
PyObject *pArgs = PyTuple_New(2);
PyObject *pArgumentString = PyUnicode_FromString(
argument.toStdString().c_str());
PyObject *pArgumentList = PyList_New(list.size());
for (size_t i = 0; i < list.size(); i++) {
PyList_SetItem(pArgumentList, i, PyBool_FromLong(list[i]));
for (auto metricPair : metricList)
{
std::string name = metricPair[0].cast<std::string>();
double value = metricPair[1].cast<double>();
metrics.addCalculatedMetric({name, value});
}
}
PyTuple_SetItem(pArgs, 0, pArgumentString);
PyTuple_SetItem(pArgs, 1, pArgumentList);
PyObject *pResult = PyObject_CallObject(function, pArgs);
Py_DECREF(pArgs);
if (!pResult) {
PyErr_Print();
throw std::runtime_error(
std::string("Error in calling " + metricFunctionName + " in module " + metricModuleName));
catch (std::exception const &err)
{
std::cout << err.what() << std::endl;
}
return pResult;
return metrics;
}
//returns a new reference to result of function call
PyObject *PythonCaller::callFunctionWithStringArgument(PyObject *function,
QString argument)
std::string PythonCaller::dumpVcd(std::string_view pathToTrace)
{
assert(PyCallable_Check(function));
PyObject *pArgs = PyTuple_New(1);
PyObject *pArgument = PyUnicode_FromString(argument.toStdString().c_str());
PyTuple_SetItem(pArgs, 0, pArgument);
PyObject *pResult = PyObject_CallObject(function, pArgs);
Py_DECREF(pArgument);
if (!pResult) {
PyErr_Print();
throw std::runtime_error(std::string("Error in calling function with string argument"));
try
{
pybind11::module_ vcdModule = pybind11::module_::import("vcdExport");
pybind11::str result = vcdModule.attr("dumpVcd")(pathToTrace);
return result.cast<std::string>();
}
catch (std::exception const &err)
{
std::cout << err.what() << std::endl;
}
return pResult;
}
PyObject *PythonCaller::callFunctionWithoutArguments(PyObject *function)
{
assert(PyCallable_Check(function));
PyObject *pResult = PyObject_CallObject(function, NULL);
if (!pResult) {
PyErr_Print();
throw std::runtime_error(std::string("Error in calling python function"));
}
return pResult;
}
TraceCalculatedMetrics PythonCaller::calculateMetricsOnTrace(QString pathToTrace, std::vector<long> list)
{
TraceCalculatedMetrics result(QFileInfo(pathToTrace).baseName());
PyObject *pResult = callMetricsFunction(pCalculateMetricsFunction, pathToTrace,
list);
for (Py_ssize_t i = 0; i < PyList_Size(pResult); ++i) {
PyObject *calculatedMetric = PyList_GetItem(pResult, i);
QString metricName(PyUnicode_AsUTF8(PyTuple_GetItem(calculatedMetric, 0)));
double value = PyFloat_AsDouble(PyTuple_GetItem(calculatedMetric, 1));
result.addCalculatedMetric(CalculatedMetric(metricName, value));
}
Py_DECREF(pResult);
return result;
}
std::vector<std::string> PythonCaller::getMetrics(QString pathToTrace)
{
std::vector<std::string> result;
PyObject *pResult = callFunctionWithStringArgument(pGetMetricsFunction,
pathToTrace);
for (Py_ssize_t i = 0; i < PyList_Size(pResult); ++i) {
PyObject *metric = PyList_GetItem(pResult, i);
QString metricName(PyUnicode_AsUTF8(metric));
result.push_back(metricName.toStdString().c_str());
}
Py_DECREF(pResult);
return result;
}
QString PythonCaller::generatePlotsOnTrace(QString pathToTrace)
{
assert(PyCallable_Check(pGenPlotsFunction));
PyObject *pResult = callFunctionWithStringArgument(pGenPlotsFunction,
pathToTrace);
QString outputFiles (PyUnicode_AsUTF8(pResult));
Py_DECREF(pResult);
return outputFiles;
}
QString PythonCaller::exportAsVcd(QString pathToTrace)
{
if (!pVcdExportFunction)
return QString();
PyObject *pResult = callFunctionWithStringArgument(pVcdExportFunction, pathToTrace);
QString dump(PyUnicode_AsUTF8(pResult));
Py_DECREF(pResult);
return dump;
}
bool PythonCaller::vcdExportDependenciesAvailable()
{
PyObject *result = callFunctionWithoutArguments(pVcdExportDependenciesFunction);
bool available = PyObject_IsTrue(result);
Py_DECREF(result);
return available;
return {};
}

39
extensions/apps/traceAnalyzer/businessObjects/pythoncaller.h
View File

@@ -45,47 +45,16 @@
#undef slots
#endif
#include <Python.h>
#include <QString>
#include <string>
#include <map>
#include "businessObjects/tracecalculatedmetrics.h"
class PythonCaller
{
public:
PythonCaller();
~PythonCaller();
TraceCalculatedMetrics calculateMetricsOnTrace(QString pathToTrace,
std::vector<long> list);
std::vector<std::string> getMetrics(QString pathToTrace);
QString generatePlotsOnTrace(QString pathToTrace);
bool vcdExportDependenciesAvailable();
QString exportAsVcd(QString pathToTrace);
private:
PyObject *pCalculateMetricsFunction, *pGetMetricsFunction;
PyObject *pGenPlotsFunction;
PyObject *pVcdExportFunction = nullptr;
PyObject *pVcdExportDependenciesFunction;
PyObject *loadFunctionFromModule(std::string moduleName,
std::string functionName);
std::string metricModuleName, metricFunctionName, getMetricFunctionName, pathToScripts;
std::string plotsModuleName;
std::string plotsFunctionName;
std::string checkDependenciesModuleName;
std::string vcdExportModuleName;
std::string vcdExportFunctionName;
std::string vcdExportDependenciesFunctionName;
PyObject *callFunctionWithStringArgument(PyObject *function, QString argument);
PyObject *callFunctionWithoutArguments(PyObject *function);
PyObject *callMetricsFunction(PyObject *function, QString argument,
std::vector<long> list);
static std::vector<std::string> availableMetrics(std::string_view pathToTrace);
static TraceCalculatedMetrics evaluateMetrics(std::string_view pathToTrace, std::vector<long> selectedMetrics);
static std::string generatePlots(std::string_view pathToTrace);
static std::string dumpVcd(std::string_view pathToTrace);
};
#endif // PYTHONCALLER_H

4
extensions/apps/traceAnalyzer/businessObjects/tracecalculatedmetrics.h
View File

@@ -65,7 +65,7 @@ public:
result.append("Trace");
for (CalculatedMetric calculatedMetric : calculatedMetrics) {
result.append(",");
result.append(calculatedMetric.getName());
result.append(calculatedMetric.name.c_str());
}
return result;
}
@@ -76,7 +76,7 @@ public:
result.append(traceName);
for (CalculatedMetric calculatedMetric : calculatedMetrics) {
result.append(",");
result.append(QString::number(calculatedMetric.getValue()));
result.append(QString::number(calculatedMetric.value));
}
return result;
}

26
extensions/apps/traceAnalyzer/evaluationtool.cpp
View File

@@ -51,11 +51,9 @@
#include "evaluationtool.h"
#include "ui_evaluationtool.h"
using namespace std;
EvaluationTool::EvaluationTool(PythonCaller &pythonCaller, QWidget *parent) :
QWidget(parent),
ui(new Ui::EvaluationTool), resourcesRelPath("/../../dram/resources/scripts"""), pythonCaller(pythonCaller)
ui(new Ui::EvaluationTool), pythonCaller(pythonCaller)
{
ui->setupUi(this);
traceFilesModel = new QStandardItemModel(this);
@@ -85,16 +83,15 @@ void EvaluationTool::showAndEvaluateMetrics(QList<QString> paths)
show();
ui->toolBox->setCurrentIndex(1);
selectMetrics->setMetrics(getMetrics());
cout << "done" << endl;
}
vector<string> EvaluationTool::getMetrics()
std::vector<std::string> EvaluationTool::getMetrics()
{
vector<string> metrics;
std::vector<std::string> metrics;
for (int row = 0; row < traceFilesModel->rowCount(); ++row) {
TraceFileItem *item = static_cast<TraceFileItem *>(traceFilesModel->item(row));
vector<string> result = pythonCaller.getMetrics(item->getPath());
if (result.size() > metrics.size())
std::vector<std::string> result = PythonCaller::availableMetrics(item->getPath().toStdString());
if (result.size() > metrics.size()) // TODO use std::set
metrics = result;
}
return metrics;
@@ -127,22 +124,21 @@ void EvaluationTool::on_btn_calculateMetrics_clicked()
void EvaluationTool::getSelectedMetrics()
{
vector<long> selectedMetrics;
std::vector<long> selectedMetrics;
for (QCheckBox *metric : selectMetrics->metrics) {
selectedMetrics.push_back(metric->isChecked());
}
calculateMetrics(selectedMetrics);
}
void EvaluationTool::calculateMetrics(vector<long> selectedMetrics)
void EvaluationTool::calculateMetrics(std::vector<long> selectedMetrics)
{
ui->traceMetricTreeWidget->clear();
for (int row = 0; row < traceFilesModel->rowCount(); ++row) {
TraceFileItem *item = static_cast<TraceFileItem *>(traceFilesModel->item(row));
if (item->checkState() == Qt::Checked)
{
TraceCalculatedMetrics result = pythonCaller.calculateMetricsOnTrace(
item->getPath(), selectedMetrics);
TraceCalculatedMetrics result = pythonCaller.evaluateMetrics(item->getPath().toStdString(), selectedMetrics);
calculatedMetrics.push_back(result);
ui->traceMetricTreeWidget->addTraceMetricResults(result);
}
@@ -194,9 +190,9 @@ void EvaluationTool::genPlots()
TraceFileItem *item = static_cast<TraceFileItem *>(traceFilesModel->item(row));
if (item->checkState() == Qt::Checked)
{
ui->traceMetricTreeWidget->addTracePlotResults(QFileInfo(
item->getPath()).baseName(),
pythonCaller.generatePlotsOnTrace(item->getPath()));
ui->traceMetricTreeWidget->addTracePlotResults(
QFileInfo(item->getPath()).baseName(),
PythonCaller::generatePlots(item->getPath().toStdString()).c_str());
}
}
ui->traceMetricTreeWidget->expandAll();

1
extensions/apps/traceAnalyzer/evaluationtool.h
View File

@@ -84,7 +84,6 @@ private:
Ui::EvaluationTool *ui;
QStandardItemModel *traceFilesModel;
std::vector<TraceCalculatedMetrics> calculatedMetrics;
QString resourcesRelPath;
SelectMetrics *selectMetrics;
PythonCaller &pythonCaller;

12
extensions/apps/traceAnalyzer/main.cpp
View File

@@ -42,7 +42,9 @@
#include <QFileInfo>
#include <QSet>
#include <QString>
#include <filesystem>
#include <iostream>
#include <pybind11/embed.h>
int main(int argc, char *argv[])
{
@@ -54,6 +56,16 @@ int main(int argc, char *argv[])
a.setApplicationName(QStringLiteral("TraceAnalyzer"));
a.setApplicationDisplayName(QStringLiteral("Trace Analyzer"));
std::filesystem::path traceAnalyzerDir = DRAMSYS_TRACEANALYZER_DIR;
std::filesystem::path modulesDir = traceAnalyzerDir / "scripts";
pybind11::scoped_interpreter guard;
// Add scripts directory to local module search path
pybind11::module_ sys = pybind11::module_::import("sys");
pybind11::list path = sys.attr("path");
path.append(modulesDir.c_str());
if (argc > 1) {
QSet<QString> arguments;
for (int i = 1; i < argc; ++i)

2
extensions/apps/traceAnalyzer/presentation/tracemetrictreewidget.cpp
View File

@@ -55,7 +55,7 @@ void TraceMetricTreeWidget::addTraceMetricResults(const TraceCalculatedMetrics
new QTreeWidgetItem(top, {QString("Number of threads: 1")});
} else {
for (CalculatedMetric calculatedMetric : result.getCalculatedMetrics()) {
new QTreeWidgetItem(top, {calculatedMetric.getName() + QString(": ") + QString::number(calculatedMetric.getValue(), 'f')});
new QTreeWidgetItem(top, {calculatedMetric.name.c_str() + QString(": ") + QString::number(calculatedMetric.value, 'f')});
}
}
}

4
extensions/apps/traceAnalyzer/scripts/requirements.txt Normal file
View File

@@ -0,0 +1,4 @@
matplotlib
numpy
pyvcd
tqdm

3
extensions/apps/traceAnalyzer/traceanalyzer.cpp
View File

@@ -127,13 +127,12 @@ void TraceAnalyzer::on_menuFile_aboutToShow()
ui->actionQuit->setEnabled(true);
bool tabsOpen = ui->traceFileTabs->count() > 0;
bool exportAsVcdAvailable = pythonCaller.vcdExportDependenciesAvailable();
ui->actionSave->setEnabled(tabsOpen);
ui->actionSave_all->setEnabled(tabsOpen);
ui->actionReload->setEnabled(tabsOpen);
ui->actionReload_all->setEnabled(tabsOpen);
ui->actionExportAsVCD->setEnabled(tabsOpen && exportAsVcdAvailable);
ui->actionExportAsVCD->setEnabled(tabsOpen);
ui->actionTest->setEnabled(tabsOpen);
ui->actionMetrics->setEnabled(tabsOpen);
ui->actionClose->setEnabled(tabsOpen);

25
extensions/apps/traceAnalyzer/tracefiletab.cpp
View File

@@ -41,7 +41,6 @@
#include "businessObjects/commentmodel.h"
#include "businessObjects/configmodels.h"
#include "businessObjects/dramTimeDependencies/phasedependenciestracker.h"
#include "businessObjects/pythoncaller.h"
#include "businessObjects/traceplotlinemodel.h"
#include "businessObjects/tracetime.h"
#include "queryeditor.h"
@@ -68,6 +67,9 @@
#include <qwt_scale_draw.h>
#include <qwt_scale_widget.h>
#include <fstream>
#include <pybind11/pybind11.h>
TraceFileTab::TraceFileTab(std::string_view traceFilePath, PythonCaller &pythonCaller, QWidget *parent)
: QWidget(parent), ui(new Ui::TraceFileTab), commentModel(new CommentModel(this)),
navigator(new TraceNavigator(traceFilePath.data(), commentModel, this)),
@@ -117,23 +119,14 @@ void TraceFileTab::commitChangesToDB()
void TraceFileTab::exportAsVCD()
{
QString filename = QFileDialog::getSaveFileName(this, "Export to VCD", "",
"VCD files (*.vcd)");
auto dumpVcd = [=]() {
QString dump = pythonCaller.exportAsVcd(traceFilePath.data());
std::string filename = QFileDialog::getSaveFileName(this, "Export to VCD", "", "VCD files (*.vcd)").toStdString();
QFile file(filename);
file.open(QIODevice::WriteOnly | QIODevice::Text);
QTextStream out(&file);
out << dump;
file.close();
auto dump = PythonCaller::dumpVcd(traceFilePath);
std::ofstream file(filename);
file << dump;
Q_EMIT statusChanged(QString("VCD export finished."));
};
if (filename != "") {
QtConcurrent::run(dumpVcd);
}
Q_EMIT statusChanged(QString("VCD export finished."));
}
void TraceFileTab::setUpTraceSelector()