Merge branch 'work/vcd_export' into 'develop'

Further improvements of the VCD export script.

See merge request ems/astdm/dram.sys!305
This commit is contained in:
Lukas Steiner
2021-08-30 12:07:59 +00:00
9 changed files with 267 additions and 67 deletions

View File

@@ -34,6 +34,7 @@
* Robert Gernhardt
* Matthias Jung
* Felipe S. Prado
* Derek Christ
*/
#include "pythoncaller.h"
@@ -46,6 +47,12 @@
using namespace std;
PythonCaller &PythonCaller::instance()
{
static PythonCaller instance;
return instance;
}
PythonCaller::PythonCaller() :
testModuleName("tests"),
testFunctionName("runTests"),
@@ -57,8 +64,10 @@ PythonCaller::PythonCaller() :
"/../../DRAMSys/traceAnalyzer/scripts/"),
plotsModuleName("plots"),
plotsFunctionName("generatePlots"),
checkDependenciesModuleName("checkDependencies"),
vcdExportModuleName("vcdExport"),
vcdExportFunctionName("dumpVcd")
vcdExportFunctionName("dumpVcd"),
vcdExportDependenciesFunctionName("checkVcd")
{
Py_Initialize();
PyObject *sysPath = PySys_GetObject((char *)"path");
@@ -79,7 +88,12 @@ PythonCaller::PythonCaller() :
pGetMetricsFunction = loadFunctionFromModule(metricModuleName,
getMetricFunctionName);
pVcdExportFunction = loadFunctionFromModule(vcdExportModuleName, vcdExportFunctionName);
pVcdExportDependenciesFunction = loadFunctionFromModule(checkDependenciesModuleName, vcdExportDependenciesFunctionName);
if (vcdExportDependenciesAvailable())
pVcdExportFunction = loadFunctionFromModule(vcdExportModuleName, vcdExportFunctionName);
else
std::cerr << "Warning: Python module pyvcd not installed! Exporting as VCD not possible." << std::endl;
}
@@ -112,6 +126,11 @@ PythonCaller::~PythonCaller()
Py_DECREF(pCalculateMetricsFunction);
Py_DECREF(pGenPlotsFunction);
Py_DECREF(pGetMetricsFunction);
if (pVcdExportFunction)
Py_DECREF(pVcdExportFunction);
Py_DECREF(pVcdExportDependenciesFunction);
Py_Finalize();
}
@@ -162,6 +181,19 @@ PyObject *PythonCaller::callFunctionWithStringArgument(PyObject *function,
return pResult;
}
PyObject *PythonCaller::callFunctionWithoutArguments(PyObject *function)
{
assert(PyCallable_Check(function));
PyObject *pResult = PyObject_CallObject(function, NULL);
if (!pResult) {
PyErr_Print();
throw runtime_error(string("Error in calling python function"));
}
return pResult;
}
TraceTestResults PythonCaller::runTestsOnTrace(QString pathToTrace)
{
TraceTestResults traceTestResult(QFileInfo(pathToTrace).baseName());
@@ -228,9 +260,21 @@ QString PythonCaller::generatePlotsOnTrace(QString pathToTrace)
QString PythonCaller::exportAsVcd(QString pathToTrace)
{
if (!pVcdExportFunction)
return QString();
PyObject *pResult = callFunctionWithStringArgument(pVcdExportFunction, pathToTrace);
QString dump(PyUnicode_AsUTF8(pResult));
Py_DECREF(pResult);
return dump;
}
bool PythonCaller::vcdExportDependenciesAvailable()
{
PyObject *result = callFunctionWithoutArguments(pVcdExportDependenciesFunction);
bool available = PyObject_IsTrue(result);
Py_DECREF(result);
return available;
}

View File

@@ -34,6 +34,7 @@
* Robert Gernhardt
* Matthias Jung
* Felipe S. Prado
* Derek Christ
*/
#ifndef PYTHONCALLER_H
@@ -54,30 +55,41 @@
class PythonCaller
{
public:
PythonCaller();
~PythonCaller();
static PythonCaller &instance();
PythonCaller(const PythonCaller &other) = delete;
TraceTestResults runTestsOnTrace(QString pathToTrace);
TraceCalculatedMetrics calculateMetricsOnTrace(QString pathToTrace,
std::vector<long> list);
std::vector<std::string> getMetrics(QString pathToTrace);
QString generatePlotsOnTrace(QString pathToTrace);
bool vcdExportDependenciesAvailable();
QString exportAsVcd(QString pathToTrace);
private:
PythonCaller();
~PythonCaller();
PyObject *pRunTestsFunction, *pCalculateMetricsFunction, *pGetMetricsFunction;
PyObject *pGenPlotsFunction;
PyObject *pVcdExportFunction;
PyObject *pVcdExportFunction = nullptr;
PyObject *pVcdExportDependenciesFunction;
PyObject *loadFunctionFromModule(std::string moduleName,
std::string functionName);
std::string testModuleName, testFunctionName, metricModuleName,
metricFunctionName, getMetricFunctionName, pathToScripts;
std::string plotsModuleName;
std::string plotsFunctionName;
std::string checkDependenciesModuleName;
std::string vcdExportModuleName;
std::string vcdExportFunctionName;
std::string vcdExportDependenciesFunctionName;
PyObject *callFunctionWithStringArgument(PyObject *function, QString argument);
PyObject *callFunctionWithoutArguments(PyObject *function);
PyObject *callMetricsFunction(PyObject *function, QString argument,
std::vector<long> list);
};

View File

@@ -35,6 +35,7 @@
* Matthias Jung
* Éder F. Zulian
* Felipe S. Prado
* Derek Christ
*/
#include <QFileInfo>
@@ -105,7 +106,7 @@ vector<string> EvaluationTool::getMetrics()
vector<string> metrics;
for (int row = 0; row < traceFilesModel->rowCount(); ++row) {
TraceFileItem *item = static_cast<TraceFileItem *>(traceFilesModel->item(row));
vector<string> result = pythonCaller.getMetrics(item->getPath());
vector<string> result = PythonCaller::instance().getMetrics(item->getPath());
if (result.size() > metrics.size())
metrics = result;
}
@@ -152,7 +153,7 @@ void EvaluationTool::runTests()
if (item->checkState() == Qt::Checked)
{
boxesChecked = true;
TraceTestResults traceTestResult = pythonCaller.runTestsOnTrace(item->getPath());
TraceTestResults traceTestResult = PythonCaller::instance().runTestsOnTrace(item->getPath());
if (!traceTestResult.hasPassedAllTests())
allTestsPassed = false;
ui->traceTestTreeWidget->addTraceTestResult(traceTestResult);
@@ -193,7 +194,7 @@ void EvaluationTool::calculateMetrics(vector<long> selectedMetrics)
TraceFileItem *item = static_cast<TraceFileItem *>(traceFilesModel->item(row));
if (item->checkState() == Qt::Checked)
{
TraceCalculatedMetrics result = pythonCaller.calculateMetricsOnTrace(
TraceCalculatedMetrics result = PythonCaller::instance().calculateMetricsOnTrace(
item->getPath(), selectedMetrics);
calculatedMetrics.push_back(result);
ui->traceMetricTreeWidget->addTraceMetricResults(result);
@@ -253,7 +254,7 @@ void EvaluationTool::genPlots()
{
ui->traceMetricTreeWidget->addTracePlotResults(QFileInfo(
item->getPath()).baseName(),
pythonCaller.generatePlotsOnTrace(item->getPath()));
PythonCaller::instance().generatePlotsOnTrace(item->getPath()));
}
}
ui->traceMetricTreeWidget->expandAll();

View File

@@ -35,6 +35,7 @@
* Matthias Jung
* Éder F. Zulian
* Felipe S. Prado
* Derek Christ
*/
#ifndef EVALUATIONTOOL_H
@@ -42,12 +43,6 @@
#include "selectmetrics.h"
// Workaround for CMAKE and Python
#ifdef slots
#undef slots
#endif
#include <Python.h>
#include <QWidget>
#include <QStandardItem>
#include <QStandardItemModel>
@@ -96,7 +91,6 @@ private:
std::vector<TraceCalculatedMetrics> calculatedMetrics;
QString resourcesRelPath;
SelectMetrics *selectMetrics;
PythonCaller pythonCaller;
class TraceFileItem : public QStandardItem
{

View File

@@ -0,0 +1,42 @@
#!/usr/bin/env python3
# Copyright (c) 2021, Technische Universität Kaiserslautern
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors:
# Derek Christ
import importlib.util
def checkVcd():
if (spec := importlib.util.find_spec("vcd")) is not None:
return True
else:
return False

View File

@@ -30,7 +30,7 @@
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors:
# Authors:
# Derek Christ
@@ -38,11 +38,14 @@ import sqlite3
import io
import sys
import enum
import math
import datetime
from abc import ABC, abstractmethod
from memUtil import *
from vcd import VCDWriter
TIME_STEP = 1_000_000_000
class Signal(ABC):
def __init__(self, name):
self.name = name
@@ -75,18 +78,37 @@ class Event():
self.value = value
class Transaction():
def __init__(self, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd):
def __init__(self, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command):
self.rank = rank
self.bankgroup = bankgroup
self.bank = bank
self.dataStrobeBegin = dataStrobeBegin
self.dataStrobeEnd = dataStrobeEnd
self.command = command
class Granularity(enum.Enum):
Bankwise = 0
Groupwise = 1
Rankwise = 2
class TimeWindow():
def __init__(self, windowSize, lastTimestamp):
self.currentTime = 0
self.windowSize = windowSize
self.lastTimestamp = lastTimestamp
def __iter__(self):
return self
def __next__(self):
currentRange = (self.currentTime, self.currentTime + self.windowSize)
if self.currentTime <= self.lastTimestamp:
self.currentTime += self.windowSize
return currentRange
else:
raise StopIteration
def getGranularity(phase):
if phase == "PRESB" or phase == "REFSB":
return Granularity.Groupwise
@@ -101,6 +123,16 @@ def getAmountOfCommandBusSpans(phase):
else:
return 1
def getUnitOfTime(connection):
_, unit = getClock(connection)
return unit.lower()
def getLastTimestamp(connection):
cursor = connection.cursor()
cursor.execute("SELECT DataStrobeEnd FROM Transactions ORDER BY DataStrobeEnd DESC LIMIT 1")
return cursor.fetchone()[0]
def getRanksBankgroupsBanks(connection):
ranks = getNumberOfRanks(connection)
bankgroups = int(getNumberOfBankGroups(connection) / ranks)
@@ -121,40 +153,46 @@ def getBankNames(ranks, bankgroups, banks):
return names
def getOccurringSignals(connection):
setOfPhases = set()
setOfSignals = set()
setOfPhases.add(NumericSignal("REQ"))
setOfPhases.add(NumericSignal("RESP"))
setOfSignals.add(StringSignal("REQ"))
setOfSignals.add(StringSignal("RESP"))
(ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection)
for name in getBankNames(ranks, bankgroups, banks):
setOfPhases.add(NumericSignal(name))
setOfSignals.add(StringSignal(name))
setOfPhases.add(StringSignal("Command_Bus"))
setOfPhases.add(NumericSignal("Data_Bus"))
setOfSignals.add(StringSignal("Command_Bus"))
setOfSignals.add(StringSignal("Data_Bus"))
return setOfPhases
return setOfSignals
def getDataBusEvents(connection, eventDict, windowRange):
beginWindow, endWindow = windowRange
def getDataBusEvents(connection, eventDict):
cursor = connection.cursor()
cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd FROM Transactions")
cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd, Command FROM Transactions " +
"WHERE DataStrobeBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) +
" AND DataStrobeEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow))
transactions = getNumberOfTransactions(connection)
for transactionId, begin, end in cursor.fetchall():
for transactionId, begin, end, command in cursor.fetchall():
if eventDict.get(begin) == None:
eventDict[begin] = []
if eventDict.get(end) == None:
eventDict[end] = []
eventDict[begin].append(Event("Data_Bus", transactionId))
eventDict[end].append(Event("Data_Bus", "z"))
eventDict[begin].append(Event("Data_Bus", command + " " + str(transactionId)))
eventDict[end].append(Event("Data_Bus", ""))
def getCommandBusEvents(connection, eventDict, transactionDict, windowRange):
beginWindow, endWindow = windowRange
def getCommandBusEvents(connection, eventDict, transactionDict):
cursor = connection.cursor()
cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases")
cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases " +
"WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) +
" AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow))
for phase, phaseBegin, phaseEnd, transactionId in cursor.fetchall():
if phase == "REQ" or phase == "RESP":
@@ -176,10 +214,11 @@ def getCommandBusEvents(connection, eventDict, transactionDict):
if eventDict.get(end) == None:
eventDict[end] = []
eventDict[begin].append(Event("Command_Bus", phase))
eventDict[begin].append(Event("Command_Bus", phase + " " + str(transactionId)))
eventDict[end].append(Event("Command_Bus", ""))
currentTransaction = transactionDict[transactionId]
rank = currentTransaction.rank
bankgroup = currentTransaction.bankgroup
bank = currentTransaction.bank
@@ -204,12 +243,36 @@ def getCommandBusEvents(connection, eventDict, transactionDict):
for _rank, _bankgroup, _bank in currentBanks:
currentBankName = getBankName(_rank, _bankgroup, _bank)
eventDict[begin].append(Event(currentBankName, transactionId))
eventDict[end].append(Event(currentBankName, "z"))
eventDict[begin].append(Event(currentBankName, phase + " " + str(transactionId)))
eventDict[end].append(Event(currentBankName, ""))
def getTransactionRange(connection, transactionRange, windowRange):
beginWindow, endWindow = windowRange
def getReqAndRespPhases(connection, eventDict):
cursor = connection.cursor()
cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases")
cursor.execute("SELECT Transact FROM Phases" +
" WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) +
" AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow))
minTransaction, maxTransaction = float('inf'), 0
for transactionId in cursor.fetchall():
maxTransaction = max(maxTransaction, transactionId[0])
minTransaction = min(minTransaction, transactionId[0])
if minTransaction == float('inf'):
minTransaction = 0
transactionRange.append(minTransaction)
transactionRange.append(maxTransaction)
def getReqAndRespPhases(connection, eventDict, transactionDict, windowRange):
beginWindow, endWindow = windowRange
cursor = connection.cursor()
cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact " +
"FROM Phases WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) +
" AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow))
for phase, begin, end, transactionId in cursor.fetchall():
if phase != "REQ" and phase != "RESP":
@@ -221,42 +284,41 @@ def getReqAndRespPhases(connection, eventDict):
if eventDict.get(end) == None:
eventDict[end] = []
eventDict[begin].append(Event(phase, transactionId))
eventDict[end].append(Event(phase, "z"))
currentTransaction = transactionDict[transactionId]
command = currentTransaction.command
eventDict[begin].append(Event(phase, command + " " + str(transactionId)))
eventDict[end].append(Event(phase, ""))
def getTransactions(connection, transactionDict, transactionRange):
minTransaction, maxTransaction = transactionRange
def getTransactions(connection, transactionDict):
cursor = connection.cursor()
cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd FROM Transactions")
cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd, Command FROM Transactions" +
" WHERE ID BETWEEN " + str(minTransaction) + " AND " + str(maxTransaction))
(ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection)
for transactionId, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command in cursor.fetchall():
(ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection)
for transactionId, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd in cursor.fetchall():
rank = rank % ranks
bankgroup = bankgroup % bankgroups
bank = bank % banks
currentTransaction = Transaction(rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd)
currentTransaction = Transaction(rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command)
transactionDict[transactionId] = currentTransaction
def dumpVcd(pathToTrace):
connection = sqlite3.connect(pathToTrace)
eventDict = {}
transactionDict = {}
getTransactions(connection, transactionDict)
signalList = getOccurringSignals(connection)
getReqAndRespPhases(connection, eventDict)
getDataBusEvents(connection, eventDict)
getCommandBusEvents(connection, eventDict, transactionDict)
# Sort the eventDict so that VCDWriter can work with it.
eventDict = sorted(eventDict.items(), key=lambda x: x[0])
window = TimeWindow(TIME_STEP, getLastTimestamp(connection))
with io.StringIO() as f:
currentDate = datetime.date.today().strftime("%B %d, %Y")
with VCDWriter(f, timescale='1 ps', date=currentDate) as writer:
unit = getUnitOfTime(connection)
with VCDWriter(f, timescale="1" + unit, date=currentDate) as writer:
variableDict = {}
for signal in signalList:
@@ -264,12 +326,30 @@ def dumpVcd(pathToTrace):
signalType = signal.getSignalType()
variableDict[signal.name] = writer.register_var("DRAMSys", signal.name, signalType, init=neutralValue)
for timestamp, eventList in eventDict:
for event in eventList:
value_to_change = variableDict.get(event.signal)
if value_to_change != None:
writer.change(value_to_change, timestamp, event.value)
for windowRange in window:
eventDict = {}
transactionDict = {}
transactionRange = []
progress = min(windowRange[0] / window.lastTimestamp, 1.0) * 100.0
print("Export progress: {0:.2f}%".format(progress), file=sys.stderr)
getTransactionRange(connection, transactionRange, windowRange)
getTransactions(connection, transactionDict, transactionRange)
getReqAndRespPhases(connection, eventDict, transactionDict, windowRange)
getCommandBusEvents(connection, eventDict, transactionDict, windowRange)
getDataBusEvents(connection, eventDict, windowRange)
# Sort the eventDict so that VCDWriter can work with it.
eventDict = sorted(eventDict.items(), key=lambda x: x[0])
for timestamp, eventList in eventDict:
for event in eventList:
value_to_change = variableDict.get(event.signal)
if value_to_change != None:
writer.change(value_to_change, timestamp, event.value)
print("Export finished.", file=sys.stderr)
f.seek(0)
return f.read()

View File

@@ -33,6 +33,7 @@
* Janik Schlemminger
* Robert Gernhardt
* Matthias Jung
* Derek Christ
*/
#include "traceanalyzer.h"
@@ -121,7 +122,10 @@ void TraceAnalyzer::openTracefile(const QString &path)
// Enable actions
ui->actionReload_all->setEnabled(true);
ui->actionSaveChangesToDB->setEnabled(true);
ui->actionExportAsVCD->setEnabled(true);
if (PythonCaller::instance().vcdExportDependenciesAvailable())
ui->actionExportAsVCD->setEnabled(true);
ui->actionClose_all->setEnabled(true);
ui->actionTest->setEnabled(true);
ui->actionMetrics->setEnabled(true);

View File

@@ -85,6 +85,9 @@
</property>
</action>
<action name="actionReload_all">
<property name="enabled">
<bool>false</bool>
</property>
<property name="text">
<string>Reload databases</string>
</property>
@@ -93,6 +96,9 @@
</property>
</action>
<action name="actionSaveChangesToDB">
<property name="enabled">
<bool>false</bool>
</property>
<property name="text">
<string>Save changes to DB</string>
</property>
@@ -101,6 +107,9 @@
</property>
</action>
<action name="actionClose_all">
<property name="enabled">
<bool>false</bool>
</property>
<property name="text">
<string>Close all</string>
</property>
@@ -127,6 +136,9 @@
</property>
</action>
<action name="actionTest">
<property name="enabled">
<bool>false</bool>
</property>
<property name="text">
<string>Test</string>
</property>
@@ -135,6 +147,9 @@
</property>
</action>
<action name="actionMetrics">
<property name="enabled">
<bool>false</bool>
</property>
<property name="text">
<string>Metrics</string>
</property>
@@ -143,6 +158,9 @@
</property>
</action>
<action name="actionExportAsVCD">
<property name="enabled">
<bool>false</bool>
</property>
<property name="text">
<string>Export as VCD</string>
</property>

View File

@@ -47,6 +47,7 @@
#include <QString>
#include <QItemDelegate>
#include <QFileDialog>
#include <QtConcurrent/QtConcurrent>
#include <QTextStream>
#include <QDebug>
#include "qwt_plot_histogram.h"
@@ -96,16 +97,20 @@ void TraceFileTab::exportAsVCD()
{
QString filename = QFileDialog::getSaveFileName(this, "Export to VCD", "",
"VCD files (*.vcd)");
auto dumpVcd = [=]() {
QString dump = PythonCaller::instance().exportAsVcd(path);
PythonCaller pythonCaller;
QString dump = pythonCaller.exportAsVcd(path);
if (filename != "") {
QFile file(filename);
file.open(QIODevice::WriteOnly | QIODevice::Text);
QTextStream out(&file);
out << dump;
file.close();
Q_EMIT statusChanged(QString("VCD export finished."));
};
if (filename != "") {
QtConcurrent::run(dumpVcd);
}
}