From a126fa86bf67f758ef253e6bc0af3aa36ec7064d Mon Sep 17 00:00:00 2001 From: Derek Christ Date: Mon, 16 Aug 2021 14:40:31 +0200 Subject: [PATCH 1/5] Check for vcd export dependencies. The TraceAnalyzer now checks if the python module pyvcd is installed and if not prints a warning and disables the export option. --- .../businessObjects/pythoncaller.cpp | 37 +++++++++++++++- .../businessObjects/pythoncaller.h | 10 ++++- .../scripts/checkDependencies.py | 42 +++++++++++++++++++ DRAMSys/traceAnalyzer/traceanalyzer.cpp | 6 ++- DRAMSys/traceAnalyzer/traceanalyzer.ui | 18 ++++++++ 5 files changed, 109 insertions(+), 4 deletions(-) create mode 100755 DRAMSys/traceAnalyzer/scripts/checkDependencies.py diff --git a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp index ce890fa5..f6490107 100644 --- a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp +++ b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp @@ -34,6 +34,7 @@ * Robert Gernhardt * Matthias Jung * Felipe S. Prado + * Derek Christ */ #include "pythoncaller.h" @@ -57,8 +58,10 @@ PythonCaller::PythonCaller() : "/../../DRAMSys/traceAnalyzer/scripts/"), plotsModuleName("plots"), plotsFunctionName("generatePlots"), + checkDependenciesModuleName("checkDependencies"), vcdExportModuleName("vcdExport"), - vcdExportFunctionName("dumpVcd") + vcdExportFunctionName("dumpVcd"), + vcdExportDependenciesFunctionName("checkVcd") { Py_Initialize(); PyObject *sysPath = PySys_GetObject((char *)"path"); @@ -79,7 +82,12 @@ PythonCaller::PythonCaller() : pGetMetricsFunction = loadFunctionFromModule(metricModuleName, getMetricFunctionName); - pVcdExportFunction = loadFunctionFromModule(vcdExportModuleName, vcdExportFunctionName); + pVcdExportDependenciesFunction = loadFunctionFromModule(checkDependenciesModuleName, vcdExportDependenciesFunctionName); + + if (vcdExportDependenciesAvailable()) + pVcdExportFunction = loadFunctionFromModule(vcdExportModuleName, vcdExportFunctionName); + else + std::cerr << "Warning: Python module pyvcd not installed! Exporting as VCD not possible." << std::endl; } @@ -162,6 +170,19 @@ PyObject *PythonCaller::callFunctionWithStringArgument(PyObject *function, return pResult; } +PyObject *PythonCaller::callFunctionWithoutArguments(PyObject *function) +{ + assert(PyCallable_Check(function)); + PyObject *pResult = PyObject_CallObject(function, NULL); + + if (!pResult) { + PyErr_Print(); + throw runtime_error(string("Error in calling python function")); + } + + return pResult; +} + TraceTestResults PythonCaller::runTestsOnTrace(QString pathToTrace) { TraceTestResults traceTestResult(QFileInfo(pathToTrace).baseName()); @@ -228,9 +249,21 @@ QString PythonCaller::generatePlotsOnTrace(QString pathToTrace) QString PythonCaller::exportAsVcd(QString pathToTrace) { + if (!pVcdExportFunction) + return QString(); + PyObject *pResult = callFunctionWithStringArgument(pVcdExportFunction, pathToTrace); QString dump(PyUnicode_AsUTF8(pResult)); Py_DECREF(pResult); return dump; } + +bool PythonCaller::vcdExportDependenciesAvailable() +{ + PyObject *result = callFunctionWithoutArguments(pVcdExportDependenciesFunction); + bool available = PyObject_IsTrue(result); + Py_DECREF(result); + + return available; +} diff --git a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h index bf50aa0d..9adab575 100644 --- a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h +++ b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h @@ -34,6 +34,7 @@ * Robert Gernhardt * Matthias Jung * Felipe S. Prado + * Derek Christ */ #ifndef PYTHONCALLER_H @@ -62,22 +63,29 @@ public: std::vector getMetrics(QString pathToTrace); QString generatePlotsOnTrace(QString pathToTrace); + bool vcdExportDependenciesAvailable(); QString exportAsVcd(QString pathToTrace); private: PyObject *pRunTestsFunction, *pCalculateMetricsFunction, *pGetMetricsFunction; PyObject *pGenPlotsFunction; - PyObject *pVcdExportFunction; + PyObject *pVcdExportFunction = nullptr; + PyObject *pVcdExportDependenciesFunction; PyObject *loadFunctionFromModule(std::string moduleName, std::string functionName); std::string testModuleName, testFunctionName, metricModuleName, metricFunctionName, getMetricFunctionName, pathToScripts; std::string plotsModuleName; std::string plotsFunctionName; + + std::string checkDependenciesModuleName; + std::string vcdExportModuleName; std::string vcdExportFunctionName; + std::string vcdExportDependenciesFunctionName; PyObject *callFunctionWithStringArgument(PyObject *function, QString argument); + PyObject *callFunctionWithoutArguments(PyObject *function); PyObject *callMetricsFunction(PyObject *function, QString argument, std::vector list); }; diff --git a/DRAMSys/traceAnalyzer/scripts/checkDependencies.py b/DRAMSys/traceAnalyzer/scripts/checkDependencies.py new file mode 100755 index 00000000..3cfe9dfd --- /dev/null +++ b/DRAMSys/traceAnalyzer/scripts/checkDependencies.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021, Technische Universität Kaiserslautern +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: +# Derek Christ + +import importlib.util + +def checkVcd(): + if (spec := importlib.util.find_spec("vcd")) is not None: + return True + else: + return False diff --git a/DRAMSys/traceAnalyzer/traceanalyzer.cpp b/DRAMSys/traceAnalyzer/traceanalyzer.cpp index bc79755c..6e599219 100644 --- a/DRAMSys/traceAnalyzer/traceanalyzer.cpp +++ b/DRAMSys/traceAnalyzer/traceanalyzer.cpp @@ -121,7 +121,11 @@ void TraceAnalyzer::openTracefile(const QString &path) // Enable actions ui->actionReload_all->setEnabled(true); ui->actionSaveChangesToDB->setEnabled(true); - ui->actionExportAsVCD->setEnabled(true); + + PythonCaller pythonCaller; + if (pythonCaller.vcdExportDependenciesAvailable()) + ui->actionExportAsVCD->setEnabled(true); + ui->actionClose_all->setEnabled(true); ui->actionTest->setEnabled(true); ui->actionMetrics->setEnabled(true); diff --git a/DRAMSys/traceAnalyzer/traceanalyzer.ui b/DRAMSys/traceAnalyzer/traceanalyzer.ui index ba36d631..6a364223 100644 --- a/DRAMSys/traceAnalyzer/traceanalyzer.ui +++ b/DRAMSys/traceAnalyzer/traceanalyzer.ui @@ -85,6 +85,9 @@ + + false + Reload databases @@ -93,6 +96,9 @@ + + false + Save changes to DB @@ -101,6 +107,9 @@ + + false + Close all @@ -127,6 +136,9 @@ + + false + Test @@ -135,6 +147,9 @@ + + false + Metrics @@ -143,6 +158,9 @@ + + false + Export as VCD From 65aa8e83e07b0a3a02efb57a9be056fd3bcb2479 Mon Sep 17 00:00:00 2001 From: Derek Christ Date: Mon, 16 Aug 2021 15:35:34 +0200 Subject: [PATCH 2/5] Make a singleton out of the PythonCaller Prevents unnecessary instances of PythonCaller. --- DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp | 6 ++++++ DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h | 8 ++++++-- DRAMSys/traceAnalyzer/evaluationtool.cpp | 9 +++++---- DRAMSys/traceAnalyzer/evaluationtool.h | 8 +------- DRAMSys/traceAnalyzer/traceanalyzer.cpp | 4 ++-- DRAMSys/traceAnalyzer/tracefiletab.cpp | 3 +-- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp index f6490107..9184a889 100644 --- a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp +++ b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp @@ -47,6 +47,12 @@ using namespace std; +PythonCaller &PythonCaller::instance() +{ + static PythonCaller instance; + return instance; +} + PythonCaller::PythonCaller() : testModuleName("tests"), testFunctionName("runTests"), diff --git a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h index 9adab575..0f843189 100644 --- a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h +++ b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.h @@ -55,8 +55,9 @@ class PythonCaller { public: - PythonCaller(); - ~PythonCaller(); + static PythonCaller &instance(); + PythonCaller(const PythonCaller &other) = delete; + TraceTestResults runTestsOnTrace(QString pathToTrace); TraceCalculatedMetrics calculateMetricsOnTrace(QString pathToTrace, std::vector list); @@ -67,6 +68,9 @@ public: QString exportAsVcd(QString pathToTrace); private: + PythonCaller(); + ~PythonCaller(); + PyObject *pRunTestsFunction, *pCalculateMetricsFunction, *pGetMetricsFunction; PyObject *pGenPlotsFunction; PyObject *pVcdExportFunction = nullptr; diff --git a/DRAMSys/traceAnalyzer/evaluationtool.cpp b/DRAMSys/traceAnalyzer/evaluationtool.cpp index 83223bdd..8820eb1f 100644 --- a/DRAMSys/traceAnalyzer/evaluationtool.cpp +++ b/DRAMSys/traceAnalyzer/evaluationtool.cpp @@ -35,6 +35,7 @@ * Matthias Jung * Éder F. Zulian * Felipe S. Prado + * Derek Christ */ #include @@ -105,7 +106,7 @@ vector EvaluationTool::getMetrics() vector metrics; for (int row = 0; row < traceFilesModel->rowCount(); ++row) { TraceFileItem *item = static_cast(traceFilesModel->item(row)); - vector result = pythonCaller.getMetrics(item->getPath()); + vector result = PythonCaller::instance().getMetrics(item->getPath()); if (result.size() > metrics.size()) metrics = result; } @@ -152,7 +153,7 @@ void EvaluationTool::runTests() if (item->checkState() == Qt::Checked) { boxesChecked = true; - TraceTestResults traceTestResult = pythonCaller.runTestsOnTrace(item->getPath()); + TraceTestResults traceTestResult = PythonCaller::instance().runTestsOnTrace(item->getPath()); if (!traceTestResult.hasPassedAllTests()) allTestsPassed = false; ui->traceTestTreeWidget->addTraceTestResult(traceTestResult); @@ -193,7 +194,7 @@ void EvaluationTool::calculateMetrics(vector selectedMetrics) TraceFileItem *item = static_cast(traceFilesModel->item(row)); if (item->checkState() == Qt::Checked) { - TraceCalculatedMetrics result = pythonCaller.calculateMetricsOnTrace( + TraceCalculatedMetrics result = PythonCaller::instance().calculateMetricsOnTrace( item->getPath(), selectedMetrics); calculatedMetrics.push_back(result); ui->traceMetricTreeWidget->addTraceMetricResults(result); @@ -253,7 +254,7 @@ void EvaluationTool::genPlots() { ui->traceMetricTreeWidget->addTracePlotResults(QFileInfo( item->getPath()).baseName(), - pythonCaller.generatePlotsOnTrace(item->getPath())); + PythonCaller::instance().generatePlotsOnTrace(item->getPath())); } } ui->traceMetricTreeWidget->expandAll(); diff --git a/DRAMSys/traceAnalyzer/evaluationtool.h b/DRAMSys/traceAnalyzer/evaluationtool.h index 509cfc3c..95a232fa 100644 --- a/DRAMSys/traceAnalyzer/evaluationtool.h +++ b/DRAMSys/traceAnalyzer/evaluationtool.h @@ -35,6 +35,7 @@ * Matthias Jung * Éder F. Zulian * Felipe S. Prado + * Derek Christ */ #ifndef EVALUATIONTOOL_H @@ -42,12 +43,6 @@ #include "selectmetrics.h" -// Workaround for CMAKE and Python -#ifdef slots -#undef slots -#endif -#include - #include #include #include @@ -96,7 +91,6 @@ private: std::vector calculatedMetrics; QString resourcesRelPath; SelectMetrics *selectMetrics; - PythonCaller pythonCaller; class TraceFileItem : public QStandardItem { diff --git a/DRAMSys/traceAnalyzer/traceanalyzer.cpp b/DRAMSys/traceAnalyzer/traceanalyzer.cpp index 6e599219..02103ce7 100644 --- a/DRAMSys/traceAnalyzer/traceanalyzer.cpp +++ b/DRAMSys/traceAnalyzer/traceanalyzer.cpp @@ -33,6 +33,7 @@ * Janik Schlemminger * Robert Gernhardt * Matthias Jung + * Derek Christ */ #include "traceanalyzer.h" @@ -122,8 +123,7 @@ void TraceAnalyzer::openTracefile(const QString &path) ui->actionReload_all->setEnabled(true); ui->actionSaveChangesToDB->setEnabled(true); - PythonCaller pythonCaller; - if (pythonCaller.vcdExportDependenciesAvailable()) + if (PythonCaller::instance().vcdExportDependenciesAvailable()) ui->actionExportAsVCD->setEnabled(true); ui->actionClose_all->setEnabled(true); diff --git a/DRAMSys/traceAnalyzer/tracefiletab.cpp b/DRAMSys/traceAnalyzer/tracefiletab.cpp index b42c743c..d8b7f457 100644 --- a/DRAMSys/traceAnalyzer/tracefiletab.cpp +++ b/DRAMSys/traceAnalyzer/tracefiletab.cpp @@ -93,8 +93,7 @@ void TraceFileTab::exportAsVCD() QString filename = QFileDialog::getSaveFileName(this, "Export to VCD", "", "VCD files (*.vcd)"); - PythonCaller pythonCaller; - QString dump = pythonCaller.exportAsVcd(path); + QString dump = PythonCaller::instance().exportAsVcd(path); if (filename != "") { QFile file(filename); From ca022a2aad0cbf795874e9686a5a817b1c4834ad Mon Sep 17 00:00:00 2001 From: Derek Christ Date: Wed, 25 Aug 2021 12:05:06 +0200 Subject: [PATCH 3/5] Sectionwise export of the VCD dump Only read in a section of the database at a time for the vcd export. This drastically reduces the memory usage for large database files. --- DRAMSys/traceAnalyzer/scripts/vcdExport.py | 122 ++++++++++++++++----- 1 file changed, 93 insertions(+), 29 deletions(-) diff --git a/DRAMSys/traceAnalyzer/scripts/vcdExport.py b/DRAMSys/traceAnalyzer/scripts/vcdExport.py index 70d03d32..cd89805e 100755 --- a/DRAMSys/traceAnalyzer/scripts/vcdExport.py +++ b/DRAMSys/traceAnalyzer/scripts/vcdExport.py @@ -30,7 +30,7 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# Authors: +# Authors: # Derek Christ @@ -38,11 +38,14 @@ import sqlite3 import io import sys import enum +import math import datetime from abc import ABC, abstractmethod from memUtil import * from vcd import VCDWriter +TIME_STEP = 1_000_000_000 + class Signal(ABC): def __init__(self, name): self.name = name @@ -87,6 +90,24 @@ class Granularity(enum.Enum): Groupwise = 1 Rankwise = 2 +class TimeWindow(): + def __init__(self, windowSize, lastTimestamp): + self.currentTime = 0 + self.windowSize = windowSize + self.lastTimestamp = lastTimestamp + + def __iter__(self): + return self + + def __next__(self): + currentRange = (self.currentTime, self.currentTime + self.windowSize) + + if self.currentTime <= self.lastTimestamp: + self.currentTime += self.windowSize + return currentRange + else: + raise StopIteration + def getGranularity(phase): if phase == "PRESB" or phase == "REFSB": return Granularity.Groupwise @@ -101,6 +122,16 @@ def getAmountOfCommandBusSpans(phase): else: return 1 +def getUnitOfTime(connection): + _, unit = getClock(connection) + return unit.lower() + +def getLastTimestamp(connection): + cursor = connection.cursor() + cursor.execute("SELECT DataStrobeEnd FROM Transactions ORDER BY DataStrobeEnd DESC LIMIT 1") + + return cursor.fetchone()[0] + def getRanksBankgroupsBanks(connection): ranks = getNumberOfRanks(connection) bankgroups = int(getNumberOfBankGroups(connection) / ranks) @@ -136,11 +167,13 @@ def getOccurringSignals(connection): return setOfPhases -def getDataBusEvents(connection, eventDict): - cursor = connection.cursor() - cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd FROM Transactions") +def getDataBusEvents(connection, eventDict, windowRange): + beginWindow, endWindow = windowRange - transactions = getNumberOfTransactions(connection) + cursor = connection.cursor() + cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd FROM Transactions " + + "WHERE DataStrobeBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + + " AND DataStrobeEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) for transactionId, begin, end in cursor.fetchall(): if eventDict.get(begin) == None: @@ -152,9 +185,13 @@ def getDataBusEvents(connection, eventDict): eventDict[begin].append(Event("Data_Bus", transactionId)) eventDict[end].append(Event("Data_Bus", "z")) -def getCommandBusEvents(connection, eventDict, transactionDict): +def getCommandBusEvents(connection, eventDict, transactionDict, windowRange): + beginWindow, endWindow = windowRange + cursor = connection.cursor() - cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases") + cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases " + + "WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + + " AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) for phase, phaseBegin, phaseEnd, transactionId in cursor.fetchall(): if phase == "REQ" or phase == "RESP": @@ -180,6 +217,7 @@ def getCommandBusEvents(connection, eventDict, transactionDict): eventDict[end].append(Event("Command_Bus", "")) currentTransaction = transactionDict[transactionId] + rank = currentTransaction.rank bankgroup = currentTransaction.bankgroup bank = currentTransaction.bank @@ -207,11 +245,20 @@ def getCommandBusEvents(connection, eventDict, transactionDict): eventDict[begin].append(Event(currentBankName, transactionId)) eventDict[end].append(Event(currentBankName, "z")) -def getReqAndRespPhases(connection, eventDict): +def getReqAndRespPhases(connection, eventDict, transactionRange, windowRange): + beginWindow, endWindow = windowRange + cursor = connection.cursor() - cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases") + cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact " + + "FROM Phases WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + + " AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) + + minTransaction, maxTransaction = float('inf'), 0 for phase, begin, end, transactionId in cursor.fetchall(): + maxTransaction = max(maxTransaction, transactionId) + minTransaction = min(minTransaction, transactionId) + if phase != "REQ" and phase != "RESP": continue @@ -224,39 +271,41 @@ def getReqAndRespPhases(connection, eventDict): eventDict[begin].append(Event(phase, transactionId)) eventDict[end].append(Event(phase, "z")) -def getTransactions(connection, transactionDict): - cursor = connection.cursor() - cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd FROM Transactions") + if minTransaction == float('inf'): + minTransaction = 0 - (ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection) + transactionRange.append(minTransaction) + transactionRange.append(maxTransaction) + +def getTransactions(connection, transactionDict, transactionRange): + minTransaction, maxTransaction = transactionRange + + cursor = connection.cursor() + cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd FROM Transactions" + + " WHERE ID BETWEEN " + str(minTransaction) + " AND " + str(maxTransaction)) for transactionId, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd in cursor.fetchall(): + (ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection) + rank = rank % ranks bankgroup = bankgroup % bankgroups bank = bank % banks currentTransaction = Transaction(rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd) - transactionDict[transactionId] = currentTransaction + def dumpVcd(pathToTrace): connection = sqlite3.connect(pathToTrace) - eventDict = {} - transactionDict = {} - - getTransactions(connection, transactionDict) signalList = getOccurringSignals(connection) - getReqAndRespPhases(connection, eventDict) - getDataBusEvents(connection, eventDict) - getCommandBusEvents(connection, eventDict, transactionDict) - # Sort the eventDict so that VCDWriter can work with it. - eventDict = sorted(eventDict.items(), key=lambda x: x[0]) + window = TimeWindow(TIME_STEP, getLastTimestamp(connection)) with io.StringIO() as f: currentDate = datetime.date.today().strftime("%B %d, %Y") - with VCDWriter(f, timescale='1 ps', date=currentDate) as writer: + unit = getUnitOfTime(connection) + with VCDWriter(f, timescale="1" + unit, date=currentDate) as writer: variableDict = {} for signal in signalList: @@ -264,12 +313,27 @@ def dumpVcd(pathToTrace): signalType = signal.getSignalType() variableDict[signal.name] = writer.register_var("DRAMSys", signal.name, signalType, init=neutralValue) - for timestamp, eventList in eventDict: - for event in eventList: - value_to_change = variableDict.get(event.signal) - if value_to_change != None: - writer.change(value_to_change, timestamp, event.value) + for windowRange in window: + eventDict = {} + transactionDict = {} + transactionRange = [] + progress = min(windowRange[0] / window.lastTimestamp, 1.0) * 100.0 + print("Export progress: {0:.2f}%".format(progress), file=sys.stderr) + + getReqAndRespPhases(connection, eventDict, transactionRange, windowRange) + getTransactions(connection, transactionDict, transactionRange) + getDataBusEvents(connection, eventDict, windowRange) + getCommandBusEvents(connection, eventDict, transactionDict, windowRange) + + # Sort the eventDict so that VCDWriter can work with it. + eventDict = sorted(eventDict.items(), key=lambda x: x[0]) + + for timestamp, eventList in eventDict: + for event in eventList: + value_to_change = variableDict.get(event.signal) + if value_to_change != None: + writer.change(value_to_change, timestamp, event.value) f.seek(0) return f.read() From 93089ce1f63f634e62e56abf81f2f584650689c0 Mon Sep 17 00:00:00 2001 From: Derek Christ Date: Thu, 26 Aug 2021 11:00:25 +0200 Subject: [PATCH 4/5] Move VCD export to seperate thread and add R/W information to all phases --- DRAMSys/traceAnalyzer/scripts/vcdExport.py | 82 +++++++++++++--------- DRAMSys/traceAnalyzer/tracefiletab.cpp | 12 +++- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/DRAMSys/traceAnalyzer/scripts/vcdExport.py b/DRAMSys/traceAnalyzer/scripts/vcdExport.py index cd89805e..c1133bd2 100755 --- a/DRAMSys/traceAnalyzer/scripts/vcdExport.py +++ b/DRAMSys/traceAnalyzer/scripts/vcdExport.py @@ -78,12 +78,13 @@ class Event(): self.value = value class Transaction(): - def __init__(self, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd): + def __init__(self, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command): self.rank = rank self.bankgroup = bankgroup self.bank = bank self.dataStrobeBegin = dataStrobeBegin self.dataStrobeEnd = dataStrobeEnd + self.command = command class Granularity(enum.Enum): Bankwise = 0 @@ -152,38 +153,38 @@ def getBankNames(ranks, bankgroups, banks): return names def getOccurringSignals(connection): - setOfPhases = set() + setOfSignals = set() - setOfPhases.add(NumericSignal("REQ")) - setOfPhases.add(NumericSignal("RESP")) + setOfSignals.add(StringSignal("REQ")) + setOfSignals.add(StringSignal("RESP")) (ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection) for name in getBankNames(ranks, bankgroups, banks): - setOfPhases.add(NumericSignal(name)) + setOfSignals.add(StringSignal(name)) - setOfPhases.add(StringSignal("Command_Bus")) - setOfPhases.add(NumericSignal("Data_Bus")) + setOfSignals.add(StringSignal("Command_Bus")) + setOfSignals.add(StringSignal("Data_Bus")) - return setOfPhases + return setOfSignals def getDataBusEvents(connection, eventDict, windowRange): beginWindow, endWindow = windowRange cursor = connection.cursor() - cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd FROM Transactions " + + cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd, Command FROM Transactions " + "WHERE DataStrobeBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + " AND DataStrobeEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) - for transactionId, begin, end in cursor.fetchall(): + for transactionId, begin, end, command in cursor.fetchall(): if eventDict.get(begin) == None: eventDict[begin] = [] if eventDict.get(end) == None: eventDict[end] = [] - eventDict[begin].append(Event("Data_Bus", transactionId)) - eventDict[end].append(Event("Data_Bus", "z")) + eventDict[begin].append(Event("Data_Bus", command + " " + str(transactionId))) + eventDict[end].append(Event("Data_Bus", "")) def getCommandBusEvents(connection, eventDict, transactionDict, windowRange): beginWindow, endWindow = windowRange @@ -213,7 +214,7 @@ def getCommandBusEvents(connection, eventDict, transactionDict, windowRange): if eventDict.get(end) == None: eventDict[end] = [] - eventDict[begin].append(Event("Command_Bus", phase)) + eventDict[begin].append(Event("Command_Bus", phase + " " + str(transactionId))) eventDict[end].append(Event("Command_Bus", "")) currentTransaction = transactionDict[transactionId] @@ -242,10 +243,30 @@ def getCommandBusEvents(connection, eventDict, transactionDict, windowRange): for _rank, _bankgroup, _bank in currentBanks: currentBankName = getBankName(_rank, _bankgroup, _bank) - eventDict[begin].append(Event(currentBankName, transactionId)) - eventDict[end].append(Event(currentBankName, "z")) + eventDict[begin].append(Event(currentBankName, phase + " " + str(transactionId))) + eventDict[end].append(Event(currentBankName, "")) -def getReqAndRespPhases(connection, eventDict, transactionRange, windowRange): +def getTransactionRange(connection, transactionRange, windowRange): + beginWindow, endWindow = windowRange + + cursor = connection.cursor() + cursor.execute("SELECT Transact FROM Phases" + + " WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + + " AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) + + minTransaction, maxTransaction = float('inf'), 0 + + for transactionId in cursor.fetchall(): + maxTransaction = max(maxTransaction, transactionId[0]) + minTransaction = min(minTransaction, transactionId[0]) + + if minTransaction == float('inf'): + minTransaction = 0 + + transactionRange.append(minTransaction) + transactionRange.append(maxTransaction) + +def getReqAndRespPhases(connection, eventDict, transactionDict, windowRange): beginWindow, endWindow = windowRange cursor = connection.cursor() @@ -253,12 +274,7 @@ def getReqAndRespPhases(connection, eventDict, transactionRange, windowRange): "FROM Phases WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + " AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) - minTransaction, maxTransaction = float('inf'), 0 - for phase, begin, end, transactionId in cursor.fetchall(): - maxTransaction = max(maxTransaction, transactionId) - minTransaction = min(minTransaction, transactionId) - if phase != "REQ" and phase != "RESP": continue @@ -268,30 +284,27 @@ def getReqAndRespPhases(connection, eventDict, transactionRange, windowRange): if eventDict.get(end) == None: eventDict[end] = [] - eventDict[begin].append(Event(phase, transactionId)) - eventDict[end].append(Event(phase, "z")) + currentTransaction = transactionDict[transactionId] + command = currentTransaction.command - if minTransaction == float('inf'): - minTransaction = 0 - - transactionRange.append(minTransaction) - transactionRange.append(maxTransaction) + eventDict[begin].append(Event(phase, command + " " + str(transactionId))) + eventDict[end].append(Event(phase, "")) def getTransactions(connection, transactionDict, transactionRange): minTransaction, maxTransaction = transactionRange cursor = connection.cursor() - cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd FROM Transactions" + + cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd, Command FROM Transactions" + " WHERE ID BETWEEN " + str(minTransaction) + " AND " + str(maxTransaction)) - for transactionId, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd in cursor.fetchall(): + for transactionId, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command in cursor.fetchall(): (ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection) rank = rank % ranks bankgroup = bankgroup % bankgroups bank = bank % banks - currentTransaction = Transaction(rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd) + currentTransaction = Transaction(rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command) transactionDict[transactionId] = currentTransaction @@ -321,10 +334,11 @@ def dumpVcd(pathToTrace): progress = min(windowRange[0] / window.lastTimestamp, 1.0) * 100.0 print("Export progress: {0:.2f}%".format(progress), file=sys.stderr) - getReqAndRespPhases(connection, eventDict, transactionRange, windowRange) + getTransactionRange(connection, transactionRange, windowRange) getTransactions(connection, transactionDict, transactionRange) - getDataBusEvents(connection, eventDict, windowRange) + getReqAndRespPhases(connection, eventDict, transactionDict, windowRange) getCommandBusEvents(connection, eventDict, transactionDict, windowRange) + getDataBusEvents(connection, eventDict, windowRange) # Sort the eventDict so that VCDWriter can work with it. eventDict = sorted(eventDict.items(), key=lambda x: x[0]) @@ -335,6 +349,8 @@ def dumpVcd(pathToTrace): if value_to_change != None: writer.change(value_to_change, timestamp, event.value) + print("Export finished.", file=sys.stderr) + f.seek(0) return f.read() diff --git a/DRAMSys/traceAnalyzer/tracefiletab.cpp b/DRAMSys/traceAnalyzer/tracefiletab.cpp index d8b7f457..f7c9be1e 100644 --- a/DRAMSys/traceAnalyzer/tracefiletab.cpp +++ b/DRAMSys/traceAnalyzer/tracefiletab.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include "qwt_plot_histogram.h" #include "qwt_plot_curve.h" #include "qwt_plot_layout.h" @@ -92,15 +93,20 @@ void TraceFileTab::exportAsVCD() { QString filename = QFileDialog::getSaveFileName(this, "Export to VCD", "", "VCD files (*.vcd)"); + auto dumpVcd = [=]() { + QString dump = PythonCaller::instance().exportAsVcd(path); - QString dump = PythonCaller::instance().exportAsVcd(path); - - if (filename != "") { QFile file(filename); file.open(QIODevice::WriteOnly | QIODevice::Text); QTextStream out(&file); out << dump; file.close(); + + Q_EMIT statusChanged(QString("VCD export finished.")); + }; + + if (filename != "") { + QtConcurrent::run(dumpVcd); } } From a73ca699611321e76063f918b188e246b0cda264 Mon Sep 17 00:00:00 2001 From: Derek Christ Date: Fri, 27 Aug 2021 10:17:37 +0200 Subject: [PATCH 5/5] Fix minor memory leak in PythonCaller --- DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp index 9184a889..f44f4a16 100644 --- a/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp +++ b/DRAMSys/traceAnalyzer/businessObjects/pythoncaller.cpp @@ -126,6 +126,11 @@ PythonCaller::~PythonCaller() Py_DECREF(pCalculateMetricsFunction); Py_DECREF(pGenPlotsFunction); Py_DECREF(pGetMetricsFunction); + + if (pVcdExportFunction) + Py_DECREF(pVcdExportFunction); + + Py_DECREF(pVcdExportDependenciesFunction); Py_Finalize(); }