From ca022a2aad0cbf795874e9686a5a817b1c4834ad Mon Sep 17 00:00:00 2001 From: Derek Christ Date: Wed, 25 Aug 2021 12:05:06 +0200 Subject: [PATCH] Sectionwise export of the VCD dump Only read in a section of the database at a time for the vcd export. This drastically reduces the memory usage for large database files. --- DRAMSys/traceAnalyzer/scripts/vcdExport.py | 122 ++++++++++++++++----- 1 file changed, 93 insertions(+), 29 deletions(-) diff --git a/DRAMSys/traceAnalyzer/scripts/vcdExport.py b/DRAMSys/traceAnalyzer/scripts/vcdExport.py index 70d03d32..cd89805e 100755 --- a/DRAMSys/traceAnalyzer/scripts/vcdExport.py +++ b/DRAMSys/traceAnalyzer/scripts/vcdExport.py @@ -30,7 +30,7 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# Authors: +# Authors: # Derek Christ @@ -38,11 +38,14 @@ import sqlite3 import io import sys import enum +import math import datetime from abc import ABC, abstractmethod from memUtil import * from vcd import VCDWriter +TIME_STEP = 1_000_000_000 + class Signal(ABC): def __init__(self, name): self.name = name @@ -87,6 +90,24 @@ class Granularity(enum.Enum): Groupwise = 1 Rankwise = 2 +class TimeWindow(): + def __init__(self, windowSize, lastTimestamp): + self.currentTime = 0 + self.windowSize = windowSize + self.lastTimestamp = lastTimestamp + + def __iter__(self): + return self + + def __next__(self): + currentRange = (self.currentTime, self.currentTime + self.windowSize) + + if self.currentTime <= self.lastTimestamp: + self.currentTime += self.windowSize + return currentRange + else: + raise StopIteration + def getGranularity(phase): if phase == "PRESB" or phase == "REFSB": return Granularity.Groupwise @@ -101,6 +122,16 @@ def getAmountOfCommandBusSpans(phase): else: return 1 +def getUnitOfTime(connection): + _, unit = getClock(connection) + return unit.lower() + +def getLastTimestamp(connection): + cursor = connection.cursor() + cursor.execute("SELECT DataStrobeEnd FROM Transactions ORDER BY DataStrobeEnd DESC LIMIT 1") + + return cursor.fetchone()[0] + def getRanksBankgroupsBanks(connection): ranks = getNumberOfRanks(connection) bankgroups = int(getNumberOfBankGroups(connection) / ranks) @@ -136,11 +167,13 @@ def getOccurringSignals(connection): return setOfPhases -def getDataBusEvents(connection, eventDict): - cursor = connection.cursor() - cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd FROM Transactions") +def getDataBusEvents(connection, eventDict, windowRange): + beginWindow, endWindow = windowRange - transactions = getNumberOfTransactions(connection) + cursor = connection.cursor() + cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd FROM Transactions " + + "WHERE DataStrobeBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + + " AND DataStrobeEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) for transactionId, begin, end in cursor.fetchall(): if eventDict.get(begin) == None: @@ -152,9 +185,13 @@ def getDataBusEvents(connection, eventDict): eventDict[begin].append(Event("Data_Bus", transactionId)) eventDict[end].append(Event("Data_Bus", "z")) -def getCommandBusEvents(connection, eventDict, transactionDict): +def getCommandBusEvents(connection, eventDict, transactionDict, windowRange): + beginWindow, endWindow = windowRange + cursor = connection.cursor() - cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases") + cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases " + + "WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + + " AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) for phase, phaseBegin, phaseEnd, transactionId in cursor.fetchall(): if phase == "REQ" or phase == "RESP": @@ -180,6 +217,7 @@ def getCommandBusEvents(connection, eventDict, transactionDict): eventDict[end].append(Event("Command_Bus", "")) currentTransaction = transactionDict[transactionId] + rank = currentTransaction.rank bankgroup = currentTransaction.bankgroup bank = currentTransaction.bank @@ -207,11 +245,20 @@ def getCommandBusEvents(connection, eventDict, transactionDict): eventDict[begin].append(Event(currentBankName, transactionId)) eventDict[end].append(Event(currentBankName, "z")) -def getReqAndRespPhases(connection, eventDict): +def getReqAndRespPhases(connection, eventDict, transactionRange, windowRange): + beginWindow, endWindow = windowRange + cursor = connection.cursor() - cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact FROM Phases") + cursor.execute("SELECT PhaseName, PhaseBegin, PhaseEnd, Transact " + + "FROM Phases WHERE PhaseBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + + " AND PhaseEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) + + minTransaction, maxTransaction = float('inf'), 0 for phase, begin, end, transactionId in cursor.fetchall(): + maxTransaction = max(maxTransaction, transactionId) + minTransaction = min(minTransaction, transactionId) + if phase != "REQ" and phase != "RESP": continue @@ -224,39 +271,41 @@ def getReqAndRespPhases(connection, eventDict): eventDict[begin].append(Event(phase, transactionId)) eventDict[end].append(Event(phase, "z")) -def getTransactions(connection, transactionDict): - cursor = connection.cursor() - cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd FROM Transactions") + if minTransaction == float('inf'): + minTransaction = 0 - (ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection) + transactionRange.append(minTransaction) + transactionRange.append(maxTransaction) + +def getTransactions(connection, transactionDict, transactionRange): + minTransaction, maxTransaction = transactionRange + + cursor = connection.cursor() + cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd FROM Transactions" + + " WHERE ID BETWEEN " + str(minTransaction) + " AND " + str(maxTransaction)) for transactionId, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd in cursor.fetchall(): + (ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection) + rank = rank % ranks bankgroup = bankgroup % bankgroups bank = bank % banks currentTransaction = Transaction(rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd) - transactionDict[transactionId] = currentTransaction + def dumpVcd(pathToTrace): connection = sqlite3.connect(pathToTrace) - eventDict = {} - transactionDict = {} - - getTransactions(connection, transactionDict) signalList = getOccurringSignals(connection) - getReqAndRespPhases(connection, eventDict) - getDataBusEvents(connection, eventDict) - getCommandBusEvents(connection, eventDict, transactionDict) - # Sort the eventDict so that VCDWriter can work with it. - eventDict = sorted(eventDict.items(), key=lambda x: x[0]) + window = TimeWindow(TIME_STEP, getLastTimestamp(connection)) with io.StringIO() as f: currentDate = datetime.date.today().strftime("%B %d, %Y") - with VCDWriter(f, timescale='1 ps', date=currentDate) as writer: + unit = getUnitOfTime(connection) + with VCDWriter(f, timescale="1" + unit, date=currentDate) as writer: variableDict = {} for signal in signalList: @@ -264,12 +313,27 @@ def dumpVcd(pathToTrace): signalType = signal.getSignalType() variableDict[signal.name] = writer.register_var("DRAMSys", signal.name, signalType, init=neutralValue) - for timestamp, eventList in eventDict: - for event in eventList: - value_to_change = variableDict.get(event.signal) - if value_to_change != None: - writer.change(value_to_change, timestamp, event.value) + for windowRange in window: + eventDict = {} + transactionDict = {} + transactionRange = [] + progress = min(windowRange[0] / window.lastTimestamp, 1.0) * 100.0 + print("Export progress: {0:.2f}%".format(progress), file=sys.stderr) + + getReqAndRespPhases(connection, eventDict, transactionRange, windowRange) + getTransactions(connection, transactionDict, transactionRange) + getDataBusEvents(connection, eventDict, windowRange) + getCommandBusEvents(connection, eventDict, transactionDict, windowRange) + + # Sort the eventDict so that VCDWriter can work with it. + eventDict = sorted(eventDict.items(), key=lambda x: x[0]) + + for timestamp, eventList in eventDict: + for event in eventList: + value_to_change = variableDict.get(event.signal) + if value_to_change != None: + writer.change(value_to_change, timestamp, event.value) f.seek(0) return f.read()