diff --git a/DRAMSys/analyzer/scripts/memUtil.py b/DRAMSys/analyzer/scripts/memUtil.py index 445cd6be..eed3fe2b 100755 --- a/DRAMSys/analyzer/scripts/memUtil.py +++ b/DRAMSys/analyzer/scripts/memUtil.py @@ -29,7 +29,9 @@ class MemSpec(object): proper format when searching for elements. """ def getValue(self, id): - return self.xmlMemSpec.findall(".//parameter[@id='{0}']".format(id))[0].attrib['value'] + match = ".//parameter[@id='{0}']".format(id) + val = self.xmlMemSpec.findall(match)[0].attrib['value'] + return val def getIntValue(self, id): return int(self.getValue(id)) @@ -55,6 +57,13 @@ def getNumberOfBanks(dbconnection): return result[0] +def getWindowSize(connection): + cursor = connection.cursor() + cursor.execute(" SELECT WindowSize FROM GeneralInfo ") + windowSize = float(cursor.fetchone()[0]) + return windowSize + + def maximum_data_rate(connection): memspec = MemSpec(connection) memoryType = memspec.getValue("memoryType") diff --git a/DRAMSys/analyzer/scripts/metrics.py b/DRAMSys/analyzer/scripts/metrics.py index da21d95c..f62084d6 100644 --- a/DRAMSys/analyzer/scripts/metrics.py +++ b/DRAMSys/analyzer/scripts/metrics.py @@ -500,7 +500,7 @@ def calculateMetrics(pathToTrace, selectedMetrics = []): if not selectedMetrics: selectedMetrics = [0] * (len(metrics) + len(getThreads(connection))*len(threadMetrics) + 1) - for i in range(len(metrics)): + for i in range(len(selectedMetrics)): selectedMetrics[i] = True if (len(getThreads(connection)) >= 1): @@ -529,8 +529,10 @@ def calculateMetrics(pathToTrace, selectedMetrics = []): if(selectedMetrics[len(metrics) + len(threadMetrics)*(thread-1) + threadMetrics.index(metric)]): mres = metric(connection, thread) mname = "Thread {0} - {1}".format(thread,metric.__name__.replace("_"," ")) - r = (mname, mres) - calculatedMetrics.append(r) + res = (mname, mres) + calculatedMetrics.append(res) + print("{0}: {1}".format(res[0], res[1])) + if(selectedMetrics[len(selectedMetrics) -1]): calculatedMetrics.extend(passRatio(connection)) # refreshMissDecision(connection, calculatedMetrics) diff --git a/DRAMSys/analyzer/scripts/plots.py b/DRAMSys/analyzer/scripts/plots.py index dc2153d3..57087064 100755 --- a/DRAMSys/analyzer/scripts/plots.py +++ b/DRAMSys/analyzer/scripts/plots.py @@ -7,34 +7,35 @@ import os plots = [] - -def plot(function): - plots.append(function) - return function - - -@plot -def memory_utilisation_window(connection, tracePath, steps): - # This function determines the average memory bandwidth over time in - # percentage and in Gbit/s. The average bandwidth over time is done - # dividing the time into windows of the same length and getting the average - # bandwidth in each window. Through data from the database, DataStrobeEnd - # and DataStrobeBegin, it is possible to access when a data transfer begins - # or ends. Hence, it is achievable to check when a data transfer happens - # and if it occupies or is inside a time window. Then, it is attainable to - # determine the average bandwidth in percentage. Besides, extracting the - # data from the memory specs, it is feasible to calculate the maximum data - # rate of the memory and then determine the bandwidth in Gbit/s. The - # bandwidth data are then plotted in two graphics. - +def getThreads(connection): + cursor = connection.cursor() + cursor.execute("SELECT DISTINCT(TThread) FROM transactions WHERE TThread != 0 ORDER BY TThread") + result = [] + for currentRow in cursor: + result.append(currentRow[0]) + return result + +def createOutputFilename(tracePath, plot_type, target_measurement, file_type): + name = ntpath.basename(tracePath) + basename, extension = os.path.splitext(name) + outputFileName = plot_type + '_' + target_measurement + basename + '.' + file_type + return outputFileName, target_measurement + basename + +def accessDatabase(connection, query): + cursor = connection.cursor() + # cursor.execute(" ") + cursor.execute(query) + resultArray = [] + while True: + result = cursor.fetchone() + if (result is not None): + resultArray.append(result[0]) + else: + break + return resultArray + +def calculate_bandwidth_util(connection, windowSize, steps, queryFull, queryEnd, queryBegin, queryPart): cursor = connection.cursor() - cursor.execute(" SELECT WindowSize FROM GeneralInfo ") - windowSize = float(cursor.fetchone()[0]) - # All possible cases of data transfers inside a time window - queryFull = """ SELECT sum(DataStrobeEnd - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeEnd <= ?""" # The data transfer begins and ends inside the time window - queryEnd = """ SELECT sum(DataStrobeEnd - ?) FROM transactions Where DataStrobeBegin < ? and DataStrobeEnd > ? and DataStrobeEnd <=?""" # Only the end of the data transfer is inside the time window - queryBegin = """ SELECT sum(? - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeBegin < ? and DataStrobeEnd > ?""" # Only the beginning of the data transfer is inside the time window - queryPart = """ SELECT DataStrobeBegin FROM transactions Where DataStrobeBegin <= ? and DataStrobeEnd >= ?""" # The data transfer occupies all the time window maxDataRate = maximum_data_rate(connection) maximumPercentage = 0 bandwidthPercentage = [0] * (steps+1) @@ -66,12 +67,60 @@ def memory_utilisation_window(connection, tracePath, steps): if(maximumPercentage < 100 and maximumPercentage < bandwidthPercentage[i+1]): maximumPercentage = bandwidthPercentage[i+1] - name = ntpath.basename(tracePath) - basename, extension = os.path.splitext(name) + return bandwidthPercentage, bandwidth, maximumPercentage - outputFileNameGBPS = 'memory_utilization_gbps_' + basename + '.pdf' - outputFileNamePercent = 'memory_utilization_percent_' + basename + '.pdf' - outputFiles = "Output files are {0},{1}".format(outputFileNameGBPS,outputFileNamePercent) +def memory_utilisation_window_thread(connection, tracePath, steps, thread_ID): + # All possible cases of data transfers inside a time window + queryFull = """ SELECT sum(DataStrobeEnd - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeEnd <= ? and TThread = {0}""" # The data transfer begins and ends inside the time window + queryEnd = """ SELECT sum(DataStrobeEnd - ?) FROM transactions Where DataStrobeBegin < ? and DataStrobeEnd > ? and DataStrobeEnd <=? and TThread = {0}""" # Only the end of the data transfer is inside the time window + queryBegin = """ SELECT sum(? - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeBegin < ? and DataStrobeEnd > ? and TThread = {0}""" # Only the beginning of the data transfer is inside the time window + queryPart = """ SELECT DataStrobeBegin FROM transactions Where DataStrobeBegin <= ? and DataStrobeEnd >= ? and TThread = {0}""" # The data transfer occupies all the time window + + queryFull = queryFull.format(thread_ID) + queryEnd = queryEnd.format(thread_ID) + queryBegin = queryBegin.format(thread_ID) + queryPart = queryPart.format(thread_ID) + + windowSize = getWindowSize(connection) + + bandwidthPercentage, bandwidth, maximumPercentage = calculate_bandwidth_util(connection, windowSize, steps, queryFull, queryEnd, queryBegin, queryPart) + + outputFileNameBWMatlab, basename = createOutputFilename(tracePath, 'memory_utilization_percent', 'thread_' + str(thread_ID) + '_', 'txt') + return bandwidthPercentage, bandwidth, outputFileNameBWMatlab + +def plot(function): + plots.append(function) + return function + +@plot +def memory_utilisation_window(connection, tracePath, steps): + # This function determines the average memory bandwidth over time in + # percentage and in Gbit/s. The average bandwidth over time is done + # dividing the time into windows of the same length and getting the average + # bandwidth in each window. Through data from the database, DataStrobeEnd + # and DataStrobeBegin, it is possible to access when a data transfer begins + # or ends. Hence, it is achievable to check when a data transfer happens + # and if it occupies or is inside a time window. Then, it is attainable to + # determine the average bandwidth in percentage. Besides, extracting the + # data from the memory specs, it is feasible to calculate the maximum data + # rate of the memory and then determine the bandwidth in Gbit/s. The + # bandwidth data are then plotted in two graphics. + + windowSize = getWindowSize(connection) + maxDataRate = maximum_data_rate(connection) + + # All possible cases of data transfers inside a time window + queryFull = """ SELECT sum(DataStrobeEnd - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeEnd <= ?""" # The data transfer begins and ends inside the time window + queryEnd = """ SELECT sum(DataStrobeEnd - ?) FROM transactions Where DataStrobeBegin < ? and DataStrobeEnd > ? and DataStrobeEnd <=?""" # Only the end of the data transfer is inside the time window + queryBegin = """ SELECT sum(? - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeBegin < ? and DataStrobeEnd > ?""" # Only the beginning of the data transfer is inside the time window + queryPart = """ SELECT DataStrobeBegin FROM transactions Where DataStrobeBegin <= ? and DataStrobeEnd >= ?""" # The data transfer occupies all the time window + + bandwidthPercentage, bandwidth, maximumPercentage = calculate_bandwidth_util(connection, windowSize, steps, queryFull, queryEnd, queryBegin, queryPart) + + outputFileNameGBPS, basename = createOutputFilename(tracePath, 'memory_utilization_gbps', '', 'pdf') + outputFileNamePercent, basename = createOutputFilename(tracePath, 'memory_utilization_percent', '', 'pdf') + outputFileNameBWMatlab, basename = createOutputFilename(tracePath, 'memory_utilization_percent', '', 'txt') + outputFiles = "{0}\n\t{1}\n\t{2}\n\t".format(outputFileNameGBPS,outputFileNamePercent,outputFileNameBWMatlab) import matplotlib.pyplot as plt import numpy as np @@ -81,41 +130,62 @@ def memory_utilisation_window(connection, tracePath, steps): time = np.arange(0, (steps+1)*windowSize/1000, windowSize/1000) maxBandwidth = [maxDataRate/1024] * (steps+1) - plt.figure() - - # Write data to a file for matlab: - outputFileNameBWMatlab = 'memory_utilization_percent_' + basename + '.txt' - f = open(outputFileNameBWMatlab, 'w') for i in range(steps): line = "{} {}\n".format(time[i], bandwidthPercentage[i]) f.write(line) - #Plot Bandwidth in Percent - plt.plot(time, bandwidthPercentage) - plt.xlabel('Time [ns]') - plt.ylabel('Bandwidth [%]') - plt.ylim(-1, maximumPercentage + (10 - maximumPercentage%10)) - plt.grid(True) + # Plot Bandwidth in Percent + BWPercentageFigure = plt.figure() + BWPercentageFigurePlot = BWPercentageFigure.add_subplot(111) + BWPercentageFigurePlot.set_xlabel('Time [ns]') + BWPercentageFigurePlot.set_ylabel('Bandwidth [%]') + BWPercentageFigurePlot.set_ylim(-1, maximumPercentage + (10 - maximumPercentage%10)) + BWPercentageFigurePlot.set_title('Memory Utilization in % ' + str(basename)) + BWPercentageFigurePlot.grid(True) + BWPercentageFigurePlot.plot(time, bandwidthPercentage, label='Total') + BWPercentageFigurePlot.legend(loc="upper left") + # Plot absolute bandwidth + BWFigure = plt.figure() + BWFigurePlot = BWFigure.add_subplot(111) + BWFigurePlot.set_xlabel('Time [ns]') + BWFigurePlot.set_ylabel('Bandwidth [Gibit/s]') + BWFigurePlot.set_title('Memory Utilization in Gbps ' + str(basename)) + BWFigurePlot.grid(True) + BWFigurePlot.plot(time, bandwidth, label='Total') + BWFigurePlot.legend(loc="upper left") + + # plt.ylim((-0.01)*float(maxDataRate)/1024, ((maximumPercentage + (10 - maximumPercentage%10))/100)*float(maxDataRate)/1024) + + threads = getThreads(connection) + if (len(threads) > 1): + for thread in threads: + threadStr = "Thread " + str(thread) + bandwidthPercentage, bandwidth, outputFileNameBWMatlab = memory_utilisation_window_thread(connection, tracePath, steps, thread) + BWPercentageFigurePlot.plot(time, bandwidthPercentage, label=threadStr) + BWPercentageFigurePlot.legend(loc="upper left") + BWFigurePlot.plot(time, bandwidth, label=threadStr) + BWFigurePlot.legend(loc="upper left") + f = open(outputFileNameBWMatlab, 'w') + for i in range(steps): + line = "{} {}\n".format(time[i], bandwidthPercentage[i]) + f.write(line) + outputFiles += "{0}\n\t".format(outputFileNameBWMatlab) + + # Save to PDF files pdf = PdfPages(outputFileNamePercent) - pdf.savefig() + pdf.savefig(BWPercentageFigure) pdf.close() - plt.close() - - #Plot absolute bandwidth - plt.plot(time, bandwidth) - plt.plot(time, maxBandwidth) - plt.xlabel('Time [ns]') - plt.ylabel('Bandwidth [Gibit/s]') - #plt.ylim((-0.01)*float(maxDataRate)/1024, ((maximumPercentage + (10 - maximumPercentage%10))/100)*float(maxDataRate)/1024) - plt.grid(True) - + BWPercentageFigure.clear() pdf = PdfPages(outputFileNameGBPS) - pdf.savefig() + BWFigurePlot.plot(time, maxBandwidth) + pdf.savefig(BWFigure) pdf.close() + BWFigurePlot.clear() plt.close() + return outputFiles @plot @@ -134,15 +204,12 @@ def power_window(connection, tracePath, steps): time[1] = float(result[0])*pow(10,9) power[1] = float(result[1]) for i in range((steps-1)): - result = cursor.fetchone() + result = cursor.fetchone() time[i+2] = float(result[0])*pow(10,9) power[i+2] = float(result[1]) - name = ntpath.basename(tracePath) - basename, extension = os.path.splitext(name) - - outputFileName = 'power_' + basename + '.pdf' - outputFile = "\n" + "Output file is {0}".format(outputFileName) + outputFileName, basename = createOutputFilename(tracePath, 'power', '', 'pdf') + outputFile = "{0}\n\t".format(outputFileName) import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages @@ -151,6 +218,7 @@ def power_window(connection, tracePath, steps): plt.xlabel('Time [ns]') plt.ylabel('Power [mW]') plt.grid(True) + plt.title('Power Consumption ' + str(basename)) pdf = PdfPages(outputFileName) pdf.savefig() pdf.close() @@ -160,39 +228,70 @@ def power_window(connection, tracePath, steps): @plot def latency_histogram(connection, tracePath, steps): # This function plots an histogram with access latencys - outputFile = "" - cursor = connection.cursor() - cursor.execute("SELECT ((p2.PhaseEnd - p1.PhaseBegin)/1000) FROM Transactions t, Phases p1, Phases p2 WHERE t.id = p1.Transact and t.id = p2.Transact and p1.PhaseName = \"REQ\" and p2.PhaseName = \"RESP\" ") - result = cursor.fetchall() - name = ntpath.basename(tracePath) - basename, extension = os.path.splitext(name) + def plot_latency_histogram(dataArray, outputFileName, basename): + # plot into PDF file + import matplotlib + matplotlib.use('TkAgg') + import matplotlib.pyplot as plt + from matplotlib.backends.backend_pdf import PdfPages - outputFileName = 'hist_' + basename + '.pdf' - outputFile = "\n" + "Output file is {0}".format(outputFileName) + numberOfBins='auto' + plt.hist(dataArray, bins=numberOfBins, histtype='barstacked', facecolor='green') + plt.grid(True) + plt.xlabel("Access Time [ns]") + plt.ylabel("Number of Accesses (Frequency)") + plt.title("Latency Histogram " + str(basename)) + pdf = PdfPages(outputFileName) + pdf.savefig() + pdf.close() + plt.close() - numberOfBins=50 + def create_latency_hist(connection, tracePath, target_measurement, query): + # form output file name + ofbname = 'access_latency_hist' + outputFileName, basename = createOutputFilename(tracePath, ofbname, target_measurement, 'pdf') + # return log string + outputFile = "{0}\n\t".format(outputFileName) + # access database + resultArray = accessDatabase(connection, query) + # plot + plot_latency_histogram(resultArray, outputFileName, basename) + return outputFile + + # create overal latency histogram + query = """ SELECT ((p2.PhaseEnd - p1.PhaseBegin)/1000) + FROM Transactions t, Phases p1, Phases p2 + WHERE t.id = p1.Transact + AND t.id = p2.Transact + AND p1.PhaseName = "REQ" + AND p2.PhaseName = "RESP" """ + outputFile = create_latency_hist(connection, tracePath, '', query) + + # create per-thread latency histogram + threads = getThreads(connection) + if (len(threads) > 1): + queryThread = """ SELECT ((p2.PhaseEnd - p1.PhaseBegin)/1000) + FROM Transactions t, Phases p1, Phases p2 + WHERE t.id = p1.Transact + AND t.id = p2.Transact + AND p1.PhaseName = "REQ" + AND p2.PhaseName = "RESP" + AND t.TThread = {0} """ + for thread in threads: + thrname = 'thread_' + str(thread) + '_' + outputFile += create_latency_hist(connection, tracePath, thrname, queryThread.format(thread)) - import matplotlib.pyplot as plt - from matplotlib.backends.backend_pdf import PdfPages - plt.hist(result, bins=numberOfBins, histtype='barstacked', facecolor='green') - plt.grid(True) - plt.xlabel("Access Time [ns]") - plt.ylabel("Number (Frequency)") - pdf = PdfPages(outputFileName) - pdf.savefig() - pdf.close() - plt.close() return outputFile def generatePlots(pathToTrace): connection = sqlite3.connect(pathToTrace) - #print("================================") - #print("Generating plots for {0}".format(pathToTrace)) + print("================================") + print("Generating plots for {0}".format(pathToTrace)) - outputFiles = "" + outputFiles = "Output files are:\n\t" cursor = connection.cursor() cursor.execute(" SELECT WindowSize FROM GeneralInfo") windowSize = float(cursor.fetchone()[0]) @@ -207,7 +306,7 @@ def generatePlots(pathToTrace): connection.close() - #print(outputFiles) + print(outputFiles) return outputFiles diff --git a/DRAMSys/simulator/src/common/DebugManager.cpp b/DRAMSys/simulator/src/common/DebugManager.cpp index ba225e00..f84146c6 100644 --- a/DRAMSys/simulator/src/common/DebugManager.cpp +++ b/DRAMSys/simulator/src/common/DebugManager.cpp @@ -69,5 +69,9 @@ DebugManager::DebugManager() : DebugManager::~DebugManager() { - debugFile.close(); + if (writeToFile) + { + debugFile.flush(); + debugFile.close(); + } } diff --git a/DRAMSys/simulator/src/common/TlmRecorder.cpp b/DRAMSys/simulator/src/common/TlmRecorder.cpp index 8fbe5a69..d7a68f5f 100644 --- a/DRAMSys/simulator/src/common/TlmRecorder.cpp +++ b/DRAMSys/simulator/src/common/TlmRecorder.cpp @@ -71,6 +71,16 @@ TlmRecorder::~TlmRecorder() { if (db) closeConnection(); + sqlite3_finalize(insertTransactionStatement); + sqlite3_finalize(insertRangeStatement); + sqlite3_finalize(updateRangeStatement); + sqlite3_finalize(insertPhaseStatement); + sqlite3_finalize(updatePhaseStatement); + sqlite3_finalize(insertGeneralInfoStatement); + sqlite3_finalize(insertDebugMessageStatement); + sqlite3_finalize(updateDataStrobeStatement); + sqlite3_finalize(insertPowerStatement); + } void TlmRecorder::recordPower(double timeInSeconds, double averagePower)