derek/DRAMSys
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change script to automatically generate per-thread plots

Browse Source
This commit is contained in:
Thanh C. Tran
2017-07-14 14:43:11 +02:00
parent e27b147634
commit 024b288f3f
2 changed files with 185 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
DRAMSys/analyzer/scripts/memUtil.py
View File

@@ -54,6 +54,11 @@ def getNumberOfBanks(dbconnection):
result = cursor.fetchone()
return result[0]
def getWindowSize(connection):
cursor = connection.cursor()
cursor.execute(" SELECT WindowSize FROM GeneralInfo ")
windowSize = float(cursor.fetchone()[0])
return windowSize
def maximum_data_rate(connection):
memspec = MemSpec(connection)

272
DRAMSys/analyzer/scripts/plots.py
View File

@@ -7,34 +7,35 @@ import os
plots = []
def plot(function):
plots.append(function)
return function
@plot
def memory_utilisation_window(connection, tracePath, steps):
# This function determines the average memory bandwidth over time in
# percentage and in Gbit/s. The average bandwidth over time is done
# dividing the time into windows of the same length and getting the average
# bandwidth in each window. Through data from the database, DataStrobeEnd
# and DataStrobeBegin, it is possible to access when a data transfer begins
# or ends. Hence, it is achievable to check when a data transfer happens
# and if it occupies or is inside a time window. Then, it is attainable to
# determine the average bandwidth in percentage. Besides, extracting the
# data from the memory specs, it is feasible to calculate the maximum data
# rate of the memory and then determine the bandwidth in Gbit/s. The
# bandwidth data are then plotted in two graphics.
def getThreads(connection):
cursor = connection.cursor()
cursor.execute("SELECT DISTINCT(TThread) FROM transactions WHERE TThread != 0 ORDER BY TThread")
result = []
for currentRow in cursor:
result.append(currentRow[0])
return result
def createOutputFilename(tracePath, plot_type, target_measurement, file_type):
name = ntpath.basename(tracePath)
basename, extension = os.path.splitext(name)
outputFileName = plot_type + '_' + target_measurement + basename + '.' + file_type
return outputFileName, target_measurement + basename
def accessDatabase(connection, query):
cursor = connection.cursor()
# cursor.execute(" ")
cursor.execute(query)
resultArray = []
while True:
result = cursor.fetchone()
if (result is not None):
resultArray.append(result[0])
else:
break
return resultArray
def calculate_bandwidth_util(connection, windowSize, steps, queryFull, queryEnd, queryBegin, queryPart):
cursor = connection.cursor()
cursor.execute(" SELECT WindowSize FROM GeneralInfo ")
windowSize = float(cursor.fetchone()[0])
# All possible cases of data transfers inside a time window
queryFull = """ SELECT sum(DataStrobeEnd - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeEnd <= ?""" # The data transfer begins and ends inside the time window
queryEnd = """ SELECT sum(DataStrobeEnd - ?) FROM transactions Where DataStrobeBegin < ? and DataStrobeEnd > ? and DataStrobeEnd <=?""" # Only the end of the data transfer is inside the time window
queryBegin = """ SELECT sum(? - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeBegin < ? and DataStrobeEnd > ?""" # Only the beginning of the data transfer is inside the time window
queryPart = """ SELECT DataStrobeBegin FROM transactions Where DataStrobeBegin <= ? and DataStrobeEnd >= ?""" # The data transfer occupies all the time window
maxDataRate = maximum_data_rate(connection)
maximumPercentage = 0
bandwidthPercentage = [0] * (steps+1)
@@ -66,12 +67,60 @@ def memory_utilisation_window(connection, tracePath, steps):
if(maximumPercentage < 100 and maximumPercentage < bandwidthPercentage[i+1]):
maximumPercentage = bandwidthPercentage[i+1]
name = ntpath.basename(tracePath)
basename, extension = os.path.splitext(name)
return bandwidthPercentage, bandwidth, maximumPercentage
outputFileNameGBPS = 'memory_utilization_gbps_' + basename + '.pdf'
outputFileNamePercent = 'memory_utilization_percent_' + basename + '.pdf'
outputFiles = "Output files are {0},{1}".format(outputFileNameGBPS,outputFileNamePercent)
def memory_utilisation_window_thread(connection, tracePath, steps, thread_ID):
# All possible cases of data transfers inside a time window
queryFull = """ SELECT sum(DataStrobeEnd - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeEnd <= ? and TThread = {0}""" # The data transfer begins and ends inside the time window
queryEnd = """ SELECT sum(DataStrobeEnd - ?) FROM transactions Where DataStrobeBegin < ? and DataStrobeEnd > ? and DataStrobeEnd <=? and TThread = {0}""" # Only the end of the data transfer is inside the time window
queryBegin = """ SELECT sum(? - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeBegin < ? and DataStrobeEnd > ? and TThread = {0}""" # Only the beginning of the data transfer is inside the time window
queryPart = """ SELECT DataStrobeBegin FROM transactions Where DataStrobeBegin <= ? and DataStrobeEnd >= ? and TThread = {0}""" # The data transfer occupies all the time window
queryFull = queryFull.format(thread_ID)
queryEnd = queryEnd.format(thread_ID)
queryBegin = queryBegin.format(thread_ID)
queryPart = queryPart.format(thread_ID)
windowSize = getWindowSize(connection)
bandwidthPercentage, bandwidth, maximumPercentage = calculate_bandwidth_util(connection, windowSize, steps, queryFull, queryEnd, queryBegin, queryPart)
outputFileNameBWMatlab, basename = createOutputFilename(tracePath, 'memory_utilization_percent_', 'thread_' + str(thread_ID) + '_', 'txt')
return bandwidthPercentage, bandwidth, outputFileNameBWMatlab
def plot(function):
plots.append(function)
return function
@plot
def memory_utilisation_window(connection, tracePath, steps):
# This function determines the average memory bandwidth over time in
# percentage and in Gbit/s. The average bandwidth over time is done
# dividing the time into windows of the same length and getting the average
# bandwidth in each window. Through data from the database, DataStrobeEnd
# and DataStrobeBegin, it is possible to access when a data transfer begins
# or ends. Hence, it is achievable to check when a data transfer happens
# and if it occupies or is inside a time window. Then, it is attainable to
# determine the average bandwidth in percentage. Besides, extracting the
# data from the memory specs, it is feasible to calculate the maximum data
# rate of the memory and then determine the bandwidth in Gbit/s. The
# bandwidth data are then plotted in two graphics.
windowSize = getWindowSize(connection)
maxDataRate = maximum_data_rate(connection)
# All possible cases of data transfers inside a time window
queryFull = """ SELECT sum(DataStrobeEnd - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeEnd <= ?""" # The data transfer begins and ends inside the time window
queryEnd = """ SELECT sum(DataStrobeEnd - ?) FROM transactions Where DataStrobeBegin < ? and DataStrobeEnd > ? and DataStrobeEnd <=?""" # Only the end of the data transfer is inside the time window
queryBegin = """ SELECT sum(? - DataStrobeBegin) FROM transactions Where DataStrobeBegin >= ? and DataStrobeBegin < ? and DataStrobeEnd > ?""" # Only the beginning of the data transfer is inside the time window
queryPart = """ SELECT DataStrobeBegin FROM transactions Where DataStrobeBegin <= ? and DataStrobeEnd >= ?""" # The data transfer occupies all the time window
bandwidthPercentage, bandwidth, maximumPercentage = calculate_bandwidth_util(connection, windowSize, steps, queryFull, queryEnd, queryBegin, queryPart)
outputFileNameGBPS, basename = createOutputFilename(tracePath, 'memory_utilization_gbps_', '', 'pdf')
outputFileNamePercent, basename = createOutputFilename(tracePath, 'memory_utilization_percent_', '', 'pdf')
outputFileNameBWMatlab, basename = createOutputFilename(tracePath, 'memory_utilization_percent_', '', 'txt')
outputFiles = "{0}\n\t{1}\n\t{2}\n\t".format(outputFileNameGBPS,outputFileNamePercent,outputFileNameBWMatlab)
import matplotlib.pyplot as plt
import numpy as np
@@ -81,41 +130,62 @@ def memory_utilisation_window(connection, tracePath, steps):
time = np.arange(0, (steps+1)*windowSize/1000, windowSize/1000)
maxBandwidth = [maxDataRate/1024] * (steps+1)
plt.figure()
# Write data to a file for matlab:
outputFileNameBWMatlab = 'memory_utilization_percent_' + basename + '.txt'
f = open(outputFileNameBWMatlab, 'w')
for i in range(steps):
line = "{} {}\n".format(time[i], bandwidthPercentage[i])
f.write(line)
#Plot Bandwidth in Percent
plt.plot(time, bandwidthPercentage)
plt.xlabel('Time [ns]')
plt.ylabel('Bandwidth [%]')
plt.ylim(-1, maximumPercentage + (10 - maximumPercentage%10))
plt.grid(True)
# Plot Bandwidth in Percent
BWPercentageFigure = plt.figure()
BWPercentageFigurePlot = BWPercentageFigure.add_subplot(111)
BWPercentageFigurePlot.set_xlabel('Time [ns]')
BWPercentageFigurePlot.set_ylabel('Bandwidth [%]')
BWPercentageFigurePlot.set_ylim(-1, maximumPercentage + (10 - maximumPercentage%10))
BWPercentageFigurePlot.set_title('Memory Utilization in % ' + str(basename))
BWPercentageFigurePlot.grid(True)
BWPercentageFigurePlot.plot(time, bandwidthPercentage, label='Total')
BWPercentageFigurePlot.legend(loc="upper left")
# Plot absolute bandwidth
BWFigure = plt.figure()
BWFigurePlot = BWFigure.add_subplot(111)
BWFigurePlot.set_xlabel('Time [ns]')
BWFigurePlot.set_ylabel('Bandwidth [Gibit/s]')
BWFigurePlot.set_title('Memory Utilization in Gbps ' + str(basename))
BWFigurePlot.grid(True)
BWFigurePlot.plot(time, bandwidth, label='Total')
BWFigurePlot.legend(loc="upper left")
# plt.ylim((-0.01)*float(maxDataRate)/1024, ((maximumPercentage + (10 - maximumPercentage%10))/100)*float(maxDataRate)/1024)
threads = getThreads(connection)
if (len(threads) > 1):
for thread in threads:
threadStr = "Thread " + str(thread)
bandwidthPercentage, bandwidth, outputFileNameBWMatlab = memory_utilisation_window_thread(connection, tracePath, steps, thread)
BWPercentageFigurePlot.plot(time, bandwidthPercentage, label=threadStr)
BWPercentageFigurePlot.legend(loc="upper left")
BWFigurePlot.plot(time, bandwidth, label=threadStr)
BWFigurePlot.legend(loc="upper left")
f = open(outputFileNameBWMatlab, 'w')
for i in range(steps):
line = "{} {}\n".format(time[i], bandwidthPercentage[i])
f.write(line)
outputFiles += "{0}\n\t".format(outputFileNameBWMatlab)
# Save to PDF files
pdf = PdfPages(outputFileNamePercent)
pdf.savefig()
pdf.savefig(BWPercentageFigure)
pdf.close()
plt.close()
#Plot absolute bandwidth
plt.plot(time, bandwidth)
plt.plot(time, maxBandwidth)
plt.xlabel('Time [ns]')
plt.ylabel('Bandwidth [Gibit/s]')
#plt.ylim((-0.01)*float(maxDataRate)/1024, ((maximumPercentage + (10 - maximumPercentage%10))/100)*float(maxDataRate)/1024)
plt.grid(True)
BWPercentageFigure.clear()
pdf = PdfPages(outputFileNameGBPS)
pdf.savefig()
BWFigurePlot.plot(time, maxBandwidth)
pdf.savefig(BWFigure)
pdf.close()
BWFigurePlot.clear()
plt.close()
return outputFiles
@plot
@@ -134,15 +204,12 @@ def power_window(connection, tracePath, steps):
time[1] = float(result[0])*pow(10,9)
power[1] = float(result[1])
for i in range((steps-1)):
result = cursor.fetchone()
result = cursor.fetchone()
time[i+2] = float(result[0])*pow(10,9)
power[i+2] = float(result[1])
name = ntpath.basename(tracePath)
basename, extension = os.path.splitext(name)
outputFileName = 'power_' + basename + '.pdf'
outputFile = "\n" + "Output file is {0}".format(outputFileName)
outputFileName, basename = createOutputFilename(tracePath, 'power', '', 'pdf')
outputFile = "{0}\n\t".format(outputFileName)
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
@@ -151,6 +218,7 @@ def power_window(connection, tracePath, steps):
plt.xlabel('Time [ns]')
plt.ylabel('Power [mW]')
plt.grid(True)
plt.title('Power Consumption ' + str(basename))
pdf = PdfPages(outputFileName)
pdf.savefig()
pdf.close()
@@ -160,49 +228,69 @@ def power_window(connection, tracePath, steps):
@plot
def latency_histogram(connection, tracePath, steps):
# This function plots an histogram with access latencys
outputFile = ""
cursor = connection.cursor()
cursor.execute("SELECT ((p2.PhaseEnd - p1.PhaseBegin)/1000) FROM Transactions t, Phases p1, Phases p2 WHERE t.id = p1.Transact and t.id = p2.Transact and p1.PhaseName = \"REQ\" and p2.PhaseName = \"RESP\" ")
resultArray = []
while True:
result = cursor.fetchone()
if (result is not None):
resultArray.append(result[0])
else:
break
def plot_latency_histogram(dataArray, outputFileName, basename):
# plot into PDF file
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
numberOfBins='auto'
plt.hist(dataArray, bins=numberOfBins, histtype='barstacked', facecolor='green')
plt.grid(True)
plt.xlabel("Access Time [ns]")
plt.ylabel("Number (Frequency)")
plt.title("Latency Histogram " + str(basename))
pdf = PdfPages(outputFileName)
pdf.savefig()
pdf.close()
plt.close()
name = ntpath.basename(tracePath)
basename, extension = os.path.splitext(name)
def create_latency_hist(connection, tracePath, target_measurement, query):
# form output file name
# outputFileName = 'hist_' + basename + '.pdf'
outputFileName, basename = createOutputFilename(tracePath, 'hist', target_measurement, 'pdf')
# return log string
outputFile = "{0}\n\t".format(outputFileName)
# access database
resultArray = accessDatabase(connection, query)
# plot
plot_latency_histogram(resultArray, outputFileName, basename)
return outputFile
outputFileName = 'hist_' + basename + '.pdf'
outputFile = "\n" + "Output file is {0}".format(outputFileName)
# create overal latency histogram
query = """ SELECT ((p2.PhaseEnd - p1.PhaseBegin)/1000)
FROM Transactions t, Phases p1, Phases p2
WHERE t.id = p1.Transact
AND t.id = p2.Transact
AND p1.PhaseName = "REQ"
AND p2.PhaseName = "RESP" """
outputFile = create_latency_hist(connection, tracePath, '', query)
numberOfBins=50
# create per-thread latency histogram
threads = getThreads(connection)
if (len(threads) > 1):
queryThread = """ SELECT ((p2.PhaseEnd - p1.PhaseBegin)/1000)
FROM Transactions t, Phases p1, Phases p2
WHERE t.id = p1.Transact
AND t.id = p2.Transact
AND p1.PhaseName = "REQ"
AND p2.PhaseName = "RESP"
AND t.TThread = {0} """
for thread in threads:
outputFile += create_latency_hist(connection, tracePath, 'thread_' + str(thread) + '_', queryThread.format(thread))
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
plt.hist(resultArray, bins=numberOfBins, histtype='barstacked', facecolor='green')
plt.grid(True)
plt.xlabel("Access Time [ns]")
plt.ylabel("Number (Frequency)")
pdf = PdfPages(outputFileName)
pdf.savefig()
pdf.close()
plt.close()
return outputFile
def generatePlots(pathToTrace):
connection = sqlite3.connect(pathToTrace)
#print("================================")
#print("Generating plots for {0}".format(pathToTrace))
print("================================")
print("Generating plots for {0}".format(pathToTrace))
outputFiles = ""
outputFiles = "Output files are:\n\t"
cursor = connection.cursor()
cursor.execute(" SELECT WindowSize FROM GeneralInfo")
windowSize = float(cursor.fetchone()[0])
@@ -217,7 +305,7 @@ def generatePlots(pathToTrace):
connection.close()
#print(outputFiles)
print(outputFiles)
return outputFiles