From d2761ce0603cda6d83be0ec4014a288ce076dff2 Mon Sep 17 00:00:00 2001 From: Derek Christ Date: Thu, 12 Oct 2023 11:57:31 +0200 Subject: [PATCH] Numerous fixes for Python scripts --- .gitignore | 1 + .../scripts/checkDependencies.py | 51 ------------ .../apps/traceAnalyzer/scripts/memUtil.py | 6 +- .../apps/traceAnalyzer/scripts/metrics.py | 80 +++++++++---------- .../apps/traceAnalyzer/scripts/plots.py | 3 +- .../apps/traceAnalyzer/scripts/vcdExport.py | 16 ++-- .../requirements.txt => requirements.txt | 0 7 files changed, 51 insertions(+), 106 deletions(-) delete mode 100644 extensions/apps/traceAnalyzer/scripts/checkDependencies.py rename extensions/apps/traceAnalyzer/scripts/requirements.txt => requirements.txt (100%) diff --git a/.gitignore b/.gitignore index 1912d553..e35a5f3d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ DRAMSys/docs/doxygen cmake-build* .idea .cache +.env diff --git a/extensions/apps/traceAnalyzer/scripts/checkDependencies.py b/extensions/apps/traceAnalyzer/scripts/checkDependencies.py deleted file mode 100644 index 883562c1..00000000 --- a/extensions/apps/traceAnalyzer/scripts/checkDependencies.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2021, RPTU Kaiserslautern-Landau -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER -# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Authors: -# Derek Christ - -import importlib.util - -def checkTqdm(): - if (spec := importlib.util.find_spec("tqdm")) is not None: - return True - else: - return False - -def checkVcd(): - if (spec := importlib.util.find_spec("vcd")) is not None: - return True - else: - return False - -def checkVcdExport(): - return checkVcd() and checkTqdm() diff --git a/extensions/apps/traceAnalyzer/scripts/memUtil.py b/extensions/apps/traceAnalyzer/scripts/memUtil.py index e6444a11..c1d8bd5e 100644 --- a/extensions/apps/traceAnalyzer/scripts/memUtil.py +++ b/extensions/apps/traceAnalyzer/scripts/memUtil.py @@ -57,7 +57,7 @@ def getNumberOfTransactions(dbconnection): def getTraceEndTime(dbconnection): cursor = dbconnection.cursor() - cursor.execute("SELECT TraceEnd FROM GeneralInfo") + cursor.execute("SELECT MAX(PhaseEnd) FROM Phases") result = cursor.fetchone() return result[0] @@ -103,7 +103,7 @@ def getPseudoChannelMode(connection): def maximum_data_rate(connection): - memspec = MemSpec(connection) + memspec = MemSpec(connection) try: width = memspec.getIntValue("memarchitecturespec", "nbrOfDevices") * memspec.getIntValue("memarchitecturespec", "width") except: @@ -162,6 +162,6 @@ def get_total_time_in_phase(connection, phase): def getCommandLengthForPhase(connection, phase): cursor = connection.cursor() - cursor.execute("SELECT " + phase + " FROM CommandLengths") + cursor.execute(f"SELECT Length FROM CommandLengths WHERE Command=\"{phase}\"") length = cursor.fetchone() return length[0] diff --git a/extensions/apps/traceAnalyzer/scripts/metrics.py b/extensions/apps/traceAnalyzer/scripts/metrics.py index 23581e9e..0da9f045 100644 --- a/extensions/apps/traceAnalyzer/scripts/metrics.py +++ b/extensions/apps/traceAnalyzer/scripts/metrics.py @@ -42,10 +42,10 @@ def command_bus_utilisation_in_percent(connection): if getRowColumnCommandBus(connection): cursor.execute(""" SELECT SUM(CommandLengths.Length) - FROM Phases - INNER JOIN CommandLengths + FROM Phases + INNER JOIN CommandLengths ON Phases.PhaseName = CommandLengths.Command - WHERE PhaseName <> 'RD' AND PhaseName <> 'RDA' AND PhaseName <> 'WR' AND PhaseName <> 'WRA' + WHERE PhaseName <> 'RD' AND PhaseName <> 'RDA' AND PhaseName <> 'WR' AND PhaseName <> 'WRA' """) rowBusUtil = cursor.fetchone()[0] if rowBusUtil is None: @@ -53,10 +53,10 @@ def command_bus_utilisation_in_percent(connection): cursor.execute(""" SELECT SUM(CommandLengths.Length) - FROM Phases - INNER JOIN CommandLengths + FROM Phases + INNER JOIN CommandLengths ON Phases.PhaseName = CommandLengths.Command - WHERE PhaseName = 'RD' OR PhaseName = 'RDA' OR PhaseName = 'WR' OR PhaseName = 'WRA' + WHERE PhaseName = 'RD' OR PhaseName = 'RDA' OR PhaseName = 'WR' OR PhaseName = 'WRA' """) columnBusUtil = cursor.fetchone()[0] if columnBusUtil is None: @@ -70,8 +70,8 @@ def command_bus_utilisation_in_percent(connection): else: cursor.execute(""" SELECT SUM(CommandLengths.Length) - FROM Phases - INNER JOIN CommandLengths + FROM Phases + INNER JOIN CommandLengths ON Phases.PhaseName = CommandLengths.Command """) util = cursor.fetchone()[0] @@ -88,8 +88,8 @@ def command_bus_utilisation_in_percent(connection): def average_response_latency_in_ns(connection): cursor = connection.cursor() cursor.execute(""" - SELECT AVG(Resp.PhaseBegin - Req.PhaseBegin) / 1000 - FROM Phases Req, Phases Resp + SELECT AVG(Resp.PhaseBegin - Req.PhaseBegin) / 1000 + FROM Phases Req, Phases Resp WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName='RESP' AND Req.Transact = Resp.Transact """) result = cursor.fetchone() @@ -104,7 +104,7 @@ def average_rd_response_latency_in_ns(connection): FROM Phases Req, Phases Resp INNER JOIN Transactions ON Req.Transact = Transactions.ID - WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' + WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' AND Req.Transact = Resp.Transact AND Transactions.Command = 'R' """) result = cursor.fetchone() @@ -119,7 +119,7 @@ def average_wr_response_latency_in_ns(connection): FROM Phases Req, Phases Resp INNER JOIN Transactions ON Req.Transact = Transactions.ID - WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' + WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' AND Req.Transact = Resp.Transact AND Transactions.Command = 'W' """) result = cursor.fetchone() @@ -130,7 +130,7 @@ def average_wr_response_latency_in_ns(connection): def max_response_latency_in_ns(connection): cursor = connection.cursor() cursor.execute(""" - SELECT MAX(Resp.PhaseBegin - Req.PhaseBegin) / 1000 + SELECT MAX(Resp.PhaseBegin - Req.PhaseBegin) / 1000 FROM Phases Req, Phases Resp WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' AND Req.Transact = Resp.Transact """) @@ -146,7 +146,7 @@ def max_rd_response_latency_in_ns(connection): FROM Phases Req, Phases Resp INNER JOIN Transactions ON Req.Transact = Transactions.ID - WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' + WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' AND Req.Transact = Resp.Transact AND Transactions.Command = 'R' """) result = cursor.fetchone() @@ -161,7 +161,7 @@ def max_wr_response_latency_in_ns(connection): FROM Phases Req, Phases Resp INNER JOIN Transactions ON Req.Transact = Transactions.ID - WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' + WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' AND Req.Transact = Resp.Transact AND Transactions.Command = 'W' """) result = cursor.fetchone() @@ -171,10 +171,10 @@ def max_wr_response_latency_in_ns(connection): @metric def trans_with_max_response_latency(connection): cursor = connection.cursor() - cursor.execute(""" - SELECT Req.Transact, MAX(Resp.PhaseBegin - Req.PhaseBegin) / 1000 - FROM Phases Req, Phases Resp - WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' AND Req.Transact = Resp.Transact + cursor.execute(""" + SELECT Req.Transact, MAX(Resp.PhaseBegin - Req.PhaseBegin) / 1000 + FROM Phases Req, Phases Resp + WHERE Req.PhaseName = 'REQ' AND Resp.PhaseName = 'RESP' AND Req.Transact = Resp.Transact """) result = cursor.fetchone() return result[0] @@ -217,7 +217,7 @@ def memory_idle_in_clks(connection): SELECT ROW_NUMBER () OVER (ORDER BY p.PhaseBegin) RowNum, p.PhaseBegin AS BeginRequest - FROM + FROM Phases p WHERE p.PhaseName = 'REQ'; @@ -234,7 +234,7 @@ def memory_idle_in_clks(connection): SELECT ROW_NUMBER () OVER (ORDER BY q.PhaseBegin) RowNum, q.PhaseEnd AS EndResponse - FROM + FROM Phases q WHERE q.PhaseName = 'RESP'; @@ -243,9 +243,9 @@ def memory_idle_in_clks(connection): # Sum up the idle times: query = """ - SELECT + SELECT SUM(c.BeginRequest - b.EndResponse) AS idle - FROM + FROM transactionsInRequestOrder AS a, transactionsInResponseOrder AS b, transactionsInRequestOrder AS c WHERE a.RowNum = b.RowNum AND @@ -291,7 +291,7 @@ def delayed_reasons(connection): Row, Bank, PhaseName AS Command - FROM + FROM Phases WHERE DataStrobeBegin <> 0 AND @@ -307,7 +307,7 @@ def delayed_reasons(connection): SELECT ROW_NUMBER () OVER (ORDER BY PhaseBegin) RowNum, PhaseBegin AS BeginRequest - FROM + FROM Phases WHERE PhaseName = 'REQ'; @@ -325,7 +325,7 @@ def delayed_reasons(connection): ROW_NUMBER () OVER (ORDER BY PhaseBegin) RowNum, Transact AS ID, PhaseEnd AS EndResponse - FROM + FROM Phases WHERE PhaseName = 'RESP'; @@ -339,9 +339,9 @@ def delayed_reasons(connection): query = """ CREATE TEMPORARY TABLE idleGaps AS - SELECT + SELECT b.ID AS ID - FROM + FROM transactionsInRequestOrder AS a, transactionsInResponseOrder AS b, transactionsInRequestOrder AS c WHERE a.RowNum = b.RowNum AND @@ -365,7 +365,7 @@ def delayed_reasons(connection): FROM dataStrobesInOrder a, dataStrobesInOrder b - WHERE + WHERE a.RowNum = b.RowNum - 1 AND b.Begin > a.End AND a.ID NOT IN (SELECT ID FROM idleGaps); @@ -388,7 +388,7 @@ def delayed_reasons(connection): query = """ SELECT COUNT(*) - FROM + FROM delayedDataBusGaps WHERE (gapBeginCommand = 'WR' OR gapBeginCommand = 'WRA') AND @@ -399,18 +399,18 @@ def delayed_reasons(connection): # Count RR Miss query = """ - SELECT COUNT(*) FROM + SELECT COUNT(*) FROM ( SELECT COUNT(*) - FROM + FROM delayedDataBusGaps d, Phases p WHERE (d.gapBeginCommand = 'RD' OR d.gapBeginCommand = 'RDA') AND (d.gapEndCommand = 'RD' OR d.gapEndCommand = 'RDA') AND p.Transact = d.gapEndID AND p.PhaseName = 'ACT' - GROUP BY + GROUP BY d.gapBeginID ) """ @@ -419,18 +419,18 @@ def delayed_reasons(connection): # Count WW Miss query = """ - SELECT COUNT(*) FROM + SELECT COUNT(*) FROM ( SELECT COUNT(*) - FROM + FROM delayedDataBusGaps d, Phases p WHERE (d.gapBeginCommand = 'WR' OR d.gapBeginCommand = 'WRA') AND (d.gapEndCommand = 'WR' OR d.gapEndCommand = 'WRA') AND p.Transact = d.gapEndID AND p.PhaseName = 'ACT' - GROUP BY + GROUP BY d.gapBeginID ) """ @@ -760,15 +760,13 @@ def bank_overlap_ratio(connection): banksPerGroup = int(numberOfBanks / numberOfBankGroups) cursor = connection.cursor() - cursor.execute("SELECT TraceEnd FROM GeneralInfo") - traceEndTMP = cursor.fetchone() - traceEnd = int(traceEndTMP[0] / clk) + traceEnd = getTraceEndTime(connection) trace = [] cursor.execute(""" - SELECT PhaseBegin, PhaseName, Rank, BankGroup, Bank + SELECT PhaseBegin, PhaseName, Rank, BankGroup, Bank FROM Phases - WHERE (PhaseName = 'ACT' OR PhaseName = 'PREPB' OR PhaseName = 'PREP2B' OR PhaseName = 'PRESB' + WHERE (PhaseName = 'ACT' OR PhaseName = 'PREPB' OR PhaseName = 'PREP2B' OR PhaseName = 'PRESB' OR PhaseName = 'PREAB' OR PhaseName = 'RDA' OR PhaseName = 'WRA') ORDER BY PhaseBegin """) diff --git a/extensions/apps/traceAnalyzer/scripts/plots.py b/extensions/apps/traceAnalyzer/scripts/plots.py index 82eb92cd..52e56460 100644 --- a/extensions/apps/traceAnalyzer/scripts/plots.py +++ b/extensions/apps/traceAnalyzer/scripts/plots.py @@ -621,8 +621,7 @@ def generatePlots(pathToTrace): if(windowSize == 0): outputFiles = "No output file created. Check WindowSize and EnableWindowing configs." else: - cursor.execute(" SELECT TraceEnd FROM GeneralInfo ") - traceEnd = float(cursor.fetchone()[0]) + traceEnd = getTraceEndTime(connection) steps = int(ceil(traceEnd/windowSize)) for p in plots: outputFiles += p(connection, pathToTrace, steps) diff --git a/extensions/apps/traceAnalyzer/scripts/vcdExport.py b/extensions/apps/traceAnalyzer/scripts/vcdExport.py index c0c88b07..2e976e30 100644 --- a/extensions/apps/traceAnalyzer/scripts/vcdExport.py +++ b/extensions/apps/traceAnalyzer/scripts/vcdExport.py @@ -79,12 +79,10 @@ class Event(): self.value = value class Transaction(): - def __init__(self, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command): + def __init__(self, rank, bankgroup, bank, command): self.rank = rank self.bankgroup = bankgroup self.bank = bank - self.dataStrobeBegin = dataStrobeBegin - self.dataStrobeEnd = dataStrobeEnd self.command = command class Granularity(enum.Enum): @@ -141,7 +139,7 @@ def getUnitOfTime(connection): def getLastTimestamp(connection): cursor = connection.cursor() - cursor.execute("SELECT DataStrobeEnd FROM Transactions ORDER BY DataStrobeEnd DESC LIMIT 1") + cursor.execute("SELECT DataStrobeEnd FROM Phases ORDER BY DataStrobeEnd DESC LIMIT 1") return cursor.fetchone()[0] @@ -189,7 +187,7 @@ def getDataBusEvents(connection, eventDict, windowRange): beginWindow, endWindow = windowRange cursor = connection.cursor() - cursor.execute("SELECT ID, DataStrobeBegin, DataStrobeEnd, Command FROM Transactions " + + cursor.execute("SELECT Transactions.ID, DataStrobeBegin, DataStrobeEnd, Command FROM Phases INNER JOIN Transactions ON Transactions.ID=Phases.Transact " + "WHERE DataStrobeBegin BETWEEN " + str(beginWindow) + " AND " + str(endWindow) + " AND DataStrobeEnd BETWEEN " + str(beginWindow) + " AND " + str(endWindow)) @@ -319,17 +317,17 @@ def getTransactions(connection, transactionDict, transactionRange): minTransaction, maxTransaction = transactionRange cursor = connection.cursor() - cursor.execute("SELECT ID, TRank, TBankgroup, TBank, DataStrobeBegin, DataStrobeEnd, Command FROM Transactions" + - " WHERE ID BETWEEN " + str(minTransaction) + " AND " + str(maxTransaction)) + cursor.execute("SELECT Transactions.ID, Rank, Bankgroup, Bank, Command FROM Transactions INNER JOIN Phases ON Transactions.ID=Phases.Transact" + + " WHERE Transactions.ID BETWEEN " + str(minTransaction) + " AND " + str(maxTransaction)) - for transactionId, rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command in cursor.fetchall(): + for transactionId, rank, bankgroup, bank, command in cursor.fetchall(): (ranks, bankgroups, banks) = getRanksBankgroupsBanks(connection) rank = rank % ranks bankgroup = bankgroup % bankgroups bank = bank % banks - currentTransaction = Transaction(rank, bankgroup, bank, dataStrobeBegin, dataStrobeEnd, command) + currentTransaction = Transaction(rank, bankgroup, bank, command) transactionDict[transactionId] = currentTransaction diff --git a/extensions/apps/traceAnalyzer/scripts/requirements.txt b/requirements.txt similarity index 100% rename from extensions/apps/traceAnalyzer/scripts/requirements.txt rename to requirements.txt