tests: Adding tests for gem5stats.

This change adds the first of many tests for the gem5stats.
The current test only considers a setup consisiting of
traffic generator, cache and memory.

Change-Id: I3ebe16131aed5bf6dad68de673f9491f1bea1c78
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/55743
Maintainer: Bobby Bruce <bbruce@ucdavis.edu>
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Bobby Bruce <bbruce@ucdavis.edu>

tests/gem5/traffic_gen/simple_traffic_run.py

@@ -32,10 +32,13 @@ set of statistics.
 """
 
 import m5
+
 import argparse
 import importlib
+from pathlib import Path
 
 from m5.objects import Root, MemorySize
+from m5.stats.gem5stats import get_simstat
 from gem5.components.boards.test_board import TestBoard
 
 
@@ -174,7 +177,6 @@ parser.add_argument(
     help="The arguments needed to instantiate the memory class.",
 )
 
-
 args = parser.parse_args()
 
 cache_hierarchy = cache_factory(args.cache_class)
@@ -207,3 +209,8 @@ exit_event = m5.simulate()
 print(
     "Exiting @ tick {} because {}.".format(m5.curTick(), exit_event.getCause())
 )
+
+simstats = get_simstat(root, prepare_stats=True)
+json_output = Path(m5.options.outdir) / "output.json"
+with open(json_output, "w") as stats_file:
+    simstats.dump(stats_file, indent=2)

tests/gem5/traffic_gen/test_memory_traffic_gen.py

@@ -61,7 +61,8 @@ def test_memory(
             "traffic_gen",
             "simple_traffic_run.py",
         ),
-        config_args=[generator, generator_cores, cache, module, memory]
+        config_args=[generator, generator_cores, cache, module]
+        + [memory]
         + list(args),
         valid_isas=(constants.all_compiled_tag,),
         valid_hosts=constants.supported_hosts,

tests/gem5/verifier.py

@@ -42,6 +42,7 @@ Built in test cases that verify particular details about a gem5 run.
 """
 import re
 import os
+import json
 
 from testlib import test_util
 from testlib.configuration import constants
@@ -149,7 +150,7 @@ class DerivedGoldStandard(MatchGoldStandard):
             standard_filename,
             test_filename=self._file,
             ignore_regex=ignore_regex,
-            **kwargs
+            **kwargs,
         )
 
 
@@ -272,6 +273,63 @@ class NoMatchRegex(MatchRegex):
                 test_util.fail("Could not match regex.")
 
 
+class MatchJSONStats(Verifier):
+    """
+    Verifer to check the correctness of stats reported by gem5. It uses
+    gem5stats to store the stastistics as json files and does the comparison.
+    """
+
+    def __init__(
+        self,
+        truth_name: str,
+        test_name: str,
+        test_name_in_outdir: bool = False,
+    ):
+        """
+        :param truth_dir: The path to the directory including the trusted_stats
+        for this test.
+        :param test_name_in_m5out: True if the 'test_name' dir is to found in
+        the `m5.options.outdir`.
+        """
+        super(MatchJSONStats, self).__init__()
+        self.truth_name = truth_name
+        self.test_name = test_name
+        self.test_name_in_outdir = test_name_in_outdir
+
+    def _compare_stats(self, trusted_file, test_file):
+        trusted_stats = json.load(trusted_file)
+        test_stats = json.load(test_file)
+        is_subset = trusted_stats.items() <= test_stats.items()
+        if is_subset:
+            err = (
+                "Following differences found between "
+                + f"{self.truth_name} and {self.test_name}.\n"
+            )
+            diffs = set(trusted_stats.items()) - set(test_stats.items())
+            for diff in diffs:
+                trusted_value = trusted_stats[diff[0]]
+                test_value = None
+                if diff[0] in test_stats.keys():
+                    test_value = test_stats[diff[0]]
+                err += f"{diff[0]}:\n"
+                err += (
+                    f"trusted_value: {trusted_value}, "
+                    + f"test_value: {test_value}"
+                )
+            test_util.fail(err)
+
+    def test(self, params):
+        trusted_file = open(self.truth_name, "r")
+        if self.test_name_in_outdir:
+            fixtures = params.fixtures
+            tempdir = fixtures[constants.tempdir_fixture_name].path
+            test_file = open(joinpath(tempdir, self.test_name), "r")
+        else:
+            test_file = open(self.test_name, "r")
+
+        return self._compare_stats(trusted_file, test_file)
+
+
 _re_type = type(re.compile(""))