stdlib: Add SparseHist to PyStats

This is inclusive of tests to ensure they have implemented correctly.

Change-Id: I5c84d5ffdb7b914936cfd86ca012a7b141eeaf42

src/python/m5/ext/pystats/statistic.py

@@ -253,3 +253,31 @@ class Distribution(Vector):
         # These check some basic conditions of a distribution.
         assert self.bin_size >= 0
         assert self.num_bins >= 1
+
+
+class SparseHist(Vector):
+    """A Sparse Histogram of values. A sparse histogram simply counts the "
+    frequency of each value in a sample. Ergo, it is, ineffect an disctionary
+    of values mapped to their count"""
+
+    def __init__(
+        self,
+        value: Dict[float, Scalar],
+        description: Optional[str] = None,
+    ):
+        super().__init__(
+            value=value,
+            type="SparseHist",
+            description=description,
+        )
+
+    def size(self) -> int:
+        """The number of unique sampled values."""
+        return len(self.value)
+
+    def count(self) -> int:
+        """
+        Returns the total number of samples.
+        """
+        assert self.value != None
+        return sum(self.value.values())

src/python/m5/stats/gem5stats.py

@@ -140,6 +140,8 @@ def __get_statistic(statistic: _m5.stats.Info) -> Optional[Statistic]:
         return __get_vector(statistic)
     elif isinstance(statistic, _m5.stats.Vector2dInfo):
         return __get_vector2d(statistic)
+    elif isinstance(statistic, _m5.stats.SparseHistInfo):
+        return __get_sparse_hist(statistic)
 
     return None
 
@@ -268,6 +270,24 @@ def __get_vector2d(statistic: _m5.stats.Vector2dInfo) -> Vector2d:
     return Vector2d(value=vector_rep, type="Vector2d", description=description)
 
 
+def __get_sparse_hist(statistic: _m5.stats.SparseHistInfo) -> SparseHist:
+    description = statistic.desc
+    value = statistic.values
+
+    parsed_values = {}
+    for val in value:
+        parsed_values[val] = Scalar(
+            value=value[val],
+            unit=statistic.unit,
+            datatype=StorageType["f64"],
+        )
+
+    return SparseHist(
+        value=parsed_values,
+        description=description,
+    )
+
+
 def _prepare_stats(group: _m5.stats.Group):
     """
     Prepares the statistics for dumping.

src/python/pybind11/stats.cc

@@ -78,6 +78,7 @@ cast_stat_info(const statistics::Info *info)
     TRY_CAST(statistics::VectorInfo);
     TRY_CAST(statistics::Vector2dInfo);
     TRY_CAST(statistics::DistInfo);
+    TRY_CAST(statistics::SparseHistInfo);
 
     return py::cast(info);
 
@@ -195,6 +196,15 @@ pybind_init_stats(py::module_ &m_native)
         .def_readonly("value", &statistics::Vector2dInfo::cvec)
         ;
 
+    py::class_<statistics::SparseHistInfo, statistics::Info,
+               std::unique_ptr<statistics::SparseHistInfo, py::nodelete>>(
+                    m, "SparseHistInfo")
+        .def_property_readonly("values", //A Dict[float, int] of sample & count
+            [](const statistics::SparseHistInfo &info) {
+                return info.data.cmap;
+            })
+        ;
+
     py::class_<statistics::FormulaInfo, statistics::VectorInfo,
                std::unique_ptr<statistics::FormulaInfo, py::nodelete>>(
                       m, "FormulaInfo")

src/test_objects/SConscript

@@ -32,5 +32,6 @@ if env['CONF']['USE_TEST_OBJECTS']:
         'ScalarStatTester',
         'VectorStatTester',
         'Vector2dStatTester',
+        'SparseHistStatTester',
     ])
     Source('stat_tester.cc')

src/test_objects/StatTester.py

@@ -87,3 +87,14 @@ class Vector2dStatTester(StatTester):
         [],
         "The vector stat's y subdescriptions. If empty, the subdescriptions ",
     )
+
+
+class SparseHistStatTester(StatTester):
+    type = "SparseHistStatTester"
+    cxx_header = "test_objects/stat_tester.hh"
+    cxx_class = "gem5::SparseHistStatTester"
+
+    samples = VectorParam.Float(
+        "The sparse histogram's sampled values, to be inserted into the "
+        "histogram."
+    )

src/test_objects/stat_tester.cc

@@ -28,6 +28,8 @@
 
 #include "test_objects/stat_tester.hh"
 
+#include <set>
+
 #include "base/stats/group.hh"
 
 namespace gem5
@@ -133,4 +135,28 @@ Vector2dStatTester::Vector2dStatTesterStats::Vector2dStatTesterStats(
 
 }
 
+void
+SparseHistStatTester::setStats()
+{
+    for (auto sample : params.samples) {
+        stats.sparse_histogram.sample(sample);
+    }
+}
+
+SparseHistStatTester::SparseHistStatTesterStats::SparseHistStatTesterStats(
+    statistics::Group *parent,
+    const SparseHistStatTesterParams &params
+) : statistics::Group(parent),
+    sparse_histogram(
+        this,
+        params.name.c_str(),
+        statistics::units::Count::get(),
+        params.description.c_str()
+    )
+{
+    sparse_histogram.init(
+        (std::set(params.samples.begin(), params.samples.end())).size()
+    );
+}
+
 } // namespace gem5

src/test_objects/stat_tester.hh

@@ -31,6 +31,7 @@
 
 #include "base/statistics.hh"
 #include "params/ScalarStatTester.hh"
+#include "params/SparseHistStatTester.hh"
 #include "params/StatTester.hh"
 #include "params/Vector2dStatTester.hh"
 #include "params/VectorStatTester.hh"
@@ -158,6 +159,27 @@ class Vector2dStatTester : public StatTester
         } stats;
 };
 
+class SparseHistStatTester : public StatTester
+{
+    private:
+        SparseHistStatTesterParams params;
+
+    public:
+        SparseHistStatTester(const SparseHistStatTesterParams &p) :
+            StatTester(p), params(p), stats(this, p) {}
+
+    protected:
+        void setStats() override;
+        struct SparseHistStatTesterStats : public statistics::Group
+        {
+            SparseHistStatTesterStats(
+                statistics::Group *parent,
+                const SparseHistStatTesterParams &params
+            );
+            statistics::SparseHistogram sparse_histogram;
+        } stats;
+};
+
 
 } // namespace gem5
 

tests/gem5/stats/configs/pystat_sparse_dist_check.py

@@ -0,0 +1,110 @@
+# Copyright (c) 2024 The Regents of the University of California
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import sys
+
+import m5
+from m5.objects import (
+    Root,
+    SparseHistStatTester,
+)
+from m5.stats.gem5stats import get_simstat
+
+parser = argparse.ArgumentParser(
+    description="Tests the output of a SparseHist Pystat."
+)
+parser.add_argument(
+    "samples",
+    help="delimited list representing the samples for the distributed "
+    "histogram.",
+    type=lambda s: [float(item) for item in s.split(",")],
+)
+
+parser.add_argument(
+    "--name",
+    type=str,
+    default="sparse_hist",
+    required=False,
+    help="The name of the Sparse Histogram statistic.",
+)
+
+parser.add_argument(
+    "--description",
+    type=str,
+    default=None,
+    required=False,
+    help="The description of the Sparse Histogram statistic.",
+)
+
+args = parser.parse_args()
+
+stat_tester = SparseHistStatTester(
+    name=args.name, description=args.description, samples=args.samples
+)
+
+root = Root(full_system=False, system=stat_tester)
+m5.instantiate()
+m5.simulate()
+
+simstats = get_simstat(root)
+output = simstats.to_json()["system"]
+
+value_dict = {}
+for sample in args.samples:
+    value_dict[sample] = (
+        1 if sample not in value_dict else value_dict[sample] + 1
+    )
+
+scaler_dict = {}
+for key in value_dict:
+    scaler_dict[key] = {
+        "unit": "Count",
+        "type": "Scalar",
+        "description": None,
+        "value": value_dict[key],
+        "datatype": "f64",
+    }
+
+expected_output = {
+    "type": "Group",
+    "time_conversion": None,
+    args.name: {
+        "value": scaler_dict,
+        "type": "SparseHist",
+        "description": str(args.description),
+    },
+}
+
+if output != expected_output:
+    print("Output statistics do not match expected:", file=sys.stderr)
+    print("", file=sys.stderr)
+    print("Expected:", file=sys.stderr)
+    print(expected_output, file=sys.stderr)
+    print("", file=sys.stderr)
+    print("Actual:", file=sys.stderr)
+    print(output, file=sys.stderr)
+    sys.exit(1)

tests/gem5/stats/test_simstats_output.py

@@ -212,3 +212,26 @@ gem5_verify_config(
     valid_isas=(constants.all_compiled_tag,),
     length=constants.quick_tag,
 )
+
+gem5_verify_config(
+    name="pystat-sparsehist-test",
+    fixtures=(),
+    verifiers=[],
+    config=joinpath(
+        config.base_dir,
+        "tests",
+        "gem5",
+        "stats",
+        "configs",
+        "pystat_sparse_dist_check.py",
+    ),
+    config_args=[
+        "1.0,1,1.00,23,23,0.2,0.2,0.2,0.2,-1,-1.0,264",
+        "--name",
+        "sparsehist_stat",
+        "--description",
+        "A sparse histogram statistic.",
+    ],
+    valid_isas=(constants.all_compiled_tag,),
+    length=constants.quick_tag,
+)