mem-ruby: Use shared and per-protocol SLICC files

This changes extends SLICC to understand two different kinds of slicc
files: files that are protocol-specific and files that are shared or
included between different protocols.

Each declaration in SLICC can now be shared or not. If it is shared,
then we can take a different action in the code generation (e.g., wrap
in a namespace).

*Developer facing change*
Removes the RubySlicc_interfaces.slicc file from the SLICC includes of
every protocol.

Changes required: If you have a custom protocol, you will need to remove
the line `include "RubySlicc_interfaces.slicc" from your .slicc file.

Change-Id: Ia6c2dafe2b8fe86749a13d17daa885bddd166855
Signed-off-by: Jason Lowe-Power <jason@lowepower.com>

RELEASE-NOTES.md

@@ -7,6 +7,15 @@
   * It returns if the stride does not match, as opposed to issuing prefetching using the new stride --- the previous, incorrect behavior.
   * Returns if the new stride is 0, indicating multiple reads from the same cache line.
 
+### Multiple Ruby Protocols in a Single Build
+
+There are many developer facing / API changes to enable Ruby multiple protocols in a single build.
+The most notable changes are:
+
+* Removes the RubySlicc_interfaces.slicc file from the SLICC includes of
+every protocol.
+  * Changes required: If you have a custom protocol, you will need to remove the line `include "RubySlicc_interfaces.slicc"` from your .slicc file.
+
 * The [behavior of the statistics `simInsts` and `simOps` has been changed](https://github.com/gem5/gem5/pull/1615).
   * They now reset to zero when m5.stats.reset() is called.
   * Previously, they incorrectly did not reset and would increase monotonically throughout the simulation.

src/learning_gem5/part3/MSI.slicc

@@ -1,5 +1,4 @@
 protocol "MSI";
-include "RubySlicc_interfaces.slicc";
 include "MSI-msg.sm";
 include "MSI-cache.sm";
 include "MSI-dir.sm";

src/mem/ruby/protocol/GPU_VIPER.slicc

@@ -1,5 +1,4 @@
 protocol "GPU_VIPER";
-include "RubySlicc_interfaces.slicc";
 include "MOESI_AMD_Base-msg.sm";
 include "MOESI_AMD_Base-dir.sm";
 include "MOESI_AMD_Base-dma.sm";

src/mem/ruby/protocol/Garnet_standalone.slicc

@@ -1,5 +1,4 @@
 protocol "Garnet_standalone";
-include "RubySlicc_interfaces.slicc";
 include "Garnet_standalone-msg.sm";
 include "Garnet_standalone-cache.sm";
 include "Garnet_standalone-dir.sm";

src/mem/ruby/protocol/MESI_Three_Level.slicc

@@ -1,5 +1,4 @@
 protocol "MESI_Three_Level";
-include "RubySlicc_interfaces.slicc";
 include "MESI_Two_Level-msg.sm";
 include "MESI_Three_Level-msg.sm";
 include "MESI_Three_Level-L0cache.sm";

src/mem/ruby/protocol/MESI_Three_Level_HTM.slicc

@@ -1,5 +1,4 @@
 protocol "MESI_Three_Level_HTM";
-include "RubySlicc_interfaces.slicc";
 include "MESI_Two_Level-msg.sm";
 include "MESI_Three_Level-msg.sm";
 include "MESI_Three_Level_HTM-L0cache.sm";

src/mem/ruby/protocol/MESI_Two_Level.slicc

@@ -1,5 +1,4 @@
 protocol "MESI_Two_Level";
-include "RubySlicc_interfaces.slicc";
 include "MESI_Two_Level-msg.sm";
 include "MESI_Two_Level-L1cache.sm";
 include "MESI_Two_Level-L2cache.sm";

src/mem/ruby/protocol/MI_example.slicc

@@ -1,5 +1,4 @@
 protocol "MI_example";
-include "RubySlicc_interfaces.slicc";
 include "MI_example-msg.sm";
 include "MI_example-cache.sm";
 include "MI_example-dir.sm";

src/mem/ruby/protocol/MOESI_AMD_Base.slicc

@@ -1,5 +1,4 @@
 protocol "MOESI_AMD_Base";
-include "RubySlicc_interfaces.slicc";
 include "MOESI_AMD_Base-msg.sm";
 include "MOESI_AMD_Base-CorePair.sm";
 include "MOESI_AMD_Base-L3cache.sm";

src/mem/ruby/protocol/MOESI_CMP_directory.slicc

@@ -1,5 +1,4 @@
 protocol "MOESI_CMP_directory";
-include "RubySlicc_interfaces.slicc";
 include "MOESI_CMP_directory-msg.sm";
 include "MOESI_CMP_directory-L1cache.sm";
 include "MOESI_CMP_directory-L2cache.sm";

src/mem/ruby/protocol/MOESI_CMP_token.slicc

@@ -1,5 +1,4 @@
 protocol "MOESI_CMP_token";
-include "RubySlicc_interfaces.slicc";
 include "MOESI_CMP_token-msg.sm";
 include "MOESI_CMP_token-L1cache.sm";
 include "MOESI_CMP_token-L2cache.sm";

src/mem/ruby/protocol/MOESI_hammer.slicc

@@ -1,5 +1,4 @@
 protocol "MOESI_hammer";
-include "RubySlicc_interfaces.slicc";
 include "MOESI_hammer-msg.sm";
 include "MOESI_hammer-cache.sm";
 include "MOESI_hammer-dir.sm";

src/mem/ruby/protocol/RubySlicc_interfaces.slicc

@@ -1,3 +1,5 @@
+// All of the sm files that are shared between all protocols
+
 include "RubySlicc_Exports.sm";
 include "RubySlicc_Types.sm";
 include "RubySlicc_Util.sm";

src/mem/ruby/protocol/SConscript

@@ -66,7 +66,12 @@ def slicc_emitter(target, source, env):
     assert len(source) == 1
     filepath = source[0].srcnode().abspath
 
-    slicc = SLICC(filepath, protocol_base.abspath, verbose=False)
+    slicc = SLICC(
+        filepath,
+        [os.path.join(protocol_base.abspath, 'RubySlicc_interfaces.slicc')],
+        protocol_base.abspath,
+        verbose=False
+    )
     slicc.process()
     slicc.writeCodeFiles(output_dir.abspath, slicc_includes)
     if env['CONF']['SLICC_HTML']:
@@ -78,8 +83,12 @@ def slicc_emitter(target, source, env):
 def slicc_action(target, source, env):
     assert len(source) == 1
     filepath = source[0].srcnode().abspath
-
-    slicc = SLICC(filepath, protocol_base.abspath, verbose=True)
+    slicc = SLICC(
+        filepath,
+        [os.path.join(protocol_base.abspath, 'RubySlicc_interfaces.slicc')],
+        protocol_base.abspath,
+        verbose=True
+    )
     slicc.process()
     slicc.writeCodeFiles(output_dir.abspath, slicc_includes)
     if env['CONF']['SLICC_HTML']:

src/mem/ruby/protocol/chi/CHI.slicc

@@ -1,6 +1,5 @@
 protocol "CHI";
 
-include "RubySlicc_interfaces.slicc";
 include "CHI-msg.sm";
 include "CHI-cache.sm";
 include "CHI-mem.sm";

src/mem/slicc/ast/DeclAST.py

@@ -31,6 +31,12 @@ from slicc.ast.AST import AST
 class DeclAST(AST):
     def __init__(self, slicc, pairs=None):
         super().__init__(slicc, pairs)
+        # If true, then this declaration is shared between protocols
+        self.shared = False
+
+    def setShared(self, shared: bool):
+        """Used to mark this declaration as shared between protocols"""
+        self.shared = shared
 
     def files(self, parent=None):
         return set()

src/mem/slicc/ast/DeclListAST.py

@@ -49,3 +49,12 @@ class DeclListAST(AST):
         for decl in self.decls:
             decl.findMachines()
             decl.generate()
+
+    def setShared(self):
+        """Mark all types in this DeclList as shared"""
+        for decl in self.decls:
+            decl.setShared(True)
+
+    def __iadd__(self, other):
+        self.decls += other.decls
+        return self

src/mem/slicc/ast/EnumDeclAST.py

@@ -58,7 +58,12 @@ class EnumDeclAST(DeclAST):
 
         # Make the new type
         t = Type(
-            self.symtab, ident, self.location, self.pairs, self.state_machine
+            self.symtab,
+            ident,
+            self.location,
+            self.pairs,
+            self.shared,
+            self.state_machine,
         )
         self.symtab.newSymbol(t)
 

src/mem/slicc/ast/StateDeclAST.py

@@ -57,7 +57,12 @@ class StateDeclAST(DeclAST):
 
         # Make the new type
         t = Type(
-            self.symtab, ident, self.location, self.pairs, self.state_machine
+            self.symtab,
+            ident,
+            self.location,
+            self.pairs,
+            self.shared,
+            self.state_machine,
         )
         self.symtab.newSymbol(t)
 

src/mem/slicc/ast/TypeDeclAST.py

@@ -55,7 +55,12 @@ class TypeDeclAST(DeclAST):
 
         # Make the new type
         new_type = Type(
-            self.symtab, ident, self.location, self.pairs, self.state_machine
+            self.symtab,
+            ident,
+            self.location,
+            self.pairs,
+            self.shared,
+            self.state_machine,
         )
 
         if machine:

src/mem/slicc/main.py

@@ -94,15 +94,11 @@ def main(args=None):
     parser.add_option("-q", "--quiet", help="don't print messages")
     opts, files = parser.parse_args(args=args)
 
-    if len(files) != 1:
-        parser.print_help()
-        sys.exit(2)
-
-    slicc_file = files[0]
-    if not slicc_file.endswith(".slicc"):
-        print("Must specify a .slicc file with a list of state machine files")
-        parser.print_help()
-        sys.exit(2)
+    for slicc_file in files:
+        if not slicc_file.endswith(".slicc"):
+            print("Must specify a .slicc file")
+            parser.print_help()
+            sys.exit(2)
 
     output = nprint if opts.quiet else eprint
 
@@ -112,27 +108,31 @@ def main(args=None):
     protocol_base = os.path.join(
         os.path.dirname(__file__), "..", "ruby", "protocol"
     )
-    slicc = SLICC(
-        slicc_file,
-        protocol_base,
-        verbose=True,
-        debug=opts.debug,
-        traceback=opts.tb,
-    )
 
-    if opts.print_files:
-        for i in sorted(slicc.files()):
-            print(f"    {i}")
-    else:
-        output("Processing AST...")
-        slicc.process()
+    for slicc_file in files:
+        output(f"Working on {slicc_file}")
+        slicc = SLICC(
+            slicc_file,
+            [os.path.join(protocol_base, "RubySlicc_interfaces.slicc")],
+            protocol_base,
+            verbose=True,
+            debug=opts.debug,
+            traceback=opts.tb,
+        )
 
-        if opts.html_path:
-            output("Writing HTML files...")
-            slicc.writeHTMLFiles(opts.html_path)
+        if opts.print_files:
+            for i in sorted(slicc.files()):
+                print(f"    {i}")
+        else:
+            output("Processing AST...")
+            slicc.process()
 
-        output("Writing C++ files...")
-        slicc.writeCodeFiles(opts.code_path, [])
+            if opts.html_path:
+                output("Writing HTML files...")
+                slicc.writeHTMLFiles(opts.html_path)
+
+            output("Writing C++ files...")
+            slicc.writeCodeFiles(opts.code_path, [])
 
     output("SLICC is Done.")
 

src/mem/slicc/parser.py

@@ -54,16 +54,36 @@ from slicc.symbols import SymbolTable
 
 class SLICC(Grammar):
     def __init__(
-        self, filename, base_dir, verbose=False, traceback=False, **kwargs
+        self,
+        protocol,
+        includes,
+        base_dir,
+        verbose=False,
+        traceback=False,
+        **kwargs,
     ):
+        """Entrypoint for SLICC parsing
+        protocol: The protocol `.slicc` file to parse
+        includes: list of `.slicc` files that are shared between all protocols
+        """
         self.protocol = None
         self.traceback = traceback
         self.verbose = verbose
         self.symtab = SymbolTable(self)
         self.base_dir = base_dir
 
+        if not includes:
+            # raise error
+            pass
+
         try:
-            self.decl_list = self.parse_file(filename, **kwargs)
+            self.decl_list = self.parse_file(includes[0], **kwargs)
+            for include in includes[1:]:
+                self.decl_list += self.parse_file(include, **kwargs)
+            # set all of the types parsed so far as shared
+            self.decl_list.setShared()
+
+            self.decl_list += self.parse_file(protocol, **kwargs)
         except ParseError as e:
             if not self.traceback:
                 sys.exit(str(e))

src/mem/slicc/symbols/SymbolTable.py

@@ -55,7 +55,7 @@ class SymbolTable:
         pairs["primitive"] = "yes"
         pairs["external"] = "yes"
         location = Location("init", 0, no_warning=not slicc.verbose)
-        void = Type(self, "void", location, pairs)
+        void = Type(self, "void", location, pairs, shared=False)
         self.newSymbol(void)
 
     def __repr__(self):

src/mem/slicc/symbols/Type.py

@@ -63,10 +63,13 @@ class Enumeration(PairContainer):
 
 
 class Type(Symbol):
-    def __init__(self, table, ident, location, pairs, machine=None):
+    def __init__(
+        self, table, ident, location, pairs, shared=False, machine=None
+    ):
         super().__init__(table, ident, location, pairs)
         self.c_ident = ident
         self.abstract_ident = ""
+        self.shared = shared
         if machine:
             if self.isExternal or self.isPrimitive:
                 if "external_name" in self: