python: Move the components lib to be compiled in the binary

There has been some debate on how best to distribute the components
library. This change builds the components library into the gem5 binary.
The components library will now function similar to the `m5` library.
There is no need for awkward imports or obtaining the library from some
third-party source.

Additional incorporated in this patch:
* Added `__init__.py` to the Python modules.
* Fixed a typo in the `abstract_ruby_cache_hierarchy.py` filename.
* Ensured that imports within the library are relative.

Issue-on: https://gem5.atlassian.net/browse/GEM5-1023
Change-Id: I3988c8710cda8dcf7b21109a2cf5c3f1608cc71a
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/49690
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-by: Austin Harris <mail@austin-harris.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

configs/example/components-library/riscv_fs.py

@@ -38,20 +38,6 @@ Characteristics
 import m5
 from m5.objects import Root
 
-import sys
-import os
-
-# This is a lame hack to get the imports working correctly.
-# TODO: This needs fixed.
-sys.path.append(
-    os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        os.pardir,
-        os.pardir,
-        os.pardir,
-    )
-)
-
 from components_library.runtime import get_runtime_isa
 from components_library.boards.riscv_board import RiscvBoard
 from components_library.memory.single_channel import SingleChannelDDR3_1600

src/python/SConscript

@@ -28,6 +28,168 @@
 
 Import('*')
 
+PySource('components_library', 'components_library/__init__.py')
+PySource('components_library', 'components_library/coherence_protocol.py')
+PySource('components_library', 'components_library/isas.py')
+PySource('components_library', 'components_library/runtime.py')
+PySource('components_library.boards', 'components_library/boards/__init__.py')
+PySource('components_library.boards',
+    'components_library/boards/abstract_board.py')
+PySource('components_library.boards',
+    'components_library/boards/mem_mode.py')
+PySource('components_library.boards',
+    'components_library/boards/riscv_board.py')
+PySource('components_library.boards',
+    'components_library/boards/simple_board.py')
+PySource('components_library.boards',
+    'components_library/boards/test_board.py')
+PySource('components_library.boards', 'components_library/boards/x86_board.py')
+PySource('components_library.cachehierarchies',
+    'components_library/cachehierarchies/__init__.py')
+PySource('components_library.cachehierarchies',
+    'components_library/cachehierarchies/abstract_cache_hierarchy.py')
+PySource('components_library.cachehierarchies',
+    'components_library/cachehierarchies/'
+    'abstract_two_level_cache_hierarchy.py')
+PySource('components_library.cachehierarchies.classic',
+    'components_library/cachehierarchies/classic/__init__.py')
+PySource('components_library.cachehierarchies.classic',
+    'components_library/cachehierarchies/classic/'
+    'abstract_classic_cache_hierarchy.py')
+PySource('components_library.cachehierarchies.classic',
+    'components_library/cachehierarchies/classic/no_cache.py')
+PySource('components_library.cachehierarchies.classic',
+    'components_library/cachehierarchies/classic/'
+    'private_l1_cache_hierarchy.py')
+PySource('components_library.cachehierarchies.classic',
+    'components_library/cachehierarchies/classic/'
+    'private_l1_private_l2_cache_hierarchy.py')
+PySource('components_library.cachehierarchies.classic.caches',
+    'components_library/cachehierarchies/classic/caches/__init__.py')
+PySource('components_library.cachehierarchies.classic.caches',
+    'components_library/cachehierarchies/classic/caches/l1dcache.py')
+PySource('components_library.cachehierarchies.classic.caches',
+    'components_library/cachehierarchies/classic/caches/l1icache.py')
+PySource('components_library.cachehierarchies.classic.caches',
+    'components_library/cachehierarchies/classic/caches/l2cache.py')
+PySource('components_library.cachehierarchies.classic.caches',
+    'components_library/cachehierarchies/classic/caches/mmu_cache.py')
+PySource('components_library.cachehierarchies.ruby',
+    'components_library/cachehierarchies/ruby/__init__.py')
+PySource('components_library.cachehierarchies.ruby',
+    'components_library/cachehierarchies/ruby/'
+    'abstract_ruby_cache_hierarchy.py')
+PySource('components_library.cachehierarchies.ruby',
+    'components_library/cachehierarchies/ruby/'
+    'mesi_two_level_cache_hierarchy.py')
+PySource('components_library.cachehierarchies.ruby',
+    'components_library/cachehierarchies/ruby/mi_example_cache_hierarchy.py')
+PySource('components_library.cachehierarchies.ruby.caches',
+    'components_library/cachehierarchies/ruby/caches/__init__.py')
+PySource('components_library.cachehierarchies.ruby.caches',
+    'components_library/cachehierarchies/ruby/caches/abstract_directory.py')
+PySource('components_library.cachehierarchies.ruby.caches',
+    'components_library/cachehierarchies/ruby/caches/'
+    'abstract_dma_controller.py')
+PySource('components_library.cachehierarchies.ruby.caches',
+    'components_library/cachehierarchies/ruby/caches/abstract_l1_cache.py')
+PySource('components_library.cachehierarchies.ruby.caches',
+    'components_library/cachehierarchies/ruby/caches/abstract_l2_cache.py')
+PySource('components_library.cachehierarchies.ruby.caches.mesi_two_level',
+    'components_library/cachehierarchies/ruby/caches/mesi_two_level/'
+    '__init__.py')
+PySource('components_library.cachehierarchies.ruby.caches.mesi_two_level',
+    'components_library/cachehierarchies/ruby/caches/mesi_two_level/'
+    'directory.py')
+PySource('components_library.cachehierarchies.ruby.caches.mesi_two_level',
+    'components_library/cachehierarchies/ruby/caches/mesi_two_level/'
+    'dma_controller.py')
+PySource('components_library.cachehierarchies.ruby.caches.mesi_two_level',
+    'components_library/cachehierarchies/ruby/caches/mesi_two_level/'
+    'l1_cache.py')
+PySource('components_library.cachehierarchies.ruby.caches.mesi_two_level',
+    'components_library/cachehierarchies/ruby/caches/mesi_two_level/'
+    'l2_cache.py')
+PySource('components_library.cachehierarchies.ruby.caches.mi_example',
+    'components_library/cachehierarchies/ruby/caches/mi_example/__init__.py')
+PySource('components_library.cachehierarchies.ruby.caches.mi_example',
+    'components_library/cachehierarchies/ruby/caches/mi_example/directory.py')
+PySource('components_library.cachehierarchies.ruby.caches.mi_example',
+    'components_library/cachehierarchies/ruby/caches/mi_example/'
+    'dma_controller.py')
+PySource('components_library.cachehierarchies.ruby.caches.mi_example',
+    'components_library/cachehierarchies/ruby/caches/mi_example/l1_cache.py')
+PySource('components_library.cachehierarchies.ruby.topologies',
+    'components_library/cachehierarchies/ruby/topologies/__init__.py')
+PySource('components_library.cachehierarchies.ruby.topologies',
+    'components_library/cachehierarchies/ruby/topologies/simple_pt2pt.py')
+PySource('components_library.memory', 'components_library/memory/__init__.py')
+PySource('components_library.memory',
+    'components_library/memory/abstract_memory_system.py')
+PySource('components_library.memory', 'components_library/memory/dramsim_3.py')
+PySource('components_library.memory',
+    'components_library/memory/single_channel.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/__init__.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/ddr3.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/ddr4.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/gddr.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/hbm.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/hmc.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/lpddr2.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/lpddr3.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/lpddr5.py')
+PySource('components_library.memory.dram_interfaces',
+    'components_library/memory/dram_interfaces/wideio.py')
+PySource('components_library.processors',
+    'components_library/processors/__init__.py')
+PySource('components_library.processors',
+    'components_library/processors/abstract_core.py')
+PySource('components_library.processors',
+    'components_library/processors/abstract_generator_core.py')
+PySource('components_library.processors',
+    'components_library/processors/abstract_processor.py')
+PySource('components_library.processors',
+    'components_library/processors/complex_generator_core.py')
+PySource('components_library.processors',
+    'components_library/processors/complex_generator.py')
+PySource('components_library.processors',
+    'components_library/processors/cpu_types.py')
+PySource('components_library.processors',
+    'components_library/processors/linear_generator_core.py')
+PySource('components_library.processors',
+    'components_library/processors/linear_generator.py')
+PySource('components_library.processors',
+    'components_library/processors/random_generator_core.py')
+PySource('components_library.processors',
+    'components_library/processors/random_generator.py')
+PySource('components_library.processors',
+    'components_library/processors/simple_core.py')
+PySource('components_library.processors',
+    'components_library/processors/simple_processor.py')
+PySource('components_library.processors',
+    'components_library/processors/simple_switchable_processor.py')
+PySource('components_library.processors',
+    'components_library/processors/switchable_processor.py')
+PySource('components_library.resources',
+    'components_library/resources/__init__.py')
+PySource('components_library.resources',
+    'components_library/resources/downloader.py')
+PySource('components_library.resources',
+    'components_library/resources/resource.py')
+PySource('components_library.utils', 'components_library/utils/__init__.py')
+PySource('components_library.utils', 'components_library/utils/filelock.py')
+PySource('components_library.utils', 'components_library/utils/override.py')
+PySource('components_library.utils', 'components_library/utils/requires.py')
+
 PySource('', 'importer.py')
 PySource('m5', 'm5/__init__.py')
 PySource('m5', 'm5/SimObject.py')

src/python/components_library/boards/simple_board.py

@@ -24,7 +24,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.resources.resource import AbstractResource
+from ..resources.resource import AbstractResource
 from m5.objects import (
     AddrRange,
     SrcClockDomain,

src/python/components_library/boards/x86_board.py

@@ -25,10 +25,10 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-from components_library.resources.resource import AbstractResource
-from components_library.utils.override import overrides
-from components_library.boards.abstract_board import AbstractBoard
-from components_library.isas import ISA
+from ..resources.resource import AbstractResource
+from ..utils.override import overrides
+from .abstract_board import AbstractBoard
+from ..isas import ISA
 
 import m5
 from m5.objects import (

src/python/components_library/cachehierarchies/classic/no_cache.py

@@ -24,10 +24,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.cachehierarchies.classic.\
-    abstract_classic_cache_hierarchy import (
-    AbstractClassicCacheHierarchy,
-)
+from .abstract_classic_cache_hierarchy import AbstractClassicCacheHierarchy
 from ..abstract_cache_hierarchy import AbstractCacheHierarchy
 from ...boards.abstract_board import AbstractBoard
 from ...isas import ISA

src/python/components_library/cachehierarchies/ruby/abstract_ruby_cache_hierarchy.py

@@ -24,7 +24,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.utils.override import overrides
+from ...utils.override import overrides
 from ..abstract_cache_hierarchy import AbstractCacheHierarchy
 
 

src/python/components_library/cachehierarchies/ruby/caches/abstract_l1_cache.py

@@ -25,9 +25,9 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from abc import abstractmethod
-from components_library.isas import ISA
-from components_library.processors.cpu_types import CPUTypes
-from components_library.processors.abstract_core import AbstractCore
+from ....isas import ISA
+from ....processors.cpu_types import CPUTypes
+from ....processors.abstract_core import AbstractCore
 
 from m5.objects import L1Cache_Controller
 

src/python/components_library/cachehierarchies/ruby/caches/mesi_two_level/directory.py

@@ -24,7 +24,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.utils.override import overrides
+from .....utils.override import overrides
 from ..abstract_directory import AbstractDirectory
 
 from m5.objects import (

src/python/components_library/cachehierarchies/ruby/caches/mesi_two_level/dma_controller.py

@@ -24,7 +24,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.utils.override import overrides
+from .....utils.override import overrides
 from ..abstract_dma_controller import AbstractDMAController
 
 from m5.objects import MessageBuffer

src/python/components_library/cachehierarchies/ruby/caches/mesi_two_level/l1_cache.py

@@ -24,10 +24,10 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.processors.abstract_core import AbstractCore
-from components_library.isas import ISA
+from .....processors.abstract_core import AbstractCore
+from .....isas import ISA
 from ..abstract_l1_cache import AbstractL1Cache
-from components_library.utils.override import *
+from .....utils.override import *
 
 from m5.objects import (
     MessageBuffer,

src/python/components_library/cachehierarchies/ruby/caches/mesi_two_level/l2_cache.py

@@ -25,7 +25,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from ..abstract_l2_cache import AbstractL2Cache
-from components_library.utils.override import *
+from .....utils.override import *
 
 from m5.objects import MessageBuffer, RubyCache
 

src/python/components_library/cachehierarchies/ruby/mesi_two_level_cache_hierarchy.py

@@ -25,13 +25,13 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-from .abstract_ruby_cache_hierarhcy import AbstractRubyCacheHierarchy
+from .abstract_ruby_cache_hierarchy import AbstractRubyCacheHierarchy
 from ..abstract_two_level_cache_hierarchy import AbstractTwoLevelCacheHierarchy
 from ...coherence_protocol import CoherenceProtocol
 from ...isas import ISA
 from ...boards.abstract_board import AbstractBoard
 from ...runtime import get_runtime_isa
-from components_library.utils.requires import requires
+from ...utils.requires import requires
 
 from .topologies.simple_pt2pt import SimplePt2Pt
 from .caches.mesi_two_level.l1_cache import L1Cache

src/python/components_library/cachehierarchies/ruby/mi_example_cache_hierarchy.py

@@ -28,14 +28,14 @@ from .caches.mi_example.l1_cache import L1Cache
 from .caches.mi_example.dma_controller import DMAController
 from .caches.mi_example.directory import Directory
 from .topologies.simple_pt2pt import SimplePt2Pt
-from .abstract_ruby_cache_hierarhcy import AbstractRubyCacheHierarchy
+from .abstract_ruby_cache_hierarchy import AbstractRubyCacheHierarchy
 from ..abstract_cache_hierarchy import AbstractCacheHierarchy
 from ...boards.abstract_board import AbstractBoard
 from ...coherence_protocol import CoherenceProtocol
 from ...isas import ISA
 from ...utils.override import overrides
 from ...runtime import get_runtime_isa
-from components_library.utils.requires import requires
+from ...utils.requires import requires
 
 
 from m5.objects import (

src/python/components_library/processors/abstract_processor.py

@@ -25,7 +25,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from abc import ABCMeta, abstractmethod
-from components_library.processors.abstract_core import AbstractCore
+from .abstract_core import AbstractCore
 
 from m5.objects import SubSystem
 

src/python/components_library/processors/simple_core.py

@@ -25,8 +25,8 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from typing import Optional
-from components_library.runtime import get_runtime_isa
-from components_library.processors.abstract_core import AbstractCore
+from ..runtime import get_runtime_isa
+from ..processors.abstract_core import AbstractCore
 
 from .cpu_types import CPUTypes
 from ..isas import ISA

src/python/components_library/processors/simple_processor.py

@@ -25,9 +25,9 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-from components_library.utils.override import overrides
-from components_library.boards.mem_mode import MemMode
-from components_library.processors.simple_core import SimpleCore
+from ..utils.override import overrides
+from ..boards.mem_mode import MemMode
+from ..processors.simple_core import SimpleCore
 
 from m5.util import warn
 

src/python/components_library/processors/simple_switchable_processor.py

@@ -24,10 +24,10 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.boards.mem_mode import MemMode
-from components_library.boards.abstract_board import AbstractBoard
-from components_library.processors.simple_core import SimpleCore
-from components_library.processors.cpu_types import CPUTypes
+from ..boards.mem_mode import MemMode
+from ..boards.abstract_board import AbstractBoard
+from ..processors.simple_core import SimpleCore
+from ..processors.cpu_types import CPUTypes
 from .switchable_processor import SwitchableProcessor
 
 from ..utils.override import *

src/python/components_library/processors/switchable_processor.py

@@ -24,9 +24,9 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from components_library.processors.simple_core import SimpleCore
-from components_library.processors.abstract_core import AbstractCore
 
+from .simple_core import SimpleCore
+from .abstract_core import AbstractCore
 from .cpu_types import CPUTypes
 
 import m5

tests/gem5/configs/components-library/boot_exit_disk_run.py

@@ -31,21 +31,6 @@ This script will run a simple boot exit test.
 import m5
 from m5.objects import Root
 
-import sys
-import os
-
-# This is a lame hack to get the imports working correctly.
-# TODO: This needs fixed.
-sys.path.append(
-    os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        os.pardir,
-        os.pardir,
-        os.pardir,
-        os.pardir,
-    )
-)
-
 from components_library.runtime import (
     get_runtime_coherence_protocol,
     get_runtime_isa,
@@ -59,7 +44,6 @@ from components_library.isas import ISA
 from components_library.coherence_protocol import CoherenceProtocol
 from components_library.resources.resource import Resource
 
-import os
 import argparse
 
 parser = argparse.ArgumentParser(

tests/gem5/configs/components-library/boot_kvm_switch_exit.py

@@ -29,24 +29,10 @@ This script boots with KVM then switches processors and exits.
 """
 
 import argparse
-import os
-import sys
 
 import m5
 from m5.objects import Root
 
-# This is a lame hack to get the imports working correctly.
-# TODO: This needs fixed.
-sys.path.append(
-    os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        os.pardir,
-        os.pardir,
-        os.pardir,
-        os.pardir,
-    )
-)
-
 from components_library.boards.x86_board import X86Board
 from components_library.coherence_protocol import CoherenceProtocol
 from components_library.isas import ISA

tests/gem5/configs/components-library/parsec_disk_run.py

@@ -39,20 +39,6 @@ import m5
 import m5.ticks
 from m5.objects import Root
 
-import sys
-import os
-
-# This is a lame hack to get the imports working correctly.
-# TODO: This needs fixed.
-sys.path.append(
-    os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        os.pardir,
-        os.pardir,
-        os.pardir,
-        os.pardir,
-    )
-)
 
 from components_library.resources.resource import Resource
 from components_library.boards.x86_board import X86Board

tests/gem5/configs/components-library/simple_binary_run.py

@@ -33,21 +33,6 @@ gem5 while still being functinal.
 import m5
 from m5.objects import Root
 
-import os
-import sys
-
-# This is a lame hack to get the imports working correctly.
-# TODO: This needs fixed.
-sys.path.append(
-    os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        os.pardir,
-        os.pardir,
-        os.pardir,
-        os.pardir,
-    )
-)
-
 from components_library.resources.resource import Resource
 from components_library.boards.simple_board import SimpleBoard
 from components_library.cachehierarchies.classic.no_cache import NoCache

tests/gem5/configs/components-library/simple_traffic_run.py

@@ -34,23 +34,9 @@ import m5
 
 from m5.objects import Root
 
-import sys
-import os
 import argparse
 import importlib
 
-# This is a lame hack to get the imports working correctly.
-# TODO: This needs fixed.
-sys.path.append(
-    os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        os.pardir,
-        os.pardir,
-        os.pardir,
-        os.pardir,
-    )
-)
-
 from components_library.boards.test_board import TestBoard
 from components_library.cachehierarchies.classic.no_cache import NoCache
 from components_library.memory.single_channel import *