arch-arm: inform bootloader of kernel position with a register

Before the commit, the bootloader had a hardcoded entry point that it
would jump to.

However, the Linux kernel arm64 v5.8 forced us to change the kernel
entry point because the required memory alignment has changed at:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/
commit/?h=v5.8&id=cfa7ede20f133cc81cef01dc3a516dda3a9721ee

Therefore the only way to have a single bootloader that boots both
pre-v5.8 and post-v5.8 kernels is to pass that information from gem5
to the bootloader, which we do in this patch via registers.

This approach was already used by the 32-bit bootloader, which passed
that value via r3, and we try to use the same register x3 in 64-bit.

Since we are now passing this information, the this patch also removes
the hardcoding of DTB and cpu-release-addr, and also passes those
values via registers.

We store the cpu-release-addr in x5 as that value appears to have a
function similar to flags_addr, which is used only in 32-bit arm and
gets stored in r5.

This commit renames atags_addr to dtb_addr, since both are mutually
exclusive, and serve a similar purpose, DTB being the newer recommended
approach.

Similarly, flags_addr is renamed to cpu_release_addr, and it is moved
from ArmSystem into ArmFsWorkload, since it is not an intrinsic system
property, and should be together with dtb_addr instead.

Before this commit, flags_addr was being set from FSConfig.py and
configs/example/arm/devices.py to self.realview.realview_io.pio_addr
+ 0x30. This commit moves that logic into RealView.py instead, and
sets the flags address 8 bytes before the start of the DTB address.

JIRA: https://gem5.atlassian.net/browse/GEM5-787
Change-Id: If70bea9690be04b84e6040e256a9b03e46710e10
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/35076
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Maintainer: Andreas Sandberg <andreas.sandberg@arm.com>
Tested-by: kokoro <noreply+kokoro@google.com>

configs/common/FSConfig.py

@@ -250,7 +250,7 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None,
     if bare_metal:
         # EOT character on UART will end the simulation
         self.realview.uart[0].end_on_eot = True
-        self.workload = ArmFsWorkload(atags_addr=0)
+        self.workload = ArmFsWorkload(dtb_addr=0)
     else:
         workload = ArmFsLinux()
 
@@ -269,8 +269,6 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None,
         if hasattr(self.realview.gic, 'cpu_addr'):
             self.gic_cpu_addr = self.realview.gic.cpu_addr
 
-        self.flags_addr = self.realview.realview_io.pio_addr + 0x30
-
         # This check is for users who have previously put 'android' in
         # the disk image filename to tell the config scripts to
         # prepare the kernel with android-specific boot options. That

configs/example/arm/devices.py

@@ -308,7 +308,6 @@ def simpleSystem(BaseSystem, caches, mem_size, platform=None, **kwargs):
 
             if hasattr(self.realview.gic, 'cpu_addr'):
                 self.gic_cpu_addr = self.realview.gic.cpu_addr
-            self.flags_addr = self.realview.realview_io.pio_addr + 0x30
 
             self.membus = MemBus()
 

configs/example/arm/workloads.py

@@ -47,7 +47,7 @@ from common.SysPaths import binary, disk
 
 class ArmBaremetal(ArmFsWorkload):
     """ Baremetal workload """
-    atags_addr = 0
+    dtb_addr = 0
 
     def __init__(self, obj, system, **kwargs):
         super(ArmBaremetal, self).__init__(**kwargs)
@@ -72,7 +72,7 @@ class ArmTrustedFirmware(ArmFsWorkload):
     https://github.com/ARM-software/arm-trusted-firmware
 
     """
-    atags_addr = 0
+    dtb_addr = 0
 
     def __init__(self, obj, system, **kwargs):
         super(ArmTrustedFirmware, self).__init__(**kwargs)

src/arch/arm/ArmFsWorkload.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2009, 2012-2013, 2015-2019 ARM Limited
+# Copyright (c) 2009, 2012-2013, 2015-2020 ARM Limited
 # All rights reserved.
 #
 # The license below extends only to copyright in the software and shall
@@ -57,11 +57,11 @@ class ArmFsWorkload(KernelWorkload):
 
     dtb_filename = Param.String("",
         "File that contains the Device Tree Blob. Don't use DTB if empty.")
+    dtb_addr = Param.Addr(0, "DTB or ATAGS address")
+    cpu_release_addr = Param.Addr(0, "cpu-release-addr property")
 
     machine_type = Param.ArmMachineType('DTOnly',
         "Machine id from http://www.arm.linux.org.uk/developer/machines/")
-    atags_addr = Param.Addr("Address where default atags structure should " \
-                                "be written")
     early_kernel_symbols = Param.Bool(False,
         "enable early kernel symbol tables before MMU")
     enable_context_switch_stats_dump = Param.Bool(False,

src/arch/arm/ArmSystem.py

@@ -48,7 +48,6 @@ class ArmSystem(System):
     cxx_header = "arch/arm/system.hh"
     multi_proc = Param.Bool(True, "Multiprocessor system?")
     gic_cpu_addr = Param.Addr(0, "Addres of the GIC CPU interface")
-    flags_addr = Param.Addr(0, "Address of the flags register for MP booting")
     have_security = Param.Bool(False,
         "True if Security Extensions are implemented")
     have_virtualization = Param.Bool(False,

src/arch/arm/freebsd/fs_workload.cc

@@ -95,7 +95,7 @@ FsFreebsd::initState()
     // Kernel supports flattened device tree and dtb file specified.
     // Using Device Tree Blob to describe system configuration.
     inform("Loading DTB file: %s at address %#x\n", params().dtb_filename,
-            params().atags_addr + _loadAddrOffset);
+            params().dtb_addr + _loadAddrOffset);
 
     auto *dtb_file = new ::Loader::DtbFile(params().dtb_filename);
 
@@ -108,7 +108,7 @@ FsFreebsd::initState()
         bootReleaseAddr = ra & ~ULL(0x7F);
 
     dtb_file->buildImage().
-        offset(params().atags_addr + _loadAddrOffset).
+        offset(params().dtb_addr + _loadAddrOffset).
         write(system->physProxy);
     delete dtb_file;
 
@@ -116,7 +116,7 @@ FsFreebsd::initState()
     for (auto *tc: system->threads) {
         tc->setIntReg(0, 0);
         tc->setIntReg(1, params().machine_type);
-        tc->setIntReg(2, params().atags_addr + _loadAddrOffset);
+        tc->setIntReg(2, params().dtb_addr + _loadAddrOffset);
     }
 }
 

src/arch/arm/fs_workload.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2012-2013, 2015,2017-2019 ARM Limited
+ * Copyright (c) 2010, 2012-2013, 2015,2017-2020 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -117,21 +117,21 @@ FsWorkload::initState()
 
         inform("Using bootloader at address %#x", bootldr->entryPoint());
 
-        // Put the address of the boot loader into r7 so we know
+        // The address of the boot loader so we know
         // where to branch to after the reset fault
         // All other values needed by the boot loader to know what to do
-        fatal_if(!arm_sys->params().flags_addr,
+        fatal_if(!params().cpu_release_addr,
-                 "flags_addr must be set with bootloader");
+                 "cpu_release_addr must be set with bootloader");
 
         fatal_if(!arm_sys->params().gic_cpu_addr && is_gic_v2,
                  "gic_cpu_addr must be set with bootloader");
 
         for (auto *tc: arm_sys->threads) {
-            if (!arm_sys->highestELIs64())
+            tc->setIntReg(3, kernelEntry);
-                tc->setIntReg(3, kernelEntry);
             if (is_gic_v2)
                 tc->setIntReg(4, arm_sys->params().gic_cpu_addr);
-            tc->setIntReg(5, arm_sys->params().flags_addr);
+            if (getArch() == Loader::Arm)
+                tc->setIntReg(5, params().cpu_release_addr);
         }
         inform("Using kernel entry physical address at %#x\n", kernelEntry);
     } else {

src/arch/arm/linux/fs_workload.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010-2013, 2016 ARM Limited
+ * Copyright (c) 2010-2013, 2016, 2020 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -92,7 +92,7 @@ FsLinux::initState()
         // Kernel supports flattened device tree and dtb file specified.
         // Using Device Tree Blob to describe system configuration.
         inform("Loading DTB file: %s at address %#x\n", params().dtb_filename,
-                params().atags_addr + _loadAddrOffset);
+                params().dtb_addr);
 
         auto *dtb_file = new ::Loader::DtbFile(params().dtb_filename);
 
@@ -102,9 +102,8 @@ FsLinux::initState()
                  params().dtb_filename);
         }
 
-        dtb_file->buildImage().
+        dtb_file->buildImage().offset(params().dtb_addr)
-            offset(params().atags_addr + _loadAddrOffset).
+            .write(system->physProxy);
-            write(system->physProxy);
         delete dtb_file;
     } else {
         // Using ATAGS
@@ -152,17 +151,27 @@ FsLinux::initState()
         DPRINTF(Loader, "Boot atags was %d bytes in total\n", size << 2);
         DDUMP(Loader, boot_data, size << 2);
 
-        system->physProxy.writeBlob(params().atags_addr + _loadAddrOffset,
+        system->physProxy.writeBlob(params().dtb_addr + _loadAddrOffset,
                                     boot_data, size << 2);
 
         delete[] boot_data;
     }
 
-    // Kernel boot requirements to set up r0, r1 and r2 in ARMv7
+    if (getArch() == Loader::Arm64) {
-    for (auto *tc: system->threads) {
+        // We inform the bootloader of the kernel entry point. This was added
-        tc->setIntReg(0, 0);
+        // originally done because the entry offset changed in kernel v5.8.
-        tc->setIntReg(1, params().machine_type);
+        // Previously the bootloader just used a hardcoded address.
-        tc->setIntReg(2, params().atags_addr + _loadAddrOffset);
+        for (auto *tc: system->threads) {
+            tc->setIntReg(0, params().dtb_addr);
+            tc->setIntReg(5, params().cpu_release_addr);
+        }
+    } else {
+        // Kernel boot requirements to set up r0, r1 and r2 in ARMv7
+        for (auto *tc: system->threads) {
+            tc->setIntReg(0, 0);
+            tc->setIntReg(1, params().machine_type);
+            tc->setIntReg(2, params().dtb_addr + _loadAddrOffset);
+        }
     }
 }
 

src/dev/arm/RealView.py

@@ -713,10 +713,11 @@ class RealView(Platform):
         self._attach_mem(self._off_chip_memory(), bus, mem_ports)
         self._attach_io(self._off_chip_devices(), bus, dma_ports)
 
-    def setupBootLoader(self, cur_sys, boot_loader, atags_addr, load_offset):
+    def setupBootLoader(self, cur_sys, boot_loader, dtb_addr, load_offset):
         cur_sys.workload.boot_loader = boot_loader
-        cur_sys.workload.atags_addr = atags_addr
         cur_sys.workload.load_addr_offset = load_offset
+        cur_sys.workload.dtb_addr = load_offset + dtb_addr
+        cur_sys.workload.cpu_release_addr = cur_sys.workload.dtb_addr - 8
 
     def generateDeviceTree(self, state):
         node = FdtNode("/") # Things in this module need to end up in the root
@@ -734,8 +735,11 @@ class RealView(Platform):
             cpu.append(FdtPropertyStrings('enable-method', 'psci'))
         else:
             cpu.append(FdtPropertyStrings("enable-method", "spin-table"))
+            # The kernel writes the entry addres of secondary CPUs to this
+            # address before waking up secondary CPUs.
+            # The gem5 bootloader then makes secondary CPUs jump to it.
             cpu.append(FdtPropertyWords("cpu-release-addr", \
-                                        state.addrCells(0x8000fff8)))
+                        state.addrCells(system.workload.cpu_release_addr)))
 
 class VExpress_EMM(RealView):
     _mem_regions = [ AddrRange('2GB', size='2GB') ]

system/arm/bootloader/arm64/boot.S

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012 ARM Limited
+ * Copyright (c) 2012, 2020 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -40,6 +40,14 @@
 
         .globl	_start
 _start:
+        /* Save some values initialized by gem5. */
+        /* DTB address. */
+        mov x21, x0
+        /* Kernel entry point. */
+        mov x20, x3
+        /* cpu-release-addr. */
+        mov x22, x5
+
         /*
          * EL3 initialisation
          */
@@ -153,8 +161,19 @@ start_ns:
          * Secondary CPUs
          */
 1:	wfe
-        ldr	x4, =PHYS_OFFSET + 0xfff8
+        /* The Linux kernel v5.8 and older writes the entry point address
-        ldr     x4, [x4]
+         * of the secondary CPUs to this address, and does a SEV, waking up
+         * the secondary CPUs.
+         *
+         * gem5 informs the kernel the desired address via cpu-release-addr
+         * of the DTB.
+         *
+         * When this is first reached immediately after the bootloader starts,
+         * the value at that address must be 0, which is the default memory
+         * value set by gem5 for otherwise uninitialized memory, leading to
+         * WFE.
+         */
+        ldr x4, [x22]
         cbz	x4, 1b
         br	x4				// branch to the given address
 
@@ -180,9 +199,13 @@ start_ns:
         /*
          * Primary CPU
          */
-        ldr	x0, =PHYS_OFFSET + 0x8000000	 // device tree blob
+        // The kernel boot protocol specifies that the DTB address is placed
-        ldr     x6, =PHYS_OFFSET + 0x80000       // kernel start address
+        // in x0.
-        br	x6
+        // https://github.com/torvalds/linux/blob/v5.7/Documentation/arm64/
+        // booting.rst#4-call-the-kernel-image
+        mov x0, x21
+        // Jump into the kernel entry point.
+        br x20
 
         .ltorg
 

system/arm/bootloader/arm64/makefile

@@ -34,11 +34,7 @@ BUILDDIR = .
 DESTDIR = $(error Please set DESTDIR to wanted installation directory)
 
 CFLAGS = -march=armv8-a
-CPPFLAGS = -DPHYS_OFFSET=0x80000000 \
+CPPFLAGS = -DUART_BASE=0x1c090000 -DSYSREGS_BASE=0x1c010000
-		   -DUART_BASE=0x1c090000 -DSYSREGS_BASE=0x1c010000 \
-		   -Dkernel=0x80080000 \
-		   -Dmbox=0x8000fff8 -Ddtb=0x80000100
-
 LDFLAGS = -N -Ttext 0x00000010 -static
 
 .PHONY: all clean install mkdir