From f7da973f34cd81f117019680c2816f6db000ae94 Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Thu, 20 Jul 2023 10:34:44 -0700 Subject: [PATCH] cpu-kvm: Make using perf when using KVM CPU optional (#95) * cpu-kvm: Add a variable signifying whether we are using perf Change-Id: Iaa081e364f85c863f781723b5524d267724ed0e4 Signed-off-by: Hoa Nguyen * cpu-kvm: Making it clear the functionalities are specific to KVM Change-Id: I982426f294d90655227dc15337bf73c42a260ded Signed-off-by: Hoa Nguyen * cpu-kvm: Make perf optional Change-Id: I8973c2a96575383976cea7ca3fda478f83e95c3f Signed-off-by: Hoa Nguyen * configs: Add an example config of using KVM without perf Change-Id: Ic69fa7dac4f1a2c8fe23712b0fa77b5b22c5f2df Signed-off-by: Hoa Nguyen * Apply suggestions from code review Co-authored-by: Jason Lowe-Power * misc: Add an example to the panic Change-Id: Ic1fdfb955e5d8b9ad1d4f0a2bf30fa8050deba70 Signed-off-by: Hoa Nguyen * misc: Add warning of not using perf when using KVM CPU Change-Id: I96c0832fb48c63a79773665ca6228da778ef0497 Signed-off-by: Hoa Nguyen * misc: Fix stuff Change-Id: Ib407ae7407955b695f0e0f2718324f41bb0d768f Signed-off-by: Hoa Nguyen * misc: style fix Change-Id: I7275942e43f46140fdd52c975f76abb3c81b8b0a Signed-off-by: Hoa Nguyen --------- Signed-off-by: Hoa Nguyen Co-authored-by: Jason Lowe-Power --- .../x86-ubuntu-run-with-kvm-no-perf.py | 138 ++++++++++++++++++ src/cpu/kvm/BaseKvmCPU.py | 5 + src/cpu/kvm/base.cc | 84 ++++++++--- src/cpu/kvm/base.hh | 7 +- src/cpu/kvm/perfevent.cc | 16 +- 5 files changed, 221 insertions(+), 29 deletions(-) create mode 100644 configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py diff --git a/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py b/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py new file mode 100644 index 0000000000..1c65357921 --- /dev/null +++ b/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py @@ -0,0 +1,138 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" + +This script demonstrates how to use KVM CPU without perf. +This simulation boots Ubuntu 18.04 using 2 KVM CPUs without using perf. + +Usage +----- + +``` +scons build/X86/gem5.opt -j`nproc` +./build/X86/gem5.opt configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py +``` +""" + +from gem5.utils.requires import requires +from gem5.components.boards.x86_board import X86Board +from gem5.components.cachehierarchies.ruby.mesi_two_level_cache_hierarchy import ( + MESITwoLevelCacheHierarchy, +) +from gem5.components.memory.single_channel import SingleChannelDDR4_2400 +from gem5.components.processors.simple_switchable_processor import ( + SimpleSwitchableProcessor, +) +from gem5.components.processors.cpu_types import CPUTypes +from gem5.isas import ISA +from gem5.coherence_protocol import CoherenceProtocol +from gem5.simulate.simulator import Simulator +from gem5.simulate.exit_event import ExitEvent +from gem5.resources.workload import Workload + +# This simulation requires using KVM with gem5 compiled for X86 simulation +# and with MESI_Two_Level cache coherence protocol. +requires( + isa_required=ISA.X86, + coherence_protocol_required=CoherenceProtocol.MESI_TWO_LEVEL, + kvm_required=True, +) + +from gem5.components.cachehierarchies.ruby.mesi_two_level_cache_hierarchy import ( + MESITwoLevelCacheHierarchy, +) + +cache_hierarchy = MESITwoLevelCacheHierarchy( + l1d_size="32KiB", + l1d_assoc=8, + l1i_size="32KiB", + l1i_assoc=8, + l2_size="512KiB", + l2_assoc=16, + num_l2_banks=1, +) + +# Main memory +memory = SingleChannelDDR4_2400(size="3GiB") + +# This is a switchable CPU. We first boot Ubuntu using KVM, then the guest +# will exit the simulation by calling "m5 exit" (see the `command` variable +# below, which contains the command to be run in the guest after booting). +# Upon exiting from the simulation, the Exit Event handler will switch the +# CPU type (see the ExitEvent.EXIT line below, which contains a map to +# a function to be called when an exit event happens). +processor = SimpleSwitchableProcessor( + starting_core_type=CPUTypes.KVM, + switch_core_type=CPUTypes.TIMING, + isa=ISA.X86, + num_cores=2, +) + +# Here we tell the KVM CPU (the starting CPU) not to use perf. +for proc in processor.start: + proc.core.usePerf = False + +# Here we setup the board. The X86Board allows for Full-System X86 simulations. +board = X86Board( + clk_freq="3GHz", + processor=processor, + memory=memory, + cache_hierarchy=cache_hierarchy, +) + +# Here we set the Full System workload. +# The `set_kernel_disk_workload` function for the X86Board takes a kernel, a +# disk image, and, optionally, a command to run. + +# This is the command to run after the system has booted. The first `m5 exit` +# will stop the simulation so we can switch the CPU cores from KVM to timing +# and continue the simulation to run the echo command, sleep for a second, +# then, again, call `m5 exit` to terminate the simulation. After simulation +# has ended you may inspect `m5out/system.pc.com_1.device` to see the echo +# output. +command = ( + "m5 exit;" + + "echo 'This is running on Timing CPU cores.';" + + "sleep 1;" + + "m5 exit;" +) + +workload = Workload("x86-ubuntu-18.04-boot") +workload.set_parameter("readfile_contents", command) +board.set_workload(workload) + +simulator = Simulator( + board=board, + on_exit_event={ + # Here we want override the default behavior for the first m5 exit + # exit event. Instead of exiting the simulator, we just want to + # switch the processor. The 2nd m5 exit after will revert to using + # default behavior where the simulator run will exit. + ExitEvent.EXIT: (func() for func in [processor.switch]) + }, +) +simulator.run() diff --git a/src/cpu/kvm/BaseKvmCPU.py b/src/cpu/kvm/BaseKvmCPU.py index f958e8126c..610663fa41 100644 --- a/src/cpu/kvm/BaseKvmCPU.py +++ b/src/cpu/kvm/BaseKvmCPU.py @@ -64,6 +64,11 @@ class BaseKvmCPU(BaseCPU): def support_take_over(cls): return True + usePerf = Param.Bool( + True, + "Use perf for gathering statistics from the guest and providing " + "statistic-related functionalities", + ) useCoalescedMMIO = Param.Bool(False, "Use coalesced MMIO (EXPERIMENTAL)") usePerfOverflow = Param.Bool( False, "Use perf event overflow counters (EXPERIMENTAL)" diff --git a/src/cpu/kvm/base.cc b/src/cpu/kvm/base.cc index e22e1628d2..eaa771d8cf 100644 --- a/src/cpu/kvm/base.cc +++ b/src/cpu/kvm/base.cc @@ -71,12 +71,15 @@ BaseKvmCPU::BaseKvmCPU(const BaseKvmCPUParams ¶ms) alwaysSyncTC(params.alwaysSyncTC), threadContextDirty(true), kvmStateDirty(false), + usePerf(params.usePerf), vcpuID(-1), vcpuFD(-1), vcpuMMapSize(0), _kvmRun(NULL), mmioRing(NULL), pageSize(sysconf(_SC_PAGE_SIZE)), tickEvent([this]{ tick(); }, "BaseKvmCPU tick", false, Event::CPU_Tick_Pri), activeInstPeriod(0), + hwCycles(nullptr), + hwInstructions(nullptr), perfControlledByTimer(params.usePerfOverflow), hostFactor(params.hostFactor), stats(this), ctrInsts(0) @@ -96,6 +99,22 @@ BaseKvmCPU::BaseKvmCPU(const BaseKvmCPUParams ¶ms) thread->setStatus(ThreadContext::Halted); tc = thread->getTC(); threadContexts.push_back(tc); + + if ((!usePerf) && perfControlledByTimer) { + panic("KVM: invalid combination of parameters: cannot use " + "perfControlledByTimer without usePerf\n"); + } + + // If we use perf, we create new PerfKVMCounters + if (usePerf) { + hwCycles = std::unique_ptr(new PerfKvmCounter()); + hwInstructions = std::unique_ptr(new PerfKvmCounter()); + } else { + inform("Using KVM CPU without perf. The stats related to the number " + "of cycles and instructions executed by the KVM CPU will not " + "be updated. The stats should not be used for performance " + "evaluation."); + } } BaseKvmCPU::~BaseKvmCPU() @@ -248,7 +267,7 @@ BaseKvmCPU::restartEqThread() setupCounters(); if (p.usePerfOverflow) { - runTimer.reset(new PerfKvmTimer(hwCycles, + runTimer.reset(new PerfKvmTimer(*hwCycles, KVM_KICK_SIGNAL, p.hostFactor, p.hostFreq)); @@ -424,8 +443,10 @@ BaseKvmCPU::notifyFork() vcpuFD = -1; _kvmRun = NULL; - hwInstructions.detach(); - hwCycles.detach(); + if (usePerf) { + hwInstructions->detach(); + hwCycles->detach(); + } } } @@ -690,7 +711,9 @@ BaseKvmCPU::kvmRunDrain() uint64_t BaseKvmCPU::getHostCycles() const { - return hwCycles.read(); + if (usePerf) + return hwCycles->read(); + return 0; } Tick @@ -746,21 +769,26 @@ BaseKvmCPU::kvmRun(Tick ticks) // Get hardware statistics after synchronizing contexts. The KVM // state update might affect guest cycle counters. uint64_t baseCycles(getHostCycles()); - uint64_t baseInstrs(hwInstructions.read()); + uint64_t baseInstrs = 0; + if (usePerf) { + baseInstrs = hwInstructions->read(); + } // Arm the run timer and start the cycle timer if it isn't // controlled by the overflow timer. Starting/stopping the cycle // timer automatically starts the other perf timers as they are in // the same counter group. runTimer->arm(ticks); - if (!perfControlledByTimer) - hwCycles.start(); + if (usePerf && (!perfControlledByTimer)) { + hwCycles->start(); + } ioctlRun(); runTimer->disarm(); - if (!perfControlledByTimer) - hwCycles.stop(); + if (usePerf && (!perfControlledByTimer)) { + hwCycles->stop(); + } // The control signal may have been delivered after we exited // from KVM. It will be pending in that case since it is @@ -771,7 +799,10 @@ BaseKvmCPU::kvmRun(Tick ticks) const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles); const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor); - const uint64_t instsExecuted(hwInstructions.read() - baseInstrs); + uint64_t instsExecuted = 0; + if (usePerf) { + instsExecuted = hwInstructions->read() - baseInstrs; + } ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted); /* Update statistics */ @@ -1288,13 +1319,14 @@ BaseKvmCPU::setupCounters() // We might be re-attaching counters due threads being // re-initialised after fork. - if (hwCycles.attached()) - hwCycles.detach(); + if (usePerf) { + if (hwCycles->attached()) { + hwCycles->detach(); + } - hwCycles.attach(cfgCycles, - 0); // TID (0 => currentThread) - - setupInstCounter(); + hwCycles->attach(cfgCycles, 0); // TID (0 => currentThread) + setupInstCounter(); + } } bool @@ -1344,10 +1376,16 @@ BaseKvmCPU::setupInstStop() void BaseKvmCPU::setupInstCounter(uint64_t period) { + // This function is for setting up instruction counter using perf + if (!usePerf) { + return; + } + // No need to do anything if we aren't attaching for the first // time or the period isn't changing. - if (period == activeInstPeriod && hwInstructions.attached()) + if (period == activeInstPeriod && hwInstructions->attached()) { return; + } PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS); @@ -1366,15 +1404,15 @@ BaseKvmCPU::setupInstCounter(uint64_t period) // We need to detach and re-attach the counter to reliably change // sampling settings. See PerfKvmCounter::period() for details. - if (hwInstructions.attached()) - hwInstructions.detach(); - assert(hwCycles.attached()); - hwInstructions.attach(cfgInstructions, + if (hwInstructions->attached()) + hwInstructions->detach(); + assert(hwCycles->attached()); + hwInstructions->attach(cfgInstructions, 0, // TID (0 => currentThread) - hwCycles); + *hwCycles); if (period) - hwInstructions.enableSignals(KVM_KICK_SIGNAL); + hwInstructions->enableSignals(KVM_KICK_SIGNAL); activeInstPeriod = period; } diff --git a/src/cpu/kvm/base.hh b/src/cpu/kvm/base.hh index 7bbf393f9b..3cf70a0bef 100644 --- a/src/cpu/kvm/base.hh +++ b/src/cpu/kvm/base.hh @@ -653,6 +653,9 @@ class BaseKvmCPU : public BaseCPU */ bool kvmStateDirty; + /** True if using perf; False otherwise*/ + bool usePerf; + /** KVM internal ID of the vCPU */ long vcpuID; @@ -763,7 +766,7 @@ class BaseKvmCPU : public BaseCPU * PerfKvmTimer (see perfControlledByTimer) to trigger exits from * KVM. */ - PerfKvmCounter hwCycles; + std::unique_ptr hwCycles; /** * Guest instruction counter. @@ -776,7 +779,7 @@ class BaseKvmCPU : public BaseCPU * @see setupInstBreak * @see scheduleInstStop */ - PerfKvmCounter hwInstructions; + std::unique_ptr hwInstructions; /** * Does the runTimer control the performance counters? diff --git a/src/cpu/kvm/perfevent.cc b/src/cpu/kvm/perfevent.cc index f9c317da41..c5e33abf82 100644 --- a/src/cpu/kvm/perfevent.cc +++ b/src/cpu/kvm/perfevent.cc @@ -173,12 +173,20 @@ PerfKvmCounter::attach(PerfKvmCounterConfig &config, { if (errno == EACCES) { - panic("PerfKvmCounter::attach recieved error EACCESS\n" + panic("PerfKvmCounter::attach received error EACCESS.\n" " This error may be caused by a too restrictive setting\n" - " in the file '/proc/sys/kernel/perf_event_paranoid'\n" - " The default value was changed to 2 in kernel 4.6\n" + " in the file '/proc/sys/kernel/perf_event_paranoid'.\n" + " The default value was changed to 2 in kernel 4.6.\n" " A value greater than 1 prevents gem5 from making\n" - " the syscall to perf_event_open"); + " the syscall to perf_event_open.\n" + " Alternatively, you can set the usePerf flag of the KVM\n" + " CPU to False. Setting this flag to False will limit some\n" + " functionalities of KVM CPU, such as counting the number of\n" + " cycles and the number of instructions, as well as the\n" + " ability of exiting to gem5 after a certain amount of cycles\n" + " or instructions when using KVM CPU. An example can be found\n" + " here, configs/example/gem5_library/" + "x86-ubuntu-run-with-kvm-no-perf.py."); } panic("PerfKvmCounter::attach failed (%i)\n", errno); }