From f467cf79fda2884f5d6d26ff616269822fac5c48 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Tue, 30 May 2023 16:44:32 -0700 Subject: [PATCH] util: Add util for GitHub runner configuration This adds files that can be used to configure Vagrant machines that will be used to test running gem5 alongside Github Actions. Change-Id: I52b0f39b6e6044c22481f02163d5fc01eab76788 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71098 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce Tested-by: kokoro --- util/github-runners-vagrant/README.md | 118 ++++++++++++++++++ .../Vagrantfile-builder | 49 ++++++++ .../github-runners-vagrant/Vagrantfile-runner | 48 +++++++ .../provision_nonroot.sh | 15 +++ util/github-runners-vagrant/provision_root.sh | 43 +++++++ util/github-runners-vagrant/vm_manager.sh | 39 ++++++ 6 files changed, 312 insertions(+) create mode 100644 util/github-runners-vagrant/README.md create mode 100644 util/github-runners-vagrant/Vagrantfile-builder create mode 100644 util/github-runners-vagrant/Vagrantfile-runner create mode 100644 util/github-runners-vagrant/provision_nonroot.sh create mode 100644 util/github-runners-vagrant/provision_root.sh create mode 100755 util/github-runners-vagrant/vm_manager.sh diff --git a/util/github-runners-vagrant/README.md b/util/github-runners-vagrant/README.md new file mode 100644 index 0000000000..86b9e1da87 --- /dev/null +++ b/util/github-runners-vagrant/README.md @@ -0,0 +1,118 @@ +# Setting up a Github Actions Runner with Vagrant + +This directory provides a way to setup a Github Actions runner using Vagrant to host the runner in a Virtual machine. + +This tutorial has been written with the assumption of running on a machine with Ubuntu 22.04. +Setting up a runner on a different OS may require some changes. + +Before anything else, copy this directory, "util/github-runners-vagrant", to the root of the location on your host system you wish to setup the VMs from. +The CWD is assumed to be this directory. + +## Install Dependencies + +```sh +sudo apt install vagrant +sudo apt-get build-dep vagrant ruby-libvirt +sudo apt-get install qemu libvirt-daemon-system libvirt-clients ebtables dnsmasq-base libxslt-dev libxml2-dev libvirt-dev zlib1g-dev ruby-dev + +# Note: The vagrant-libvirt APT package does not work as intended. We must +# remove it from the system otherwise errors will occur (we will install it +# later using the vagrant plugin command). +sudo apt purge vagrant-libvirt +``` + +## Set up the Vagrantfiles for the GitHub repository + +First, generate a Personal Access Token, which you can create [here](https://github.com/settings/tokens) +Make sure to set admin permissions on this token, then replace instances of `` in the Vagrantfiles ("Vagrantfile-builder" and "Vagrant-runner") with your token. + +Next, replace instances of `` with your GitHub account name and the repository name, separated by a forward slash. +For example, if your GitHub account name is `example` and your repository name is `example-repo`, you would replace `` with `example/example-repo`. + +## Install Vagrant Plugins + +Once everything is set properly, set the `VAGRANT_HOME` environment variable to the directory in which the Vagrant files and other scripts are stored (i.e., the CWD). +For example: + +```sh +export VAGRANT_HOME=`pwd` +``` + +After this, install the relevant vagrant plugins: + +``` sh +vagrant plugin install dotenv +vagrant plugin install vagrant-libvirt +vagrant plugin install vagrant-reload +``` + +## The "builder" and "runner" VMs + +The number of CPUs and the memory size differs between the "Vagrantfile-builder" and "Vagrantfile-runner". + +In our work we have two types of machines "runners" and "builders". +Runners are single core machines with 8GB of memory, and builders are 4 core machines with 16GB of memory. +The latter is used for building gem5 binaries while the former is used for running instances of gem5. +You can expect each machine to take up approximately 60GB of disk space though VMs will consume the disk space they require. + +The "Vagrantfile-builder" file is set to create a runner machine and the "Vagrantfile-builder" file is set to create a builder machine. + +Specifying which Vagrantfile to use is done by setting the `VAGRANT_VAGRANTFILE` environment variable. + +## Creating the virtual machine + +Each VM on your host system must have a unique name. +Give the VM to be created a unique name by setting the `` variables in the Vagrantfile you wish to utilize. + +Then run: + +```sh +VAGRANT_VAGRANTFILE= vagrant up --provider=libvirt +``` + +This should automatically create your machine, as well as configure and start up a Github Actions runner. +You can check the status of the runner here: https://github.com///settings/actions/runners + +If the runner ever shows as offline, you can rerun the `vagrant up --provider=libvirt` command to make sure everything is working properly. + +If you wish to create more than one runner you must edit the `` in the Vagrant file. + +## Helper scripts + +The "vm_manager" script can be used to set up multiple builder and runner VMs. +To use this script simply modify the `NUM_RUNNERS`, `NUM_BUILDERS`, `RUNNER_PREFIX`, and `BUILDER_PREFIX` variables to the desired values. +Then run the script with: + +```sh +./vm_manager.sh +``` + +This script will create any VMs that don't already exist and ensure those that do exists are running. + +If you wish to destroy all the VMs you can run: + +```sh +./vm_manager.sh destroy +``` + +**Note:** This script assumes "VAGRANT_HOME" is set to the CWD. + +## Troubleshooting + +### The default libvirt disk image storage pool is on the wrong drive + +By default libvirt will store disk images in "/var/lib/libvirt/images". +This is not ideal as it is on a small root partition. +A solution to this is to change the default storage location. +To do so, do the following: + +```sh +virsh pool-list --all # Confirm here a "default" pool exist. We'll modify this. +virsh pool-dumpxml default >default-pool.xml # We take a dump of the default then removed it. +virsh pool-destroy default +virsh pool-undefine default +vim default-pool.xml # Change the image path to the desired path +virsh pool-define default-pool.xml # From here we re-add the default. +virsh pool-start default +virsh pool-autostart default +``` diff --git a/util/github-runners-vagrant/Vagrantfile-builder b/util/github-runners-vagrant/Vagrantfile-builder new file mode 100644 index 0000000000..242e094d19 --- /dev/null +++ b/util/github-runners-vagrant/Vagrantfile-builder @@ -0,0 +1,49 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "generic/ubuntu2204" + config.vm.box_check_update = true + config.vm.define "" + config.vm.hostname = "" + # allows us to ssh into the machine, addressing the problem below + # https://www.reddit.com/r/vagrant/comments/sb7hfl/new_to_vagrant_getting_efault_warning/ + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider "libvirt" do |vb| + # Customize the amount of cpus and memory on the VM: + vb.cpus = "4".to_i + vb.memory = "16384".to_i + end + + # sets up vm + config.vm.provision :shell, path: "provision_root.sh" + config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" + # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. + config.vm.provision :reload + config.vm.provision :shell, privileged: false, run: 'always', inline: <<-SHELL + if [ -d ~/actions-runner ]; then + # This will be run everytime the VM is run (once created). + cd actions-runner + nohup ./run.sh & + else + # This will be run the first time the VM is created. + mkdir ~/actions-runner && cd ~/actions-runner + curl -so actions-runner-linux-x64-2.304.0.tar.gz -L \ + https://github.com/actions/runner/releases/download/v2.304.0/actions-runner-linux-x64-2.304.0.tar.gz + tar xzf ./actions-runner-linux-x64-2.304.0.tar.gz + + # configure the runner + # echo automatically sets the name of the runner, and the tags + # create a personal access token with admin permission and copy it into the curl command + echo -ne '\n\nbuild\n\n' | ./config.sh --url https://github.com/ --token $(curl -L \ + -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer " \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos//actions/runners/registration-token | jq -r '.token') + # start the runner + nohup ./run.sh & + fi + SHELL + +end diff --git a/util/github-runners-vagrant/Vagrantfile-runner b/util/github-runners-vagrant/Vagrantfile-runner new file mode 100644 index 0000000000..ab601cd9c8 --- /dev/null +++ b/util/github-runners-vagrant/Vagrantfile-runner @@ -0,0 +1,48 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "generic/ubuntu2204" + config.vm.box_check_update = true + config.vm.define "" + config.vm.hostname = "" + # allows us to ssh into the machine, addressing the problem below + # https://www.reddit.com/r/vagrant/comments/sb7hfl/new_to_vagrant_getting_efault_warning/ + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider "libvirt" do |vb| + # Customize the amount of cpus and memory on the VM: + vb.cpus = "1".to_i + vb.memory = "8192".to_i + end + + # sets up vm + config.vm.provision :shell, path: "provision_root.sh" + config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" + # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. + config.vm.provision :reload + config.vm.provision :shell, privileged: false, run: 'always', inline: <<-SHELL + if [ -d ~/actions-runner ]; then + # This will be run everytime the VM is run (once created). + cd actions-runner + nohup ./run.sh & + else + # This will be run the first time the VM is created. + mkdir ~/actions-runner && cd ~/actions-runner + curl -so actions-runner-linux-x64-2.304.0.tar.gz -L \ + https://github.com/actions/runner/releases/download/v2.304.0/actions-runner-linux-x64-2.304.0.tar.gz + tar xzf ./actions-runner-linux-x64-2.304.0.tar.gz + + # configure the runner + # echo automatically sets the name of the runner, and the tags + # create a personal access token with admin permission and copy it into the curl command + echo -ne '\n\nrun\n\n' | ./config.sh --url https://github.com/ --token $(curl -L \ + -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer " \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos//actions/runners/registration-token | jq -r '.token') + # start the runner + nohup ./run.sh & + fi + SHELL +end diff --git a/util/github-runners-vagrant/provision_nonroot.sh b/util/github-runners-vagrant/provision_nonroot.sh new file mode 100644 index 0000000000..4465b5a192 --- /dev/null +++ b/util/github-runners-vagrant/provision_nonroot.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# fail on unset variables and command errors +set -eu -o pipefail # -x: is for debugging + +# Install deno +curl -fsSL https://deno.land/x/install/install.sh | sh +echo "export PATH=\"\${HOME}/.deno/bin:\${PATH}\"" >> ~/.profile +echo "export PATH=\"\${HOME}/.deno/bin:\${PATH}\"" >> ~/.bash_profile + +# Install docker compose +DOCKER_COMPOSE_VERSION=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.tag_name') +mkdir -p "${HOME}/.docker/cli-plugins" +curl -sL "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o "${HOME}/.docker/cli-plugins/docker-compose" +chmod +x "${HOME}/.docker/cli-plugins/docker-compose" diff --git a/util/github-runners-vagrant/provision_root.sh b/util/github-runners-vagrant/provision_root.sh new file mode 100644 index 0000000000..660064f8ca --- /dev/null +++ b/util/github-runners-vagrant/provision_root.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# fail on unset variables and command errors +set -eu -o pipefail # -x: is for debugging + +apt-get update +apt-get upgrade -y +apt-get install -y software-properties-common +add-apt-repository --yes --update ppa:git-core/ppa +apt-get install -y \ + bash \ + build-essential \ + clang-format \ + git \ + git-lfs \ + jq \ + libffi-dev \ + libssl-dev \ + nkf \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + shellcheck \ + tree \ + wget \ + yamllint \ + zstd +snap install jq + +# Install docker +apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null +apt-get update -y +apt-get install -y docker-ce docker-ce-cli containerd.io +groupadd docker || true +gpasswd -a vagrant docker +newgrp docker +systemctl restart docker + +# Cleanup +apt-get autoremove -y diff --git a/util/github-runners-vagrant/vm_manager.sh b/util/github-runners-vagrant/vm_manager.sh new file mode 100755 index 0000000000..1cae8d2686 --- /dev/null +++ b/util/github-runners-vagrant/vm_manager.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +NUM_RUNNERS=20 +NUM_BUILDERS=3 +RUNNER_PREFIX="$(hostname)-runner-" +BUILDER_PREFIX="$(hostname)-builder-" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +export VAGRANT_HOME=${SCRIPT_DIR} + +param="up" +if [[ $# -eq 1 ]]; then + param=$1 + if [[ "${param}" != "destroy" ]]; then + echo "Only valid parameter is 'destroy', to destroy all VMs" + exit 1 + fi +fi + + +for (( i=1; i<=NUM_RUNNERS; i++ )); do + sed -i "s/ config.vm.define.*/ config.vm.define \"${RUNNER_PREFIX}${i}\"/g" Vagrantfile-runner + sed -i "s/ config.vm.hostname.*/ config.vm.hostname = \"${RUNNER_PREFIX}${i}\"/g" Vagrantfile-runner + if [[ "${param}" == "destroy" ]]; then + VAGRANT_VAGRANTFILE=Vagrantfile-runner vagrant destroy -f + else + VAGRANT_VAGRANTFILE=Vagrantfile-runner vagrant up --provider=libvirt + fi +done + +for (( i=1; i<=NUM_BUILDERS; i++ )); do + sed -i "s/ config.vm.define.*/ config.vm.define \"${BUILDER_PREFIX}${i}\"/g" Vagrantfile-builder + sed -i "s/ config.vm.hostname.*/ config.vm.hostname = \"${BUILDER_PREFIX}${i}\"/g" Vagrantfile-builder + if [[ "${param}" == "destroy" ]]; then + VAGRANT_VAGRANTFILE=Vagrantfile-builder vagrant destroy -f + else + VAGRANT_VAGRANTFILE=Vagrantfile-builder vagrant up --provider=libvirt + fi +done