diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46c69860..53c44f12 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ image: ubuntu before_script: - apt-get update --yes - - apt-get install --yes build-essential git cmake python3 python3-dev qtbase5-dev libqwt-qt5-dev sqlite3 lcov + - apt-get install --yes build-essential git cmake python3 python3-dev qtbase5-dev libqwt-qt5-dev sqlite3 sqlite3-tools lcov stages: - build diff --git a/CMakePresets.json b/CMakePresets.json index 96d88df9..6acf65ad 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -22,7 +22,8 @@ "inherits": "cmake-pedantic", "cacheVariables": { "DRAMSYS_BUILD_TESTS": "ON", - "DRAMSYS_ENABLE_EXTENSIONS": "ON" + "DRAMSYS_ENABLE_EXTENSIONS": "ON", + "DRAMSYS_WITH_DRAMPOWER": "ON" } }, { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cd1d5ece..4702149b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,4 +2,5 @@ set(DRAMSYS_TEST_RESOURCES_DIR "${CMAKE_CURRENT_SOURCE_DIR}/resources") add_subdirectory(tests_configuration) add_subdirectory(tests_dramsys) +add_subdirectory(tests_regression) add_subdirectory(tests_util) diff --git a/tests/tests_regression/CMakeLists.txt b/tests/tests_regression/CMakeLists.txt new file mode 100644 index 00000000..779e2a42 --- /dev/null +++ b/tests/tests_regression/CMakeLists.txt @@ -0,0 +1,89 @@ +# Copyright (c) 2023, Technische Universität Kaiserslautern +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER +# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: +# Derek Christ + +############################################### +### tests_regression ### +############################################### + +cmake_minimum_required(VERSION 3.1.0) + +project(tests_regression) + +find_program(Bash bash) +find_program(SqlDiff sqldiff) + +if(NOT Bash OR NOT SqlDiff) + message(WARNING "Regression tests require bash and sqldiff to be installed") + return() +endif() + +set(TABLES_TO_COMPARE + Phases + Transactions + Power +) + +function(test_standard standard base_config resource_dir output_filename) + configure_file(compare.sh compare-${standard}.sh) + + # Test to create database + add_test( + NAME Regression${standard}.CreateDatabase + COMMAND $ ${base_config} ${resource_dir} + ) + set_tests_properties(Regression${standard}.CreateDatabase PROPERTIES FIXTURES_SETUP Regression${standard}.CreateDatabase) + + # Test to diff the whole database + add_test( + NAME Regression${standard}.SqlDiff + COMMAND compare-${standard}.sh + ) + set_tests_properties(Regression${standard}.SqlDiff PROPERTIES FIXTURES_REQUIRED Regression${standard}.CreateDatabase) + + # Tests to diff individual tables + foreach(table IN LISTS TABLES_TO_COMPARE) + configure_file(compare_table.sh compare_table-${table}-${standard}.sh) + + add_test( + NAME Regression${standard}.SqlDiff.${table} + COMMAND compare_table-${table}-${standard}.sh + ) + set_tests_properties(Regression${standard}.SqlDiff.${table} PROPERTIES FIXTURES_REQUIRED Regression${standard}.CreateDatabase) + endforeach() +endfunction() + +test_standard(DDR3 ${CMAKE_CURRENT_SOURCE_DIR}/DDR3/ddr3-example.json ${CMAKE_CURRENT_SOURCE_DIR}/DDR3 DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb) +test_standard(DDR4 ${CMAKE_CURRENT_SOURCE_DIR}/DDR4/ddr4-example.json ${CMAKE_CURRENT_SOURCE_DIR}/DDR4 DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb) +test_standard(LPDDR4 ${CMAKE_CURRENT_SOURCE_DIR}/LPDDR4/lpddr4-example.json ${CMAKE_CURRENT_SOURCE_DIR}/LPDDR4 DRAMSys_lpddr4-example_lpddr4_ch0.tdb) +test_standard(HBM2.Ch0 ${CMAKE_CURRENT_SOURCE_DIR}/HBM2/hbm2-example.json ${CMAKE_CURRENT_SOURCE_DIR}/HBM2 DRAMSys_hbm2-example_hbm2_ch0.tdb) +test_standard(HBM2.Ch1 ${CMAKE_CURRENT_SOURCE_DIR}/HBM2/hbm2-example.json ${CMAKE_CURRENT_SOURCE_DIR}/HBM2 DRAMSys_hbm2-example_hbm2_ch1.tdb) diff --git a/tests/tests_regression/DDR3/ci.yml b/tests/tests_regression/DDR3/ci.yml deleted file mode 100644 index 903121f4..00000000 --- a/tests/tests_regression/DDR3/ci.yml +++ /dev/null @@ -1,30 +0,0 @@ -# DDR3 Dual Rank Test with Staggered Power Down Policy and Scheduler FrFcfsGrp -test_DDR3: - stage: test - needs: - - build - script: - - export GCOV_PREFIX=$(pwd) - - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}') - - cd build/bin - - ./DRAMSys ../../tests/tests_regression/DDR3/ddr3-example.json ../../tests/tests_regression/DDR3/ - - ls -lah - - ls -lah ../../tests/tests_regression/DDR3/expected/ - - sqldiff ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb - - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - # Run Code Coverage - - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out - - cache: - key: build - paths: - - build/ - policy: pull - - artifacts: - paths: - - build/bin/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb - - coverage/${CI_JOB_NAME}.out - expire_in: 2 days diff --git a/tests/tests_regression/DDR4/ci.yml b/tests/tests_regression/DDR4/ci.yml deleted file mode 100644 index 4d7dab12..00000000 --- a/tests/tests_regression/DDR4/ci.yml +++ /dev/null @@ -1,30 +0,0 @@ -# DDR4 with 4 bank groups, flexible rankwise refresh and FrFcfs scheduler: -test_DDR4: - stage: test - needs: - - build - script: - - export GCOV_PREFIX=$(pwd) - - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}') - - cd build/bin - - ./DRAMSys ../../tests/tests_regression/DDR4/ddr4-example.json ../../tests/tests_regression/DDR4/ - - ls -lah - - ls -lah ../../tests/tests_regression/DDR4/expected/ - - sqldiff ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb - - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - # Run Code Coverage - - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out - - cache: - key: build - paths: - - build/ - policy: pull - - artifacts: - paths: - - build/bin/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb - - coverage/${CI_JOB_NAME}.out - expire_in: 2 days diff --git a/tests/tests_regression/HBM2/ci.yml b/tests/tests_regression/HBM2/ci.yml deleted file mode 100644 index 8a604ca0..00000000 --- a/tests/tests_regression/HBM2/ci.yml +++ /dev/null @@ -1,32 +0,0 @@ -test_HBM2: - stage: test - needs: - - build - script: - - export GCOV_PREFIX=$(pwd) - - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}') - - cd build/bin - - ./DRAMSys ../../tests/tests_regression/HBM2/hbm2-example.json ../../tests/tests_regression/HBM2/ - - ls -lah - - ls -lah ../../tests/tests_regression/HBM2/expected/ - - sqldiff ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb - - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - sqldiff ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb - - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}' - - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out - cache: - key: build - paths: - - build/ - policy: pull - - artifacts: - paths: - - build/bin/DRAMSys_hbm2-example_hbm2_ch0.tdb - - build/bin/DRAMSys_hbm2-example_hbm2_ch1.tdb - - coverage/${CI_JOB_NAME}.out - expire_in: 2 days diff --git a/tests/tests_regression/LPDDR4/ci.yml b/tests/tests_regression/LPDDR4/ci.yml deleted file mode 100644 index 4c1d7261..00000000 --- a/tests/tests_regression/LPDDR4/ci.yml +++ /dev/null @@ -1,30 +0,0 @@ -# LPDDR4 with Bankwise Flexible Refresh and FIFO Scheduler: -test_LPDDR4: - stage: test - needs: - - build - script: - - export GCOV_PREFIX=$(pwd) - - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}') - - cd build/bin - - ./DRAMSys ../../tests/tests_regression/LPDDR4/lpddr4-example.json ../../tests/tests_regression/LPDDR4/ - - ls -lah - - ls -lah ../../tests/tests_regression/LPDDR4/expected/ - - sqldiff ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb - - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}' - # Run Code Coverage - - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out - - cache: - key: build - paths: - - build/ - policy: pull - - artifacts: - paths: - - build/bin/DRAMSys_lpddr4-example_lpddr4_ch0.tdb - - coverage/${CI_JOB_NAME}.out - expire_in: 2 days diff --git a/tests/tests_regression/compare.sh b/tests/tests_regression/compare.sh new file mode 100755 index 00000000..e5262576 --- /dev/null +++ b/tests/tests_regression/compare.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ $(sqldiff ${CMAKE_CURRENT_SOURCE_DIR}/${standard}/expected/${output_filename} ${output_filename}) ]]; then + exit -1 +else + exit 0 +fi diff --git a/tests/tests_regression/compare_table.sh b/tests/tests_regression/compare_table.sh new file mode 100755 index 00000000..8978e49d --- /dev/null +++ b/tests/tests_regression/compare_table.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ $(sqldiff --table ${table} ${CMAKE_CURRENT_SOURCE_DIR}/${standard}/expected/${output_filename} ${output_filename}) ]]; then + exit -1 +else + exit 0 +fi