Integrate regression tests with CTest
This commit is contained in:
@@ -3,7 +3,7 @@ image: ubuntu
|
||||
|
||||
before_script:
|
||||
- apt-get update --yes
|
||||
- apt-get install --yes build-essential git cmake python3 python3-dev qtbase5-dev libqwt-qt5-dev sqlite3 lcov
|
||||
- apt-get install --yes build-essential git cmake python3 python3-dev qtbase5-dev libqwt-qt5-dev sqlite3 sqlite3-tools lcov
|
||||
|
||||
stages:
|
||||
- build
|
||||
|
||||
@@ -22,7 +22,8 @@
|
||||
"inherits": "cmake-pedantic",
|
||||
"cacheVariables": {
|
||||
"DRAMSYS_BUILD_TESTS": "ON",
|
||||
"DRAMSYS_ENABLE_EXTENSIONS": "ON"
|
||||
"DRAMSYS_ENABLE_EXTENSIONS": "ON",
|
||||
"DRAMSYS_WITH_DRAMPOWER": "ON"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -2,4 +2,5 @@ set(DRAMSYS_TEST_RESOURCES_DIR "${CMAKE_CURRENT_SOURCE_DIR}/resources")
|
||||
|
||||
add_subdirectory(tests_configuration)
|
||||
add_subdirectory(tests_dramsys)
|
||||
add_subdirectory(tests_regression)
|
||||
add_subdirectory(tests_util)
|
||||
|
||||
89
tests/tests_regression/CMakeLists.txt
Normal file
89
tests/tests_regression/CMakeLists.txt
Normal file
@@ -0,0 +1,89 @@
|
||||
# Copyright (c) 2023, Technische Universität Kaiserslautern
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
|
||||
# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
# Authors:
|
||||
# Derek Christ
|
||||
|
||||
###############################################
|
||||
### tests_regression ###
|
||||
###############################################
|
||||
|
||||
cmake_minimum_required(VERSION 3.1.0)
|
||||
|
||||
project(tests_regression)
|
||||
|
||||
find_program(Bash bash)
|
||||
find_program(SqlDiff sqldiff)
|
||||
|
||||
if(NOT Bash OR NOT SqlDiff)
|
||||
message(WARNING "Regression tests require bash and sqldiff to be installed")
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(TABLES_TO_COMPARE
|
||||
Phases
|
||||
Transactions
|
||||
Power
|
||||
)
|
||||
|
||||
function(test_standard standard base_config resource_dir output_filename)
|
||||
configure_file(compare.sh compare-${standard}.sh)
|
||||
|
||||
# Test to create database
|
||||
add_test(
|
||||
NAME Regression${standard}.CreateDatabase
|
||||
COMMAND $<TARGET_FILE:DRAMSys> ${base_config} ${resource_dir}
|
||||
)
|
||||
set_tests_properties(Regression${standard}.CreateDatabase PROPERTIES FIXTURES_SETUP Regression${standard}.CreateDatabase)
|
||||
|
||||
# Test to diff the whole database
|
||||
add_test(
|
||||
NAME Regression${standard}.SqlDiff
|
||||
COMMAND compare-${standard}.sh
|
||||
)
|
||||
set_tests_properties(Regression${standard}.SqlDiff PROPERTIES FIXTURES_REQUIRED Regression${standard}.CreateDatabase)
|
||||
|
||||
# Tests to diff individual tables
|
||||
foreach(table IN LISTS TABLES_TO_COMPARE)
|
||||
configure_file(compare_table.sh compare_table-${table}-${standard}.sh)
|
||||
|
||||
add_test(
|
||||
NAME Regression${standard}.SqlDiff.${table}
|
||||
COMMAND compare_table-${table}-${standard}.sh
|
||||
)
|
||||
set_tests_properties(Regression${standard}.SqlDiff.${table} PROPERTIES FIXTURES_REQUIRED Regression${standard}.CreateDatabase)
|
||||
endforeach()
|
||||
endfunction()
|
||||
|
||||
test_standard(DDR3 ${CMAKE_CURRENT_SOURCE_DIR}/DDR3/ddr3-example.json ${CMAKE_CURRENT_SOURCE_DIR}/DDR3 DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb)
|
||||
test_standard(DDR4 ${CMAKE_CURRENT_SOURCE_DIR}/DDR4/ddr4-example.json ${CMAKE_CURRENT_SOURCE_DIR}/DDR4 DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb)
|
||||
test_standard(LPDDR4 ${CMAKE_CURRENT_SOURCE_DIR}/LPDDR4/lpddr4-example.json ${CMAKE_CURRENT_SOURCE_DIR}/LPDDR4 DRAMSys_lpddr4-example_lpddr4_ch0.tdb)
|
||||
test_standard(HBM2.Ch0 ${CMAKE_CURRENT_SOURCE_DIR}/HBM2/hbm2-example.json ${CMAKE_CURRENT_SOURCE_DIR}/HBM2 DRAMSys_hbm2-example_hbm2_ch0.tdb)
|
||||
test_standard(HBM2.Ch1 ${CMAKE_CURRENT_SOURCE_DIR}/HBM2/hbm2-example.json ${CMAKE_CURRENT_SOURCE_DIR}/HBM2 DRAMSys_hbm2-example_hbm2_ch1.tdb)
|
||||
@@ -1,30 +0,0 @@
|
||||
# DDR3 Dual Rank Test with Staggered Power Down Policy and Scheduler FrFcfsGrp
|
||||
test_DDR3:
|
||||
stage: test
|
||||
needs:
|
||||
- build
|
||||
script:
|
||||
- export GCOV_PREFIX=$(pwd)
|
||||
- export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
|
||||
- cd build/bin
|
||||
- ./DRAMSys ../../tests/tests_regression/DDR3/ddr3-example.json ../../tests/tests_regression/DDR3/
|
||||
- ls -lah
|
||||
- ls -lah ../../tests/tests_regression/DDR3/expected/
|
||||
- sqldiff ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb
|
||||
- perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
# Run Code Coverage
|
||||
- lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
|
||||
|
||||
cache:
|
||||
key: build
|
||||
paths:
|
||||
- build/
|
||||
policy: pull
|
||||
|
||||
artifacts:
|
||||
paths:
|
||||
- build/bin/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb
|
||||
- coverage/${CI_JOB_NAME}.out
|
||||
expire_in: 2 days
|
||||
@@ -1,30 +0,0 @@
|
||||
# DDR4 with 4 bank groups, flexible rankwise refresh and FrFcfs scheduler:
|
||||
test_DDR4:
|
||||
stage: test
|
||||
needs:
|
||||
- build
|
||||
script:
|
||||
- export GCOV_PREFIX=$(pwd)
|
||||
- export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
|
||||
- cd build/bin
|
||||
- ./DRAMSys ../../tests/tests_regression/DDR4/ddr4-example.json ../../tests/tests_regression/DDR4/
|
||||
- ls -lah
|
||||
- ls -lah ../../tests/tests_regression/DDR4/expected/
|
||||
- sqldiff ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb
|
||||
- perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
# Run Code Coverage
|
||||
- lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
|
||||
|
||||
cache:
|
||||
key: build
|
||||
paths:
|
||||
- build/
|
||||
policy: pull
|
||||
|
||||
artifacts:
|
||||
paths:
|
||||
- build/bin/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb
|
||||
- coverage/${CI_JOB_NAME}.out
|
||||
expire_in: 2 days
|
||||
@@ -1,32 +0,0 @@
|
||||
test_HBM2:
|
||||
stage: test
|
||||
needs:
|
||||
- build
|
||||
script:
|
||||
- export GCOV_PREFIX=$(pwd)
|
||||
- export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
|
||||
- cd build/bin
|
||||
- ./DRAMSys ../../tests/tests_regression/HBM2/hbm2-example.json ../../tests/tests_regression/HBM2/
|
||||
- ls -lah
|
||||
- ls -lah ../../tests/tests_regression/HBM2/expected/
|
||||
- sqldiff ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb
|
||||
- perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- sqldiff ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb
|
||||
- perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
|
||||
cache:
|
||||
key: build
|
||||
paths:
|
||||
- build/
|
||||
policy: pull
|
||||
|
||||
artifacts:
|
||||
paths:
|
||||
- build/bin/DRAMSys_hbm2-example_hbm2_ch0.tdb
|
||||
- build/bin/DRAMSys_hbm2-example_hbm2_ch1.tdb
|
||||
- coverage/${CI_JOB_NAME}.out
|
||||
expire_in: 2 days
|
||||
@@ -1,30 +0,0 @@
|
||||
# LPDDR4 with Bankwise Flexible Refresh and FIFO Scheduler:
|
||||
test_LPDDR4:
|
||||
stage: test
|
||||
needs:
|
||||
- build
|
||||
script:
|
||||
- export GCOV_PREFIX=$(pwd)
|
||||
- export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
|
||||
- cd build/bin
|
||||
- ./DRAMSys ../../tests/tests_regression/LPDDR4/lpddr4-example.json ../../tests/tests_regression/LPDDR4/
|
||||
- ls -lah
|
||||
- ls -lah ../../tests/tests_regression/LPDDR4/expected/
|
||||
- sqldiff ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb
|
||||
- perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
- perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
|
||||
# Run Code Coverage
|
||||
- lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
|
||||
|
||||
cache:
|
||||
key: build
|
||||
paths:
|
||||
- build/
|
||||
policy: pull
|
||||
|
||||
artifacts:
|
||||
paths:
|
||||
- build/bin/DRAMSys_lpddr4-example_lpddr4_ch0.tdb
|
||||
- coverage/${CI_JOB_NAME}.out
|
||||
expire_in: 2 days
|
||||
7
tests/tests_regression/compare.sh
Executable file
7
tests/tests_regression/compare.sh
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ $(sqldiff ${CMAKE_CURRENT_SOURCE_DIR}/${standard}/expected/${output_filename} ${output_filename}) ]]; then
|
||||
exit -1
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
7
tests/tests_regression/compare_table.sh
Executable file
7
tests/tests_regression/compare_table.sh
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ $(sqldiff --table ${table} ${CMAKE_CURRENT_SOURCE_DIR}/${standard}/expected/${output_filename} ${output_filename}) ]]; then
|
||||
exit -1
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user