Integrate regression tests with CTest

.gitlab-ci.yml

@@ -3,7 +3,7 @@ image: ubuntu
 
 before_script:
     - apt-get update --yes
-    - apt-get install --yes build-essential git cmake python3 python3-dev qtbase5-dev libqwt-qt5-dev sqlite3 lcov
+    - apt-get install --yes build-essential git cmake python3 python3-dev qtbase5-dev libqwt-qt5-dev sqlite3 sqlite3-tools lcov
 
 stages:
     - build

CMakePresets.json

@@ -22,7 +22,8 @@
             "inherits": "cmake-pedantic",
             "cacheVariables": {
                 "DRAMSYS_BUILD_TESTS": "ON",
-                "DRAMSYS_ENABLE_EXTENSIONS": "ON"
+                "DRAMSYS_ENABLE_EXTENSIONS": "ON",
+                "DRAMSYS_WITH_DRAMPOWER": "ON"
             }
         },
         {

tests/CMakeLists.txt

@@ -2,4 +2,5 @@ set(DRAMSYS_TEST_RESOURCES_DIR "${CMAKE_CURRENT_SOURCE_DIR}/resources")
 
 add_subdirectory(tests_configuration)
 add_subdirectory(tests_dramsys)
+add_subdirectory(tests_regression)
 add_subdirectory(tests_util)

tests/tests_regression/CMakeLists.txt

@@ -0,0 +1,89 @@
+# Copyright (c) 2023, Technische Universität Kaiserslautern
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
+# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: 
+#    Derek Christ
+
+###############################################
+###             tests_regression            ###
+###############################################
+
+cmake_minimum_required(VERSION 3.1.0)
+
+project(tests_regression)
+
+find_program(Bash bash)
+find_program(SqlDiff sqldiff)
+
+if(NOT Bash OR NOT SqlDiff)
+    message(WARNING "Regression tests require bash and sqldiff to be installed")
+    return()
+endif()
+
+set(TABLES_TO_COMPARE
+    Phases
+    Transactions
+    Power
+)
+
+function(test_standard standard base_config resource_dir output_filename)
+    configure_file(compare.sh compare-${standard}.sh)
+
+    # Test to create database
+    add_test(
+        NAME Regression${standard}.CreateDatabase
+        COMMAND $<TARGET_FILE:DRAMSys> ${base_config} ${resource_dir}
+    )
+    set_tests_properties(Regression${standard}.CreateDatabase PROPERTIES FIXTURES_SETUP Regression${standard}.CreateDatabase)
+
+    # Test to diff the whole database
+    add_test(
+        NAME Regression${standard}.SqlDiff
+        COMMAND compare-${standard}.sh
+    )
+    set_tests_properties(Regression${standard}.SqlDiff PROPERTIES FIXTURES_REQUIRED Regression${standard}.CreateDatabase)
+
+    # Tests to diff individual tables
+    foreach(table IN LISTS TABLES_TO_COMPARE)
+        configure_file(compare_table.sh compare_table-${table}-${standard}.sh)
+
+        add_test(
+            NAME Regression${standard}.SqlDiff.${table}
+            COMMAND compare_table-${table}-${standard}.sh
+        )
+        set_tests_properties(Regression${standard}.SqlDiff.${table} PROPERTIES FIXTURES_REQUIRED Regression${standard}.CreateDatabase)
+    endforeach()
+endfunction()
+
+test_standard(DDR3 ${CMAKE_CURRENT_SOURCE_DIR}/DDR3/ddr3-example.json ${CMAKE_CURRENT_SOURCE_DIR}/DDR3 DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb)
+test_standard(DDR4 ${CMAKE_CURRENT_SOURCE_DIR}/DDR4/ddr4-example.json ${CMAKE_CURRENT_SOURCE_DIR}/DDR4 DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb)
+test_standard(LPDDR4 ${CMAKE_CURRENT_SOURCE_DIR}/LPDDR4/lpddr4-example.json ${CMAKE_CURRENT_SOURCE_DIR}/LPDDR4 DRAMSys_lpddr4-example_lpddr4_ch0.tdb)
+test_standard(HBM2.Ch0 ${CMAKE_CURRENT_SOURCE_DIR}/HBM2/hbm2-example.json ${CMAKE_CURRENT_SOURCE_DIR}/HBM2 DRAMSys_hbm2-example_hbm2_ch0.tdb)
+test_standard(HBM2.Ch1 ${CMAKE_CURRENT_SOURCE_DIR}/HBM2/hbm2-example.json ${CMAKE_CURRENT_SOURCE_DIR}/HBM2 DRAMSys_hbm2-example_hbm2_ch1.tdb)

tests/tests_regression/DDR3/ci.yml

@@ -1,30 +0,0 @@
-# DDR3 Dual Rank Test with Staggered Power Down Policy and Scheduler FrFcfsGrp
-test_DDR3:
-    stage: test
-    needs:
-        - build
-    script:
-        - export GCOV_PREFIX=$(pwd)
-        - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
-        - cd build/bin
-        - ./DRAMSys ../../tests/tests_regression/DDR3/ddr3-example.json ../../tests/tests_regression/DDR3/
-        - ls -lah
-        - ls -lah ../../tests/tests_regression/DDR3/expected/
-        - sqldiff ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb
-        - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/DDR3/expected/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        # Run Code Coverage
-        - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
- 
-    cache:
-        key: build
-        paths:
-            - build/
-        policy: pull
-
-    artifacts:
-        paths:
-          - build/bin/DRAMSys_ddr3-dual-rank_ddr3_ch0.tdb
-          - coverage/${CI_JOB_NAME}.out
-        expire_in: 2 days

tests/tests_regression/DDR4/ci.yml

@@ -1,30 +0,0 @@
-# DDR4 with 4 bank groups, flexible rankwise refresh and FrFcfs scheduler:
-test_DDR4:
-    stage: test
-    needs:
-        - build
-    script:
-        - export GCOV_PREFIX=$(pwd)
-        - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
-        - cd build/bin
-        - ./DRAMSys ../../tests/tests_regression/DDR4/ddr4-example.json ../../tests/tests_regression/DDR4/
-        - ls -lah
-        - ls -lah ../../tests/tests_regression/DDR4/expected/
-        - sqldiff ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb
-        - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/DDR4/expected/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        # Run Code Coverage
-        - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
- 
-    cache:
-        key: build
-        paths:
-            - build/
-        policy: pull
-
-    artifacts:
-        paths:
-          - build/bin/DRAMSys_ddr4-bankgrp_ddr4_ch0.tdb
-          - coverage/${CI_JOB_NAME}.out
-        expire_in: 2 days

tests/tests_regression/HBM2/ci.yml

@@ -1,32 +0,0 @@
-test_HBM2:
-    stage: test
-    needs:
-        - build
-    script: 
-        - export GCOV_PREFIX=$(pwd)
-        - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
-        - cd build/bin
-        - ./DRAMSys ../../tests/tests_regression/HBM2/hbm2-example.json ../../tests/tests_regression/HBM2/
-        - ls -lah
-        - ls -lah ../../tests/tests_regression/HBM2/expected/
-        - sqldiff ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb
-        - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch0.tdb DRAMSys_hbm2-example_hbm2_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - sqldiff ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb
-        - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/HBM2/expected/DRAMSys_hbm2-example_hbm2_ch1.tdb DRAMSys_hbm2-example_hbm2_ch1.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
-    cache: 
-        key: build
-        paths: 
-            - build/
-        policy: pull
-  
-    artifacts: 
-        paths: 
-            - build/bin/DRAMSys_hbm2-example_hbm2_ch0.tdb
-            - build/bin/DRAMSys_hbm2-example_hbm2_ch1.tdb
-            - coverage/${CI_JOB_NAME}.out
-        expire_in: 2 days

tests/tests_regression/LPDDR4/ci.yml

@@ -1,30 +0,0 @@
-# LPDDR4 with Bankwise Flexible Refresh and FIFO Scheduler:
-test_LPDDR4:
-    stage: test
-    needs:
-        - build
-    script:
-        - export GCOV_PREFIX=$(pwd)
-        - export GCOV_PREFIX_STRIP=$(pwd | awk -F"/" '{print NF-1}')
-        - cd build/bin
-        - ./DRAMSys ../../tests/tests_regression/LPDDR4/lpddr4-example.json ../../tests/tests_regression/LPDDR4/
-        - ls -lah
-        - ls -lah ../../tests/tests_regression/LPDDR4/expected/
-        - sqldiff ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb
-        - perl -e 'if(`sqldiff --table Phases ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Transactions ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        - perl -e 'if(`sqldiff --table Power ../../tests/tests_regression/LPDDR4/expected/DRAMSys_lpddr4-example_lpddr4_ch0.tdb DRAMSys_lpddr4-example_lpddr4_ch0.tdb` eq "") {exit(0)} else {exit(-1)}'
-        # Run Code Coverage
-        - lcov -q -c --rc geninfo_adjust_src_path=$GCOV_PREFIX -d ${CI_PROJECT_DIR}/build/ -o ${CI_PROJECT_DIR}/coverage/${CI_JOB_NAME}.out
- 
-    cache:
-        key: build
-        paths:
-            - build/
-        policy: pull
-
-    artifacts:
-        paths:
-          - build/bin/DRAMSys_lpddr4-example_lpddr4_ch0.tdb
-          - coverage/${CI_JOB_NAME}.out
-        expire_in: 2 days

tests/tests_regression/compare.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ $(sqldiff ${CMAKE_CURRENT_SOURCE_DIR}/${standard}/expected/${output_filename} ${output_filename}) ]]; then
+    exit -1
+else
+    exit 0
+fi

tests/tests_regression/compare_table.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ $(sqldiff --table ${table} ${CMAKE_CURRENT_SOURCE_DIR}/${standard}/expected/${output_filename} ${output_filename}) ]]; then
+    exit -1
+else
+    exit 0
+fi