From 279d3fca5c26fbaa94abf4bc6640a6082e1bf25f Mon Sep 17 00:00:00 2001
From: Jan Eitzinger <jan.eitzinger@fau.de>
Date: Tue, 8 Dec 2020 15:20:56 +0100
Subject: [PATCH] Pull out likwid intrumentation from benchmark functions.

---
 src/copy.c            | 14 ++++----------
 src/daxpy.c           |  2 +-
 src/init.c            | 14 ++++----------
 src/main.c            | 23 +++++++++++++++--------
 src/sdaxpy.c          | 14 ++++----------
 src/striad.c          | 14 ++++----------
 src/sum.c             | 14 ++++----------
 src/triad.c           | 14 ++++----------
 src/update.c          | 14 ++++----------
 util/bwBench-likwid.c |  4 ++--
 util/bwBench.c        |  5 +++--
 11 files changed, 49 insertions(+), 83 deletions(-)

diff --git a/src/copy.c b/src/copy.c
index 468db8a..b5f20f8 100644
--- a/src/copy.c
+++ b/src/copy.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,6 @@
  */
 
 #include <timing.h>
-#include <likwid-marker.h>
 
 double copy(
         double * restrict a,
@@ -37,14 +36,9 @@ double copy(
     double S, E;
 
     S = getTimeStamp();
-#pragma omp parallel
-    {
-        LIKWID_MARKER_START("COPY");
-#pragma omp for
-        for (int i=0; i<N; i++) {
-            a[i] = b[i];
-        }
-        LIKWID_MARKER_STOP("COPY");
+#pragma omp parallel for schedule(static)
+    for (int i=0; i<N; i++) {
+        a[i] = b[i];
     }
     E = getTimeStamp();
 
diff --git a/src/daxpy.c b/src/daxpy.c
index 81a1215..0faa4e0 100644
--- a/src/daxpy.c
+++ b/src/daxpy.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
diff --git a/src/init.c b/src/init.c
index 501fbc5..fe7e20c 100644
--- a/src/init.c
+++ b/src/init.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,6 @@
  */
 
 #include <timing.h>
-#include <likwid-marker.h>
 
 double init(
         double * restrict a,
@@ -37,14 +36,9 @@ double init(
     double S, E;
 
     S = getTimeStamp();
-#pragma omp parallel
-    {
-        LIKWID_MARKER_START("INIT");
-#pragma omp for schedule(static)
-        for (int i=0; i<N; i++) {
-            a[i] = scalar;
-        }
-        LIKWID_MARKER_STOP("INIT");
+#pragma omp parallel for schedule(static)
+    for (int i=0; i<N; i++) {
+        a[i] = scalar;
     }
     E = getTimeStamp();
 
diff --git a/src/main.c b/src/main.c
index a1282b3..fa763ec 100644
--- a/src/main.c
+++ b/src/main.c
@@ -52,6 +52,13 @@
 #define ABS(a) ((a) >= 0 ? (a) : -(a))
 #endif
 
+#define LIKWID_PROFILE(tag,call) \
+    _Pragma ("omp parallel") \
+    {LIKWID_MARKER_START("tag");} \
+    times[tag][k]  = call; \
+    _Pragma ("omp parallel") \
+    {LIKWID_MARKER_STOP("tag");}
+
 typedef enum benchmark {
     INIT = 0,
     SUM,
@@ -169,16 +176,16 @@ int main (int argc, char** argv)
     scalar = 3.0;
 
     for ( int k=0; k < NTIMES; k++) {
-        times[INIT][k]   = init(b, scalar, N);
+        LIKWID_PROFILE(INIT,init(b, scalar, N));
         tmp = a[10];
-        times[SUM][k]    = sum(a, N);
+        LIKWID_PROFILE(SUM,sum(a, N));
         a[10] = tmp;
-        times[COPY][k]   = copy(c, a, N);
-        times[UPDATE][k] = update(a, scalar, N);
-        times[TRIAD][k]  = triad(a, b, c, scalar, N);
-        times[DAXPY][k]  = daxpy(a, b, scalar, N);
-        times[STRIAD][k] = striad(a, b, c, d, N);
-        times[SDAXPY][k] = sdaxpy(a, b, c, N);
+        LIKWID_PROFILE(COPY,copy(c, a, N));
+        LIKWID_PROFILE(UPDATE,update(a, scalar, N));
+        LIKWID_PROFILE(TRIAD,triad(a, b, c, scalar, N));
+        LIKWID_PROFILE(DAXPY,daxpy(a, b, scalar, N));
+        LIKWID_PROFILE(STRIAD,striad(a, b, c, d, N));
+        LIKWID_PROFILE(SDAXPY,sdaxpy(a, b, c, N));
     }
 
     for (int j=0; j<NUMBENCH; j++) {
diff --git a/src/sdaxpy.c b/src/sdaxpy.c
index 2f8f0d9..730d5d2 100644
--- a/src/sdaxpy.c
+++ b/src/sdaxpy.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,6 @@
  */
 
 #include <timing.h>
-#include <likwid-marker.h>
 
 double sdaxpy(
         double * restrict a,
@@ -38,14 +37,9 @@ double sdaxpy(
     double S, E;
 
     S = getTimeStamp();
-#pragma omp parallel
-    {
-        LIKWID_MARKER_START("SDAXPY");
-#pragma omp for schedule(static)
-        for (int i=0; i<N; i++) {
-            a[i] = a[i] + b[i] * c[i];
-        }
-        LIKWID_MARKER_STOP("SDAXPY");
+#pragma omp parallel for schedule(static)
+    for (int i=0; i<N; i++) {
+        a[i] = a[i] + b[i] * c[i];
     }
     E = getTimeStamp();
 
diff --git a/src/striad.c b/src/striad.c
index f13e533..1c8a041 100644
--- a/src/striad.c
+++ b/src/striad.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,6 @@
  */
 
 #include <timing.h>
-#include <likwid-marker.h>
 
 double striad(
         double * restrict a,
@@ -39,14 +38,9 @@ double striad(
     double S, E;
 
     S = getTimeStamp();
-#pragma omp parallel
-    {
-        LIKWID_MARKER_START("STRIAD");
-#pragma omp for schedule(static)
-        for (int i=0; i<N; i++) {
-            a[i] = b[i] + d[i] * c[i];
-        }
-        LIKWID_MARKER_STOP("STRIAD");
+#pragma omp parallel for schedule(static)
+    for (int i=0; i<N; i++) {
+        a[i] = b[i] + d[i] * c[i];
     }
     E = getTimeStamp();
 
diff --git a/src/sum.c b/src/sum.c
index 8b973e2..b91ee68 100644
--- a/src/sum.c
+++ b/src/sum.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,6 @@
  */
 
 #include <timing.h>
-#include <likwid-marker.h>
 
 double sum(
         double * restrict a,
@@ -37,14 +36,9 @@ double sum(
     double sum = 0.0;
 
     S = getTimeStamp();
-#pragma omp parallel
-    {
-        LIKWID_MARKER_START("SUM");
-#pragma omp for reduction(+:sum) schedule(static)
-        for (int i=0; i<N; i++) {
-            sum += a[i];
-        }
-        LIKWID_MARKER_STOP("SUM");
+#pragma omp parallel for reduction(+:sum) schedule(static)
+    for (int i=0; i<N; i++) {
+        sum += a[i];
     }
     E = getTimeStamp();
 
diff --git a/src/triad.c b/src/triad.c
index 5f6bae2..a46c192 100644
--- a/src/triad.c
+++ b/src/triad.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,6 @@
  */
 
 #include <timing.h>
-#include <likwid-marker.h>
 
 double triad(
         double * restrict a,
@@ -39,14 +38,9 @@ double triad(
     double S, E;
 
     S = getTimeStamp();
-#pragma omp parallel
-    {
-        LIKWID_MARKER_START("TRIAD");
-#pragma omp for schedule(static)
-        for (int i=0; i<N; i++) {
-            a[i] = b[i] + scalar * c[i];
-        }
-        LIKWID_MARKER_STOP("TRIAD");
+#pragma omp parallel for schedule(static)
+    for (int i=0; i<N; i++) {
+        a[i] = b[i] + scalar * c[i];
     }
     E = getTimeStamp();
 
diff --git a/src/update.c b/src/update.c
index c094252..f74c020 100644
--- a/src/update.c
+++ b/src/update.c
@@ -2,7 +2,7 @@
  * =======================================================================================
  *
  *      Author:   Jan Eitzinger (je), jan.eitzinger@fau.de
- *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *      Copyright (c) 2020 RRZE, University Erlangen-Nuremberg
  *
  *      Permission is hereby granted, free of charge, to any person obtaining a copy
  *      of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +26,6 @@
  */
 
 #include <timing.h>
-#include <likwid-marker.h>
 
 double update(
         double * restrict a,
@@ -37,14 +36,9 @@ double update(
     double S, E;
 
     S = getTimeStamp();
-#pragma omp parallel
-    {
-        LIKWID_MARKER_START("UPDATE");
-#pragma omp for schedule(static)
-        for (int i=0; i<N; i++) {
-            a[i] = a[i] * scalar;
-        }
-        LIKWID_MARKER_STOP("UPDATE");
+#pragma omp parallel for schedule(static)
+    for (int i=0; i<N; i++) {
+        a[i] = a[i] * scalar;
     }
     E = getTimeStamp();
 
diff --git a/util/bwBench-likwid.c b/util/bwBench-likwid.c
index c2d3129..06b015a 100644
--- a/util/bwBench-likwid.c
+++ b/util/bwBench-likwid.c
@@ -38,8 +38,8 @@
 
 #include <likwid-marker.h>
 
-#define SIZE 40000000ull
-#define NTIMES 10
+#define SIZE 120000000ull
+#define NTIMES 5
 #define ARRAY_ALIGNMENT 64
 #define HLINE "----------------------------------------------------------------------------\n"
 
diff --git a/util/bwBench.c b/util/bwBench.c
index 94673c3..d5e9a3a 100644
--- a/util/bwBench.c
+++ b/util/bwBench.c
@@ -25,6 +25,7 @@
  * =======================================================================================
  */
 
+#define _GNU_SOURCE
 #include <stdlib.h>
 #include <stdio.h>
 #include <unistd.h>
@@ -36,8 +37,8 @@
 #include <omp.h>
 #endif
 
-#define SIZE 40000000ull
-#define NTIMES 10
+#define SIZE 120000000ull
+#define NTIMES 5
 #define ARRAY_ALIGNMENT 64
 #define HLINE "----------------------------------------------------------------------------\n"