From 6d1012f71e6a780e8b4c0b461ecaf90eee8ade8e Mon Sep 17 00:00:00 2001
From: Jan Eitzinger <jan.treibig@gmail.com>
Date: Sat, 29 Jun 2019 10:09:03 +0200
Subject: [PATCH 1/6] Fif error in flop calculation.

---
 src/main.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main.c b/src/main.c
index f19cb92..53e11af 100644
--- a/src/main.c
+++ b/src/main.c
@@ -180,7 +180,7 @@ int main (int argc, char** argv)
     for (int j=0; j<NUMBENCH; j++) {
         avgtime[j] = avgtime[j]/(double)(NTIMES-1);
         double bytes = (double) benchmarks[j].words * sizeof(double) * N;
-        double flops = (double) benchmarks[j].flops * sizeof(double) * N;
+        double flops = (double) benchmarks[j].flops *  N;
 
         if (flops > 0){
             printf("%s%11.2f %11.2f %11.4f  %11.4f  %11.4f\n", benchmarks[j].label,

From df88c45a4c4d9f8acb45e31d12bd5971011af54c Mon Sep 17 00:00:00 2001
From: Jan Eitzinger <jan.treibig@gmail.com>
Date: Sat, 29 Jun 2019 10:24:53 +0200
Subject: [PATCH 2/6] Move benchmarks to MainMemory subdirectory

---
 Makefile => MainMemory/Makefile                 | 0
 bench.pl => MainMemory/bench.pl                 | 0
 config.mk => MainMemory/config.mk               | 0
 include_CLANG.mk => MainMemory/include_CLANG.mk | 0
 include_GCC.mk => MainMemory/include_GCC.mk     | 0
 include_ICC.mk => MainMemory/include_ICC.mk     | 0
 {src => MainMemory/src}/affinity.c              | 0
 {src => MainMemory/src}/allocate.c              | 0
 {src => MainMemory/src}/copy.c                  | 0
 {src => MainMemory/src}/daxpy.c                 | 0
 {src => MainMemory/src}/includes/affinity.h     | 0
 {src => MainMemory/src}/includes/allocate.h     | 0
 {src => MainMemory/src}/includes/timing.h       | 0
 {src => MainMemory/src}/init.c                  | 0
 {src => MainMemory/src}/main.c                  | 0
 {src => MainMemory/src}/sdaxpy.c                | 0
 {src => MainMemory/src}/striad.c                | 0
 {src => MainMemory/src}/sum.c                   | 0
 {src => MainMemory/src}/timing.c                | 0
 {src => MainMemory/src}/triad.c                 | 0
 {src => MainMemory/src}/update.c                | 0
 21 files changed, 0 insertions(+), 0 deletions(-)
 rename Makefile => MainMemory/Makefile (100%)
 rename bench.pl => MainMemory/bench.pl (100%)
 rename config.mk => MainMemory/config.mk (100%)
 rename include_CLANG.mk => MainMemory/include_CLANG.mk (100%)
 rename include_GCC.mk => MainMemory/include_GCC.mk (100%)
 rename include_ICC.mk => MainMemory/include_ICC.mk (100%)
 rename {src => MainMemory/src}/affinity.c (100%)
 rename {src => MainMemory/src}/allocate.c (100%)
 rename {src => MainMemory/src}/copy.c (100%)
 rename {src => MainMemory/src}/daxpy.c (100%)
 rename {src => MainMemory/src}/includes/affinity.h (100%)
 rename {src => MainMemory/src}/includes/allocate.h (100%)
 rename {src => MainMemory/src}/includes/timing.h (100%)
 rename {src => MainMemory/src}/init.c (100%)
 rename {src => MainMemory/src}/main.c (100%)
 rename {src => MainMemory/src}/sdaxpy.c (100%)
 rename {src => MainMemory/src}/striad.c (100%)
 rename {src => MainMemory/src}/sum.c (100%)
 rename {src => MainMemory/src}/timing.c (100%)
 rename {src => MainMemory/src}/triad.c (100%)
 rename {src => MainMemory/src}/update.c (100%)

diff --git a/Makefile b/MainMemory/Makefile
similarity index 100%
rename from Makefile
rename to MainMemory/Makefile
diff --git a/bench.pl b/MainMemory/bench.pl
similarity index 100%
rename from bench.pl
rename to MainMemory/bench.pl
diff --git a/config.mk b/MainMemory/config.mk
similarity index 100%
rename from config.mk
rename to MainMemory/config.mk
diff --git a/include_CLANG.mk b/MainMemory/include_CLANG.mk
similarity index 100%
rename from include_CLANG.mk
rename to MainMemory/include_CLANG.mk
diff --git a/include_GCC.mk b/MainMemory/include_GCC.mk
similarity index 100%
rename from include_GCC.mk
rename to MainMemory/include_GCC.mk
diff --git a/include_ICC.mk b/MainMemory/include_ICC.mk
similarity index 100%
rename from include_ICC.mk
rename to MainMemory/include_ICC.mk
diff --git a/src/affinity.c b/MainMemory/src/affinity.c
similarity index 100%
rename from src/affinity.c
rename to MainMemory/src/affinity.c
diff --git a/src/allocate.c b/MainMemory/src/allocate.c
similarity index 100%
rename from src/allocate.c
rename to MainMemory/src/allocate.c
diff --git a/src/copy.c b/MainMemory/src/copy.c
similarity index 100%
rename from src/copy.c
rename to MainMemory/src/copy.c
diff --git a/src/daxpy.c b/MainMemory/src/daxpy.c
similarity index 100%
rename from src/daxpy.c
rename to MainMemory/src/daxpy.c
diff --git a/src/includes/affinity.h b/MainMemory/src/includes/affinity.h
similarity index 100%
rename from src/includes/affinity.h
rename to MainMemory/src/includes/affinity.h
diff --git a/src/includes/allocate.h b/MainMemory/src/includes/allocate.h
similarity index 100%
rename from src/includes/allocate.h
rename to MainMemory/src/includes/allocate.h
diff --git a/src/includes/timing.h b/MainMemory/src/includes/timing.h
similarity index 100%
rename from src/includes/timing.h
rename to MainMemory/src/includes/timing.h
diff --git a/src/init.c b/MainMemory/src/init.c
similarity index 100%
rename from src/init.c
rename to MainMemory/src/init.c
diff --git a/src/main.c b/MainMemory/src/main.c
similarity index 100%
rename from src/main.c
rename to MainMemory/src/main.c
diff --git a/src/sdaxpy.c b/MainMemory/src/sdaxpy.c
similarity index 100%
rename from src/sdaxpy.c
rename to MainMemory/src/sdaxpy.c
diff --git a/src/striad.c b/MainMemory/src/striad.c
similarity index 100%
rename from src/striad.c
rename to MainMemory/src/striad.c
diff --git a/src/sum.c b/MainMemory/src/sum.c
similarity index 100%
rename from src/sum.c
rename to MainMemory/src/sum.c
diff --git a/src/timing.c b/MainMemory/src/timing.c
similarity index 100%
rename from src/timing.c
rename to MainMemory/src/timing.c
diff --git a/src/triad.c b/MainMemory/src/triad.c
similarity index 100%
rename from src/triad.c
rename to MainMemory/src/triad.c
diff --git a/src/update.c b/MainMemory/src/update.c
similarity index 100%
rename from src/update.c
rename to MainMemory/src/update.c

From 9bad3340b63ed97f188d6313e073d569c58718c1 Mon Sep 17 00:00:00 2001
From: Jan Eitzinger <jan.treibig@gmail.com>
Date: Sat, 29 Jun 2019 10:25:36 +0200
Subject: [PATCH 3/6] Add MemoryHierarchy benchmark

---
 MemoryHierarchy/Makefile             |  45 ++++++
 MemoryHierarchy/bench.pl             |  33 ++++
 MemoryHierarchy/src/affinity.c       |  89 +++++++++++
 MemoryHierarchy/src/affinity.h       |  36 +++++
 MemoryHierarchy/src/allocate.c       |  58 +++++++
 MemoryHierarchy/src/allocate.h       |  33 ++++
 MemoryHierarchy/src/likwid_markers.h |  44 ++++++
 MemoryHierarchy/src/main.c           | 216 +++++++++++++++++++++++++++
 MemoryHierarchy/src/striad_seq.c     |  57 +++++++
 MemoryHierarchy/src/striad_tp.c      |  62 ++++++++
 MemoryHierarchy/src/striad_ws.c      |  57 +++++++
 MemoryHierarchy/src/timing.c         |  49 ++++++
 MemoryHierarchy/src/timing.h         |  35 +++++
 13 files changed, 814 insertions(+)
 create mode 100644 MemoryHierarchy/Makefile
 create mode 100755 MemoryHierarchy/bench.pl
 create mode 100644 MemoryHierarchy/src/affinity.c
 create mode 100644 MemoryHierarchy/src/affinity.h
 create mode 100644 MemoryHierarchy/src/allocate.c
 create mode 100644 MemoryHierarchy/src/allocate.h
 create mode 100644 MemoryHierarchy/src/likwid_markers.h
 create mode 100644 MemoryHierarchy/src/main.c
 create mode 100644 MemoryHierarchy/src/striad_seq.c
 create mode 100644 MemoryHierarchy/src/striad_tp.c
 create mode 100644 MemoryHierarchy/src/striad_ws.c
 create mode 100644 MemoryHierarchy/src/timing.c
 create mode 100644 MemoryHierarchy/src/timing.h

diff --git a/MemoryHierarchy/Makefile b/MemoryHierarchy/Makefile
new file mode 100644
index 0000000..3753fcc
--- /dev/null
+++ b/MemoryHierarchy/Makefile
@@ -0,0 +1,45 @@
+#CONFIGURE TOOL CHAIN
+CC  = gcc
+OPENMP   =# -fopenmp
+#CFLAGS   = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP)
+CFLAGS   = -O3 -std=c99 $(OPENMP)
+LFLAGS   = $(OPENMP)
+DEFINES  = -D_GNU_SOURCE
+#DEFINES += -DVERBOSE
+#DEFINES += -DLIKWID_PERFMON
+DEFINES += -DNTIMES=5
+DEFINES += -DARRAY_ALIGNMENT=64
+
+#CONFIGURE BUILD SYSTEM
+TARGET	   = striad
+BUILD_DIR  = ./build
+SRC_DIR    = ./src
+INCLUDES  += -I./src/
+
+#DO NOT EDIT BELOW
+VPATH     = $(SRC_DIR)
+ASM       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.s,$(wildcard $(SRC_DIR)/*.c))
+OBJ       = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o,$(wildcard $(SRC_DIR)/*.c))
+CPPFLAGS := $(CPPFLAGS) $(DEFINES) $(INCLUDES)
+
+${TARGET}: $(BUILD_DIR) $(OBJ)
+	${CC} ${LFLAGS} -o $(TARGET) $(OBJ) $(LIBS)
+
+asm:  $(BUILD_DIR) $(ASM)
+
+$(BUILD_DIR)/%.o:  %.c
+	$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.s:  %.c
+	$(CC) -S $(CPPFLAGS) $(CFLAGS) $< -o $@
+
+$(BUILD_DIR):
+	@mkdir $(BUILD_DIR)
+
+.PHONY: clean
+
+clean:
+	@echo "===>  CLEAN"
+	@rm -rf $(BUILD_DIR)
+	@rm -f $(TARGET)
+
diff --git a/MemoryHierarchy/bench.pl b/MemoryHierarchy/bench.pl
new file mode 100755
index 0000000..7deccc7
--- /dev/null
+++ b/MemoryHierarchy/bench.pl
@@ -0,0 +1,33 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use utf8;
+
+my $numCores = $ARGV[0];
+my $type =  0;
+my $SMT = $ARGV[2] ? $ARGV[2] : 2;
+my $N = 100;
+
+if ( $ARGV[1] eq 'seq' ){
+    $type = 0;
+} elsif (  $ARGV[1] eq 'tp'  ){
+    $type = 1;
+} elsif (  $ARGV[1] eq 'ws'  ){
+    $type = 2;
+}
+
+
+while ( $N < 8000000 ) {
+    my $result;
+    my $performance = '0.00';
+
+    while ( $performance eq '0.00' ){
+        $result =  `likwid-pin -c E:S0:$numCores:1:$SMT -q ./striad $type $N`;
+        $result =~ /([0-9.]+) ([0-9.]+)/;
+        my $size = $1;
+        $performance = $2;
+    }
+
+    print $result;
+    $N = int($N * 1.2);
+}
diff --git a/MemoryHierarchy/src/affinity.c b/MemoryHierarchy/src/affinity.c
new file mode 100644
index 0000000..cc361c2
--- /dev/null
+++ b/MemoryHierarchy/src/affinity.c
@@ -0,0 +1,89 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#ifdef __linux__
+#ifdef _OPENMP
+#include <stdlib.h>
+#include <stdio.h>
+#include <sched.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <pthread.h>
+#include <sys/syscall.h>
+
+#define MAX_NUM_THREADS 128
+#define gettid() syscall(SYS_gettid)
+
+static int
+getProcessorID(cpu_set_t* cpu_set)
+{
+    int processorId;
+
+    for ( processorId = 0; processorId < MAX_NUM_THREADS; processorId++ )
+    {
+        if ( CPU_ISSET(processorId,cpu_set) )
+        {
+            break;
+        }
+    }
+    return processorId;
+}
+
+
+int
+affinity_getProcessorId()
+{
+    cpu_set_t  cpu_set;
+    CPU_ZERO(&cpu_set);
+    sched_getaffinity(gettid(),sizeof(cpu_set_t), &cpu_set);
+
+    return getProcessorID(&cpu_set);
+}
+
+void
+affinity_pinThread(int processorId)
+{
+    cpu_set_t cpuset;
+    pthread_t thread;
+
+    thread = pthread_self();
+    CPU_ZERO(&cpuset);
+    CPU_SET(processorId, &cpuset);
+    pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
+}
+
+void
+affinity_pinProcess(int processorId)
+{
+    cpu_set_t cpuset;
+
+    CPU_ZERO(&cpuset);
+    CPU_SET(processorId, &cpuset);
+    sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+}
+#endif /*__linux__*/
+#endif /*_OPENMP*/
diff --git a/MemoryHierarchy/src/affinity.h b/MemoryHierarchy/src/affinity.h
new file mode 100644
index 0000000..9c7eecf
--- /dev/null
+++ b/MemoryHierarchy/src/affinity.h
@@ -0,0 +1,36 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#ifndef AFFINITY_H
+#define AFFINITY_H
+
+extern int affinity_getProcessorId();
+extern void affinity_pinProcess(int);
+extern void affinity_pinThread(int);
+
+#endif /*AFFINITY_H*/
+
diff --git a/MemoryHierarchy/src/allocate.c b/MemoryHierarchy/src/allocate.c
new file mode 100644
index 0000000..eaaa5e3
--- /dev/null
+++ b/MemoryHierarchy/src/allocate.c
@@ -0,0 +1,58 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+
+void* allocate (int alignment, size_t bytesize)
+{
+    int errorCode;
+    void* ptr;
+
+    errorCode =  posix_memalign(&ptr, alignment, bytesize);
+
+    if (errorCode) {
+        if (errorCode == EINVAL) {
+            fprintf(stderr,
+                    "Error: Alignment parameter is not a power of two\n");
+            exit(EXIT_FAILURE);
+        }
+        if (errorCode == ENOMEM) {
+            fprintf(stderr,
+                    "Error: Insufficient memory to fulfill the request\n");
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    if (ptr == NULL) {
+        fprintf(stderr, "Error: posix_memalign failed!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    return ptr;
+}
diff --git a/MemoryHierarchy/src/allocate.h b/MemoryHierarchy/src/allocate.h
new file mode 100644
index 0000000..a6ee561
--- /dev/null
+++ b/MemoryHierarchy/src/allocate.h
@@ -0,0 +1,33 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#ifndef __ALLOCATE_H_
+#define __ALLOCATE_H_
+
+extern void* allocate (int alignment, size_t bytesize);
+
+#endif
diff --git a/MemoryHierarchy/src/likwid_markers.h b/MemoryHierarchy/src/likwid_markers.h
new file mode 100644
index 0000000..76351d2
--- /dev/null
+++ b/MemoryHierarchy/src/likwid_markers.h
@@ -0,0 +1,44 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#ifndef LIKWID_MARKERS_H
+#define LIKWID_MARKERS_H
+
+#ifdef LIKWID_PERFMON
+#include <likwid.h>
+#else
+#define LIKWID_MARKER_INIT
+#define LIKWID_MARKER_THREADINIT
+#define LIKWID_MARKER_SWITCH
+#define LIKWID_MARKER_REGISTER(regionTag)
+#define LIKWID_MARKER_START(regionTag)
+#define LIKWID_MARKER_STOP(regionTag)
+#define LIKWID_MARKER_CLOSE
+#define LIKWID_MARKER_GET(regionTag, nevents, events, time, count)
+#endif
+
+#endif /*LIKWID_MARKERS_H*/
diff --git a/MemoryHierarchy/src/main.c b/MemoryHierarchy/src/main.c
new file mode 100644
index 0000000..b634e68
--- /dev/null
+++ b/MemoryHierarchy/src/main.c
@@ -0,0 +1,216 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <limits.h>
+#include <float.h>
+
+#ifdef _OPENMP
+#include <omp.h>
+#endif
+
+#include <likwid_markers.h>
+#include <timing.h>
+#include <allocate.h>
+#include <affinity.h>
+
+#define HLINE "----------------------------------------------------------------------------\n"
+
+
+#ifndef MIN
+#define MIN(x,y) ((x)<(y)?(x):(y))
+#endif
+#ifndef MAX
+#define MAX(x,y) ((x)>(y)?(x):(y))
+#endif
+#ifndef ABS
+#define ABS(a) ((a) >= 0 ? (a) : -(a))
+#endif
+
+extern double striad_seq(double*, double*, double*, double*, int, int);
+extern double striad_tp(double*, double*, double*, double*, int, int);
+extern double striad_ws(double*, double*, double*, double*, int, int);
+
+typedef double (*testFunc)(double*, double*, double*, double*, int, int);
+
+int main (int argc, char** argv)
+{
+    size_t bytesPerWord = sizeof(double);
+    size_t N;
+    int type;
+    size_t iter = 1;
+    size_t scale = 1;
+    double *a, *b, *c, *d;
+    double E, S;
+    double	avgtime, maxtime, mintime;
+    double times[NTIMES];
+    double dataSize;
+    testFunc func;
+    char* testname;
+
+
+    if ( argc > 2 ) {
+        type = atoi(argv[1]);
+        N = atoi(argv[2]);
+    } else {
+        printf("Usage: %s <test type>  <N>\n",argv[0]);
+        printf("Test types: 0 - sequential, 1 - OpenMP throughput, 2 - OpenMP worksharing\n");
+        exit(EXIT_SUCCESS);
+    }
+
+    LIKWID_MARKER_INIT;
+
+    switch ( type ) {
+        case 0:
+            func = striad_seq;
+            testname = "striad_seq";
+            break;
+        case 1:
+            func = striad_tp;
+            testname = "striad_tp";
+#ifdef _OPENMP
+#pragma omp parallel
+            {
+#pragma omp single
+                scale = omp_get_num_threads();
+
+
+                LIKWID_MARKER_REGISTER("BENCH");
+            }
+#endif
+            break;
+        case 2:
+            func = striad_ws;
+            testname = "striad_ws";
+            break;
+        default:
+            printf("Unknown test type: %d\n", type);
+            exit(EXIT_FAILURE);
+    }
+
+    a = (double*) allocate( ARRAY_ALIGNMENT, N * bytesPerWord );
+    b = (double*) allocate( ARRAY_ALIGNMENT, N * bytesPerWord );
+    c = (double*) allocate( ARRAY_ALIGNMENT, N * bytesPerWord );
+    d = (double*) allocate( ARRAY_ALIGNMENT, N * bytesPerWord );
+
+#ifdef VERBOSE
+    printf(HLINE);
+    dataSize = 4.0 * bytesPerWord * N;
+
+    if ( dataSize < 1.0E06 ) {
+        printf ("Total allocated datasize: %8.2f KB\n", dataSize * 1.0E-03);
+    } else {
+        printf ("Total allocated datasize: %8.2f MB\n", dataSize * 1.0E-06);
+    }
+#endif
+        avgtime = 0;
+        maxtime = 0;
+        mintime = FLT_MAX;
+
+#ifdef VERBOSE
+#ifdef _OPENMP
+    printf(HLINE);
+#pragma omp parallel
+    {
+        int k = omp_get_num_threads();
+        int i = omp_get_thread_num();
+
+#pragma omp single
+        printf ("OpenMP enabled, running with %d threads\n", k);
+
+        printf ("\tThread %d running on processor %d\n", i, affinity_getProcessorId());
+    }
+#endif
+#endif
+
+    S = getTimeStamp();
+#pragma omp parallel for
+    for (int i=0; i<N; i++) {
+        a[i] = 2.0;
+        b[i] = 1.0;
+        c[i] = 0.8;
+        d[i] = 1.01;
+    }
+    E = getTimeStamp();
+#ifdef VERBOSE
+    printf ("Timer resolution %.2e ", getTimeResolution());
+    printf ("Ticks used %.0e\n", (E-S) / getTimeResolution());
+#endif
+
+    iter = 5;
+    times[0] = 0.0;
+    times[1] = 0.0;
+
+    while ( times[0] < 0.2 ){
+        times[0] = func(a, b, c, d, N, iter);
+        if ( times[0] > 0.1 ) break;
+        double factor = 0.2 / (times[0] - times[1]);
+        iter *= (int) factor;
+        times[1] = times[0];
+    }
+
+#ifdef VERBOSE
+    printf ("Using %d iterations \n", iter);
+#endif
+
+    for ( int k=0; k < NTIMES; k++) {
+        times[k] = func(a, b, c, d, N, iter);
+    }
+
+    for (int k=1; k<NTIMES; k++) {
+        avgtime = avgtime + times[k];
+        mintime = MIN(mintime, times[k]);
+        maxtime = MAX(maxtime, times[k]);
+    }
+
+#ifdef VERBOSE
+    printf(HLINE);
+    printf("Function      Rate(MB/s)  Rate(MFlop/s)  Avg time     Min time     Max time\n");
+    avgtime = avgtime/(double)(NTIMES-1);
+    double bytes = (double) 4.0 * sizeof(double) * N * iter * scale;
+    double flops = (double) 2.0 * N * iter * scale;
+
+        printf("%s %11.2f %11.2f %11.4f  %11.4f  %11.4f\n",
+                testname,
+                1.0E-06 * bytes/mintime,
+                1.0E-06 * flops/mintime,
+                avgtime,
+                mintime,
+                maxtime);
+    printf("Flops %e\n", flops);
+    printf(HLINE);
+#else
+    double flops = (double) 2 * N * iter * scale;
+    printf("%d %.2f\n", N, 1.0E-06 * flops/mintime);
+#endif
+
+    LIKWID_MARKER_CLOSE;
+    return EXIT_SUCCESS;
+}
+
diff --git a/MemoryHierarchy/src/striad_seq.c b/MemoryHierarchy/src/striad_seq.c
new file mode 100644
index 0000000..400d1a4
--- /dev/null
+++ b/MemoryHierarchy/src/striad_seq.c
@@ -0,0 +1,57 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#include <stdio.h>
+
+#include <timing.h>
+#include <likwid_markers.h>
+
+double striad_seq(
+        double * restrict a,
+        const double * restrict b,
+        const double * restrict c,
+        const double * restrict d,
+        int N,
+        int iter
+        )
+{
+    double S, E;
+
+    S = getTimeStamp();
+    LIKWID_MARKER_START("BENCH");
+    for(int j = 0; j < iter; j++) {
+        for (int i=0; i<N; i++) {
+            a[i] = b[i] + d[i] * c[i];
+        }
+
+        if (a[N-1] > 2000) printf("Ai = %f\n",a[N-1]);
+    }
+    LIKWID_MARKER_STOP("BENCH");
+    E = getTimeStamp();
+
+    return E-S;
+}
diff --git a/MemoryHierarchy/src/striad_tp.c b/MemoryHierarchy/src/striad_tp.c
new file mode 100644
index 0000000..cfe4748
--- /dev/null
+++ b/MemoryHierarchy/src/striad_tp.c
@@ -0,0 +1,62 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#include <stdio.h>
+
+#include <timing.h>
+#include <allocate.h>
+
+double striad_tp(
+        double * restrict a,
+        const double * restrict b,
+        const double * restrict c,
+        const double * restrict d,
+        int N,
+        int iter
+        )
+{
+    double S, E;
+
+#pragma omp parallel
+    {
+        double* al = (double*) allocate( ARRAY_ALIGNMENT, N * sizeof(double) );
+
+#pragma omp single
+        S = getTimeStamp();
+        for(int j = 0; j < iter; j++) {
+            for (int i=0; i<N; i++) {
+                al[i] = b[i] + d[i] * c[i];
+            }
+
+            if (al[N-1] > 2000) printf("Ai = %f\n",al[N-1]);
+        }
+#pragma omp single
+        E = getTimeStamp();
+    }
+
+    return E-S;
+}
diff --git a/MemoryHierarchy/src/striad_ws.c b/MemoryHierarchy/src/striad_ws.c
new file mode 100644
index 0000000..9ba1374
--- /dev/null
+++ b/MemoryHierarchy/src/striad_ws.c
@@ -0,0 +1,57 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#include <stdio.h>
+
+#include <timing.h>
+
+double striad_ws(
+        double * restrict a,
+        const double * restrict b,
+        const double * restrict c,
+        const double * restrict d,
+        int N,
+        int iter
+        )
+{
+    double S, E;
+
+    S = getTimeStamp();
+#pragma omp parallel
+    {
+        for(int j = 0; j < iter; j++) {
+#pragma omp for
+            for (int i=0; i<N; i++) {
+                a[i] = b[i] + d[i] * c[i];
+            }
+            if (a[N-1] > 2000) printf("Ai = %f\n",a[N-1]);
+        }
+    }
+    E = getTimeStamp();
+
+    return E-S;
+}
diff --git a/MemoryHierarchy/src/timing.c b/MemoryHierarchy/src/timing.c
new file mode 100644
index 0000000..0ed2617
--- /dev/null
+++ b/MemoryHierarchy/src/timing.c
@@ -0,0 +1,49 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#include <stdlib.h>
+#include <time.h>
+
+double getTimeStamp()
+{
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
+}
+
+double getTimeResolution()
+{
+    struct timespec ts;
+    clock_getres(CLOCK_MONOTONIC, &ts);
+    return (double)ts.tv_sec + (double)ts.tv_nsec * 1.e-9;
+}
+
+double getTimeStamp_()
+{
+    return getTimeStamp();
+}
+
diff --git a/MemoryHierarchy/src/timing.h b/MemoryHierarchy/src/timing.h
new file mode 100644
index 0000000..79bdf95
--- /dev/null
+++ b/MemoryHierarchy/src/timing.h
@@ -0,0 +1,35 @@
+/*
+ * =======================================================================================
+ *
+ *      Author:   Jan Eitzinger (je), jan.treibig@gmail.com
+ *      Copyright (c) 2019 RRZE, University Erlangen-Nuremberg
+ *
+ *      Permission is hereby granted, free of charge, to any person obtaining a copy
+ *      of this software and associated documentation files (the "Software"), to deal
+ *      in the Software without restriction, including without limitation the rights
+ *      to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *      copies of the Software, and to permit persons to whom the Software is
+ *      furnished to do so, subject to the following conditions:
+ *
+ *      The above copyright notice and this permission notice shall be included in all
+ *      copies or substantial portions of the Software.
+ *
+ *      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *      IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *      FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *      AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *      LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *      OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ *      SOFTWARE.
+ *
+ * =======================================================================================
+ */
+
+#ifndef __TIMING_H_
+#define __TIMING_H_
+
+extern double getTimeStamp();
+extern double getTimeResolution();
+extern double getTimeStamp_();
+
+#endif

From d8b4feb50ade5a4aea024dc69d868e596174f12b Mon Sep 17 00:00:00 2001
From: Jan Eitzinger <jan.treibig@gmail.com>
Date: Mon, 1 Jul 2019 09:59:10 +0200
Subject: [PATCH 4/6] Small fixes.

---
 MemoryHierarchy/Makefile        | 2 +-
 MemoryHierarchy/bench.pl        | 7 ++++++-
 MemoryHierarchy/src/striad_tp.c | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/MemoryHierarchy/Makefile b/MemoryHierarchy/Makefile
index 3753fcc..dee5aed 100644
--- a/MemoryHierarchy/Makefile
+++ b/MemoryHierarchy/Makefile
@@ -1,6 +1,6 @@
 #CONFIGURE TOOL CHAIN
 CC  = gcc
-OPENMP   =# -fopenmp
+OPENMP   = -fopenmp
 #CFLAGS   = -O3 -xHost -qopt-zmm-usage=high -std=c99 $(OPENMP)
 CFLAGS   = -O3 -std=c99 $(OPENMP)
 LFLAGS   = $(OPENMP)
diff --git a/MemoryHierarchy/bench.pl b/MemoryHierarchy/bench.pl
index 7deccc7..6c29456 100755
--- a/MemoryHierarchy/bench.pl
+++ b/MemoryHierarchy/bench.pl
@@ -3,11 +3,17 @@ use strict;
 use warnings;
 use utf8;
 
+if ( $#ARGV < 2 ){
+    print "Usage: ./bench.pl <numcores> <seq|tp|ws> <SMT>\n";
+    exit;
+}
+
 my $numCores = $ARGV[0];
 my $type =  0;
 my $SMT = $ARGV[2] ? $ARGV[2] : 2;
 my $N = 100;
 
+
 if ( $ARGV[1] eq 'seq' ){
     $type = 0;
 } elsif (  $ARGV[1] eq 'tp'  ){
@@ -24,7 +30,6 @@ while ( $N < 8000000 ) {
     while ( $performance eq '0.00' ){
         $result =  `likwid-pin -c E:S0:$numCores:1:$SMT -q ./striad $type $N`;
         $result =~ /([0-9.]+) ([0-9.]+)/;
-        my $size = $1;
         $performance = $2;
     }
 
diff --git a/MemoryHierarchy/src/striad_tp.c b/MemoryHierarchy/src/striad_tp.c
index cfe4748..6b5297a 100644
--- a/MemoryHierarchy/src/striad_tp.c
+++ b/MemoryHierarchy/src/striad_tp.c
@@ -43,7 +43,7 @@ double striad_tp(
 
 #pragma omp parallel
     {
-        double* al = (double*) allocate( ARRAY_ALIGNMENT, N * sizeof(double) );
+        double* al = (double*) allocate( ARRAY_ALIGNMENT, N * sizeof(double));
 
 #pragma omp single
         S = getTimeStamp();

From 1d2063065f9cb65e6e41cb40e0fae5650477d96c Mon Sep 17 00:00:00 2001
From: moebiusband73 <jan.treibig@gmail.com>
Date: Mon, 1 Jul 2019 12:00:23 +0200
Subject: [PATCH 5/6] Update README.md

---
 README.md | 114 +-----------------------------------------------------
 1 file changed, 2 insertions(+), 112 deletions(-)

diff --git a/README.md b/README.md
index ad0e064..82e5bab 100644
--- a/README.md
+++ b/README.md
@@ -1,117 +1,7 @@
 # The Bandwidth Benchmark
 
 This is a collection of simple streaming kernels for teaching purposes.
-It is heavily inspired by John McCalpin's https://www.cs.virginia.edu/stream/ benchmark.
 
-It contains the following streaming kernels with corresponding data access pattern (Notation: S - store, L - load, WA - write allocate). All variables are vectors, s is a scalar:
+It consists of two banchmark applications:
 
-* init (S1, WA): Initilize an array: `a = s`. Store only.
-* sum (L1): Vector reduction: `s += a`. Load only.
-* copy  (L1, S1, WA): Classic memcopy: `a = b`.
-* update (L1, S1): Update vector: `a = a * scalar`. Also load + store but without write allocate.
-* triad (L2, S1, WA): Stream triad: `a = b + c * scalar`.
-* daxpy (L2, S1): Daxpy: `a = a + b * scalar`.
-* striad (L3, S1, WA): Schoenauer triad: `a = b + c * d`.
-* sdaxpy (L3, S1): Schoenauer triad without write allocate: `a = a + b * c`.
-
-As added benefit the code is a blueprint for a minimal benchmarking application with a generic makefile and modules for aligned array allocation, accurate timing and affinity settings. Those components can be used standalone in your own project.
-
-## Build
-
-1. Configure the toolchain and additional options in `config.mk`:
-```
-# Supported: GCC, CLANG, ICC
-TAG ?= GCC
-ENABLE_OPENMP ?= false
-
-OPTIONS  =  -DSIZE=40000000ull
-OPTIONS +=  -DNTIMES=10
-OPTIONS +=  -DARRAY_ALIGNMENT=64
-#OPTIONS +=  -DVERBOSE_AFFINITY
-#OPTIONS +=  -DVERBOSE_DATASIZE
-#OPTIONS +=  -DVERBOSE_TIMER
-```
-
-The verbosity options enable detailed output about affinity settings, allocation sizes and timer resolution.
-
-2. Build with:
-```
-make
-```
-
-You can build multiple toolchains in the same directory, but notice that the Makefile is only acting on the one currently set. Intermediate build results are located in the `<TOOLCHAIN>` directory.
-
-To output the executed commands use:
-```
-make Q=
-```
-
-3. Clean up with:
-```
-make clean
-```
-to clean intermediate build results.
-
-```
-make distclean
-```
-to clean intermediate build results and binary.
-
-4. (Optional) Generate assembler:
-```
-make asm
-```
-The assembler files will also be located in the `<TOOLCHAIN>` directory.
-
-## Usage
-
-To run the benchmark call:
-```
-./bwBench-<TOOLCHAIN>
-```
-
-The benchmark will output the results similar to the stream benchmark. Results are validated.
-For threaded execution it is recommended to control thread affinity.
-
-We recommend to use likwid-pin for benchmarking:
-```
-likwid-pin -c 0-3 ./bwbench-GCC  
-```
-
-Example output for threaded execution:
-```
--------------------------------------------------------------
-[pthread wrapper] 
-[pthread wrapper] MAIN -> 0
-[pthread wrapper] PIN_MASK: 0->1  1->2  2->3  
-[pthread wrapper] SKIP MASK: 0x0
-        threadid 140271463495424 -> core 1 - OK
-        threadid 140271455102720 -> core 2 - OK
-        threadid 140271446710016 -> core 3 - OK
-OpenMP enabled, running with 4 threads
-----------------------------------------------------------------------------
-Function      Rate(MB/s)  Rate(MFlop/s)  Avg time     Min time     Max time
-Init:          22111.53    -             0.0148       0.0145       0.0165
-Sum:           46808.59    46808.59      0.0077       0.0068       0.0140
-Copy:          30983.06    -             0.0207       0.0207       0.0208
-Update:        43778.69    21889.34      0.0147       0.0146       0.0148
-Triad:         34476.64    22984.43      0.0282       0.0278       0.0305
-Daxpy:         45908.82    30605.88      0.0214       0.0209       0.0242
-STriad:        37502.37    18751.18      0.0349       0.0341       0.0388
-SDaxpy:        46822.63    23411.32      0.0281       0.0273       0.0325
-----------------------------------------------------------------------------
-Solution Validates
-```
-
-A perl wrapper script (bench.pl) is also provided to scan ranges of thread counts and determine the absolute highest sustained main memory bandwidth. In order to use it `likwid-pin` has to be in your path. The script has three required and one optional command line arguments:
-```
-$./bench.pl <executable> <thread count range>  <repititions> [<SMT setting>] 
-```
-Example usage:
-```
-$./bench.pl ./bwbench-GCC 2-8 6
-```
-The script will always use physical cores only, where two SMT threads is the default. For different SMT thread counts use the 4th command line argument. Example for a processor without SMT:
-```
-$./bench.pl ./bwbench-GCC 14-24  10  1
-```
+* [[MainMemory|https://github.com/RRZE-HPC/TheBandwidthBenchmark/wiki/MainMemory]] 

From 57667c3b870cfc1b943681583e126b0dc4c23cab Mon Sep 17 00:00:00 2001
From: moebiusband73 <jan.treibig@gmail.com>
Date: Mon, 1 Jul 2019 12:01:32 +0200
Subject: [PATCH 6/6] Update README.md

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 82e5bab..c0abdac 100644
--- a/README.md
+++ b/README.md
@@ -4,4 +4,5 @@ This is a collection of simple streaming kernels for teaching purposes.
 
 It consists of two banchmark applications:
 
-* [[MainMemory|https://github.com/RRZE-HPC/TheBandwidthBenchmark/wiki/MainMemory]] 
+* [MainMemory](https://github.com/RRZE-HPC/TheBandwidthBenchmark/wiki/MainMemory)
+* [MemoryHierarchy](https://github.com/RRZE-HPC/TheBandwidthBenchmark/wiki/MemoryHierarchy)