Various tool overhauls

.gitignore

@@ -6,5 +6,6 @@ build*/
 !scripts/build
 *.swp
 __pycache__
+.cache/
 examples/*/daxpy
 examples/*/chop.db

.gitmodules

@@ -2,3 +2,6 @@
 	path = external/fmt
 	url = https://github.com/fmtlib/fmt
 	branch = master
+[submodule "external/cjson"]
+	path = external/cjson
+	url = https://github.com/DaveGamble/cJSON.git

.travis.yml

@@ -43,23 +43,23 @@ jobs:
               - CACHE_NAME=amd64
           script:
               - set -e
-              - shellcheck -x -s sh $(find . -name \*.sh) $(find . -name \*.sh.in) $(/bin/ls ./tools/chop-* | grep -v lib.c) ./test/drivers/* ./test/daxpy/*spec
+              - shellcheck -x -s sh $(find . -name \*.sh | grep -v ./external/ | grep -v ./build) $(find . -name \*.sh.in) $(/bin/ls ./tools/chop-* | grep -v lib.c) ./test/drivers/* ./test/daxpy/*spec
         - stage: shell_conventions
           name: shell conventions ppc64le
           arch: ppc64le
           env:
               - CACHE_NAME=ppc64le
           script:
               - set -e
-              - shellcheck -x -s sh $(find . -name \*.sh) $(find . -name \*.sh.in) $(/bin/ls ./tools/chop-* | grep -v lib.c) ./test/drivers/* ./test/daxpy/*spec
+              - shellcheck -x -s sh $(find . -name \*.sh | grep -v ./external/ | grep -v ./build) $(find . -name \*.sh.in) $(/bin/ls ./tools/chop-* | grep -v lib.c) ./test/drivers/* ./test/daxpy/*spec
         - stage: shell_conventions
           name: shell conventions s390x
           arch: s390x
           env:
               - CACHE_NAME=s390x
           script:
               - set -e
-              - shellcheck -x -s sh $(find . -name \*.sh) $(find . -name \*.sh.in) $(/bin/ls ./tools/chop-* | grep -v lib.c) ./test/drivers/* ./test/daxpy/*spec
+              - shellcheck -x -s sh $(find . -name \*.sh | grep -v ./external/ | grep -v ./build) $(find . -name \*.sh.in) $(/bin/ls ./tools/chop-* | grep -v lib.c) ./test/drivers/* ./test/daxpy/*spec
         - stage: build_and_test
           name: Build and test amd64
           arch: amd64

cmake/Modules/BuildConfig.cmake

@@ -21,3 +21,6 @@ set_env (CHOPSTIX_SQLITE_PREFIX "" CACHE PATH "Path to sqlite3 installation.")
 set_env (CHOPSTIX_PERFMON_PREFIX "" CACHE PATH "Path to Perfmon2 installation.")
 
 set_env (CHOPSTIX_PROFILE_CPU OFF CACHE BOOL "Enable profiling of CPU performance.")
+
+set_env (CHOPSTIX_PIP_INSTALL ON CACHE BOOL "Enable the installation of python dependencies via pip.")
+set_env (CHOPSTIX_VALGRIND ON CACHE BOOL "Enable Valgrind support.")

cmake/Modules/CheckArchSupport.cmake

@@ -26,7 +26,7 @@ if (NOT(CHOPSTIX_POWER_SUPPORT OR CHOPSTIX_POWERLE_SUPPORT
     elseif (${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
         SET(CHOPSTIX_X86_SUPPORT ON)
         message (STATUS "Current architecture is x86_64")
-    elseif (${CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "riscv64")
+    elseif (${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "riscv64")
         SET(CHOPSTIX_RISCV_SUPPORT ON)
         message (STATUS "Current architecture is riscv64")
     else()

cmake/Modules/CompileFlags.cmake

@@ -7,7 +7,7 @@ set (CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
 
 set (CXX_STANDARD 11)
 set (CXX_STANDARD_REQUIRED ON)
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall -Wno-unused -Wno-sequence-point -std=gnu++11 -Werror")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall -Wno-unused -Wno-sequence-point -std=gnu++11 -Werror -fno-stack-protector")
 set (CMAKE_CXX_FLAGS_DEBUG "-g -O0 -fno-omit-frame-pointer")
 set (CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
 

configure

@@ -31,6 +31,8 @@ Options:
   --no-build-examples  Do not generate targets for examples
   --build-type=TYPE    Debug|Release|RelWithDebInfo (default: RelWithDebInfo)
   --profile-cpu        Build with profiling support
+  --skip-pip-install   Skip installing python dependencies
+  --disable-valgrind   Disable valgrind support
 
 You can also use CMake directly. For more information see docs/installation.md.
 --enable-* options have the corresponding --disable-* ones.
@@ -42,7 +44,7 @@ cmake --version >/dev/null || { echo "You need to install CMake to build ChopSti
 
 PROJECT_DIR=$(dirname "$0")
 
-TEMP=$(getopt -o 'h' -l 'help,prefix:,enable-x86,enable-sysz,enable-power,enable-powerle,enable-riscv,disable-x86,disable-sysz,disable-power,disable-powerle,disable-riscv,build-sqlite,with-sqlite:,with-perfmon:,with-zlib:,build-tests,build-examples,no-build-tests,no-build-examples,build-type:,profile-cpu' -n 'configure' -- "$@")
+TEMP=$(getopt -o 'h' -l 'help,prefix:,enable-x86,enable-sysz,enable-power,enable-powerle,enable-riscv,disable-x86,disable-sysz,disable-power,disable-powerle,disable-riscv,build-sqlite,with-sqlite:,with-perfmon:,with-zlib:,build-tests,build-examples,no-build-tests,no-build-examples,build-type:,profile-cpu,skip-pip-install,disable-valgrind' -n 'configure' -- "$@")
 
 eval set -- "$TEMP"
 
@@ -78,6 +80,8 @@ while true ; do
         --no-build-examples) BUILD_OPTIONS="$BUILD_OPTIONS -DCHOPSTIX_BUILD_EXAMPLES=OFF" ; shift ;;
         --build-type) BUILD_OPTIONS="$BUILD_OPTIONS -DCMAKE_BUILD_TYPE=$2" ; shift 2 ;;
         --profile-cpu) BUILD_OPTIONS="$BUILD_OPTIONS -DCHOPSTIX_PROFILE_CPU=ON" ; shift ;;
+        --skip-pip-install) BUILD_OPTIONS="$BUILD_OPTIONS -DCHOPSTIX_PIP_INSTALL=OFF" ; shift ;;
+        --disable-valgrind) BUILD_OPTIONS="$BUILD_OPTIONS -DCHOPSTIX_VALGRIND=OFF" ; shift ;;
         --) shift ; break ;;
         *) echo "Internal error!" ; exit 1 ;;
     esac

docs/clustering_json_layout.md

@@ -0,0 +1,55 @@
+# Layout of the JSON file describing the clustering information
+
+The `cti_cluster` tool generates a JSON describing the clusters found of the
+invocations to a particular function of a benchmark. This document describes
+its layout.
+
+The root of the document is an object, having the following mandatory fields:
+
+- `epsilon`: The epsilon parameter used when clustering. It is a number.
+- `clusters`: The clusters found when clustering. It is an array in which every
+              element is a cluster. Each element is, in turn, an array in which
+              every element is the index of an invocation set found in the
+              `invocation_sets` field described later. Clusters are 0-indexed.
+- `noise_invocations`: The invocation sets which are considered noise points
+                       when clustering. It is an array in which each element is
+                       the index of an invocation set found in the
+                       `invocation_sets` field described later. Noise
+                       invocations are 0-indexed.
+- `invocation_sets`: The invocation sets found within all the invocations to a
+                     function. It is an array in which each element is an
+                     invocation set. Invocation sets are 0-indexed. Each
+                     element is, in turn, an array in which each element is the
+                     index of an invocation to the function. Invocations are
+                     0-indexed.
+
+
+The root of the document also has the following optional fields:
+
+- `extra`: TODO document this field
+
+Example:
+
+```json
+{
+    "epsilon": 0.5,
+    "clusters": [[1, 3], [0]],
+    "noise_invocations": [2, 4],
+    "invocation_sets": [
+        [0, 1, 2],
+        [3, 4, 5],
+        [6, 7, 8],
+        [9, 10, 11],
+        [12, 13, 14]
+    ]
+}
+```
+
+In this example there are two clusters, 0 and 1.
+
+- Cluster 0 has 6 invocations, corresponding to those found in invocation sets
+  1 and 3: 3, 4, 5, 9, 10, 11.
+- Cluster 1 has 3 invocations, corresponding to invocation set 0: 0, 1, 2.
+
+In this example there are 6 noise invocations, found in invocation sets 2 and 4.
+

external/CMakeLists.txt

@@ -50,13 +50,14 @@ else ()
     find_package(Perfmon2 REQUIRED)
 endif ()
 
-
-
 # {fmt} https://github.com/fmtlib/fmt
 add_subdirectory (fmt)
+add_subdirectory (cjson)
+set (CJSON_INCLUDE_DIRS "${EXTERNAL_DIR}" PARENT_SCOPE)
 
 set (EXTERNAL_INCLUDE_DIRS
     ${EXTERNAL_DIR}/fmt
+    ${EXTERNAL_DIR}/cjson
     ${PERFMON2_INCLUDE_DIRS}
     ${SQLITE3_INCLUDE_DIRS}
     PARENT_SCOPE

src/client/text.cpp

@@ -57,6 +57,7 @@ std::unique_ptr<TextFormat> get_format() {
         return format(new MptFormat(arch->get_endianess()));
     }
     CHECK_USAGE(text, 0, "Unknown format '{}'", opt);
+    exit(1); // Tell compiler to shut up
 }
 
 std::unique_ptr<std::ostream> get_output() {

src/client/view.cpp

@@ -140,6 +140,7 @@ formatter get_format() {
     FMT_CASE(compact)
     FMT_CASE(detail)
     CHECK_USAGE(view, 0, "Unknown format '{}'", fmt);
+    exit(1); // Tell compiler to shut up
 }
 
 void view_function() {

src/core/tracer/tracer.cpp

@@ -249,7 +249,9 @@ void Tracer::track_mmap() {
 void Tracer::init(int argc, char **argv) {
     log::debug("Tracer:: init start");
     setenv("LD_BIND_NOW", "1", 1);
-    preload(library_path());
+    //preload();
+    // TODO: Fix this platform-specific hardcoded path
+    preload("/lib/riscv64-linux-gnu/libdl.so.2:" + library_path());
     log::debug("Tracer:: preload set");
     child.exec(argv, argc);
     child.ready();

src/support/filesystem.cpp

@@ -180,7 +180,7 @@ void copy(const std::string &from, const std::string &to) {
         ofs.write(buf, ifs.gcount());
     }
 
-    delete buf;
+    delete[] buf;
 }
 
 std::vector<std::string> list(const std::string &path) {

src/trace/CMakeLists.txt

@@ -40,7 +40,7 @@ install (
 target_link_libraries(cxtrace cx-support)
 target_link_libraries(cxtrace dl)
 
-SET(SUPPORT_LIBRARY_CFLAGS "-DLOG_CHILD")
+SET(SUPPORT_LIBRARY_CFLAGS "-DLOG_CHILD -fno-stack-protector")
 get_target_property(TEMP cxtrace COMPILE_FLAGS)
 if(TEMP STREQUAL "TEMP-NOTFOUND")
   SET(TEMP "") # Set to empty string

src/trace/memory.cpp

@@ -164,15 +164,18 @@ static int filter_region(mem_region *reg, const char *perm,
     if (streq(reg->path, "")) return 1;
     if (streq(reg->path, "[stack]")) return 1;
     if (streq(reg->path, "[heap]")) return 1;
+    if (strstr(reg->path, "/libcxtrace.so")) return 0;
     if (strstr(reg->path, "/libc-") && strstr(reg->perm, "r--p")) return 1;
     if (strstr(reg->path, "/libc-") && strstr(reg->perm, "rw-p")) return 1;
     if (strstr(reg->path, "/libc-") != NULL) return 1;
+    if (strstr(reg->path, "/libc.") && strstr(reg->perm, "r--p")) return 1;
+    if (strstr(reg->path, "/libc.") && strstr(reg->perm, "rw-p")) return 1;
+    if (strstr(reg->path, "/libc.") != NULL) return 1;
     if (strstr(reg->path, "/libpthread-") != NULL) return 1;
     if (strstr(reg->path, "/libgfortran") != NULL) return 1;
     if (strstr(reg->path, "/libm-") != NULL) return 1;
     if (strstr(reg->path, "/libgcc") != NULL) return 1;
     if (strstr(reg->path, "/libstdc++") != NULL) return 1;
-    if (strstr(reg->path, "/libcxtrace.so")) return 0;
     if (strstr(reg->path, "/ld-") != NULL) return 0;
     if (strstr(reg->path, "[v")) return 0;
     return 1;
@@ -329,7 +332,7 @@ void Memory::update() {
             if (streq(map_[n - 1].path, "[stack]")) {
                 ++stack_cnt;
             }
-            if ((strstr(map_[n - 1].path, "/libc-") ||
+            if ((strstr(map_[n - 1].path, "/libc.") || strstr(map_[n - 1].path, "/libc-") ||
                  strstr(map_[n - 1].path, "/libpthread-")) &&
                 map_[n - 1].perm[2] == 'x') {
                 //log::debug(

src/trace/memory.h

@@ -90,6 +90,6 @@ struct Memory {
     mem_region prot_[REGIONS_MAX + 1];
     long prot_siz_ = 0;
     stack_type alt_stack_;
-    char stack_buf_[SIGSTKSZ * 2];
+    char stack_buf_[2048]; // TODO: Find adequate size
 };
 }  // namespace chopstix

tools/CMakeLists.txt

@@ -49,8 +49,22 @@ else ()
     )
 endif ()
 
+# find_package(cJSON REQUIRED)
+
 target_link_libraries(chop-trace2mpt ${ZLIB_LIBRARIES})
 
+add_executable(chop-weight microweight.c)
+
+set (CJSON_LIBRARIES "${CMAKE_CURRENT_BINARY_DIR}/../external/cjson/libcjson.so")
+
+add_dependencies(chop-weight cjson)
+target_link_libraries(chop-weight ${CJSON_LIBRARIES})
+target_include_directories(chop-weight PUBLIC ${CJSON_INCLUDE_DIRS} ${EXTERNAL_INCLUDE_DIRS})
+
+add_executable(chop-perf-avg perfavg.c)
+
+target_link_libraries(chop-perf-avg m)
+
 install(PROGRAMS
     chop-marks-dyn-addr
     chop-marks-x86_64
@@ -66,6 +80,8 @@ install (TARGETS
     chop-detrace
     chop-detrace-mem
     chop-trace2mpt
+    chop-weight
+    chop-perf-avg
     DESTINATION bin
 )
 
@@ -76,6 +92,8 @@ install (TARGETS
 
 add_subdirectory (clustering)
 add_subdirectory (perf-invok)
-add_subdirectory (valgrind)
+if (CHOPSTIX_VALGRIND)
+    add_subdirectory (valgrind)
+endif ()
 add_subdirectory (bp-checker)
 # add_subdirectory (loader)

tools/chop-marks-riscv

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 #
 # ----------------------------------------------------------------------------
 #
@@ -25,7 +25,7 @@ EXECUTABLE=$1
 FUNCTION=$2
 
 find_ret_address () {
-    objdump --disassemble="$1" "$EXECUTABLE" | grep -w "ret" | cut -d":" -f 1 | awk '{ printf "%s%s", sep, $0 ; sep="," } END { print "" }' | tr -d ' '
+    objdump --disassemble="$1" "$EXECUTABLE" | grep -w "ret" | cut -d":" -f 1 | awk '{ printf "%s %s", sep, $0 ; sep="," } END { print "" }' | tr -d ' '
 }
 
 BEGIN=$(nm "$EXECUTABLE" | grep -w "$FUNCTION" -m 1 | cut -d" " -f 1)
@@ -38,4 +38,11 @@ then
 fi
 
 # shellcheck disable=SC2039
-echo "-begin $BEGIN -end $END"
+printf -- "-begin %s" "$BEGIN"
+
+# shellcheck disable=SC3060,SC2039
+for addr in ${END//,/ }; do
+    printf -- " -end %s" "$addr"
+done
+
+printf "\n"

tools/chop-score-table

@@ -61,6 +61,7 @@ tmp="$(mktemp)"
 tmp2="$(mktemp)"
 
 cleanup() {
+    # shellcheck disable=SC2317
     rm -f "$tmp" "$tmp2" "$db"
 }
 trap cleanup EXIT

tools/chop-trace-mem

@@ -147,6 +147,7 @@ echo ""
 tmp=$(mktemp)
 tmp2=$(mktemp)
 cleanup() {
+    # shellcheck disable=SC2317
     rm -f "$tmp" "$tmp2" 
 }
 trap cleanup EXIT

tools/clustering/CMakeLists.txt

@@ -20,8 +20,6 @@
 
 find_program(PYTHON "python3" REQUIRED)
 message (STATUS "Python3 program found: ${PYTHON}")
-find_program(VIRTUALENV "virtualenv" REQUIRED)
-message (STATUS "Virtualenv program found: ${VIRTUALENV}")
 
 set(SETUP_PY "${CMAKE_CURRENT_SOURCE_DIR}/setup.py")
 set(DEPS     "${CMAKE_CURRENT_SOURCE_DIR}/*.py" "${CMAKE_CURRENT_SOURCE_DIR}/src/*.py" "${CMAKE_CURRENT_SOURCE_DIR}/*.txt" "${CMAKE_CURRENT_SOURCE_DIR}/*.md")
@@ -35,8 +33,10 @@ add_custom_command(OUTPUT ${OUTPUT}/clustering_package
 
 add_custom_target(clustering ALL DEPENDS ${OUTPUT}/clustering_package ${DEPS})
 
-install(CODE "execute_process(COMMAND ${PYTHON} -m virtualenv -p ${PYTHON} --prompt=\\(chopstix\\) ${CMAKE_INSTALL_PREFIX})")
-install(CODE "execute_process(COMMAND sh -c \"wget http://yktgsa.ibm.com/gsa/yktgsa/projects/m/microprobe/pip.conf.devel.http -O ${CMAKE_INSTALL_PREFIX}/pip.conf || rm -f ~/chopstix-install/pip.conf\")")
-install(CODE "execute_process(COMMAND sh -c \". ${CMAKE_INSTALL_PREFIX}/bin/activate && pip3 install --trusted-host yktgsa.ibm.com -U -r ${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt\")")
-install(CODE "execute_process(COMMAND sh -c \". ${CMAKE_INSTALL_PREFIX}/bin/activate && test -f ${CMAKE_INSTALL_PREFIX}/pip.conf && pip3 install --trusted-host yktgsa.ibm.com -U microprobe_target_z microprobe_target_private microprobe_target_power_priv\")")
-install(CODE "execute_process(COMMAND sh -c \". ${CMAKE_INSTALL_PREFIX}/bin/activate && pip3 install --trusted-host yktgsa.ibm.com --no-deps --prefix ${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR}\" WORKING_DIRECTORY ${OUTPUT})")
+if (CHOPSTIX_PIP_INSTALL)
+    install(CODE "execute_process(COMMAND ${PYTHON} -m virtualenv -p ${PYTHON} --prompt=\\(chopstix\\) ${CMAKE_INSTALL_PREFIX})")
+    install(CODE "execute_process(COMMAND sh -c \"wget http://yktgsa.ibm.com/gsa/yktgsa/projects/m/microprobe/pip.conf.devel.http -O ${CMAKE_INSTALL_PREFIX}/pip.conf || rm -f ~/chopstix-install/pip.conf\")")
+    install(CODE "execute_process(COMMAND sh -c \". ${CMAKE_INSTALL_PREFIX}/bin/activate && pip3 install --trusted-host yktgsa.ibm.com -U -r ${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt\")")
+    install(CODE "execute_process(COMMAND sh -c \". ${CMAKE_INSTALL_PREFIX}/bin/activate && test -f ${CMAKE_INSTALL_PREFIX}/pip.conf && pip3 install --trusted-host yktgsa.ibm.com -U microprobe_target_z microprobe_target_private microprobe_target_power_priv\")")
+    install(CODE "execute_process(COMMAND sh -c \". ${CMAKE_INSTALL_PREFIX}/bin/activate && pip3 install --trusted-host yktgsa.ibm.com --no-deps --prefix ${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR}\" WORKING_DIRECTORY ${OUTPUT})")
+endif ()