Fixed artifacts handling + added tests logs aggregation (#445)

* Snapshot packages are placed into http://redismodules.s3.amazonaws.com/redisai/snapshots. Private branches (i.e., non-master) are also supported, just need to enable deploy-snapshot in config.yml to fire on on-any-branch rather than just on-master.
* Release packages are placed into http://redismodules.s3.amazonaws.com/redisai, and it's going to be crowded in there, so we may want to consider putting each version into its own directory.
* `BB` in-source breakpoints are now supported (in `DEBUG=1` builds): put `BB;` inside the code and it will stop if you're running under gdb.
* Tests results in CircleCI are now aggregated as artifacts.

Author: rafie

.circleci/config.yml

@@ -46,12 +46,17 @@ commands:
           no_output_timeout: 20m
       - run:
           name: Package
-          command: make -C opt pack SHOW=1
+          command: |
+            make -C opt pack SHOW=1
+            (cd bin/artifacts; tar -cf snapshots-<<parameters.platform>>.tar snapshots/)
       - persist_to_workspace:
           root: bin/
           paths:
             - artifacts/*.zip
             - artifacts/*.tgz
+            - artifacts/*.tar
+            # - artifacts/shapshots/*.zip
+            # - artifacts/shapshots/*.tgz
       - store_artifacts:
           path: test/logs
 
@@ -71,16 +76,27 @@ commands:
           name: Build for platform
           command: |
             docker login -u redisfab -p $DOCKER_REDISFAB_PWD
-            cd opt/build/docker
+            pushd opt/build/docker
             #@@ make build publish $(./version-params) CPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 TEST=1 VERBOSE=1
             make build publish $(./version-params) CPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 VERBOSE=1
             make build publish $(./version-params) GPU=1 OSNICK=<<parameters.platform>> X64=1 ARTIFACTS=1 VERBOSE=1
+            popd > /dev/null
+            logstar=bin/artifacts/tests-logs-cpu.tgz
+            logsdir=tests/logs/cpu
+            mkdir -p $logsdir
+            if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi
+            (cd bin/artifacts; tar -cf snapshots-<<parameters.platform>>.tar snapshots/)
           no_output_timeout: 40m
       - persist_to_workspace:
           root: bin/
           paths:
             - artifacts/*.zip
             - artifacts/*.tgz
+            - artifacts/*.tar
+            # - artifacts/shapshots/*.zip
+            # - artifacts/shapshots/*.tgz
+      - store_artifacts:
+          path: test/logs
 
   deploy-steps:
     parameters:
@@ -202,11 +218,11 @@ jobs:
             docker run --gpus all -v $HOME/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test
           no_output_timeout: 40m
       - store_artifacts:
-          path: tests
+          path: test/log
 
   deploy-artifacts:
     parameters:
-      package:
+      location:
         type: string
     docker:
       - image: redisfab/rmbuilder:6.0.5-x64-buster
@@ -216,8 +232,42 @@ jobs:
       - run:
           name: Deploy to S3
           command: |
-            cd workspace
-            aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz"
+            cd "workspace/artifacts/<<parameters.location>>"
+            for f in *.zip snapshot/*.tgz; do
+              aws s3 cp $f s3://redismodules/$PACKAGE_NAME/<<parameters.location>>/ --acl public-read
+            done
+
+  deploy-snapshot:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - attach_workspace:
+          at: workspace
+      - run:
+          name: Deploy Snapshots to S3
+          command: |
+            cd workspace/artifacts
+            for f in snapshots-*.tar; do
+                tar xf $f
+            done
+            cd snapshots
+            for f in *.zip *.tgz; do
+              aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/snapshots/ --acl public-read
+            done
+
+  deploy-release:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - attach_workspace:
+          at: workspace
+      - run:
+          name: Deploy Releases to S3
+          command: |
+            cd workspace/artifacts
+            for f in *.zip *.tgz; do
+              aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/ --acl public-read
+            done
 
 
 on-any-branch: &on-any-branch
@@ -296,14 +346,10 @@ workflows:
       - build-multiarch-docker:
           <<: *never # temporarily disabled
           # <<: *on-version-tags
-      - deploy-artifacts:
-          name: deploy-branch
-          package: branch
+      - deploy-snapshot:
           <<: *after-platform-builds
-          <<: *on-any-branch
-      - deploy-artifacts:
-          name: deploy-release
-          package: release
+          <<: *on-master
+      - deploy-release:
           <<: *after-platform-builds
           <<: *on-version-tags
 

CMakeLists.txt

@@ -39,8 +39,10 @@ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
 
 # Add -fno-omit-frame-pointer to avoid seeing incomplete stack traces
-set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND")
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -ggdb -fno-omit-frame-pointer -DVALGRIND")
+set(CMAKE_COMMON_FLAGS_DEBUG "-g -ggdb -fno-omit-frame-pointer -D_DEBUG -DVALGRIND -include \
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/common.h -I${CMAKE_CURRENT_SOURCE_DIR}/opt")
+set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}")
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}")
 
 #----------------------------------------------------------------------------------------------
 

Dockerfile

@@ -31,7 +31,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy3
+RUN PIP=19.3.1 FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""
@@ -40,17 +40,24 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN set -e ;\
+RUN bash -c "set -e ;\
     . ./opt/readies/bin/sourced ./profile.d ;\
-    make -C opt build $BUILD_ARGS SHOW=1
+    make -C opt build $BUILD_ARGS SHOW=1"
 
 ARG PACK
 ARG TEST
 
 RUN mkdir -p bin/artifacts
-RUN if [ "$PACK" = "1" ]; then make -C opt pack; fi
+RUN set -e ;\
+    if [ "$PACK" = "1" ]; then make -C opt pack; fi
 
-RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test $BUILD_ARGS NO_LFS=1; fi
+RUN set -e ;\
+    if [ "$TEST" = "1" ]; then \
+        TEST= make -C opt test $BUILD_ARGS NO_LFS=1 ;\
+        if [[ -d test/logs ]]; then \
+            tar -C test/logs -czf bin/artifacts/test-logs-cpu.tgz . ;\
+        fi ;\
+    fi
 
 #----------------------------------------------------------------------------------------------
 FROM redisfab/redis:${REDIS_VER}-${ARCH}-${OSNICK}

Dockerfile.gpu

@@ -5,12 +5,12 @@ ARG REDIS_VER=6.0.5
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic
 
-# ARCH=x64|arm64v8|arm32v7
-ARG ARCH=x64
-
 # OS=ubuntu18.04|ubuntu16.04|centos7
 ARG OS=ubuntu18.04
 
+# ARCH=x64|arm64v8|arm32v7
+ARG ARCH=x64
+
 ARG CUDA_VER=10.1-cudnn7
 
 ARG PACK=0
@@ -22,10 +22,11 @@ FROM nvidia/cuda:${CUDA_VER}-devel-${OS} AS builder
 
 ARG OSNICK
 ARG OS
+ARG ARCH
 ARG REDIS_VER
 ARG CUDA_VER
 
-RUN echo "Building for ${OSNICK} (${OS}) [with Redis ${REDIS_VER}]"
+RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]"
 
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
@@ -36,7 +37,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./test/test_requirements.txt test/
 
-RUN ./opt/readies/bin/getpy3
+RUN PIP=19.3.1 FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""
@@ -45,17 +46,24 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh gpu; else env $DEPS_ARGS ./get_
 
 ARG BUILD_ARGS=""
 ADD ./ /build
-RUN set -e ;\
+RUN bash -c "set -e ;\
     . ./opt/readies/bin/sourced ./profile.d ;\
-    make -C opt build GPU=1 $BUILD_ARGS SHOW=1
+    make -C opt build GPU=1 $BUILD_ARGS SHOW=1"
 
 ARG PACK
 ARG TEST
 
 RUN mkdir -p bin/artifacts
-RUN if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
+RUN set -e ;\
+    if [ "$PACK" = "1" ]; then make -C opt pack GPU=1; fi
 
-RUN if [ "$TEST" = "1" ]; then TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1; fi
+RUN set -e ;\
+    if [ "$TEST" = "1" ]; then \
+        TEST= make -C opt test GPU=1 $BUILD_ARGS NO_LFS=1 ;\
+        if [[ -d test/logs ]]; then \
+            tar -C test/logs -czf bin/artifacts/test-logs-gpu.tgz . ;\
+        fi ;\
+    fi
 
 #----------------------------------------------------------------------------------------------
 FROM nvidia/cuda:${CUDA_VER}-runtime-${OS}

opt/pack.sh

@@ -112,8 +112,6 @@ pack_ramp() {
 		>&2 cat /tmp/ramp.err
 		exit 1
 	fi
-
-	echo "Done."
 }
 
 #----------------------------------------------------------------------------------------------
@@ -130,13 +128,13 @@ pack_deps() {
 	local backends_prefix_dir=""
 
 	if [[ $depname == all ]]; then
-		local backens_dir=.
+		local backends_dir=.
 	else
-		local backens_dir=${PRODUCT}_$depname
+		local backends_dir=${PRODUCT}_$depname
 	fi
 
 	cd $INSTALL_DIR/backends
-	{ find $backens_dir -name "*.so*" | \
+	{ find $backends_dir -name "*.so*" | \
 	  xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' --transform "s,^,$backends_prefix_dir," 2>> /tmp/pack.err | \
 	  gzip -n - > $tar_path ; E=$?; } || true
 	sha256sum $tar_path | gawk '{print $1}' > $tar_path.sha256

opt/readies

@@ -31,23 +31,29 @@ def debian_compat(self):
         self.install("gawk")
         self.install("build-essential cmake")
         self.install("python3-regex")
-        self.install("python3-venv python3-psutil python3-networkx python3-numpy") # python3-skimage
+        self.install("python3-psutil python3-networkx python3-numpy") # python3-skimage
         self.install_git_lfs_on_linux()
 
     def redhat_compat(self):
         self.install("redhat-lsb-core")
         self.run("%s/readies/bin/enable-utf8" % HERE)
-        
+
         self.group_install("'Development Tools'")
         self.install("cmake3")
-        self.run("ln -s `command -v cmake3` /usr/local/bin/cmake")
-        
+        self.run("ln -sf `command -v cmake3` /usr/local/bin/cmake")
+
         self.install("centos-release-scl")
         self.install("devtoolset-8")
         self.run("cp /opt/rh/devtoolset-8/enable /etc/profile.d/scl-devtoolset-8.sh")
         paella.mkdir_p("%s/profile.d" % ROOT)
         self.run("cp /opt/rh/devtoolset-8/enable %s/profile.d/scl-devtoolset-8.sh" % ROOT)
 
+        self.run("""
+            dir=$(mktemp -d /tmp/tar.XXXXXX)
+            (cd $dir; wget -q -O tar.tgz http://redismodules.s3.amazonaws.com/gnu/gnu-tar-1.32-x64-centos7.tgz; tar -xzf tar.tgz -C /; )
+            rm -rf $dir
+            """)
+
         if not self.dist == "amzn":
             self.install("epel-release")
             self.install("python3-devel libaec-devel")
@@ -62,8 +68,8 @@ def redhat_compat(self):
     def fedora(self):
         self.group_install("'Development Tools'")
         self.install("cmake")
-        self.run("ln -s `command -v cmake3` /usr/local/bin/cmake")
-        self.install("python3-venv python3-psutil python3-networkx")
+        self.run("ln -sf `command -v cmake3` /usr/local/bin/cmake")
+        self.install("python3 python3-psutil python3-networkx")
         self.install_git_lfs_on_linux()
 
     def macosx(self):

opt/system-setup.py

@@ -1,3 +1,7 @@
+if (CMAKE_BUILD_TYPE STREQUAL Debug)
+	SET(DEBUG_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../opt/readies/cetara/diag/gdb.c")
+endif()
+
 ADD_LIBRARY(redisai_obj OBJECT
         util/dict.c
         util/queue.c
@@ -19,7 +23,8 @@ ADD_LIBRARY(redisai_obj OBJECT
         rmutil/args.c
         rmutil/heap.c
         rmutil/priority_queue.c
-        rmutil/vector.c run_info.c)
+        rmutil/vector.c run_info.c
+		${DEBUG_SRC})
 
 IF(BUILD_TF)
     ADD_LIBRARY(redisai_tensorflow_obj OBJECT

src/CMakeLists.txt

@@ -0,0 +1,4 @@
+
+#if defined(DEBUG) || defined(_DEBUG)
+#include "readies/cetara/diag/gdb.h"
+#endif

src/common.h

@@ -24,8 +24,8 @@
 COV = os.environ.get("COV") != "0" and os.environ.get("COV") != "0"
 DEVICE = os.environ.get('DEVICE', 'CPU').upper().encode('utf-8', 'ignore').decode('utf-8')
 VALGRIND = os.environ.get("VALGRIND") == "1"
-print(f"Running tests on {DEVICE}\n")
-print(f"Using a max of {MAX_ITERATIONS} iterations per test\n")
+print("Running tests on {}\n".format(DEVICE))
+print("Using a max of {} iterations per test\n".format(MAX_ITERATIONS))
 # change this to make inference tests longer
 MAX_TRANSACTIONS=100