Merge branch 'master' into redis-cli-modelget-limitation

Author: DvirDukhan

.circleci/config.yml

@@ -12,6 +12,14 @@ commands:
       - setup_remote_docker:
           docker_layer_caching: true
 
+  setup-automation:
+    steps:
+      - run:
+          name: Setup automation
+          command: |
+            ./opt/readies/bin/getpy3
+            ./opt/system-setup.py
+
   build-steps:
     parameters:
       platform:
@@ -25,11 +33,10 @@ commands:
           keys:
           - v1-dependencies-{{ checksum "get_deps.sh" }}
           # If no exact match is found will get dependencies from source
+      - setup-automation
       - run:
           name: Install dependencies
           command: |
-            ./opt/readies/bin/getpy3
-            BREW_NO_UPDATE=1 ./opt/system-setup.py
             ./opt/readies/bin/getredis -v 6 --force
             ./get_deps.sh cpu
       - save_cache:
@@ -39,6 +46,11 @@ commands:
       - run:
           name: Build
           command: make -C opt all SHOW=1
+      - run:
+          name: Unit Tests
+          command: |
+            make -C opt unit_tests SHOW=1
+          no_output_timeout: 5m
       - run:
           name: Test
           command: |
@@ -55,10 +67,8 @@ commands:
             - artifacts/*.zip
             - artifacts/*.tgz
             - artifacts/*.tar
-            # - artifacts/shapshots/*.zip
-            # - artifacts/shapshots/*.tgz
       - store_artifacts:
-          path: test/logs
+          path: tests/logs
 
   platform-build-steps:
     parameters:
@@ -93,8 +103,6 @@ commands:
             - artifacts/*.zip
             - artifacts/*.tgz
             - artifacts/*.tar
-            # - artifacts/shapshots/*.zip
-            # - artifacts/shapshots/*.tgz
       - store_artifacts:
           path: test/logs
 
@@ -109,6 +117,20 @@ commands:
             aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz"
 
 jobs:
+  lint:
+    docker:
+      - image: redislabsmodules/llvm-toolset:latest
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - setup-automation
+      - run:
+          name: lint
+          command: |
+            make -C opt lint
+
   build-debian:
     docker:
       - image: redisfab/rmbuilder:6.0.5-x64-buster
@@ -133,16 +155,15 @@ jobs:
       - checkout
       - run:
           name: Submodule checkout
-          command: git submodule update --init --recursive 
+          command: git submodule update --init --recursive
       - restore_cache:
           keys:
           - build-dependencies-{{ checksum "get_deps.sh" }}
           # If no exact match is found will get dependencies from source
+      - setup-automation
       - run:
           name: Install dependencies
           command: |
-            ./opt/readies/bin/getpy3
-            ./opt/system-setup.py
             ./opt/readies/bin/getredis -v 6 --valgrind --force
             ./get_deps.sh cpu
       - run:
@@ -154,7 +175,91 @@ jobs:
           command: |
             make -C opt test SHOW=1 COV=1 CLUSTER=1
             make -C opt cov-upload
-          no_output_timeout: 20m
+          no_output_timeout: 30m
+
+  valgrind:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive 
+      - restore_cache:
+          keys:
+          - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind
+          command: |
+            make -C opt test VALGRIND=1 CLUSTER=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-cluster:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with cluster
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and cluster
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-AOF:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with AOF
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and AOF
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 CLUSTER=0
+          no_output_timeout: 120m
 
   build-macos:
     macos:
@@ -175,7 +280,7 @@ jobs:
       - checkout
       - run:
           name: Submodule checkout
-          command: git submodule update --init --recursive 
+          command: git submodule update --init --recursive
       - run:
           name: Checkout LFS
           command: |
@@ -206,7 +311,7 @@ jobs:
       - checkout
       - run:
           name: Submodule checkout
-          command: git submodule update --init --recursive 
+          command: git submodule update --init --recursive
       - run:
           name: Build
           command: |
@@ -215,7 +320,7 @@ jobs:
           name: Test
           command: |
             mkdir -p $HOME/tests
-            docker run --gpus all -v $HOME/tests:/build/test/logs -it --rm redisai-gpu:latest-x64-bionic-test
+            docker run --gpus all -v $HOME/tests:/build/tests/logs -it --rm redisai-gpu:latest-x64-bionic-test
           no_output_timeout: 40m
       - store_artifacts:
           path: test/log
@@ -288,6 +393,17 @@ on-master: &on-master
   filters:
     branches:
       only: master
+    tags:
+      ignore: /.*/
+
+on-integ-branch: &on-integ-branch
+  filters:
+    branches:
+      only:
+        - master
+        - /^\d+\.\d+.*$/
+    tags:
+      ignore: /.*/
 
 on-version-tags: &on-version-tags
   filters:
@@ -296,38 +412,45 @@ on-version-tags: &on-version-tags
     tags:
       only: /^v[0-9].*/
 
-on-master-and-version-tags: &on-master-and-version-tags
+on-integ-and-version-tags: &on-integ-and-version-tags
   filters:
     branches:
       only:
         - master
+        - /^\d+\.\d+.*$/
     tags:
       only: /^v[0-9].*/
 
 
 platform-build-defs: &platform-build-defs
   requires:
     - build-debian
-  <<: *on-master-and-version-tags
+  <<: *on-integ-and-version-tags
   # <<: *on-any-branch # debugging
 
 after-platform-builds: &after-platform-builds
   requires:
-    - build-centos7
+    # - build-centos7
     - build-bionic
     - build-xenial
 
+after-linter: &after-linter
+  requires:
+    - lint
 
 workflows:
   version: 2
   build_and_package:
     jobs:
+      - lint:
+          <<: *on-any-branch
       - build-debian:
           <<: *on-any-branch
+          <<: *after-linter
       - platform-build:
           name: build-centos7
           platform: centos7
-          <<: *platform-build-defs
+          <<: *never # temporarily disabled
       - platform-build:
           name: build-bionic
           platform: bionic
@@ -338,8 +461,19 @@ workflows:
           <<: *platform-build-defs
       - coverage:
           <<: *on-any-branch
+          <<: *after-linter
+      - valgrind:
+          <<: *on-any-branch
+          <<: *after-linter
+      - valgrind-cluster:
+          <<: *on-integ-branch
+          <<: *after-linter
+      - valgrind-AOF:
+          <<: *on-integ-branch
+          <<: *after-linter
       - build-and-test-gpu:
           <<: *on-any-branch
+          <<: *after-linter
       - build-macos:
           <<: *never # temporarily disabled
           # <<: *on-version-tags
@@ -348,7 +482,7 @@ workflows:
           # <<: *on-version-tags
       - deploy-snapshot:
           <<: *after-platform-builds
-          <<: *on-master
+          <<: *on-integ-branch
       - deploy-release:
           <<: *after-platform-builds
           <<: *on-version-tags
@@ -357,7 +491,9 @@ workflows:
     triggers:
       - schedule:
           cron: "20 17 * * *"
-          <<: *on-master
+          filters:
+            branches:
+              only: master
     jobs:
       - build-macos:
           <<: *never # temporarily disabled

.clang-format

@@ -0,0 +1,6 @@
+IndentWidth: 4
+ColumnLimit: 100
+SortIncludes: false
+AlignEscapedNewlinesLeft: false
+AlignConsecutiveMacros: true
+SpacesBeforeTrailingComments: 1

.dockerignore

@@ -1,5 +1,6 @@
 /bin/
 /deps/
+/docs/
 /install*
 /build/
 .venv/

.gitignore

@@ -2,9 +2,9 @@
 /deps/
 /build/
 /install*
-/test/venv/
+/tests/venv/
 logs/
-/test/logs/
+/tests/logs/
 .env/
 env*/
 .venv/

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "opt/readies"]
 	path = opt/readies
 	url = https://github.com/RedisLabsModules/readies.git
+[submodule "opt/googletest"]
+	path = opt/googletest
+	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -3,6 +3,7 @@ PROJECT(RedisAI)
 
 # CMake modules should be included in ${PROJECT_SOURCE_DIR}/opt/cmake/modules
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/opt/cmake/modules)
+list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/opt/GoogleTest)
 
 # Set a default build type if none was specified
 set(default_build_type "Release")
@@ -23,6 +24,9 @@ ENDIF()
 #----------------------------------------------------------------------------------------------
 
 SET(CMAKE_CC_COMMON_FLAGS "-fPIC")
+IF (USE_PROFILE)
+    SET(CMAKE_CC_COMMON_FLAGS "${CMAKE_CC_COMMON_FLAGS} -g -ggdb -fno-omit-frame-pointer")
+ENDIF()
 
 IF (USE_COVERAGE)
     IF (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -46,6 +50,7 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CMAKE_COMMON_FLAGS_DEBUG}"
 
 #----------------------------------------------------------------------------------------------
 
+option(PACKAGE_UNIT_TESTS "Build unit tests" ON)
 option(BUILD_TF "Build the TensorFlow backend" ON)
 option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON)
 option(BUILD_ORT "Build the ONNXRuntime backend" ON)
@@ -309,3 +314,10 @@ ENDIF()
 IF (NOT ${installAbs} STREQUAL ${CMAKE_SOURCE_DIR}/install-${DEVICE})
     INSTALL_SYMLINK(${installAbs} ${CMAKE_SOURCE_DIR}/install-${DEVICE})
 ENDIF()
+
+
+if(PACKAGE_UNIT_TESTS)
+    enable_testing()
+    include(GoogleTest)
+    add_subdirectory(tests/unit)
+endif()