Merge master

Author: alonre24

.circleci/config.yml

@@ -17,7 +17,7 @@ commands:
       - run:
           name: Setup automation
           command: |
-            PIP=1 ./opt/readies/bin/getpy3
+            ./opt/readies/bin/getpy3
             ./opt/system-setup.py
 
   build-steps:
@@ -125,6 +125,7 @@ jobs:
       - run:
           name: Submodule checkout
           command: git submodule update --init --recursive
+      - setup-automation
       - run:
           name: lint
           command: |
@@ -201,7 +202,63 @@ jobs:
       - run:
           name: Test with valgrind
           command: |
-            make -C opt test VALGRIND=1
+            make -C opt test VALGRIND=1 CLUSTER=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-cluster:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with cluster
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and cluster
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 AOF=0
+          no_output_timeout: 120m
+
+  valgrind-AOF:
+    docker:
+      - image: redisfab/rmbuilder:6.0.5-x64-buster
+    steps:
+      - checkout
+      - run:
+          name: Submodule checkout
+          command: git submodule update --init --recursive
+      - restore_cache:
+          keys:
+            - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-automation
+      - run:
+          name: Install dependencies
+          command: |
+            ./opt/readies/bin/getredis -v 6 --valgrind --force
+            ./get_deps.sh cpu
+      - run:
+          name: Build for valgrind with AOF
+          command: |
+            make -C opt all VALGRIND=1 SHOW=1
+      - run:
+          name: Test with valgrind and AOF
+          command: |
+            make -C opt test VALGRIND=1 GEN=0 CLUSTER=0
           no_output_timeout: 120m
 
   build-macos:
@@ -385,7 +442,8 @@ workflows:
   version: 2
   build_and_package:
     jobs:
-      - lint
+      - lint:
+          <<: *on-any-branch
       - build-debian:
           <<: *on-any-branch
           <<: *after-linter
@@ -405,7 +463,13 @@ workflows:
           <<: *on-any-branch
           <<: *after-linter
       - valgrind:
-          <<: *on-master
+          <<: *on-any-branch
+          <<: *after-linter
+      - valgrind-cluster:
+          <<: *on-integ-branch
+          <<: *after-linter
+      - valgrind-AOF:
+          <<: *on-integ-branch
           <<: *after-linter
       - build-and-test-gpu:
           <<: *on-any-branch

docs/intro.md

@@ -8,6 +8,10 @@ This introduction is intended to present the core concepts it uses and the funct
 !!! important "Prerequisites"
     Before diving into RedisAI please make sure that you are familiar with the basic concepts of machine learning and Redis.
 
+This video will introduce you to RedisAI at a high level. Give it a watch, then keep reading for more details.
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/gbiqF-eyTW4" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
 In broad strokes, RedisAI's looks as follows:
 
 ```

opt/system-setup.py

@@ -18,33 +18,28 @@ def __init__(self, nop=False):
 
     def common_first(self):
         self.install_downloaders()
-        self.setup_pip()
-        self.pip_install("wheel virtualenv")
+        self.pip_install("wheel")
 
-        if self.os == 'linux':
-            self.install("ca-certificates")
-        self.install("git unzip wget patchelf")
+        self.install("git unzip patchelf")
         self.install("coreutils") # for realpath
 
     def debian_compat(self):
         self.install("gawk")
         self.install("build-essential")
         self.install("libssl-dev")
-        self.install("clang-format")
         self.install("python3-regex")
         self.install("python3-psutil python3-networkx python3-numpy")
-        if self.arch == 'arm64v8' or self.arch == 'arm32v7':
+        if self.platform.is_arm():
             self.install("python3-dev") # python3-skimage
         self.install("libmpich-dev libopenblas-dev") # for libtorch
         self.install_git_lfs_on_linux()
 
     def redhat_compat(self):
         self.install("redhat-lsb-core")
-        self.run("%s/readies/bin/enable-utf8" % HERE)
+        self.run("%s/bin/enable-utf8" % READIES)
 
-        self.group_install("'Development Tools'")
         self.run("%s/bin/getgcc --modern" % READIES)
-        self.install("llvm-toolset-7")
+        # self.install("llvm-toolset-7")
 
         if self.arch == 'x64':
             self.install_linux_gnu_tar()
@@ -63,7 +58,6 @@ def redhat_compat(self):
     def fedora(self):
         self.group_install("'Development Tools'")
         self.install("python3 python3-psutil python3-networkx")
-        self.install("clang")
         self.install_git_lfs_on_linux()
 
     def linux_last(self):
@@ -73,18 +67,13 @@ def macos(self):
         self.install_gnu_utils()
         self.install("git-lfs")
         self.install("redis")
-        self.install("clang-format")
 
     def common_last(self):
+        self.run("%s/bin/getclang --format" % READIES)
         self.run("%s/bin/getcmake" % READIES)
 
-        self.run("python3 -m pip uninstall -y ramp-packer RLTest || true")
-        # redis-py-cluster should be installed from git due to redis-py dependency
-        self.pip_install("--no-cache-dir git+https://github.com/Grokzen/redis-py-cluster.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master")
-        self.pip_install("--no-cache-dir git+https://github.com/RedisLabs/RAMP@master")
+        self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES))
 
-        self.pip_install("-r %s/readies/paella/requirements.txt" % HERE)
         self.pip_install("-r %s/tests/flow/test_requirements.txt" % ROOT)
 
         self.pip_install("awscli")

src/CMakeLists.txt

@@ -16,6 +16,7 @@ file (GLOB BACKEND_COMMON_SRC
         err.c
         util/dict.c
         tensor.c
+        util/string_utils.c
         serialization/ai_datatypes.c)
 
 ADD_LIBRARY(redisai_obj OBJECT

src/backends.c

@@ -122,7 +122,7 @@ int RAI_LoadBackend_TensorFlow(RedisModuleCtx *ctx, const char *path) {
     }
 
     backend.model_serialize = (int (*)(RAI_Model *, char **, size_t *, RAI_Error *))(
-        unsigned long)dlsym(handle, "RAI_ModelSerializeTF");
+        (unsigned long)dlsym(handle, "RAI_ModelSerializeTF"));
     if (backend.model_serialize == NULL) {
         dlclose(handle);
         RedisModule_Log(ctx, "warning",

src/libtorch_c/torch_extensions/torch_redis.cpp

@@ -56,8 +56,8 @@ torch::IValue redisExecute(std::string fn_name, std::vector<std::string> args )
     RedisModuleCallReply *reply = RedisModule_Call(ctx, fn_name.c_str(), "!v", arguments, len);
     RedisModule_ThreadSafeContextUnlock(ctx);
     torch::IValue value = IValueFromRedisReply(reply);
-    RedisModule_FreeThreadSafeContext(ctx);
     RedisModule_FreeCallReply(reply);
+    RedisModule_FreeThreadSafeContext(ctx);
     for(int i= 0; i < len; i++){
         RedisModule_FreeString(NULL, arguments[i]);
     }

src/tensor.c

@@ -98,10 +98,14 @@ int Tensor_DataTypeStr(DLDataType dtype, char *dtypestr) {
 RAI_Tensor *RAI_TensorCreateWithDLDataType(DLDataType dtype, long long *dims, int ndims,
                                            int tensorAllocMode) {
 
+    size_t dtypeSize = Tensor_DataTypeSize(dtype);
+    if (dtypeSize == 0) {
+        return NULL;
+    }
+
     RAI_Tensor *ret = RedisModule_Alloc(sizeof(*ret));
     int64_t *shape = RedisModule_Alloc(ndims * sizeof(*shape));
     int64_t *strides = RedisModule_Alloc(ndims * sizeof(*strides));
-    size_t dtypeSize = Tensor_DataTypeSize(dtype);
 
     size_t len = 1;
     for (int64_t i = 0; i < ndims; ++i) {
@@ -189,6 +193,7 @@ RAI_Tensor *_TensorCreateWithDLDataTypeAndRString(DLDataType dtype, size_t dtype
     }
     char *data = RedisModule_Alloc(nbytes);
     memcpy(data, blob, nbytes);
+    RAI_HoldString(NULL, rstr);
 
     RAI_Tensor *ret = RedisModule_Alloc(sizeof(*ret));
     ret->tensor = (DLManagedTensor){.dl_tensor = (DLTensor){.ctx = ctx,
@@ -686,7 +691,6 @@ int RAI_parseTensorSetArgs(RedisModuleString **argv, int argc, RAI_Tensor **t, i
 
     if (datafmt == TENSOR_BLOB) {
         RedisModuleString *rstr = argv[argpos];
-        RedisModule_RetainString(NULL, rstr);
         *t = _TensorCreateWithDLDataTypeAndRString(datatype, datasize, dims, ndims, rstr, error);
     } else {
         *t = RAI_TensorCreateWithDLDataType(datatype, dims, ndims, tensorAllocMode);

tests/flow/tests_dag.py

@@ -230,6 +230,7 @@ def test_dagrun_ro_modelrun_scriptrun_resnet(env):
     if(VALGRIND):
         env.debugPrint("skipping {} since it's hanging CI".format(sys._getframe().f_code.co_name), force=True)
         env.skip()
+
     con = env.getConnection()
     model_name = 'imagenet_model{{1}}'
     script_name = 'imagenet_script{{1}}'

tests/flow/tests_tensorflow.py

@@ -84,10 +84,6 @@ def test_run_mobilenet(env):
 
 @skip_if_no_TF
 def test_run_mobilenet_multiproc(env):
-    if VALGRIND:
-        env.debugPrint("skipping {} since VALGRIND=1".format(
-            sys._getframe().f_code.co_name), force=True)
-        return
 
     con = env.getConnection()
 
@@ -634,10 +630,7 @@ def test_tensorflow_modelrun_disconnect(env):
 
 @skip_if_no_TF
 def test_tensorflow_modelrun_with_batch_and_minbatch(env):
-    if VALGRIND:
-        env.debugPrint("skipping {} since VALGRIND=1".format(
-            sys._getframe().f_code.co_name), force=True)
-        return
+
     con = env.getConnection()
     batch_size = 2
     minbatch_size = 2