initial ci changes (#695)

Author: chayim

.circleci/config.yml

@@ -1,12 +1,20 @@
 version: 2.1
 
 commands:
+
+  abort_for_valgrind:
+    steps:
+      - run:
+          name: Avoid running valgrind unless explicitly called
+          command: |
+            git log --format=oneline -n 1 $CIRCLE_SHA1|grep 'run valgrind' || circleci step halt
+
   abort_for_docs:
     steps:
       - run:
           name: Avoid tests for docs
           command: |
-            if [[ $CIRCLE_BRANCH == *_docs ]]; then
+            if [[ $CIRCLE_BRANCH == *docs* ]]; then
               echo "Identifies as documents PR, no testing required"
               circleci step halt
             fi
@@ -29,8 +37,8 @@ commands:
       - run:
           name: Setup executor
           command: |
-            apt-get -qq update
-            apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
+            sudo apt-get -qq update
+            sudo apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
             bash <(curl -fsSL https://get.docker.com)
       - setup_remote_docker:
           docker_layer_caching: true
@@ -40,19 +48,19 @@ commands:
       - run:
           name: Relocate docker overlay2 dir
           command: |
-            sudo systemctl stop docker
+            sudo service docker stop
             sudo mkdir -p /var2/lib/docker
             sudo mv /var/lib/docker/overlay2 /var2/lib/docker
             sudo mkdir /var/lib/docker/overlay2
             sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
-            sudo systemctl start docker
+            sudo service docker start
 
   setup-automation:
     steps:
       - run:
           name: Setup automation
           command: |
-            git submodule update --init opt/readies 
+            git submodule update --init opt/readies
             ./opt/readies/bin/getpy3
 
   setup-build-system:
@@ -121,8 +129,8 @@ commands:
   platforms-build-steps:
     steps:
       - abort_for_docs
+      - setup-executor
       - checkout-all
-      - relocate-docker-storage
       - setup-automation
       - run:
           name: Build for platform
@@ -191,12 +199,11 @@ jobs:
           platform: debian
 
   platforms-build:
-    machine:
-      enabled: true
-      docker_layer_caching: true
-      resource_class: gpu.nvidia.small
-      image: ubuntu-1604-cuda-11.1:202012-01
+    docker:
+      - image: cimg/base:2020.01
     steps:
+      - setup-executor
+      - relocate-docker-storage
       - platforms-build-steps
 
   coverage:
@@ -256,51 +263,84 @@ jobs:
             make -C opt test VALGRIND=1 <<parameters.test_args>>
           no_output_timeout: 120m
 
-  build-macos:
-    macos:
-      xcode: 11.3.0
-    steps:
-      - abort_for_docs
-      - run:
-          name: Fix macOS Python installation
-          command: |
-            brew reinstall -f python2
-      - build-steps:
-          platform: macos
-
-  build-multiarch-docker:
-    machine:
-      enabled: true
-      image: cimg/base:2020.01
+  valgrind-general:
+    parameters:
+      test_args:
+        type: string
+        default: "CLUSTER=0 AOF=0"
+    docker:
+      - image: redisfab/rmbuilder:6.2.1-x64-buster
     steps:
+      - early_return_for_forked_pull_requests
       - abort_for_docs
       - checkout-all
+      - restore_cache:
+          keys:
+          - build-dependencies-{{ checksum "get_deps.sh" }}
+          # If no exact match is found will get dependencies from source
+      - setup-build-system
       - run:
-          name: Checkout LFS
+          name: Install dependencies
           command: |
-            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
-            sudo apt-get install -y git-lfs
-            git lfs install
-            git lfs pull
+            ./opt/readies/bin/getredis -v 6.0 --valgrind --force
+            ./get_deps.sh cpu
       - run:
-          name: Setup Docker client experimental features
+          name: Build for valgrind
           command: |
-            sudo ./opt/readies/bin/getdocker --just-enable-exp
-            docker version
+            make -C opt all VALGRIND=1 SHOW=1
       - run:
-          name: Build
+          name: Test with valgrind
           command: |
-            sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
-            cd opt/build/docker
-            make build
-            sudo make publish
+            make -C opt test VALGRIND=1 <<parameters.test_args>>
+          no_output_timeout: 120m
+
+
+#  build-macos:
+#    macos:
+#      xcode: 11.3.0
+#    steps:
+#      - abort_for_docs
+#      - run:
+#          name: Fix macOS Python installation
+#          command: |
+#            brew reinstall -f python2
+#      - build-steps:
+#          platform: macos
+#
+#  build-multiarch-docker:
+#    machine:
+#      enabled: true
+#      image: cimg/base:2020.01
+#    steps:
+#      - abort_for_docs
+#      - checkout-all
+#      - run:
+#          name: Checkout LFS
+#          command: |
+#            curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
+#            sudo apt-get install -y git-lfs
+#            git lfs install
+#            git lfs pull
+#      - run:
+#          name: Setup Docker client experimental features
+#          command: |
+#            sudo ./opt/readies/bin/getdocker --just-enable-exp
+#            docker version
+#      - run:
+#          name: Build
+#          command: |
+#            sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
+#            cd opt/build/docker
+#            make build
+#            sudo make publish
 
   build-and-test-gpu:
     machine:
       enabled: true
       docker_layer_caching: true
       resource_class: gpu.nvidia.small
       image: ubuntu-1604-cuda-11.1:202012-01
+
     steps:
       - abort_for_docs
       - checkout-all
@@ -414,6 +454,13 @@ on-any-branch: &on-any-branch
     tags:
       only: /.*/
 
+on-any-branch-but-tags: &on-any-branch-but-tags
+  filters:
+    branches:
+      only: /.*/
+    tags:
+      ignore: /.*/
+
 never: &never
   filters:
     branches:
@@ -434,17 +481,15 @@ on-integ-branch: &on-integ-branch
       only:
         - master
         - /^\d+\.\d+.*$/
-        - /^feature-.*$/
     tags:
       ignore: /.*/
 
-not-on-integ-branch: &not-on-integ-branch
+on-dev-branches: &on-dev-branches
   filters:
     branches:
       ignore:
         - master
         - /^\d+\.\d+.*$/
-        - /^feature-.*$/
     tags:
       ignore: /.*/
 
@@ -455,13 +500,11 @@ on-version-tags: &on-version-tags
     tags:
       only: /^v[0-9].*/
 
-on-integ-and-version-tags: &on-integ-and-version-tags
+on-master-and-version-tags: &on-master-and-version-tags
   filters:
     branches:
       only:
         - master
-        - /^\d+\.\d+.*$/
-        - /^feature-.*$/
     tags:
       only: /^v[0-9].*/
 
@@ -478,30 +521,22 @@ after-platform-builds: &after-platform-builds
   requires:
     - platforms-build
 
-platform-build-defs: &platform-build-defs
-  <<: *after-build-and-test
-  <<: *on-integ-and-version-tags
-  # <<: *on-any-branch # debugging
-
-
+#### define workflows
 workflows:
   version: 2
   build_and_package:
     jobs:
       - lint:
           <<: *on-any-branch
       - build-and-test:
-          <<: *on-any-branch
+          <<: *on-any-branch-but-tags
           <<: *after-linter
       - platforms-build:
-          <<: *platform-build-defs
+          <<: *after-build-and-test
+          <<: *on-master-and-version-tags
       - coverage:
           context: common
-          <<: *on-any-branch
-          <<: *after-linter
-      - valgrind:
-          test_args: CLUSTER=0 AOF=0
-          <<: *on-any-branch
+          <<: *on-dev-branches
           <<: *after-linter
       - valgrind:
           name: valgrind-cluster
@@ -514,15 +549,8 @@ workflows:
           <<: *on-integ-branch
           <<: *after-linter
       - build-and-test-gpu:
-          <<: *on-any-branch
+          <<: *on-integ-branch
           <<: *after-linter
-      - build-macos:
-          <<: *never # temporarily disabled
-          # <<: *on-version-tags
-      - build-multiarch-docker:
-          context: common
-          <<: *never # temporarily disabled
-          # <<: *on-version-tags
       - deploy-snapshot:
           context: common
           <<: *after-platform-builds
@@ -545,7 +573,7 @@ workflows:
             branches:
               only: master
     jobs:
-      - build-macos:
-          <<: *never # temporarily disabled
+#      - build-macos:
+#          <<: *never # temporarily disabled
       - nightly-automation:
           context: common