new docker system (#783)

Author: chayim

.circleci/config.yml

@@ -174,19 +174,12 @@ commands:
       - abort_for_docs
       - abort_for_noci
       - checkout-all
-      - run:
-          name: Relocate docker overlay2 dir
-          command: |
-            sudo systemctl stop docker
-            sudo mkdir -p /var2/lib/docker
-            sudo mv /var/lib/docker/overlay2 /var2/lib/docker
-            sudo mkdir /var/lib/docker/overlay2
-            sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
-            sudo systemctl start docker
+      - relocate-docker-storage
       - run:
           name: Build
           command: |
-            docker build -f Dockerfile.gpu-test --no-cache -t redisai-gpu:latest-x64-bionic-test .
+            pushd opt/build/docker
+            make DOCKER_SUFFIX=".gpu-test" DOCKER_ARGS="-s dockerfile-gpu-test.tmpl" DOCKER_OPTS=--no-cache DEFAULT_TAG="redisai-gpu:latest-test"
       - run:
           name: Test
           command: |
@@ -196,7 +189,6 @@ commands:
       - store_artifacts:
           path: /home/circleci/tests
 
-
 jobs:
   lint:
     docker:
@@ -235,13 +227,15 @@ jobs:
       - setup_remote_docker
       - checkout-all
       - setup-automation
+
+      # since we run in parallel, we need to generate docker files with different suffixes hence the DOCKER_SUFFIX
       - run:
           name: Build for platform
           command: |
             bash <(curl -fsSL https://get.docker.com)
             pushd opt/build/docker
             docker login -u redisfab -p $DOCKER_REDISFAB_PWD
-            make build <<parameters.target>> OSNICK=<<parameters.osnick>> ARTIFACTS=1 VERBOSE=1 <<parameters.lite>>
+            make build DOCKER_SUFFIX="$$" <<parameters.target>> OSNICK=<<parameters.osnick>> PACK=1 VERBOSE=1 <<parameters.lite>>
             popd > /dev/null
             logstar=bin/artifacts/tests-logs-cpu.tgz
             logsdir=tests/logs/cpu
@@ -574,7 +568,7 @@ workflows:
                 - xenial
                 - bionic
               lite:
-                - "REDISAI_LITE=0 publish PUSH_GENERAL=1 OFFICIAL=1"
+                - "REDISAI_LITE=0 PUBLISH=1"
                 - "REDISAI_LITE=1"
               target:
                 - "CPU=1"

Dockerfile

@@ -113,12 +113,15 @@ You can compile and build the module from its source code - refer to the [Buildi
 
 **Building in a docker (x86_64)**
 
-The RedisAI source code can be mounted in a docker, and built there, but edited from the external operating system. This assumes that you are running a modern version of docker, and that you are making a recursive clone of this repository and all of its submodules.
+The RedisAI source code can be mounted in a docker, and built there, but edited from the external operating system. This assumes that you are running a modern version of docker, and that you are making a recursive clone of this repository and all of its submodules. This assumes that you have jinja installed, as the docker files are geneated from the dockerfile.tmpl in the *opt/build/docker* directory.
 
 ```
 git clone --recursive https://github.com/RedisAI/RedisAI
-cd RedisAI
-docker build -t redisai:latest -f Dockerfile --build-arg OSNICK=bionic OS=ubuntu:18.04 .
+cd RedisAI/opt/build/docker
+make
+
+# note, to build with GPU support instead
+make GPU=1
 ```
 
 After this, you can run the create docker and mount your source code with the following command, from within the RedisAI folder.

Dockerfile.gpu

@@ -1,46 +1,68 @@
+# Feel free to change these
+PRODUCT=redisai
+DOCKER_ORG=redislabs
+REDIS_VERSION=6.2.4
+REDIS_CUDA_VERSION=11.0-cudnn8
+REDISAI_LITE=0
 
-ROOT=../../..
-MK.pyver:=3
-include $(ROOT)/opt/readies/mk/main
-
-# OSNICK=buster|stretch|bionic|xenial|centos7
-OSNICK ?= buster
-
-REPO=redisai
-
-# LATEST_BRANCH=1.2
-INT_BRANCHES=1.2
-
-REDIS_VER:=6.2.4
-REDISAI_LITE:=0
+PACK=1 # to fetch the artifacts
 
-export ART_DIR=$(ROOT)/bin/artifacts
-export ART_INT_DIR=/var/opt/redislabs/artifacts
+DOCKER_OPTS=  # set, to pass custom options to docker, but remember to quote them
+OSNICK ?= bionic
 
-DOCKER_BUILD_ARGS=\
-	REDISAI_LITE=$(REDISAI_LITE)
+# set to pass multiple directories as docker sources to include in templates
+DOCKER_SOURCES=""
 
-#----------------------------------------------------------------------------------------------
+# set, to generate a file named Dockerfile<suffix>, i.e useful when you want to run things in parallel
+DOCKER_SUFFIX=
 
-ifeq ($(CPU),1)
-VARIANT=cpu
+# Add space delimited docker tags, and they'll all be applied
+DOCKER_TAGS=
 
-else ifeq ($(GPU),1)
-VARIANT=gpu
-DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.gpu
+DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-cpu-${OSNICK}
 
-OS.custom:=1
-
-OS.centos7=centos7
-OS.bionic=ubuntu18.04
-OS.xenial=ubuntu16.04
-
-else ifeq ($(JETSON),1)
-VARIANT=jetson
-DOCKERFILE_STEM ?= $(ROOT)/Dockerfile.jetson
+# set gpu arguments
+ifeq ($(GPU),1)
+DOCKER_ARGS = gpu_build=1
+DOCKER_SUFFIX=gpu
+DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-gpu-${OSNICK}
+endif
 
+# remap ubuntu versions because of nvidia cuda
+ifeq ($(OSNICK),xenial)
+REDIS_CUDA_UBUNTUVERSION=16.04
+endif
+ifeq ($(OSNICK),bionic)
+REDIS_CUDA_UBUNTUVERSION=18.04
 endif
 
-#----------------------------------------------------------------------------------------------
+DOCKERWRAPPER_EXTRA_VARS=\
+	REDISAI_LITE=${REDISAI_LITE} \
+	REDIS_CUDA_VERSION=${REDIS_CUDA_VERSION} \
+	REDIS_CUDA_UBUNTUVERSION=${REDIS_CUDA_UBUNTUVERSION}
 
-include $(MK)/docker.native
+### Defaults ###
+ROOT=../../..
+READIES=${ROOT}/opt/readies
+
+#------------------------------------------------------------------------------
+define HELP
+make build		# build and optionally publish the docker, from a template file
+	OSNICK=<>					# base operating system
+	REDISAI_LITE=1					# enable RedisAI lite builds
+	DOCKER_SUFFIX=<>				# optional suffix for the generated dockerfile
+	DOCKER_ARGS=FOO=BAR				# key-value pairs of variables to pass into the docker build
+	PACK=1						# fetch generated artifacts
+	DOCKER_TAGS=a,b,c				# tags to append and push to dockerhub
+	DEFAULT_TAG=redislabs/redisai			# default docker tag to build, and push
+	TEST=1						# run tests, if specified
+	VERSION=x.y.z					# set the docker version
+	NOP=1						# set to echo files in docker generation, and not run
+	DOCKER_SOURCES=/a/path				# append paths to the template generator
+	PUBLISH=1					# if set, push to dockerhub (requires docker login)
+	DOCKER_OPTS=XXX					# Options to pass to the docker build command
+endef
+#------------------------------------------------------------------------------
+
+# rules, for building
+include ${READIES}/mk/docker.rules