From e9f6d177a7285decdd22367c37d49ed6fc9a3160 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Wed, 28 Mar 2018 13:45:30 -0700 Subject: [PATCH 1/4] Add travis test for DockerHub images * Add a new Travis job to test DockerHub images and modify Travis to run 2 test jobs in parallel. * Move the test script commands under etc/travis. * Add new helper make function pull-dockerhub-images to pull the DockerHub images with tag 0.0.1-master. --- .travis.yml | 18 ++++++------------ Makefile | 17 ++++++++++++++++- README.md | 14 +++++--------- etc/travis/test-images-dockerhub.sh | 10 ++++++++++ etc/travis/test-images-local.sh | 11 +++++++++++ lcm/service/lcm/container_helper_extensions.go | 2 +- 6 files changed, 49 insertions(+), 23 deletions(-) create mode 100755 etc/travis/test-images-dockerhub.sh create mode 100755 etc/travis/test-images-local.sh diff --git a/.travis.yml b/.travis.yml index 067c7a01..bf9708fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,18 +27,12 @@ before_script: # install aws CLI - (sudo pip install -q awscli; which aws > /dev/null) & -script: - # fail fast - - set -e - - export MAKE_ARGS=--no-print-directory - # compile and build Docker images - - glide -q install - - make $MAKE_ARGS build - - make $MAKE_ARGS docker-build - # deploy services - - make $MAKE_ARGS deploy - # submit a test job - - make $MAKE_ARGS test-submit +jobs: + include: + - stage: test + script: ./etc/travis/test-images-local.sh + - if: type IN (push, api, cron) + script: ./etc/travis/test-images-dockerhub.sh notifications: email: false diff --git a/Makefile b/Makefile index a9185064..1baa1107 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,10 @@ UI_REPO = git@github.com:IBM/FfDL-dashboard.git CLI_CMD = $(shell pwd)/cli/bin/ffdl-$(UNAME_SHORT) CLUSTER_NAME ?= mycluster PUBLIC_IP ?= 127.0.0.1 +TRAVIS_IMAGES_VERSION ?= 0.0.1-master +TEST_IMAGES = $(addprefix $(DOCKER_NAMESPACE)/, $(TEST_IMAGES_SUBFIX)) +TEST_IMAGES_SUBFIX = ffdl-lcm ffdl-trainer ffdl-metrics ffdl-databroker_s3 ffdl-ui ffdl-restapi ffdl-jobmonitor ffdl-controller tensorboard_extract_1.3-py3 log_collector ffdl-databroker_objectstorage tensorboard_extract + IMAGE_DIR := $(IMAGE_NAME) ifneq ($(filter $(IMAGE_NAME),controller ),) @@ -203,8 +207,19 @@ test-submit: ## Submit test training job kubectl get pods | grep learner- | awk '{print $$1}' | xargs -I '{}' kubectl logs '{}' -c load-data; \ exit 1); +pull-dockerhub-images: ## Pull FfDL images from dockerhub +pull-dockerhub-images: $(addprefix pull-, $(TEST_IMAGES)) + +$(addprefix pull-, $(TEST_IMAGES)): pull-%: % + @TRAVIS_IMAGES=$< make .pull-dockerhub-images + +$(TEST_IMAGES): ; + # Helper targets +.pull-dockerhub-images: + docker pull $(TRAVIS_IMAGES):$(TRAVIS_IMAGES_VERSION) + .build-service: (cd ./$(SERVICE_NAME)/ && (test ! -e main.go || CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -a -installsuffix cgo -o bin/$(BINARY_NAME))) @@ -233,7 +248,7 @@ install-minikube-in-ci: chmod +x kubectl && sudo mv kubectl /usr/local/bin/ @curl -s -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && \ chmod +x minikube && sudo mv minikube /usr/local/bin/ - @sudo minikube start --vm-driver=none --feature-gates=ReadOnlyAPIDataVolumes=false + @sudo minikube start --vm-driver=none @minikube update-context @JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; \ until kubectl get nodes -o jsonpath="$$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 1; done diff --git a/README.md b/README.md index b046ee29..f7a24960 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,7 @@ FfDL is a collaboration platform for: * `S3 CLI`: The [command-line interface](https://aws.amazon.com/cli/) to configure your Object Storage * An existing Kubernetes cluster (e.g., [Minikube](https://github.com/kubernetes/minikube) for local testing). - Once installed, use the command `make minikube` to start Minikube and set up local network routes. Alternatively, - use the Vagrant based setup to automatically install a local Kubernetes cluster. + Once installed, use the command `make minikube` to start Minikube and set up local network routes. * Follow the appropriate instructions for standing up your Kubernetes cluster using [IBM Cloud Public](https://github.com/IBM/container-journey-template/blob/master/README.md) or [IBM Cloud Private](https://github.com/IBM/deploy-ibm-cloud-private/blob/master/README.md) @@ -32,13 +31,13 @@ FfDL is a collaboration platform for: ## Usage Scenarios -* If you already have a FfDL deployment up and running, you can jump to [FfDL User Guide](docs/user-guide.md) to use FfDL for training your models. +* If you already have a FfDL deployment up and running, you can jump to [FfDL User Guide](docs/user-guide.md) to use FfDL for training your deep learning models. * If you have FfDL confiugured to use GPUs, and want to train using GPUs, follow steps [here](docs/gpu-guide.md) -* If you have used FfDL to train your models, and want to use a GPU enabled public cloud hosted service for further training and serving, please follow instructions [here](etc/converter/ffdl-wml.md) to train and serve your models using [Watson Studio Deep Learning](https://www.ibm.com/cloud/machine-learning) service +* If you have used FfDL to train your models, and want to use a GPU enabled public cloud hosted service for further training and serving, please follow instructions [here](etc/converter/ffdl-wml.md) to train and serve your models using [Watson Studio Deep Learning](https://www.ibm.com/cloud/deep-learning) service -* If you are starting and want to setup your own FfDL deployment, please follow the steps below. +* If you are getting started and want to setup your own FfDL deployment, please follow the steps below. ## Steps @@ -59,8 +58,7 @@ FfDL is a collaboration platform for: ## 1. Quick Start -There are multiple installation paths for installing FfDL locally ("1-click-install") or -into an existing Kubernetes cluster. +There are multiple installation paths for installing FfDL locally ("1-click-install") or into an existing Kubernetes cluster. > Note: If your Kubernetes Cluster version is 1.7 or below, please go to the [values.yaml](values.yaml) and change `k8s_1dot8_or_above` to **false**. @@ -403,8 +401,6 @@ helm delete $(helm list | grep ffdl | awk '{print $1}' | head -n 1) * To remove FfDL on your Cluster, simply run `make undeploy` -* Since the current implementation of FfDL needs write access for its mounted volume, if you are using Kubernetes 1.9.4 or above, please modify the feature gate `ReadOnlyAPIDataVolumes=false`. - ## 9. References Based on IBM Research work in Deep Learning. diff --git a/etc/travis/test-images-dockerhub.sh b/etc/travis/test-images-dockerhub.sh new file mode 100755 index 00000000..0374c337 --- /dev/null +++ b/etc/travis/test-images-dockerhub.sh @@ -0,0 +1,10 @@ +# fail fast +set -e +export MAKE_ARGS=--no-print-directory +# pull images from dockerhub +make $MAKE_ARGS pull-dockerhub-images + +# deploy services +make $MAKE_ARGS deploy +# submit a test job +make $MAKE_ARGS test-submit diff --git a/etc/travis/test-images-local.sh b/etc/travis/test-images-local.sh new file mode 100755 index 00000000..333e6ca4 --- /dev/null +++ b/etc/travis/test-images-local.sh @@ -0,0 +1,11 @@ +# fail fast +set -e +export MAKE_ARGS=--no-print-directory +# compile and build Docker images +glide -q install +make $MAKE_ARGS build +make $MAKE_ARGS docker-build +# deploy services +make $MAKE_ARGS deploy +# submit a test job +make $MAKE_ARGS test-submit diff --git a/lcm/service/lcm/container_helper_extensions.go b/lcm/service/lcm/container_helper_extensions.go index 04de99fc..cb61a5ab 100644 --- a/lcm/service/lcm/container_helper_extensions.go +++ b/lcm/service/lcm/container_helper_extensions.go @@ -35,7 +35,7 @@ func extendLearnerContainer(learner *v1core.Container, req *service.JobDeploymen // TODO! } - extCmd := "export PATH=" + learnerEntrypointFilesPath + ":$PATH; chmod +x " + learnerEntrypointFilesPath + "/*.sh; " + extCmd := "export PATH=/usr/local/bin/:$PATH; cp " + learnerEntrypointFilesPath + "/*.sh /usr/local/bin/; chmod +x /usr/local/bin/*.sh;" extMount := v1core.VolumeMount{ Name: learnerEntrypointFilesVolume, MountPath: learnerEntrypointFilesPath, From ee6249bc24a3bff2a1421e693c6fbbb6c2c46a37 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Wed, 28 Mar 2018 14:03:20 -0700 Subject: [PATCH 2/4] modify travis to tag pulled images as latest for helm install --- Makefile | 8 ++++++++ etc/travis/test-images-dockerhub.sh | 1 + 2 files changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 1baa1107..f43a5919 100644 --- a/Makefile +++ b/Makefile @@ -215,11 +215,19 @@ $(addprefix pull-, $(TEST_IMAGES)): pull-%: % $(TEST_IMAGES): ; +tag-dockerhub-images-to-latest: $(addprefix tag-, $(TEST_IMAGES)) + +$(addprefix tag-, $(TEST_IMAGES)): tag-%: % + @TRAVIS_IMAGES=$< make .tag-dockerhub-latest + # Helper targets .pull-dockerhub-images: docker pull $(TRAVIS_IMAGES):$(TRAVIS_IMAGES_VERSION) +.tag-dockerhub-latest: + docker tag $(TRAVIS_IMAGES):$(TRAVIS_IMAGES_VERSION) $(TRAVIS_IMAGES):latest + .build-service: (cd ./$(SERVICE_NAME)/ && (test ! -e main.go || CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -a -installsuffix cgo -o bin/$(BINARY_NAME))) diff --git a/etc/travis/test-images-dockerhub.sh b/etc/travis/test-images-dockerhub.sh index 0374c337..891fc615 100755 --- a/etc/travis/test-images-dockerhub.sh +++ b/etc/travis/test-images-dockerhub.sh @@ -3,6 +3,7 @@ set -e export MAKE_ARGS=--no-print-directory # pull images from dockerhub make $MAKE_ARGS pull-dockerhub-images +make $MAKE_ARGS tag-dockerhub-images-to-latest # deploy services make $MAKE_ARGS deploy From f5acf3d373ddfa0ec02ad0d205a3f493fd72e1cf Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 9 Apr 2018 10:42:41 -0700 Subject: [PATCH 3/4] update script and makefile as suggested --- Makefile | 8 ++++---- etc/travis/test-images-dockerhub.sh | 1 + etc/travis/test-images-local.sh | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f43a5919..14d9eb61 100644 --- a/Makefile +++ b/Makefile @@ -211,22 +211,22 @@ pull-dockerhub-images: ## Pull FfDL images from dockerhub pull-dockerhub-images: $(addprefix pull-, $(TEST_IMAGES)) $(addprefix pull-, $(TEST_IMAGES)): pull-%: % - @TRAVIS_IMAGES=$< make .pull-dockerhub-images + @TRAVIS_IMAGE=$< make .pull-dockerhub-images $(TEST_IMAGES): ; tag-dockerhub-images-to-latest: $(addprefix tag-, $(TEST_IMAGES)) $(addprefix tag-, $(TEST_IMAGES)): tag-%: % - @TRAVIS_IMAGES=$< make .tag-dockerhub-latest + @TRAVIS_IMAGE=$< make .tag-dockerhub-latest # Helper targets .pull-dockerhub-images: - docker pull $(TRAVIS_IMAGES):$(TRAVIS_IMAGES_VERSION) + docker pull $(TRAVIS_IMAGE):$(TRAVIS_IMAGES_VERSION) .tag-dockerhub-latest: - docker tag $(TRAVIS_IMAGES):$(TRAVIS_IMAGES_VERSION) $(TRAVIS_IMAGES):latest + docker tag $(TRAVIS_IMAGE):$(TRAVIS_IMAGES_VERSION) $(TRAVIS_IMAGE):latest .build-service: (cd ./$(SERVICE_NAME)/ && (test ! -e main.go || CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -a -installsuffix cgo -o bin/$(BINARY_NAME))) diff --git a/etc/travis/test-images-dockerhub.sh b/etc/travis/test-images-dockerhub.sh index 891fc615..87dcc3dc 100755 --- a/etc/travis/test-images-dockerhub.sh +++ b/etc/travis/test-images-dockerhub.sh @@ -1,3 +1,4 @@ +#!/bin/bash # fail fast set -e export MAKE_ARGS=--no-print-directory diff --git a/etc/travis/test-images-local.sh b/etc/travis/test-images-local.sh index 333e6ca4..3668b38e 100755 --- a/etc/travis/test-images-local.sh +++ b/etc/travis/test-images-local.sh @@ -1,3 +1,4 @@ +#!/bin/bash # fail fast set -e export MAKE_ARGS=--no-print-directory From 6d94dcfea1b7c172c361d73cf4f93cbbecf8f4db Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 9 Apr 2018 10:44:58 -0700 Subject: [PATCH 4/4] update script and makefile as suggested --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 14d9eb61..6e667b2d 100644 --- a/Makefile +++ b/Makefile @@ -22,9 +22,9 @@ UI_REPO = git@github.com:IBM/FfDL-dashboard.git CLI_CMD = $(shell pwd)/cli/bin/ffdl-$(UNAME_SHORT) CLUSTER_NAME ?= mycluster PUBLIC_IP ?= 127.0.0.1 -TRAVIS_IMAGES_VERSION ?= 0.0.1-master -TEST_IMAGES = $(addprefix $(DOCKER_NAMESPACE)/, $(TEST_IMAGES_SUBFIX)) -TEST_IMAGES_SUBFIX = ffdl-lcm ffdl-trainer ffdl-metrics ffdl-databroker_s3 ffdl-ui ffdl-restapi ffdl-jobmonitor ffdl-controller tensorboard_extract_1.3-py3 log_collector ffdl-databroker_objectstorage tensorboard_extract +TRAVIS_IMAGE_VERSION ?= 0.0.1-master +TEST_IMAGES = $(addprefix $(DOCKER_NAMESPACE)/, $(TEST_IMAGES_SUFFIX)) +TEST_IMAGES_SUFFIX = ffdl-lcm ffdl-trainer ffdl-metrics ffdl-databroker_s3 ffdl-ui ffdl-restapi ffdl-jobmonitor ffdl-controller tensorboard_extract_1.3-py3 log_collector ffdl-databroker_objectstorage tensorboard_extract IMAGE_DIR := $(IMAGE_NAME) @@ -223,10 +223,10 @@ $(addprefix tag-, $(TEST_IMAGES)): tag-%: % # Helper targets .pull-dockerhub-images: - docker pull $(TRAVIS_IMAGE):$(TRAVIS_IMAGES_VERSION) + docker pull $(TRAVIS_IMAGE):$(TRAVIS_IMAGE_VERSION) .tag-dockerhub-latest: - docker tag $(TRAVIS_IMAGE):$(TRAVIS_IMAGES_VERSION) $(TRAVIS_IMAGE):latest + docker tag $(TRAVIS_IMAGE):$(TRAVIS_IMAGE_VERSION) $(TRAVIS_IMAGE):latest .build-service: (cd ./$(SERVICE_NAME)/ && (test ! -e main.go || CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -a -installsuffix cgo -o bin/$(BINARY_NAME)))