Skip to content

Commit bacadd7

Browse files
alonre24alonre24
authored
Merge branch 'master' into redis-cli-modelget-limitation
2 parents 2c283d1 + fda8e30 commit bacadd7

36 files changed

+1755
-1221
lines changed

‎.circleci/config.yml‎

Lines changed: 98 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,20 @@
11
version: 2.1
22

33
commands:
4+
5+
abort_for_valgrind:
6+
steps:
7+
- run:
8+
name: Avoid running valgrind unless explicitly called
9+
command: |
10+
git log --format=oneline -n 1 $CIRCLE_SHA1|grep 'run valgrind' || circleci step halt
11+
412
abort_for_docs:
513
steps:
614
- run:
715
name: Avoid tests for docs
816
command: |
9-
if [[ $CIRCLE_BRANCH == *_docs ]]; then
17+
if [[ $CIRCLE_BRANCH == *docs* ]]; then
1018
echo "Identifies as documents PR, no testing required"
1119
circleci step halt
1220
fi
@@ -29,8 +37,8 @@ commands:
2937
- run:
3038
name: Setup executor
3139
command: |
32-
apt-get -qq update
33-
apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
40+
sudo apt-get -qq update
41+
sudo apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
3442
bash <(curl -fsSL https://get.docker.com)
3543
- setup_remote_docker:
3644
docker_layer_caching: true
@@ -40,19 +48,19 @@ commands:
4048
- run:
4149
name: Relocate docker overlay2 dir
4250
command: |
43-
sudo systemctl stop docker
51+
sudo service docker stop
4452
sudo mkdir -p /var2/lib/docker
4553
sudo mv /var/lib/docker/overlay2 /var2/lib/docker
4654
sudo mkdir /var/lib/docker/overlay2
4755
sudo mount --bind /var2/lib/docker/overlay2 /var/lib/docker/overlay2
48-
sudo systemctl start docker
56+
sudo service docker start
4957
5058
setup-automation:
5159
steps:
5260
- run:
5361
name: Setup automation
5462
command: |
55-
git submodule update --init opt/readies
63+
git submodule update --init opt/readies
5664
./opt/readies/bin/getpy3
5765
5866
setup-build-system:
@@ -121,8 +129,8 @@ commands:
121129
platforms-build-steps:
122130
steps:
123131
- abort_for_docs
132+
- setup-executor
124133
- checkout-all
125-
- relocate-docker-storage
126134
- setup-automation
127135
- run:
128136
name: Build for platform
@@ -191,12 +199,11 @@ jobs:
191199
platform: debian
192200

193201
platforms-build:
194-
machine:
195-
enabled: true
196-
docker_layer_caching: true
197-
resource_class: gpu.nvidia.small
198-
image: ubuntu-1604-cuda-11.1:202012-01
202+
docker:
203+
- image: cimg/base:2020.01
199204
steps:
205+
- setup-executor
206+
- relocate-docker-storage
200207
- platforms-build-steps
201208

202209
coverage:
@@ -256,51 +263,84 @@ jobs:
256263
make -C opt test VALGRIND=1 <<parameters.test_args>>
257264
no_output_timeout: 120m
258265

259-
build-macos:
260-
macos:
261-
xcode: 11.3.0
262-
steps:
263-
- abort_for_docs
264-
- run:
265-
name: Fix macOS Python installation
266-
command: |
267-
brew reinstall -f python2
268-
- build-steps:
269-
platform: macos
270-
271-
build-multiarch-docker:
272-
machine:
273-
enabled: true
274-
image: cimg/base:2020.01
266+
valgrind-general:
267+
parameters:
268+
test_args:
269+
type: string
270+
default: "CLUSTER=0 AOF=0"
271+
docker:
272+
- image: redisfab/rmbuilder:6.2.1-x64-buster
275273
steps:
274+
- early_return_for_forked_pull_requests
276275
- abort_for_docs
277276
- checkout-all
277+
- restore_cache:
278+
keys:
279+
- build-dependencies-{{ checksum "get_deps.sh" }}
280+
# If no exact match is found will get dependencies from source
281+
- setup-build-system
278282
- run:
279-
name: Checkout LFS
283+
name: Install dependencies
280284
command: |
281-
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
282-
sudo apt-get install -y git-lfs
283-
git lfs install
284-
git lfs pull
285+
./opt/readies/bin/getredis -v 6.0 --valgrind --force
286+
./get_deps.sh cpu
285287
- run:
286-
name: Setup Docker client experimental features
288+
name: Build for valgrind
287289
command: |
288-
sudo ./opt/readies/bin/getdocker --just-enable-exp
289-
docker version
290+
make -C opt all VALGRIND=1 SHOW=1
290291
- run:
291-
name: Build
292+
name: Test with valgrind
292293
command: |
293-
sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
294-
cd opt/build/docker
295-
make build
296-
sudo make publish
294+
make -C opt test VALGRIND=1 <<parameters.test_args>>
295+
no_output_timeout: 120m
296+
297+
298+
# build-macos:
299+
# macos:
300+
# xcode: 11.3.0
301+
# steps:
302+
# - abort_for_docs
303+
# - run:
304+
# name: Fix macOS Python installation
305+
# command: |
306+
# brew reinstall -f python2
307+
# - build-steps:
308+
# platform: macos
309+
#
310+
# build-multiarch-docker:
311+
# machine:
312+
# enabled: true
313+
# image: cimg/base:2020.01
314+
# steps:
315+
# - abort_for_docs
316+
# - checkout-all
317+
# - run:
318+
# name: Checkout LFS
319+
# command: |
320+
# curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash
321+
# sudo apt-get install -y git-lfs
322+
# git lfs install
323+
# git lfs pull
324+
# - run:
325+
# name: Setup Docker client experimental features
326+
# command: |
327+
# sudo ./opt/readies/bin/getdocker --just-enable-exp
328+
# docker version
329+
# - run:
330+
# name: Build
331+
# command: |
332+
# sudo docker login -u redisfab -p $DOCKER_REDISFAB_PWD
333+
# cd opt/build/docker
334+
# make build
335+
# sudo make publish
297336

298337
build-and-test-gpu:
299338
machine:
300339
enabled: true
301340
docker_layer_caching: true
302341
resource_class: gpu.nvidia.small
303342
image: ubuntu-1604-cuda-11.1:202012-01
343+
304344
steps:
305345
- abort_for_docs
306346
- checkout-all
@@ -414,6 +454,13 @@ on-any-branch: &on-any-branch
414454
tags:
415455
only: /.*/
416456

457+
on-any-branch-but-tags: &on-any-branch-but-tags
458+
filters:
459+
branches:
460+
only: /.*/
461+
tags:
462+
ignore: /.*/
463+
417464
never: &never
418465
filters:
419466
branches:
@@ -434,17 +481,15 @@ on-integ-branch: &on-integ-branch
434481
only:
435482
- master
436483
- /^\d+\.\d+.*$/
437-
- /^feature-.*$/
438484
tags:
439485
ignore: /.*/
440486

441-
not-on-integ-branch: &not-on-integ-branch
487+
on-dev-branches: &on-dev-branches
442488
filters:
443489
branches:
444490
ignore:
445491
- master
446492
- /^\d+\.\d+.*$/
447-
- /^feature-.*$/
448493
tags:
449494
ignore: /.*/
450495

@@ -455,13 +500,11 @@ on-version-tags: &on-version-tags
455500
tags:
456501
only: /^v[0-9].*/
457502

458-
on-integ-and-version-tags: &on-integ-and-version-tags
503+
on-master-and-version-tags: &on-master-and-version-tags
459504
filters:
460505
branches:
461506
only:
462507
- master
463-
- /^\d+\.\d+.*$/
464-
- /^feature-.*$/
465508
tags:
466509
only: /^v[0-9].*/
467510

@@ -478,30 +521,22 @@ after-platform-builds: &after-platform-builds
478521
requires:
479522
- platforms-build
480523

481-
platform-build-defs: &platform-build-defs
482-
<<: *after-build-and-test
483-
<<: *on-integ-and-version-tags
484-
# <<: *on-any-branch # debugging
485-
486-
524+
#### define workflows
487525
workflows:
488526
version: 2
489527
build_and_package:
490528
jobs:
491529
- lint:
492530
<<: *on-any-branch
493531
- build-and-test:
494-
<<: *on-any-branch
532+
<<: *on-any-branch-but-tags
495533
<<: *after-linter
496534
- platforms-build:
497-
<<: *platform-build-defs
535+
<<: *after-build-and-test
536+
<<: *on-master-and-version-tags
498537
- coverage:
499538
context: common
500-
<<: *on-any-branch
501-
<<: *after-linter
502-
- valgrind:
503-
test_args: CLUSTER=0 AOF=0
504-
<<: *on-any-branch
539+
<<: *on-dev-branches
505540
<<: *after-linter
506541
- valgrind:
507542
name: valgrind-cluster
@@ -514,15 +549,8 @@ workflows:
514549
<<: *on-integ-branch
515550
<<: *after-linter
516551
- build-and-test-gpu:
517-
<<: *on-any-branch
552+
<<: *on-integ-branch
518553
<<: *after-linter
519-
- build-macos:
520-
<<: *never # temporarily disabled
521-
# <<: *on-version-tags
522-
- build-multiarch-docker:
523-
context: common
524-
<<: *never # temporarily disabled
525-
# <<: *on-version-tags
526554
- deploy-snapshot:
527555
context: common
528556
<<: *after-platform-builds
@@ -545,7 +573,7 @@ workflows:
545573
branches:
546574
only: master
547575
jobs:
548-
- build-macos:
549-
<<: *never # temporarily disabled
576+
# - build-macos:
577+
# <<: *never # temporarily disabled
550578
- nightly-automation:
551579
context: common

‎.github/workflows/trigger-build-and-test-gpu.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# https://github.com/marketplace/actions/run-circle-ci-on-label
2+
name: build-and-test-gpu
3+
4+
on:
5+
pull_request:
6+
types:
7+
- labeled
8+
9+
jobs:
10+
valgrind_general:
11+
if: github.event.label.name == 'ci-test'
12+
runs-on: ubuntu-latest
13+
name: Run build and test cpu
14+
15+
steps:
16+
- uses: actions/checkout@v2
17+
- name: circle-ci job runner
18+
id: curl-circle-ci
19+
uses: Open-Source-Contrib/circle-ci-trigger-action@latest
20+
with:
21+
circle_ci_token: ${{ secrets.CIRCLE_CI_SECRET}}
22+
circle_ci_job: build-and-test-gpu
23+
circle_ci_project_url: ${{ github.event.pull_request.head.ref }}

‎.github/workflows/trigger-valgrind.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# https://github.com/marketplace/actions/run-circle-ci-on-label
2+
name: valgrind-general-run
3+
4+
on:
5+
pull_request:
6+
types:
7+
- labeled
8+
9+
jobs:
10+
valgrind_general:
11+
if: github.event.label.name == 'ci-test'
12+
runs-on: ubuntu-latest
13+
name: Run valgrind general
14+
15+
steps:
16+
- uses: actions/checkout@v2
17+
- name: circle-ci job runner
18+
id: curl-circle-ci
19+
uses: Open-Source-Contrib/circle-ci-trigger-action@latest
20+
with:
21+
circle_ci_token: ${{ secrets.CIRCLE_CI_SECRET}}
22+
circle_ci_job: valgrind-general
23+
circle_ci_project_url: ${{ github.event.pull_request.head.ref }}

‎README.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,13 @@ If you want to run examples, make sure you have [git-lfs](https://git-lfs.github
2323
To quickly tryout RedisAI, launch an instance using docker:
2424

2525
```sh
26-
docker run -p 6379:6379 -it --rm redislabs/redisai
26+
docker run -p 6379:6379 -it --rm redislabs/redisai:edge-cpu
2727
```
2828

2929
For docker instance with GPU support, you can launch it from `tensorwerk/redisai-gpu`
3030

3131
```sh
32-
docker run -p 6379:6379 --gpus all -it --rm redislabs/redisai:latest-gpu
32+
docker run -p 6379:6379 --gpus all -it --rm redislabs/redisai:edge-gpu
3333
```
3434

3535
But if you'd like to build the docker image, you need a machine that has Nvidia driver (CUDA 10.0), nvidia-container-toolkit and Docker 19.03+ installed. For detailed information, checkout [nvidia-docker documentation](https://github.com/NVIDIA/nvidia-docker)
@@ -46,7 +46,7 @@ Note that Redis config is located at `/usr/local/etc/redis/redis.conf` which can
4646

4747
On the client, set the model
4848
```sh
49-
redis-cli -x AI.MODELSET foo TF CPU INPUTS a b OUTPUTS c BLOB < tests/test_data/graph.pb
49+
redis-cli -x AI.MODELSTORE foo TF CPU INPUTS 2 a b OUTPUTS 1 c BLOB < tests/test_data/graph.pb
5050
```
5151

5252
Then create the input tensors, run the computation graph and get the output tensor (see `load_model.sh`). Note the signatures:
@@ -122,6 +122,9 @@ Some languages have client libraries that provide support for RedisAI's commands
122122
| redisai-py | Python | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-py) |
123123
| redisai-go | Go | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-go) |
124124
| redisai-js | Typescript/Javascript | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-js) |
125+
| redis-modules-sdk | TypeScript | BSD-3-Clause | [Dani Tseitlin](https://github.com/danitseitlin) | [Github](https://github.com/danitseitlin/redis-modules-sdk) |
126+
| redis-modules-java | Java | Apache-2.0 | [dengliming](https://github.com/dengliming) | [Github](https://github.com/dengliming/redis-modules-java) |
127+
125128

126129
## Backend Dependancy
127130

‎docs/clients.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,8 @@ Some languages already have client libraries that provide support for RedisAI's
88
| redisai-py | Python | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-py) |
99
| redisai-go | Go | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-go) |
1010
| redisai-js | Typescript/Javascript | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-js) |
11+
| redis-modules-sdk | TypeScript | BSD-3-Clause | [Dani Tseitlin](https://github.com/danitseitlin) | [Github](https://github.com/danitseitlin/redis-modules-sdk) |
12+
| redis-modules-java | Java | Apache-2.0 | [dengliming](https://github.com/dengliming) | [Github](https://github.com/dengliming/redis-modules-java) |
13+
1114

1215
The full documentation for RedisAI's API can be found at the [Commands page](commands.md).

0 commit comments

Comments
 (0)