Skip to content

Commit 80c9906

Browse files
committed
Merge branch 'master' into Create_tensor_through_gears-refactor_and_test
2 parents 93fbce4 + 9fe3b7d commit 80c9906

File tree

8 files changed

+77
-61
lines changed

8 files changed

+77
-61
lines changed

docs/requirements.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
mkdocs==1.0.4
2-
mkdocs-material==4.6.3
1+
mkdocs==1.1.2
2+
mkdocs-material==6.2.2
33
-e git+https://github.com/RedisLabs/mkdocs-versions-menu.git#egg=mkdocs-versions-menu
44
-e git+https://github.com/RedisLabs/mkdocs-include.git#egg=mkdocs-include
5+
-e git+https://github.com/RedisLabs/mkdocs-modules-template.git#egg=mkdocs-modules-template
56
# uncomment requirement to build LL API
67
# -e git+https://github.com/matusnovak/doxybook.git#egg=doxybook

get_deps.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,13 @@ MKL=mkl
7070
ONNXRUNTIME=onnxruntime
7171

7272
######################################################################################## DLPACK
73-
73+
DLPACK_VERSION="v0.3"
7474
if [[ $WITH_DLPACK != 0 ]]; then
7575
[[ $FORCE == 1 ]] && rm -rf $DLPACK
7676

7777
if [[ ! -d $DLPACK ]]; then
7878
echo "Cloning dlpack ..."
79-
git clone --depth 1 https://github.com/dmlc/dlpack.git $DLPACK
79+
git clone --depth 1 --branch $DLPACK_VERSION https://github.com/dmlc/dlpack.git $DLPACK
8080
echo "Done."
8181
else
8282
echo "dlpack is in place."

mkdocs.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,24 @@ plugins:
5353
- search
5454
- include:
5555
src_path: docs/snippets
56+
- modules-template:
57+
try-for-free: false
58+
modules:
59+
- name: 'Search'
60+
link: '/redisearch'
61+
62+
- name: 'Time Series'
63+
link: '/redistimeseries'
64+
65+
- name: 'Bloom'
66+
link: '/redisbloom'
67+
68+
- name: 'Graph'
69+
link: '/redisgraph'
70+
71+
- name: 'JSON'
72+
link: '/redisjson'
73+
74+
- name: 'Gears'
75+
link: '/redisgears'
5676
- versions-menu:

opt/system-setup.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,10 @@ def macos(self):
7070

7171
def common_last(self):
7272
self.run("%s/bin/getclang --format" % READIES)
73-
self.run("%s/bin/getcmake" % READIES)
73+
if self.platform == "arm":
74+
self.run("%s/bin/getcmake" % READIES)
75+
else:
76+
self.run("%s/bin/getcmake --no-repo" % READIES)
7477

7578
self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES))
7679

tests/flow/Install_RedisGears.sh

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
#!/bin/bash
22

3+
set -e
4+
35
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
4-
ROOT=$(cd $HERE/../..; pwd)
6+
ROOT=$(cd $HERE/../.. && pwd)
57
READIES=$ROOT/opt/readies
68
. $READIES/shibumi/defs
79

8-
if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then
10+
if [[ "$1" == "--help" || "$1" == "help" || "$HELP" == "1" ]]; then
911
cat <<-END
1012
Obtain RedisGears module binaries
1113
1214
Install_RedisGears.sh [--help|help]
13-
15+
1416
Argument variables:
1517
GEARS_OSNICK=nick Get binaries for give osnick
1618
GEARS_PATH=dir Get binaries from given Gears repo
@@ -22,47 +24,47 @@ if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then
2224
fi
2325

2426
OP=""
25-
[[ $NOP == 1 ]] && OP=echo
27+
[[ "$NOP" == "1" ]] && OP=echo
2628

27-
os=$($READIES/bin/platform --os)
28-
arch=$($READIES/bin/platform --arch)
29+
os="$($READIES/bin/platform --os)"
30+
arch="$($READIES/bin/platform --arch)"
2931

30-
if [[ ! -z $GEARS_PATH ]]; then
31-
platform=$($READIES/bin/platform -t)
32+
if [[ ! -z "$GEARS_PATH" ]]; then
33+
platform="$($READIES/bin/platform -t)"
3234
else
33-
if [[ $os != linux || $arch != x64 ]]; then
35+
if [[ "$os" != "linux" || "$arch" != "x64" ]]; then
3436
eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
3537
exit 1
3638
fi
3739

38-
dist=$($READIES/bin/platform --dist)
39-
nick=$($READIES/bin/platform --osnick)
40+
dist="$($READIES/bin/platform --dist)"
41+
nick="$($READIES/bin/platform --osnick)"
4042

41-
if [[ $dist == ubuntu ]]; then
42-
if [[ $nick != bionic && $nick != xenial && $nick != trusty ]]; then
43-
nick=bionic
43+
if [[ $dist == "ubuntu" ]]; then
44+
if [[ $nick != "bionic" && $nick != "xenial" && $nick != "trusty" ]]; then
45+
nick="bionic"
4446
fi
4547
elif [[ $dist == debian ]]; then
4648
nick=bionic
4749
elif [[ $dist == centos || $dist == redhat || $dist == fedora ]]; then
4850
nick=centos7
49-
elif [[ ! -z $GEARS_OSNICK ]]; then
50-
nick="$GEARS_OSNICK"
51+
elif [[ ! -z "$GEARS_OSNICK" ]]; then
52+
nick=$GEARS_OSNICK
5153
else
5254
eprint "Cannot match binary artifacts - build RedisGears and set GEARS_PATH"
5355
exit 1
5456
fi
5557
platform="${os}-${nick}-${arch}"
5658
fi
5759

58-
GEARS_S3_URL=http://redismodules.s3.amazonaws.com/redisgears/snapshots
59-
GEARS_MOD=redisgears.$platform.master.zip
60-
GEARS_DEPS=redisgears-python.$platform.master.tgz
60+
GEARS_S3_URL="http://redismodules.s3.amazonaws.com/redisgears/snapshots"
61+
GEARS_MOD="redisgears.${platform}.master.zip"
62+
GEARS_DEPS="redisgears-python.${platform}.master.tgz"
6163

62-
FINAL_WORK_DIR=$ROOT/bin/$($READIES/bin/platform -t)/RedisGears
64+
FINAL_WORK_DIR="$ROOT/bin/$($READIES/bin/platform -t)/RedisGears"
6365

6466
if [[ -d $FINAL_WORK_DIR && -f $FINAL_WORK_DIR/redisgears.so ]]; then
65-
echo "RedisGears is in $FINAL_WORK_DIR"
67+
echo "RedisGears is in ${FINAL_WORK_DIR}"
6668
exit 0
6769
fi
6870

@@ -72,20 +74,20 @@ WORK_DIR=$(mktemp -d ${FINAL_WORK_DIR}.XXXXXX)
7274
$OP mkdir -p $WORK_DIR
7375

7476
if [[ -z $GEARS_PATH ]]; then
75-
F_GEARS_MOD=$WORK_DIR/$GEARS_MOD
77+
F_GEARS_MOD="$WORK_DIR/$GEARS_MOD"
7678
if [[ ! -f $F_GEARS_MOD ]]; then
7779
echo "Download RedisGears ..."
7880
$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_MOD
7981
fi
8082

81-
F_GEARS_DEPS=$WORK_DIR/$GEARS_DEPS
83+
F_GEARS_DEPS="$WORK_DIR/$GEARS_DEPS"
8284
if [[ ! -f $F_GEARS_DEPS ]]; then
8385
echo "Download RedisGears deps ..."
8486
$OP wget -q -P $WORK_DIR $GEARS_S3_URL/$GEARS_DEPS
8587
fi
8688
else
87-
F_GEARS_MOD=$GEARS_PATH/artifacts/snapshot/$GEARS_MOD
88-
F_GEARS_DEPS=$GEARS_PATH/artifacts/snapshot/$GEARS_DEPS
89+
F_GEARS_MOD="${GEARS_PATH}/artifacts/snapshot/${GEARS_MOD}"
90+
F_GEARS_DEPS="${GEARS_PATH}/artifacts/snapshot/${GEARS_DEPS}"
8991
[[ ! -f $F_GEARS_MOD ]] && { eprint "$F_GEARS_MOD is missing"; exit 1; }
9092
[[ ! -f $F_GEARS_DEPS ]] && { eprint "$F_GEARS_DEPS is missing"; exit 1; }
9193
fi

tests/flow/includes.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../opt/readies"))
1515
import paella
1616

17-
ROOT = os.environ.get("ROOT")
17+
ROOT = os.environ.get("ROOT", None)
18+
if ROOT is None:
19+
sys.stderr.write("ROOT was not defined in the environment.\n")
20+
sys.exit(1)
1821
MAX_ITERATIONS = 2 if os.environ.get("MAX_ITERATIONS") == None else os.environ.get("MAX_ITERATIONS")
1922
TEST_TF = os.environ.get("TEST_TF") != "0" and os.environ.get("WITH_TF") != "0"
2023
TEST_TFLITE = os.environ.get("TEST_TFLITE") != "0" and os.environ.get("WITH_TFLITE") != "0"

tests/flow/tests_withGears.py

Lines changed: 16 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -12,38 +12,24 @@ def wrapper(env, *args, **kwargs):
1212
modules = con.execute_command("MODULE", "LIST")
1313
if b'rg' in [module[1] for module in modules]:
1414
return f(env, *args, **kwargs)
15+
platform = paella.Platform()
16+
redisgears_dir = "{ROOT}/bin/{PLATFORM}/RedisGears".format(ROOT=ROOT, PLATFORM=platform.triplet())
17+
if not os.path.isdir(redisgears_dir):
18+
env.debugPrint("RedisGears directory does not exist", force=True)
19+
return
20+
redisgears_path = os.path.join(redisgears_dir, 'redisgears.so')
21+
python_plugin_path = os.path.join(redisgears_dir, 'plugin/gears_python.so')
1522
try:
16-
platform = paella.Platform()
17-
redisgears_dir = "{ROOT}/bin/{PLATFORM}/RedisGears".format(ROOT=ROOT, PLATFORM=platform.triplet())
18-
redisgears_path = os.path.join(redisgears_dir, 'redisgears.so')
19-
python_plugin_path = os.path.join(redisgears_dir, 'plugin/gears_python.so')
2023
ret = con.execute_command('MODULE', 'LOAD', redisgears_path, 'Plugin', python_plugin_path, 'CreateVenv',
2124
0, 'PythonInstallationDir', redisgears_dir)
2225
env.assertEqual(ret, b'OK')
23-
except Exception as e:
24-
env.debugPrint(str(e), force=True)
26+
except Exception:
2527
env.debugPrint("skipping since RedisGears not loaded", force=True)
2628
return
2729
return f(env, *args, **kwargs)
2830
return wrapper
2931

3032

31-
@skip_if_gears_not_loaded
32-
def test_ping_gears(env):
33-
34-
script = '''
35-
def ping(record):
36-
return "pong"
37-
38-
GB("CommandReader").map(ping).register(trigger="ping_test")
39-
'''
40-
con = env.getConnection()
41-
ret = con.execute_command('rg.pyexecute', script)
42-
env.assertEqual(ret, b'OK')
43-
ret = con.execute_command('rg.trigger', 'ping_test')
44-
env.assertEqual(ret[0], b'pong')
45-
46-
4733
@skip_if_gears_not_loaded
4834
def test_model_run(env):
4935
script = '''
@@ -56,7 +42,7 @@ def ModelRun_oldAPI(record):
5642
modelRunner = redisAI.createModelRunner('m{1}')
5743
redisAI.modelRunnerAddInput(modelRunner, 'a', tensors[0])
5844
redisAI.modelRunnerAddInput(modelRunner, 'b', tensors[1])
59-
redisAI.modelRunnerAddOutput(modelRunner, 'mul')
45+
redisAI.modelRunnerAddOutput(modelRunner, 'c')
6046
res = redisAI.modelRunnerRun(modelRunner)
6147
redisAI.setTensorInKey('c{1}', res[0])
6248
return "ModelRun_oldAPI_OK"
@@ -67,7 +53,7 @@ async def ModelRun_Async(record):
6753
modelRunner = redisAI.createModelRunner('m{1}')
6854
redisAI.modelRunnerAddInput(modelRunner, 'a', tensors[0])
6955
redisAI.modelRunnerAddInput(modelRunner, 'b', tensors[1])
70-
redisAI.modelRunnerAddOutput(modelRunner, 'mul')
56+
redisAI.modelRunnerAddOutput(modelRunner, 'c')
7157
res = await redisAI.modelRunnerRunAsync(modelRunner)
7258
redisAI.setTensorInKey('c{1}', res[0])
7359
return "ModelRun_Async_OK"
@@ -108,6 +94,8 @@ async def ModelRun_AsyncRunError(record):
10894
env.assertEqual(ret[0], b'ModelRun_oldAPI_OK')
10995
values = con.execute_command('AI.TENSORGET', 'c{1}', 'VALUES')
11096
env.assertEqual(values, [b'4', b'9', b'4', b'9'])
97+
ret = con.execute_command('DEL', 'c{1}')
98+
env.assertEqual(ret, 1)
11199

112100
ret = con.execute_command('rg.trigger', 'ModelRun_Async_test2')
113101
env.assertEqual(ret[0], b'ModelRun_Async_OK')
@@ -186,6 +174,8 @@ async def ScriptRun_AsyncRunError(record):
186174
env.assertEqual(ret[0], b'ScriptRun_oldAPI_OK')
187175
values = con.execute_command('AI.TENSORGET', 'c{1}', 'VALUES')
188176
env.assertEqual(values, [b'4', b'6', b'4', b'6'])
177+
ret = con.execute_command('DEL', 'c{1}')
178+
env.assertEqual(ret, 1)
189179

190180
ret = con.execute_command('rg.trigger', 'ScriptRun_Async_test2')
191181
env.assertEqual(ret[0], b'ScriptRun_Async_OK')
@@ -194,8 +184,7 @@ async def ScriptRun_AsyncRunError(record):
194184

195185
ret = con.execute_command('rg.trigger', 'ScriptRun_AsyncRunError_test3')
196186
# This should raise an exception
197-
error_string = b'attempted to get undefined function bad_func'
198-
env.assertEqual(str(ret[0])[:len(error_string)+2]+"'", "{}".format(error_string))
187+
env.assertTrue(str(ret[0]).startswith("b'attempted to get undefined function bad_func"))
199188

200189

201190
@skip_if_gears_not_loaded
@@ -314,9 +303,7 @@ async def DAGRun_addOpsFromString(record):
314303

315304
ret = con.execute_command('rg.trigger', 'DAGRun_test4')
316305
# This should raise an exception
317-
318-
error_string = b'attempted to get undefined function no_func'
319-
env.assertEqual(str(ret[0])[:len(error_string)+2]+"'", "{}".format(error_string))
306+
env.assertTrue(str(ret[0]).startswith("b'attempted to get undefined function no_func"))
320307

321308
ret = con.execute_command('rg.trigger', 'DAGRun_test5')
322309
env.assertEqual(ret[0], b'test5_OK')

0 commit comments

Comments
 (0)