\n",
@@ -9470,90 +9485,100 @@
" | rerank | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float16 | \n",
- " 0.291734 | \n",
+ " 0.264426 | \n",
" 0.143587 | \n",
" 0.303922 | \n",
+ " 911.6829833984375 | \n",
" \n",
" \n",
" | 8 | \n",
" rerank | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float32 | \n",
- " 0.203621 | \n",
+ " 0.186951 | \n",
" 0.143587 | \n",
" 0.303922 | \n",
+ " 807.8170166015625 | \n",
"
\n",
" \n",
" | 4 | \n",
" weighted_rrf | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float16 | \n",
- " 0.002931 | \n",
+ " 0.002152 | \n",
" 0.134273 | \n",
" 0.286378 | \n",
+ " 911.6829833984375 | \n",
"
\n",
" \n",
" | 9 | \n",
" weighted_rrf | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float32 | \n",
- " 0.003677 | \n",
+ " 0.002076 | \n",
" 0.134273 | \n",
" 0.286378 | \n",
+ " 807.8170166015625 | \n",
"
\n",
" \n",
" | 1 | \n",
" vector | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float16 | \n",
- " 0.002028 | \n",
+ " 0.001888 | \n",
" 0.126754 | \n",
" 0.284314 | \n",
+ " 911.6829833984375 | \n",
"
\n",
" \n",
" | 2 | \n",
" hybrid | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float16 | \n",
- " 0.001111 | \n",
+ " 0.000828 | \n",
" 0.126754 | \n",
" 0.284314 | \n",
+ " 911.6829833984375 | \n",
"
\n",
" \n",
" | 6 | \n",
" vector | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float32 | \n",
- " 0.000751 | \n",
+ " 0.000575 | \n",
" 0.126754 | \n",
" 0.284314 | \n",
+ " 807.8170166015625 | \n",
"
\n",
" \n",
" | 7 | \n",
" hybrid | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float32 | \n",
- " 0.001082 | \n",
+ " 0.000895 | \n",
" 0.126754 | \n",
" 0.284314 | \n",
+ " 807.8170166015625 | \n",
"
\n",
" \n",
" | 0 | \n",
" bm25 | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float16 | \n",
- " 0.000890 | \n",
+ " 0.000606 | \n",
" 0.122591 | \n",
" 0.313880 | \n",
+ " 911.6829833984375 | \n",
"
\n",
" \n",
" | 5 | \n",
" bm25 | \n",
" sentence-transformers/all-MiniLM-L6-v2 | \n",
" float32 | \n",
- " 0.001118 | \n",
+ " 0.001010 | \n",
" 0.122591 | \n",
" 0.313880 | \n",
+ " 807.8170166015625 | \n",
"
\n",
" \n",
"\n",
@@ -9572,17 +9597,17 @@
"0 bm25 sentence-transformers/all-MiniLM-L6-v2 float16 \n",
"5 bm25 sentence-transformers/all-MiniLM-L6-v2 float32 \n",
"\n",
- " avg_query_time recall precision \n",
- "3 0.291734 0.143587 0.303922 \n",
- "8 0.203621 0.143587 0.303922 \n",
- "4 0.002931 0.134273 0.286378 \n",
- "9 0.003677 0.134273 0.286378 \n",
- "1 0.002028 0.126754 0.284314 \n",
- "2 0.001111 0.126754 0.284314 \n",
- "6 0.000751 0.126754 0.284314 \n",
- "7 0.001082 0.126754 0.284314 \n",
- "0 0.000890 0.122591 0.313880 \n",
- "5 0.001118 0.122591 0.313880 "
+ " avg_query_time recall precision total_indexing_time \n",
+ "3 0.264426 0.143587 0.303922 911.6829833984375 \n",
+ "8 0.186951 0.143587 0.303922 807.8170166015625 \n",
+ "4 0.002152 0.134273 0.286378 911.6829833984375 \n",
+ "9 0.002076 0.134273 0.286378 807.8170166015625 \n",
+ "1 0.001888 0.126754 0.284314 911.6829833984375 \n",
+ "2 0.000828 0.126754 0.284314 911.6829833984375 \n",
+ "6 0.000575 0.126754 0.284314 807.8170166015625 \n",
+ "7 0.000895 0.126754 0.284314 807.8170166015625 \n",
+ "0 0.000606 0.122591 0.313880 911.6829833984375 \n",
+ "5 0.001010 0.122591 0.313880 807.8170166015625 "
]
},
"execution_count": 8,
@@ -9593,11 +9618,19 @@
"source": [
"metrics[[\"search_method\", \"model\", \"vector_data_type\", \"avg_query_time\", \"recall\", \"precision\"]].sort_values(by=\"recall\", ascending=False)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6567d631",
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "3.11.9",
"language": "python",
"name": "python3"
},
diff --git a/docs/examples/search_study/search_study_walkthrough.ipynb b/docs/examples/search_study/search_study_walkthrough.ipynb
new file mode 100644
index 0000000..faeb4f1
--- /dev/null
+++ b/docs/examples/search_study/search_study_walkthrough.ipynb
@@ -0,0 +1,847 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "7b65e29b",
+ "metadata": {},
+ "source": [
+ "# Search study\n",
+ "\n",
+ "Let's say you have an existing Redis database with a search index seeded. You may wish to quickly test different search method against the existing index without having to recreate data and/or recreate the index. This demo will walk you though how to set this up and get going.\n",
+ "\n",
+ "# Installation"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f6daae6e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install redis-retrieval-optimizer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "8aa48190",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'0.4.1'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import redis_retrieval_optimizer\n",
+ "\n",
+ "redis_retrieval_optimizer.__version__"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d50d98ee",
+ "metadata": {},
+ "source": [
+ "# Load data\n",
+ "\n",
+ "We will load our custom car dataset for this example. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "d67af4f3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import json\n",
+ "\n",
+ "with open('../resources/cars/car_corpus.json', 'r') as f:\n",
+ " corpus = json.load(f)\n",
+ "\n",
+ "with open('../resources/cars/car_queries.json', 'r') as f:\n",
+ " queries = json.load(f)\n",
+ "\n",
+ "with open('../resources/cars/car_qrels.json', 'r') as f:\n",
+ " qrels = json.load(f)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8ba127a8",
+ "metadata": {},
+ "source": [
+ "# Create the index with redisvl\n",
+ "\n",
+ "For the search_study we are assuming that the search index already exists. The cell below will create a Redis search index and populate it with our test data for example purposes but is assumed with a search study is populated and running within your data. \n",
+ "\n",
+ "Note: the demo assumes you have a instance of redis running on localhost:6379. If this is not the case, update the redis_url to direct to your running instance or start a local instance with the following command. \n",
+ "\n",
+ "`docker run -d --name redis-stack-server -p 6379:6379 redis/redis-stack-server:latest`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "3e09361e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# assuming you have a redis instance running on localhost:6379\n",
+ "redis_url = \"redis://localhost:6379\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "1de89fc9",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "15:37:06 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n",
+ "15:37:06 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: sentence-transformers/all-mpnet-base-v2\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "['cars:01K3KY4XKBMD7B5VWDYMBPT8CV',\n",
+ " 'cars:01K3KY4XKBJBADHWYSSVKYXFA3',\n",
+ " 'cars:01K3KY4XKBAFQXKJA8D24XTN4V',\n",
+ " 'cars:01K3KY4XKB4YK5J8M2BJ3RG2ZZ',\n",
+ " 'cars:01K3KY4XKBJ9BEN9KA5MY5B17F',\n",
+ " 'cars:01K3KY4XKB1YN12H0E8RYF908V',\n",
+ " 'cars:01K3KY4XKBBCQRKZF2WKN35N1M',\n",
+ " 'cars:01K3KY4XKBCFNP86PB1X20BKP6',\n",
+ " 'cars:01K3KY4XKBB1TW18MW7FXQHPGV',\n",
+ " 'cars:01K3KY4XKBNSAD3KQKWHY8ZEZV',\n",
+ " 'cars:01K3KY4XKBZASFPS1QAFT8AB9M',\n",
+ " 'cars:01K3KY4XKB6WS6EXC6D29HGB2B',\n",
+ " 'cars:01K3KY4XKBG7BDE2W5096JCPV1',\n",
+ " 'cars:01K3KY4XKBTJPHE8DJJDZ0F25N',\n",
+ " 'cars:01K3KY4XKB5G1J1HVVZNEHMD1G',\n",
+ " 'cars:01K3KY4XKB0G2NJSFABD1C0V2V',\n",
+ " 'cars:01K3KY4XKBE15Y724Q1MFRT1JD',\n",
+ " 'cars:01K3KY4XKB9BET7ZPS9PWS9FAM',\n",
+ " 'cars:01K3KY4XKBKK7N32WVSACBRDAH',\n",
+ " 'cars:01K3KY4XKB8WN4FZQHYN7N5M3Y',\n",
+ " 'cars:01K3KY4XKB3NFV4KCC3ZR9S1W3',\n",
+ " 'cars:01K3KY4XKBS0HT7SQPHKCZJ7K1',\n",
+ " 'cars:01K3KY4XKB9YAVWSK2EHPKCMJW',\n",
+ " 'cars:01K3KY4XKB26NXA6V8NY5DFS28',\n",
+ " 'cars:01K3KY4XKBKRQFSYB64XT263TZ',\n",
+ " 'cars:01K3KY4XKBA1G1N5GV81TQBG4V',\n",
+ " 'cars:01K3KY4XKBAZ9F12ANSMDMXY89',\n",
+ " 'cars:01K3KY4XKBHR8EYYS9S8KWZXR9',\n",
+ " 'cars:01K3KY4XKB7MS6SPWW4DKQJX7W',\n",
+ " 'cars:01K3KY4XKBMSCD9E477ZQ17YVZ',\n",
+ " 'cars:01K3KY4XKBJPG3D0807NNP8WC3',\n",
+ " 'cars:01K3KY4XKBWWXFH18Y473W7G6B',\n",
+ " 'cars:01K3KY4XKBS5D809VQY7PQ8QRC',\n",
+ " 'cars:01K3KY4XKBJ8ZDE8AK3AKWDZ6Z',\n",
+ " 'cars:01K3KY4XKBNN6P0668D815NEC1',\n",
+ " 'cars:01K3KY4XKBE0TBJ01VTT1TFHZS',\n",
+ " 'cars:01K3KY4XKCPCVH83FK2EA71Z48',\n",
+ " 'cars:01K3KY4XKC73XMV3GBFCT7J0WW',\n",
+ " 'cars:01K3KY4XKC5KV2AKX0CGXVH5RZ',\n",
+ " 'cars:01K3KY4XKCX1H1N5GXK2PY1HHD',\n",
+ " 'cars:01K3KY4XKC2D62TXN60TFZZVY5',\n",
+ " 'cars:01K3KY4XKCH1BDH5Z90PSNF5S4',\n",
+ " 'cars:01K3KY4XKCHWZK9FW9CVKS1ASM',\n",
+ " 'cars:01K3KY4XKCJ94D3SBZF1F1YYH3',\n",
+ " 'cars:01K3KY4XKCBAFQ964BAV94YAN0',\n",
+ " 'cars:01K3KY4XKC2NY2N5XYAW7K6YZ6',\n",
+ " 'cars:01K3KY4XKCAPFHNRHMJ7BW0CME',\n",
+ " 'cars:01K3KY4XKCDB5YYVT0776P48R0',\n",
+ " 'cars:01K3KY4XKC1TSCSCKJ67WZ04SV',\n",
+ " 'cars:01K3KY4XKCE0GXB8CHSJP1A00Q',\n",
+ " 'cars:01K3KY4XKCN4B08WG2PYSM70BJ',\n",
+ " 'cars:01K3KY4XKC50XEKZPM2FMRXCKG',\n",
+ " 'cars:01K3KY4XKCZRN029JGZ9JE0JQQ',\n",
+ " 'cars:01K3KY4XKCWR2KYMJVYV0VVGWK',\n",
+ " 'cars:01K3KY4XKCCWRMW2TDY5B2FZ7Q',\n",
+ " 'cars:01K3KY4XKCKSDQZHJ02W2H2B6G',\n",
+ " 'cars:01K3KY4XKCFN5YDRA3HNNNXFDB',\n",
+ " 'cars:01K3KY4XKCXJ5XNSFY197P9M27',\n",
+ " 'cars:01K3KY4XKC6SF9HZSBRX70VSH8',\n",
+ " 'cars:01K3KY4XKCWTZ9ERQHFD9WG9PM',\n",
+ " 'cars:01K3KY4XKCY3ZD176718N45V71',\n",
+ " 'cars:01K3KY4XKC4XZN931SSWJP74P0',\n",
+ " 'cars:01K3KY4XKC9Z1GPP9ABWW297M1',\n",
+ " 'cars:01K3KY4XKCE6Z0EPJKWG7N7GGQ',\n",
+ " 'cars:01K3KY4XKCYJ57RSQDJM7K6EFG',\n",
+ " 'cars:01K3KY4XKCBMKCPCT2KRDVGEG5',\n",
+ " 'cars:01K3KY4XKCXC7D9M2G1BXX2R8F',\n",
+ " 'cars:01K3KY4XKCG6KERMEM80A3RAAZ',\n",
+ " 'cars:01K3KY4XKCNCX9F4G1Y7H44S3S',\n",
+ " 'cars:01K3KY4XKCF153RPXWFRBK0QG5',\n",
+ " 'cars:01K3KY4XKCHXNB6WMF9HRRZ412',\n",
+ " 'cars:01K3KY4XKCMXEDN1JQHTXKKTDN',\n",
+ " 'cars:01K3KY4XKCSX846C67FYSXN6ZR',\n",
+ " 'cars:01K3KY4XKCGJY54D9T45BQPY55',\n",
+ " 'cars:01K3KY4XKCMEG7R2QQM5MYE3PP',\n",
+ " 'cars:01K3KY4XKC6NBQRQ4ETHKD090S',\n",
+ " 'cars:01K3KY4XKCDTB4AVHNFY9CX34N',\n",
+ " 'cars:01K3KY4XKC2JFZAJ878A3S9GZ9',\n",
+ " 'cars:01K3KY4XKCVCQVM1V9NJV4B6RC',\n",
+ " 'cars:01K3KY4XKC7330H6JC31MBPZJH',\n",
+ " 'cars:01K3KY4XKC9ZXV3ADZR3TDHG1Z',\n",
+ " 'cars:01K3KY4XKC884QN3FVXT2AW067',\n",
+ " 'cars:01K3KY4XKC7Q4TJ5CYV4GNXWNP',\n",
+ " 'cars:01K3KY4XKC20J1XK5NBZ0F9QCJ',\n",
+ " 'cars:01K3KY4XKCD2E0XSQ1B1257ZMX',\n",
+ " 'cars:01K3KY4XKCEYZ9RENGCEQAGZ6C',\n",
+ " 'cars:01K3KY4XKC97SWS7D34VXR3BTA',\n",
+ " 'cars:01K3KY4XKC8SD33PCV73D5GJHV',\n",
+ " 'cars:01K3KY4XKCRCH40DGW9K8DM6BR',\n",
+ " 'cars:01K3KY4XKCM6SGFZB2J3ZRSVXJ',\n",
+ " 'cars:01K3KY4XKCHXM25926JZRXFFRT',\n",
+ " 'cars:01K3KY4XKCRZPSW28J8SNAF29E',\n",
+ " 'cars:01K3KY4XKCV73EW2WE2MV6P28J',\n",
+ " 'cars:01K3KY4XKC5N07JHPDVVYCG8WA',\n",
+ " 'cars:01K3KY4XKCXS49NSXQV5APQM8C',\n",
+ " 'cars:01K3KY4XKCZW34RRWNKBV6CWHX',\n",
+ " 'cars:01K3KY4XKCE38EJGPQ664HR0S8',\n",
+ " 'cars:01K3KY4XKCG6STKQ9Q1CA7A793',\n",
+ " 'cars:01K3KY4XKCF04XXK204CK571FE',\n",
+ " 'cars:01K3KY4XKCMXMMHV39N4T24ZKE',\n",
+ " 'cars:01K3KY4XKCNZM085V5FMQASBBT',\n",
+ " 'cars:01K3KY4XKCZ6RJYR61QEQ5HN1B',\n",
+ " 'cars:01K3KY4XKCXW0GEXZAFJB0VGRX',\n",
+ " 'cars:01K3KY4XKCB4CVKRHX8XMWGFP1',\n",
+ " 'cars:01K3KY4XKC6KQA5ZDPPZ41KYQD',\n",
+ " 'cars:01K3KY4XKCEQ86M8NWYMK3QK12',\n",
+ " 'cars:01K3KY4XKCRD8MSEJFPB4DQ79B',\n",
+ " 'cars:01K3KY4XKCD13MDN1D72DZ62MX',\n",
+ " 'cars:01K3KY4XKC4XQ7CRD6G1XNHYEH',\n",
+ " 'cars:01K3KY4XKCRFCPFE2K240S1FDJ',\n",
+ " 'cars:01K3KY4XKC4PGY540BSVNXMNC5',\n",
+ " 'cars:01K3KY4XKCEA5MDNVWD0WVPYVN',\n",
+ " 'cars:01K3KY4XKCM1MCRJG6G5QWHDPB',\n",
+ " 'cars:01K3KY4XKCNKGZ9237T41MNGZ3',\n",
+ " 'cars:01K3KY4XKCVP7FDDS77KFV0TFG',\n",
+ " 'cars:01K3KY4XKCNYNE6RAYRQ06BQVT',\n",
+ " 'cars:01K3KY4XKC172KHV4BGB284FMZ',\n",
+ " 'cars:01K3KY4XKCYJNDKFKW5X6NFJZW',\n",
+ " 'cars:01K3KY4XKC3PFKDSNJSWCJ2CV8',\n",
+ " 'cars:01K3KY4XKCQD4B7CGSFYYDH99G',\n",
+ " 'cars:01K3KY4XKCYB7V4F4K54N9EB3E',\n",
+ " 'cars:01K3KY4XKCQH16DCMY0J4M2SW5',\n",
+ " 'cars:01K3KY4XKC8SW4K7TTAW2BSSQP',\n",
+ " 'cars:01K3KY4XKC1MWF5C7J277HXP9Q',\n",
+ " 'cars:01K3KY4XKCQ58467MAJRT7CGHJ',\n",
+ " 'cars:01K3KY4XKC6QN3TNWWN5SDZQRH',\n",
+ " 'cars:01K3KY4XKC8CT7ZYDBJWFTKKKP',\n",
+ " 'cars:01K3KY4XKCJGY153DBG59F4EW2',\n",
+ " 'cars:01K3KY4XKCTP4MS8EC299XAQE3',\n",
+ " 'cars:01K3KY4XKCMPDSRGRKNY8X4AV3',\n",
+ " 'cars:01K3KY4XKC2R4M324MQNZCFQCA',\n",
+ " 'cars:01K3KY4XKCBMDVXTM40NHRF596',\n",
+ " 'cars:01K3KY4XKCK16Z8WR8QSC8CSG3',\n",
+ " 'cars:01K3KY4XKCDB955M4PS11EN6R3',\n",
+ " 'cars:01K3KY4XKCRR27A0FM30DPSESX',\n",
+ " 'cars:01K3KY4XKCGH7V8Z5MMSV3TF61',\n",
+ " 'cars:01K3KY4XKCQ1CD5CEQRXR3PYPX',\n",
+ " 'cars:01K3KY4XKCKAMBSTXPMAJA6GR4',\n",
+ " 'cars:01K3KY4XKCWAA1FY1V8A81X420',\n",
+ " 'cars:01K3KY4XKCVFCCHBF5SQ416CW7',\n",
+ " 'cars:01K3KY4XKCTKBAGZYM48JYZTY4',\n",
+ " 'cars:01K3KY4XKC4E8RM1AGW73K0FBV',\n",
+ " 'cars:01K3KY4XKCT4ANPK0ZWVM0VZF3',\n",
+ " 'cars:01K3KY4XKCW76ZH894B4HBBZGY',\n",
+ " 'cars:01K3KY4XKCBPNZR56CT7TE7C9C',\n",
+ " 'cars:01K3KY4XKCAQ6KMSG8FWJ0AKRY',\n",
+ " 'cars:01K3KY4XKCTJKQFGD1209R054M',\n",
+ " 'cars:01K3KY4XKC6M4880W411F2A8BS',\n",
+ " 'cars:01K3KY4XKC1G6BSHFM95VK8ANJ',\n",
+ " 'cars:01K3KY4XKCNX68N90BTTM47KBM',\n",
+ " 'cars:01K3KY4XKC1J94G7B2JRAV6N8F',\n",
+ " 'cars:01K3KY4XKCRHFHJ35CPKQ1349W',\n",
+ " 'cars:01K3KY4XKCKP425BNQPQ37BX0D',\n",
+ " 'cars:01K3KY4XKCFWQ93YX3MPWX1T8Y',\n",
+ " 'cars:01K3KY4XKCQ87H8E759BA01DQ6',\n",
+ " 'cars:01K3KY4XKCNSYEW2ZF7ASH6X8Z',\n",
+ " 'cars:01K3KY4XKC2PF4D6NYJEVSEN73',\n",
+ " 'cars:01K3KY4XKCWYBG935J96Z1Y5GR',\n",
+ " 'cars:01K3KY4XKCJJP2MGCMK23RYMS0',\n",
+ " 'cars:01K3KY4XKC44WF6AY0JPJGCDYQ',\n",
+ " 'cars:01K3KY4XKC19CPKGC56T4Z2N8T',\n",
+ " 'cars:01K3KY4XKCF6Z8VTJH848KQXN8',\n",
+ " 'cars:01K3KY4XKC9DCG85FZQ32R2J5M',\n",
+ " 'cars:01K3KY4XKCM96NG95K9CKM8H9W',\n",
+ " 'cars:01K3KY4XKCNW70SZVACJ4F0BBJ',\n",
+ " 'cars:01K3KY4XKCWZ7X6CQM0QVK0677',\n",
+ " 'cars:01K3KY4XKC1TS1GM8YCPEC05D1',\n",
+ " 'cars:01K3KY4XKCSRRVDCH7VZ08YMZV',\n",
+ " 'cars:01K3KY4XKCRSB68MQY0S85K1N0',\n",
+ " 'cars:01K3KY4XKCDFDYM74T72J6B22X',\n",
+ " 'cars:01K3KY4XKCSD5SDNJNKVXZ1MM8',\n",
+ " 'cars:01K3KY4XKC9EW20CBS0B54CFCT',\n",
+ " 'cars:01K3KY4XKCZ4JX4M8X2B6JX709',\n",
+ " 'cars:01K3KY4XKCNBR86KZJR16TJVJF',\n",
+ " 'cars:01K3KY4XKCRZER39BJDXM27FMF',\n",
+ " 'cars:01K3KY4XKC3SD9E22X0YV2PZQ6',\n",
+ " 'cars:01K3KY4XKCPMY5Z6Q43J18Q27D',\n",
+ " 'cars:01K3KY4XKCJ4S7BJ3QW9KAAMTB',\n",
+ " 'cars:01K3KY4XKCEM0CABENAKZDFSZT',\n",
+ " 'cars:01K3KY4XKCP4S4QABNR620NB39',\n",
+ " 'cars:01K3KY4XKCH400MVKT09EDJNF6',\n",
+ " 'cars:01K3KY4XKCK82A3DK8ZBPSAZFE',\n",
+ " 'cars:01K3KY4XKCRYGF4MHT5FZCS3YN',\n",
+ " 'cars:01K3KY4XKCA2N41T4MDNSKXSB9',\n",
+ " 'cars:01K3KY4XKC298V6G8W0XX3C2VW',\n",
+ " 'cars:01K3KY4XKCB27X1RM491PKXMVV',\n",
+ " 'cars:01K3KY4XKCX33B5MFTEFGBRG41',\n",
+ " 'cars:01K3KY4XKCF1D59C951RFRHFAD',\n",
+ " 'cars:01K3KY4XKC8D9F8AMAWG2TZ2ZP',\n",
+ " 'cars:01K3KY4XKC3WTQVTAMVD8YMHJB',\n",
+ " 'cars:01K3KY4XKCV19X7CRYT7305JTW',\n",
+ " 'cars:01K3KY4XKC1PZ5NQWF5CBAM8PX',\n",
+ " 'cars:01K3KY4XKCK5PZNYTG0NGE4JMH',\n",
+ " 'cars:01K3KY4XKCFE0PVC7QVEVMJ7QS',\n",
+ " 'cars:01K3KY4XKC1WR798DNJQ0MY34G',\n",
+ " 'cars:01K3KY4XKCMVTSFJF83M6DW19Q',\n",
+ " 'cars:01K3KY4XKCBX7NG7YBMYQC5CWZ',\n",
+ " 'cars:01K3KY4XKCD2KHA69GVJ4WS6Y9',\n",
+ " 'cars:01K3KY4XKCFJRH9Q65A3FPP7XC',\n",
+ " 'cars:01K3KY4XKCXVG31SNKFDKJM4MH',\n",
+ " 'cars:01K3KY4XKCA7NBF9429HY9YC2K',\n",
+ " 'cars:01K3KY4XKCEG4NET6NQE9MWA5M',\n",
+ " 'cars:01K3KY4XKCBPV5FDYS26M7Z3XJ',\n",
+ " 'cars:01K3KY4XKCYX1S1DGNBHQAJ0ZV',\n",
+ " 'cars:01K3KY4XKCW4V75SZNKFPQT7Y1',\n",
+ " 'cars:01K3KY4XKCTEGDGNA0WH5PWJDT',\n",
+ " 'cars:01K3KY4XKCETTPNFS2R2KJ9BGV',\n",
+ " 'cars:01K3KY4XKCKBREY2SMC9W8NGM3',\n",
+ " 'cars:01K3KY4XKCSYP79NCDY2T636PA',\n",
+ " 'cars:01K3KY4XKCQ0TQ5MCG0MGHW352',\n",
+ " 'cars:01K3KY4XKC3KWKPXFAT91BTHAR',\n",
+ " 'cars:01K3KY4XKCH12X1MGMYNYYBJ5A',\n",
+ " 'cars:01K3KY4XKCGQ7SYKPZRPC3NFE4',\n",
+ " 'cars:01K3KY4XKCE9Q9FC2RP98AQ4DY',\n",
+ " 'cars:01K3KY4XKC6PC65FQSZ0HR468J',\n",
+ " 'cars:01K3KY4XKCEXE7V5M9F8D851MC',\n",
+ " 'cars:01K3KY4XKCYY3YRC9M1VQPVAK1',\n",
+ " 'cars:01K3KY4XKC2ZYRVQ4HW1VT31T3',\n",
+ " 'cars:01K3KY4XKCMNV6JQJXD1RKSYSZ',\n",
+ " 'cars:01K3KY4XKC4WQ0YEXQ3VQHFG21',\n",
+ " 'cars:01K3KY4XKCNWYZT0Z3QHXAWST7',\n",
+ " 'cars:01K3KY4XKCABRR08VZXCS8BW8C',\n",
+ " 'cars:01K3KY4XKC0KG7QAR5Q2FDRVAB',\n",
+ " 'cars:01K3KY4XKCRCJFN0F5ZJAZG6ZQ',\n",
+ " 'cars:01K3KY4XKCBQP96D1P76HRXT3P',\n",
+ " 'cars:01K3KY4XKCBHKNGE42TDACG28T',\n",
+ " 'cars:01K3KY4XKCWFHY9J0CT2RVHCC6',\n",
+ " 'cars:01K3KY4XKCK69SF474Z1C9511E',\n",
+ " 'cars:01K3KY4XKCT3WVJD0VFENHZF80',\n",
+ " 'cars:01K3KY4XKCAJY0BF0BDBQ6SKFF',\n",
+ " 'cars:01K3KY4XKC6HYPEG68D459H9QK',\n",
+ " 'cars:01K3KY4XKCAB5NKQKH7PKVGQBB',\n",
+ " 'cars:01K3KY4XKCJG0M4RJDV6Y9G6S0',\n",
+ " 'cars:01K3KY4XKCMJ0WKBYC1M1TTH1B',\n",
+ " 'cars:01K3KY4XKCB05JZZN0C2DHJXBK',\n",
+ " 'cars:01K3KY4XKCJK2TSB2A5104TWJF',\n",
+ " 'cars:01K3KY4XKCNMMA21VGTWF9WKFC',\n",
+ " 'cars:01K3KY4XKC5MTBCZ321MV5GPBW',\n",
+ " 'cars:01K3KY4XKC5XM1XTEBQJGSBYHQ',\n",
+ " 'cars:01K3KY4XKCWQXABHGH18KGD2ZG',\n",
+ " 'cars:01K3KY4XKCCC3FZ7YKTB25HQC7',\n",
+ " 'cars:01K3KY4XKC0V1J3WJ19YYVN2HQ',\n",
+ " 'cars:01K3KY4XKCF1DKKDKQ2G902GQA',\n",
+ " 'cars:01K3KY4XKC8ZYSYGWK83W13FNF',\n",
+ " 'cars:01K3KY4XKC8H8CY57Q4XT2RQYF',\n",
+ " 'cars:01K3KY4XKC935ZZKMAAEC84QVC',\n",
+ " 'cars:01K3KY4XKCT6EFAQXGVH3NJ8G0',\n",
+ " 'cars:01K3KY4XKC10G7YVTA57A0J34S',\n",
+ " 'cars:01K3KY4XKCTV935RCG4QGXYFE2',\n",
+ " 'cars:01K3KY4XKCB6WXE0720TJ5MTX4',\n",
+ " 'cars:01K3KY4XKCCC1FW86A1TMDFNPP',\n",
+ " 'cars:01K3KY4XKCA2WF7SNPNS1H8CTR',\n",
+ " 'cars:01K3KY4XKCPDMTQFW46ZX84N0A',\n",
+ " 'cars:01K3KY4XKCYXJKKS86VZE7N2RZ',\n",
+ " 'cars:01K3KY4XKC0VBRSEJMKNVSASEN',\n",
+ " 'cars:01K3KY4XKCZ1XPKSYF8H4F0QV4',\n",
+ " 'cars:01K3KY4XKCR9ZDAQWHYTGGX1H3',\n",
+ " 'cars:01K3KY4XKCVZS0KXNW1QZ7ENC7',\n",
+ " 'cars:01K3KY4XKCDRF7X3QX2PH7NVBW',\n",
+ " 'cars:01K3KY4XKC8VMSBP284WFGRBDC',\n",
+ " 'cars:01K3KY4XKC0CRKEMEZASHTXBKQ',\n",
+ " 'cars:01K3KY4XKC1P3AC0J7GCMJMBVY',\n",
+ " 'cars:01K3KY4XKCW6TGFDWN12RNC2F4',\n",
+ " 'cars:01K3KY4XKCHWHS9A7YD95QWNY6',\n",
+ " 'cars:01K3KY4XKC1HF10HGN4FYC2KDZ',\n",
+ " 'cars:01K3KY4XKCNWB95NHWRQGGDHHR',\n",
+ " 'cars:01K3KY4XKCJG6WR7QKHK814K3Y',\n",
+ " 'cars:01K3KY4XKCQH9JTPEY7Q5KEMPX',\n",
+ " 'cars:01K3KY4XKCM48TPJSAPFH67YE0',\n",
+ " 'cars:01K3KY4XKDE8YNCKAWTVHQ0M6G',\n",
+ " 'cars:01K3KY4XKDK912DF3HC4ENQ6G6',\n",
+ " 'cars:01K3KY4XKD5A16Y0GQF81D4M5K',\n",
+ " 'cars:01K3KY4XKDC689CN5WSKV146B4',\n",
+ " 'cars:01K3KY4XKDC529GCDR22499DYF',\n",
+ " 'cars:01K3KY4XKDYRR8V2EVRMMB7ZKJ',\n",
+ " 'cars:01K3KY4XKD6EHWD7EMNJETM1QT',\n",
+ " 'cars:01K3KY4XKDF8ZB3KGAQ2JKWK8N',\n",
+ " 'cars:01K3KY4XKD7M2PE10V9VA4KCD5',\n",
+ " 'cars:01K3KY4XKDRMPYCRZNVMNA937N',\n",
+ " 'cars:01K3KY4XKDRXTX3PRP9EFW7GSZ',\n",
+ " 'cars:01K3KY4XKDSQ3P6TS7GEF3Z2FT',\n",
+ " 'cars:01K3KY4XKDFXY3X2BC32VFER85',\n",
+ " 'cars:01K3KY4XKDH39YKHZGDWT41N2Q',\n",
+ " 'cars:01K3KY4XKD9EDP55GPQHCT5KP6',\n",
+ " 'cars:01K3KY4XKD4YBD24SAFGWTSYR2',\n",
+ " 'cars:01K3KY4XKD8F2RBYGCE9AYFPNJ',\n",
+ " 'cars:01K3KY4XKDWG5HDDF1DJ215GAE',\n",
+ " 'cars:01K3KY4XKDZ9YVM2RS1VBFHABG',\n",
+ " 'cars:01K3KY4XKDFRWDP7QS3B738A4Z',\n",
+ " 'cars:01K3KY4XKDW2FSNPWAR0SDGK3B',\n",
+ " 'cars:01K3KY4XKDKXZHDBSDVY0YQ936',\n",
+ " 'cars:01K3KY4XKDPT8SGKCDT662S5FY',\n",
+ " 'cars:01K3KY4XKDR8YF76RE280EFG7V',\n",
+ " 'cars:01K3KY4XKDB6BNZ6VP4963Y4S0',\n",
+ " 'cars:01K3KY4XKD12N299F4NNTN22P2',\n",
+ " 'cars:01K3KY4XKDBKFZBHFRES8N0P6E',\n",
+ " 'cars:01K3KY4XKDYQBC28S08HG847NM',\n",
+ " 'cars:01K3KY4XKD4B36PHQ7JWXM44XE',\n",
+ " 'cars:01K3KY4XKD6HNSV9JKJ4WDBXX1',\n",
+ " 'cars:01K3KY4XKD84AN6DC314NHDEZ2',\n",
+ " 'cars:01K3KY4XKD7KSFXSQ53ZEK9Z6G',\n",
+ " 'cars:01K3KY4XKDQ7VC2P299DK3NB2Q',\n",
+ " 'cars:01K3KY4XKD57ZGSD9RANAJ2DKQ',\n",
+ " 'cars:01K3KY4XKD9S5GAGB36JFPKV3V',\n",
+ " 'cars:01K3KY4XKD69TZJ7FKKGWMEMB7',\n",
+ " 'cars:01K3KY4XKD97MCTSWSH1161MVN',\n",
+ " 'cars:01K3KY4XKDRQ2SC82AWQ8MNCWN',\n",
+ " 'cars:01K3KY4XKDJRJM8NENDTDX93YX',\n",
+ " 'cars:01K3KY4XKD4VHBPN9Q85TFQHTB',\n",
+ " 'cars:01K3KY4XKD7J6585F4PQ967CN8',\n",
+ " 'cars:01K3KY4XKDYG7E75AZGDN60632',\n",
+ " 'cars:01K3KY4XKDAZPQXQ0YC8JR2CAV',\n",
+ " 'cars:01K3KY4XKD4YH6Y8R0KN9K31CA',\n",
+ " 'cars:01K3KY4XKD5FFDEQXKCQ3T5F9K',\n",
+ " 'cars:01K3KY4XKDV1JYTC7S725S92PE',\n",
+ " 'cars:01K3KY4XKDD47XTEBXMBAQDX8T',\n",
+ " 'cars:01K3KY4XKDBQJ7YCDQMB3VGTQR',\n",
+ " 'cars:01K3KY4XKDZWGQFDFNK7YQ30N8',\n",
+ " 'cars:01K3KY4XKDZAW9FE973WW1XHP9',\n",
+ " 'cars:01K3KY4XKDGFKDMCBD5H7AFQ5Z',\n",
+ " 'cars:01K3KY4XKDDW5WXXS48XXKHTKG',\n",
+ " 'cars:01K3KY4XKDWBRPVG204010W2FP',\n",
+ " 'cars:01K3KY4XKDFQP36J45YHM8Q6QX',\n",
+ " 'cars:01K3KY4XKD9XA88MZQK25PB1YC',\n",
+ " 'cars:01K3KY4XKDJWR9NK38FNE5YCH5',\n",
+ " 'cars:01K3KY4XKD1WGN3FE0NPKS0YAX',\n",
+ " 'cars:01K3KY4XKDQWDX2FJMHJNVBKYM',\n",
+ " 'cars:01K3KY4XKDTJ5DP85A6VPSRE92',\n",
+ " 'cars:01K3KY4XKDDDF48MJRBC68XZ0F',\n",
+ " 'cars:01K3KY4XKDEB0KEB9YNWW8HXRX',\n",
+ " 'cars:01K3KY4XKD3X0P80W09J7V05F1',\n",
+ " 'cars:01K3KY4XKD4E5KSSX71S94NJXB',\n",
+ " 'cars:01K3KY4XKD5CB6XSJM9VD82A3R',\n",
+ " 'cars:01K3KY4XKD3QN7XEY3JF49TXGK',\n",
+ " 'cars:01K3KY4XKD9Z1S8Y761VFH8WP3',\n",
+ " 'cars:01K3KY4XKDMQHT6GTRVKS8KJX3',\n",
+ " 'cars:01K3KY4XKDM00MSBWRZ06KDZ2Y',\n",
+ " 'cars:01K3KY4XKD7CH4JXM0FMW9BP6T',\n",
+ " 'cars:01K3KY4XKD1YCGKPBTPA40F2NE',\n",
+ " 'cars:01K3KY4XKDSMS8D1AP49RDD7DG',\n",
+ " 'cars:01K3KY4XKD8RS0YWZDG97R7SBT',\n",
+ " 'cars:01K3KY4XKDT5BBZCQ826KQWPED',\n",
+ " 'cars:01K3KY4XKDFQMJ1QK7XYNFQVNS',\n",
+ " 'cars:01K3KY4XKDQYSJ4Q2A2GKGP1F0',\n",
+ " 'cars:01K3KY4XKDB23S7YJJ5735DZ68',\n",
+ " 'cars:01K3KY4XKDG2B4G5J0YHF1RFFG',\n",
+ " 'cars:01K3KY4XKD3TYCHSDA21Z9Y39B',\n",
+ " 'cars:01K3KY4XKD7MWG2STZ61BCW038',\n",
+ " 'cars:01K3KY4XKDM53DKNQHE0YT7WAP',\n",
+ " 'cars:01K3KY4XKDW10C3TSB52XRJP0Q',\n",
+ " 'cars:01K3KY4XKD9Y5EDR1Y2YYY8MNZ',\n",
+ " 'cars:01K3KY4XKD6RNFDQ5XVEW5JK83',\n",
+ " 'cars:01K3KY4XKD3VZWQQW2ZAGDDJQA',\n",
+ " 'cars:01K3KY4XKDNKD3AMKX6SY7PHEW',\n",
+ " 'cars:01K3KY4XKDTAJ08RS37JYQAFPH',\n",
+ " 'cars:01K3KY4XKDTWC9R96K2FWBDHJA',\n",
+ " 'cars:01K3KY4XKDAC8W91XC88WTXG85',\n",
+ " 'cars:01K3KY4XKD6G9NK3E6WZRDWAN0',\n",
+ " 'cars:01K3KY4XKDJ36ND2QAP5HD2G1Q',\n",
+ " 'cars:01K3KY4XKDJWPCA18WX8BKVN0Z',\n",
+ " 'cars:01K3KY4XKDNFY1QHYTB50NF9X0',\n",
+ " 'cars:01K3KY4XKD2P0WJNR0J2ECE4F3',\n",
+ " 'cars:01K3KY4XKD7DHXNN721X2Z4901',\n",
+ " 'cars:01K3KY4XKDX7WJMXCQHN01ZWEW',\n",
+ " 'cars:01K3KY4XKD3N61BPZYM9RHKB76',\n",
+ " 'cars:01K3KY4XKDT39H578M5SCDRS4W',\n",
+ " 'cars:01K3KY4XKD36GVZZPHR5HW6BTE',\n",
+ " 'cars:01K3KY4XKDH46WXPQ411RE05AA',\n",
+ " 'cars:01K3KY4XKD77E8MREKF5BTTQWC',\n",
+ " 'cars:01K3KY4XKDZ6PAEKNKGHA49MDY',\n",
+ " 'cars:01K3KY4XKDZ4VCWP13S9JYQ1KE',\n",
+ " 'cars:01K3KY4XKDM4X9T68MVY8XEC77',\n",
+ " 'cars:01K3KY4XKDNE3P2VC8XSPTK2FB',\n",
+ " 'cars:01K3KY4XKDZPKNV2A7FHZ8Q15Y',\n",
+ " 'cars:01K3KY4XKD3JGBMPGY8QWJCSBF',\n",
+ " 'cars:01K3KY4XKD5JC2E4FWPTEXYXP9',\n",
+ " 'cars:01K3KY4XKDCP8TKVWM2CPGR46Y',\n",
+ " 'cars:01K3KY4XKDCGWA1SBNMP9ZQ8CY',\n",
+ " 'cars:01K3KY4XKD4GGBS7RFN3RTDG7W',\n",
+ " 'cars:01K3KY4XKD8EV0T32GA88XFD5M',\n",
+ " 'cars:01K3KY4XKDSGCNKV26B7G00SM1',\n",
+ " 'cars:01K3KY4XKDNQ04PETXE4EB1NCW',\n",
+ " 'cars:01K3KY4XKDY4B7492NE2F19RHN',\n",
+ " 'cars:01K3KY4XKD37BN33XTMZBQAYZD',\n",
+ " 'cars:01K3KY4XKD4X39XY2YJ6WFGVMT',\n",
+ " 'cars:01K3KY4XKDVV33A45E0KWMANJ7',\n",
+ " 'cars:01K3KY4XKDYHDVPWPWV7T8SD51',\n",
+ " 'cars:01K3KY4XKDKJB31DCPNB8219DV',\n",
+ " 'cars:01K3KY4XKDWPAVH6V4WCZQHS7V',\n",
+ " 'cars:01K3KY4XKDSZ4VQX8SCJH5MW7J',\n",
+ " 'cars:01K3KY4XKDE4JS9Z6B8F48CR79',\n",
+ " 'cars:01K3KY4XKDRR161G9WEBXZ7N6Q',\n",
+ " 'cars:01K3KY4XKDT19Z8HWXNPFY7N6A',\n",
+ " 'cars:01K3KY4XKD24CJ2Z6X5MXXS7AV',\n",
+ " 'cars:01K3KY4XKDGK035B9J5ZCE849C',\n",
+ " 'cars:01K3KY4XKD7TW2TKNVHHG3FC8Q',\n",
+ " 'cars:01K3KY4XKDWD641Z3X3D1EGZ4Z',\n",
+ " 'cars:01K3KY4XKDB3PKF9M9KJW70GC4',\n",
+ " 'cars:01K3KY4XKD54GN408MR4WX8DPA',\n",
+ " 'cars:01K3KY4XKDEM3J8QT58F5CWQZB',\n",
+ " 'cars:01K3KY4XKD8Q9NMC2TTVCK55QS',\n",
+ " 'cars:01K3KY4XKDXKB7WYHPBY96VDDN',\n",
+ " 'cars:01K3KY4XKDGG9D6B6QFAVEQ1C7',\n",
+ " 'cars:01K3KY4XKDC71F92HZE9X58NVR',\n",
+ " 'cars:01K3KY4XKDH8X7ZW7YWV2999BM',\n",
+ " 'cars:01K3KY4XKDV0H38VXE9FPTCM1W',\n",
+ " 'cars:01K3KY4XKDV8FS1Q4NBWQGRC0M',\n",
+ " 'cars:01K3KY4XKDFD4T99R4PV9EY39G',\n",
+ " 'cars:01K3KY4XKDPGP7596P0GXBHTY9',\n",
+ " 'cars:01K3KY4XKDB7D2EH5R96PQA55J',\n",
+ " 'cars:01K3KY4XKD3DCAM7SQWPZHZ64P',\n",
+ " 'cars:01K3KY4XKDD7H9YJBBP1XRFAPR',\n",
+ " 'cars:01K3KY4XKDER9B9BSV73R38F8S',\n",
+ " 'cars:01K3KY4XKDWBXSVWGC9444B4VE',\n",
+ " 'cars:01K3KY4XKDBQ1Q0Q55HXVZZ0HY',\n",
+ " 'cars:01K3KY4XKDE73WGNGHACSC2PB1',\n",
+ " 'cars:01K3KY4XKDPT3C3G9DFQFPS62Y',\n",
+ " 'cars:01K3KY4XKDASPNHXQ7ST7ZTGYW',\n",
+ " 'cars:01K3KY4XKDQ3NEJH6QH8BQ8HDB',\n",
+ " 'cars:01K3KY4XKDGA3XH8ECSGH3CDDM',\n",
+ " 'cars:01K3KY4XKDQHPBN13JCRGDK6PG',\n",
+ " 'cars:01K3KY4XKD4PGSYVY4KM6A8QNT',\n",
+ " 'cars:01K3KY4XKDK5AZPXPF97AVAMXB',\n",
+ " 'cars:01K3KY4XKDFHYDDEDAWSF9KEDG',\n",
+ " 'cars:01K3KY4XKDK179MH8N2GQBHFS5',\n",
+ " 'cars:01K3KY4XKDC7SZZRYPY90X8BFW',\n",
+ " 'cars:01K3KY4XKDCP8D1KY85QYS8XDB',\n",
+ " 'cars:01K3KY4XKDJ79YE2WX8NKG6F6D',\n",
+ " 'cars:01K3KY4XKDTB6TD1F9EN8RPEJB',\n",
+ " 'cars:01K3KY4XKDR6NMT8QS9FY9A763',\n",
+ " 'cars:01K3KY4XKDAGGQF224PKWC9XFA',\n",
+ " 'cars:01K3KY4XKDKRNE8JGCW95NYQSV',\n",
+ " 'cars:01K3KY4XKD7GTZB416X46EH4X4',\n",
+ " 'cars:01K3KY4XKDGYXTRP2N18DWH2R9',\n",
+ " 'cars:01K3KY4XKD9R3D2RNFNNAQYRSC',\n",
+ " 'cars:01K3KY4XKDS3VV5R6ZYP3484HN',\n",
+ " 'cars:01K3KY4XKD7HRQY1AQK7SX8P4P',\n",
+ " 'cars:01K3KY4XKDZE3YKQ3CV5KT6190',\n",
+ " 'cars:01K3KY4XKD7ZB8H21AT1DNQD9J',\n",
+ " 'cars:01K3KY4XKD96RQY82KB0ZS4WV8',\n",
+ " 'cars:01K3KY4XKD264VZAYDHC12D5VF',\n",
+ " 'cars:01K3KY4XKDWK2QSCYWPTEZ5CQ1',\n",
+ " 'cars:01K3KY4XKDWB0D3JAWG7Q722ZK',\n",
+ " 'cars:01K3KY4XKDEGZ5W1XY8DD9Z10R',\n",
+ " 'cars:01K3KY4XKDZZC893XST16VYSFS',\n",
+ " 'cars:01K3KY4XKDTB9ARMC4WGSM1VYA',\n",
+ " 'cars:01K3KY4XKDSJ9RFNFMKTSXZCMH',\n",
+ " 'cars:01K3KY4XKDB9QKVFRJF4D3QH4C',\n",
+ " 'cars:01K3KY4XKDJSEZ271HMD9RNC83',\n",
+ " 'cars:01K3KY4XKDQ2M91N0WHATCHC2P',\n",
+ " 'cars:01K3KY4XKDRRMT4DN86SNTW1MP',\n",
+ " 'cars:01K3KY4XKDEFK8MR6ZAEYXQNHY',\n",
+ " 'cars:01K3KY4XKD7ZFVGF8EJZJCZ7H9',\n",
+ " 'cars:01K3KY4XKD5DRTP7M0TYAHC89A',\n",
+ " 'cars:01K3KY4XKDJMN8E6VRB1RJWAZ6',\n",
+ " 'cars:01K3KY4XKDS01E74C0G339KHVA',\n",
+ " 'cars:01K3KY4XKDFECPKH7WTF61WST0',\n",
+ " 'cars:01K3KY4XKD6QZS69S9CDG6TZ71',\n",
+ " 'cars:01K3KY4XKDR5BAZJJF9450G164',\n",
+ " 'cars:01K3KY4XKD75R40XPN0HT4EQ8Y',\n",
+ " 'cars:01K3KY4XKDVFMGW0HDHGB8XXEE',\n",
+ " 'cars:01K3KY4XKDB6X456JHX74SEWH0',\n",
+ " 'cars:01K3KY4XKDPSD0YNVA196X7XM8',\n",
+ " 'cars:01K3KY4XKDZQW3WFYRT15YXM02']"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from redisvl.index import SearchIndex\n",
+ "from redisvl.utils.vectorize import HFTextVectorizer\n",
+ "\n",
+ "emb_model = HFTextVectorizer()\n",
+ "\n",
+ "# define schema\n",
+ "car_schema = {\n",
+ " \"index\": {\n",
+ " \"name\": \"cars\",\n",
+ " \"prefix\": \"cars\"\n",
+ " },\n",
+ " \"fields\": [\n",
+ " {\"name\": \"item_id\", \"type\": \"tag\"},\n",
+ " {\"name\": \"text\", \"type\": \"text\"},\n",
+ " {\"name\": \"make\", \"type\": \"tag\"},\n",
+ " {\"name\": \"model\", \"type\": \"tag\"},\n",
+ " {\n",
+ " \"name\": \"vector\",\n",
+ " \"type\": \"vector\",\n",
+ " \"attrs\": {\n",
+ " \"dims\": 768,\n",
+ " \"distance_metric\": \"cosine\",\n",
+ " \"algorithm\": \"FLAT\",\n",
+ " \"datatype\": \"float32\"\n",
+ " },\n",
+ " },\n",
+ " ]\n",
+ "}\n",
+ "\n",
+ "# create index\n",
+ "index = SearchIndex.from_dict(car_schema, redis_url=redis_url)\n",
+ "index.create(overwrite=True)\n",
+ "\n",
+ "embeddings = emb_model.embed_many([c[\"text\"] for c in corpus], as_buffer=True)\n",
+ "\n",
+ "# vectorize corpus data\n",
+ "corpus_data = [\n",
+ " {\n",
+ " \"text\": c[\"text\"],\n",
+ " \"item_id\": c[\"item_id\"],\n",
+ " \"make\": c[\"query_metadata\"][\"make\"],\n",
+ " \"model\": c[\"query_metadata\"][\"model\"],\n",
+ " \"vector\": embeddings[i]\n",
+ " }\n",
+ " for i, c in enumerate(corpus)\n",
+ "]\n",
+ "\n",
+ "index.load(corpus_data)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f9bc0270",
+ "metadata": {},
+ "source": [
+ "# Check index created successfully"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "200bbeb9",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "464"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "index.info()[\"num_docs\"]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "58036fff",
+ "metadata": {},
+ "source": [
+ "# Review search study config\n",
+ "\n",
+ "- index_name should point to index created above\n",
+ "- qrels and queries should point to the queries and set of labeled queries under test\n",
+ "- search methods should match with the custom methods defined below\n",
+ "- embedding_model should match with the one used to create the index"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "ef6c1802",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "SearchStudyConfig(study_id='test-search-study', index_name='cars', qrels='../resources/cars/car_qrels.json', queries='../resources/cars/car_queries.json', search_methods=['base_vector', 'pre_filter_vector'], ret_k=3, id_field_name='_id', vector_field_name='vector', text_field_name='text', embedding_model=EmbeddingModel(type='hf', model='sentence-transformers/all-mpnet-base-v2', dim=768, embedding_cache_name='vec-cache', dtype='float32'))"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from redis_retrieval_optimizer.utils import load_search_study_config\n",
+ "\n",
+ "search_study_config = load_search_study_config(\"search_study_config.yaml\")\n",
+ "search_study_config"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3a0db837",
+ "metadata": {},
+ "source": [
+ "# Define search methods for search study\n",
+ "\n",
+ "A search method can be anything as long as it takes a `SearchMethodInput` and returns a `SearchMethodOutput`. Below we will compare a basic vector search to a vector search with a pre-filter. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bf495d1d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ranx import Run\n",
+ "from redis_retrieval_optimizer.search_methods.base import run_search_w_time\n",
+ "from redisvl.query import VectorQuery\n",
+ "from redisvl.query.filter import Tag\n",
+ "\n",
+ "from redis_retrieval_optimizer.schema import SearchMethodInput, SearchMethodOutput\n",
+ "from redis_retrieval_optimizer.search_methods.vector import make_score_dict_vec\n",
+ "\n",
+ "def vector_query(query_info, num_results: int, emb_model) -> VectorQuery:\n",
+ " vector = emb_model.embed(query_info[\"query\"], as_buffer=True)\n",
+ "\n",
+ " return VectorQuery(\n",
+ " vector=vector,\n",
+ " vector_field_name=\"vector\",\n",
+ " num_results=num_results,\n",
+ " return_fields=[\"item_id\", \"make\", \"model\", \"text\"]\n",
+ " )\n",
+ "\n",
+ "def pre_filter_query(query_info, num_results, emb_model) -> VectorQuery:\n",
+ " vec = emb_model.embed(query_info[\"query\"])\n",
+ " make = query_info[\"query_metadata\"][\"make\"]\n",
+ " model = query_info[\"query_metadata\"][\"model\"]\n",
+ "\n",
+ " filter = (Tag(\"make\") == make) & (Tag(\"model\") == model)\n",
+ "\n",
+ " # Create a vector query\n",
+ " query = VectorQuery(\n",
+ " vector=vec,\n",
+ " vector_field_name=\"vector\",\n",
+ " num_results=num_results,\n",
+ " filter_expression=filter,\n",
+ " return_fields=[\"item_id\", \"make\", \"model\", \"text\"]\n",
+ " )\n",
+ "\n",
+ " return query\n",
+ "\n",
+ "def gather_pre_filter_results(search_method_input: SearchMethodInput) -> SearchMethodOutput:\n",
+ " redis_res_vector = {}\n",
+ "\n",
+ " for key, query_info in search_method_input.raw_queries.items():\n",
+ " # create the query\n",
+ " query = pre_filter_query(query_info, search_method_input.ret_k, search_method_input.emb_model)\n",
+ "\n",
+ " # run with timing helper function\n",
+ " res = run_search_w_time(\n",
+ " search_method_input.index, query, search_method_input.query_metrics\n",
+ " )\n",
+ "\n",
+ " # format into scores dict\n",
+ " score_dict = make_score_dict_vec(res, id_field_name=\"item_id\")\n",
+ "\n",
+ " redis_res_vector[key] = score_dict\n",
+ "\n",
+ " # return search method output\n",
+ " return SearchMethodOutput(\n",
+ " run=Run(redis_res_vector),\n",
+ " query_metrics=search_method_input.query_metrics,\n",
+ " )\n",
+ "\n",
+ "\n",
+ "def gather_vector_results(search_method_input: SearchMethodInput) -> SearchMethodOutput:\n",
+ " redis_res_vector = {}\n",
+ "\n",
+ " for key, query_info in search_method_input.raw_queries.items():\n",
+ " # get query\n",
+ " vec_query = vector_query(query_info, search_method_input.ret_k, search_method_input.emb_model)\n",
+ "\n",
+ " # run with timing helper function\n",
+ " res = run_search_w_time(\n",
+ " search_method_input.index, vec_query, search_method_input.query_metrics\n",
+ " )\n",
+ "\n",
+ " # format into scores dict\n",
+ " score_dict = make_score_dict_vec(res, id_field_name=\"item_id\")\n",
+ " redis_res_vector[key] = score_dict\n",
+ " \n",
+ " # return search method output\n",
+ " return SearchMethodOutput(\n",
+ " run=Run(redis_res_vector),\n",
+ " query_metrics=search_method_input.query_metrics,\n",
+ " )\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bfbf4bb9",
+ "metadata": {},
+ "source": [
+ "# Run the search study"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9f7ecda0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from redis_retrieval_optimizer.search_study import run_search_study\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/redis_retrieval_optimizer/__init__.py b/redis_retrieval_optimizer/__init__.py
index 1b236d6..a2fb2ed 100644
--- a/redis_retrieval_optimizer/__init__.py
+++ b/redis_retrieval_optimizer/__init__.py
@@ -1,3 +1,3 @@
-__version__ = "0.4.2"
+__version__ = "0.4.3"
all = ["__version__"]
diff --git a/redis_retrieval_optimizer/bayes_study.py b/redis_retrieval_optimizer/bayes_study.py
index fd8c563..1e16945 100644
--- a/redis_retrieval_optimizer/bayes_study.py
+++ b/redis_retrieval_optimizer/bayes_study.py
@@ -122,6 +122,7 @@ def objective(trial, study_config, redis_url, corpus_processor, search_method_ma
dtype=trial_settings.index_settings.vector_data_type,
)
+
if recreate_data:
logging.info("Recreating index...")
corpus = utils.load_json(study_config.corpus)
@@ -129,15 +130,30 @@ def objective(trial, study_config, redis_url, corpus_processor, search_method_ma
corpus_size = len(corpus_data)
logging.info(f"Corpus size: {corpus_size}")
- # reload data
+ # reload data and measure wall-clock time until indexing completes
+ indexing_start_time = time.time()
trial_index.load(corpus_data)
- while float(trial_index.info()["percent_indexed"]) < 1:
- time.sleep(1)
- logging.info(f"Indexing progress: {trial_index.info()['percent_indexed']}")
+ while float(trial_index.info()["percent_indexed"]) < 1:
+ time.sleep(1)
+ logging.info(f"Indexing progress: {trial_index.info()['percent_indexed']}")
+ else:
+ # Only wait if index is not fully indexed
+ if float(trial_index.info()["percent_indexed"]) < 1:
+ while float(trial_index.info()["percent_indexed"]) < 1:
+ time.sleep(1)
+ logging.info(f"Indexing progress: {trial_index.info()['percent_indexed']}")
+
+ if recreate_data:
+ assert indexing_start_time is not None
+ total_indexing_time = time.time() - indexing_start_time
+ utils.set_last_indexing_time(redis_url, total_indexing_time)
+ else:
+ last_indexing_time = utils.get_last_indexing_time(redis_url)
+ total_indexing_time = (
+ last_indexing_time if last_indexing_time is not None else 0.0
+ )
- # capture index metrics
- total_indexing_time = float(trial_index.info()["total_indexing_time"])
num_docs = trial_index.info()["num_docs"]
logging.info(f"Data indexed {total_indexing_time=}s, {num_docs=}")
diff --git a/redis_retrieval_optimizer/grid_study.py b/redis_retrieval_optimizer/grid_study.py
index 0618327..bc12fc0 100644
--- a/redis_retrieval_optimizer/grid_study.py
+++ b/redis_retrieval_optimizer/grid_study.py
@@ -112,12 +112,16 @@ def init_index_from_grid_settings(
# corpus processing functions should be user defined
corpus_data = corpus_processor(corpus, emb_model)
+ indexing_start_time = time.time()
index.load(corpus_data)
while float(index.info()["percent_indexed"]) < 1:
time.sleep(1)
logging.info(f"Indexing progress: {index.info()['percent_indexed']}")
+ total_indexing_time = time.time() - indexing_start_time
+ utils.set_last_indexing_time(redis_url, total_indexing_time)
+
index_settings["embedding"] = embed_settings.model_dump()
utils.set_last_index_settings(redis_url, index_settings)
@@ -188,6 +192,7 @@ def run_grid_study(
# corpus processing functions should be user defined
corpus_data = corpus_processor(corpus, emb_model)
+ indexing_start_time = time.time()
index.load(corpus_data)
while float(index.info()["percent_indexed"]) < 1:
@@ -196,6 +201,9 @@ def run_grid_study(
f"Indexing progress: {index.info()['percent_indexed']}"
)
+ total_indexing_time = time.time() - indexing_start_time
+ utils.set_last_indexing_time(redis_url, total_indexing_time)
+
# Get embedding model with current dtype
emb_model = utils.get_embedding_model(
embedding_model, redis_url, dtype=dtype
@@ -220,9 +228,11 @@ def run_grid_study(
qrels, search_method_output.run
)
- index_info = index.info()
+ last_indexing_time = utils.get_last_indexing_time(redis_url)
- trial_metrics["total_indexing_time"] = index_info["total_indexing_time"]
+ trial_metrics["total_indexing_time"] = (
+ last_indexing_time if last_indexing_time is not None else 0.0
+ )
memory_stats = utils.get_index_memory_stats(
grid_study_config.index_settings.name,
diff --git a/redis_retrieval_optimizer/search_study.py b/redis_retrieval_optimizer/search_study.py
index 9e691b0..7bd92aa 100644
--- a/redis_retrieval_optimizer/search_study.py
+++ b/redis_retrieval_optimizer/search_study.py
@@ -90,7 +90,10 @@ def run_search_study(
trial_metrics = utils.eval_trial_metrics(qrels, search_method_output.run)
- trial_metrics["total_indexing_time"] = index_info["total_indexing_time"]
+ last_indexing_time = utils.get_last_indexing_time(redis_url)
+ trial_metrics["total_indexing_time"] = (
+ last_indexing_time if last_indexing_time is not None else 0.0
+ )
memory_stats = utils.get_index_memory_stats(
search_study_config.index_name,
diff --git a/redis_retrieval_optimizer/utils.py b/redis_retrieval_optimizer/utils.py
index 695f242..c3aa9a4 100644
--- a/redis_retrieval_optimizer/utils.py
+++ b/redis_retrieval_optimizer/utils.py
@@ -69,6 +69,27 @@ def set_last_index_settings(redis_url, index_settings):
client.json().set("ret-opt:last_schema", Path.root_path(), index_settings)
+def get_last_indexing_time(redis_url: str) -> float | None:
+ """Return the last recorded total indexing time in seconds, if any.
+
+ This is stored under a dedicated JSON key so we can reuse the
+ indexing time across runs where we do not reload data.
+ """
+ client = Redis.from_url(redis_url)
+ value = client.json().get("ret-opt:last_indexing_time")
+ return float(value) if value is not None else None
+
+
+def set_last_indexing_time(redis_url: str, indexing_time: float) -> None:
+ """Persist the total indexing time (in seconds) for the current index.
+
+ This is used when subsequent runs reuse the existing indexed data
+ and therefore should reuse the previously measured indexing time.
+ """
+ client = Redis.from_url(redis_url)
+ client.json().set("ret-opt:last_indexing_time", Path.root_path(), indexing_time)
+
+
def check_recreate(index_settings, last_index_settings):
embedding_settings = index_settings.pop("embedding") if index_settings else None
last_embedding_settings = (
diff --git a/tests/integration/test_bayes.py b/tests/integration/test_bayes.py
index 9ee1a83..9a9b7d1 100644
--- a/tests/integration/test_bayes.py
+++ b/tests/integration/test_bayes.py
@@ -1,5 +1,6 @@
import os
+import pytest
import yaml
from redisvl.index import SearchIndex
@@ -31,6 +32,16 @@ def test_run_bayes_study(redis_url):
assert metrics.shape[0] == study_config["optimization_settings"]["n_trials"]
+ # total_indexing_time should be recorded for each trial and persisted
+ assert "total_indexing_time" in metrics.columns
+
+ last_indexing_time = utils.get_last_indexing_time(redis_url)
+ assert last_indexing_time is not None
+ assert last_indexing_time > 0.0
+
+ # The last trial's recorded indexing time should match the persisted value
+ assert metrics["total_indexing_time"].iloc[-1] == pytest.approx(last_indexing_time)
+
for score in metrics["f1"].tolist():
assert score > 0.0
@@ -43,4 +54,5 @@ def test_run_bayes_study(redis_url):
# clean up
index.client.json().delete("ret-opt:last_schema")
+ index.client.json().delete("ret-opt:last_indexing_time")
index.delete(drop=True)
diff --git a/tests/integration/test_grid.py b/tests/integration/test_grid.py
index 5454e5e..82e0d76 100644
--- a/tests/integration/test_grid.py
+++ b/tests/integration/test_grid.py
@@ -1,5 +1,6 @@
import os
+import pytest
import yaml
from redisvl.index import SearchIndex
@@ -44,6 +45,19 @@ def test_run_grid_study(redis_url):
for score in metrics["f1"].tolist():
assert score > 0.0
+ # total_indexing_time should be recorded and reused across trials
+ assert "total_indexing_time" in metrics.columns
+
+ # With a single vector data type, all trials should share the same
+ # positive indexing time value.
+ unique_times = metrics["total_indexing_time"].unique()
+ assert len(unique_times) == 1
+ assert unique_times[0] > 0.0
+
+ last_indexing_time = utils.get_last_indexing_time(redis_url)
+ assert last_indexing_time is not None
+ assert unique_times[0] == pytest.approx(last_indexing_time)
+
last_schema = utils.get_last_index_settings(redis_url)
assert last_schema is not None
@@ -53,6 +67,7 @@ def test_run_grid_study(redis_url):
# clean up
index.client.json().delete("ret-opt:last_schema")
+ index.client.json().delete("ret-opt:last_indexing_time")
index.delete(drop=True)
@@ -96,6 +111,17 @@ def test_run_grid_study_with_multiple_dtypes(redis_url):
for score in metrics["f1"].tolist():
assert score > 0.0
+ # total_indexing_time should be recorded for each dtype and reused
+ # across search methods for that dtype.
+ assert "total_indexing_time" in metrics.columns
+
+ for dtype in unique_dtypes:
+ dtype_times = metrics.loc[
+ metrics["vector_data_type"] == dtype, "total_indexing_time"
+ ]
+ assert dtype_times.nunique() == 1
+ assert dtype_times.iloc[0] > 0.0
+
last_schema = utils.get_last_index_settings(redis_url)
assert last_schema is not None
@@ -105,4 +131,5 @@ def test_run_grid_study_with_multiple_dtypes(redis_url):
# clean up
index.client.json().delete("ret-opt:last_schema")
+ index.client.json().delete("ret-opt:last_indexing_time")
index.delete(drop=True)
diff --git a/tests/integration/test_search_study.py b/tests/integration/test_search_study.py
index 09b450c..ae00bef 100644
--- a/tests/integration/test_search_study.py
+++ b/tests/integration/test_search_study.py
@@ -1,5 +1,7 @@
import os
+import time
+import pytest
import yaml
from redisvl.index import SearchIndex
from redisvl.utils.vectorize.text.huggingface import HFTextVectorizer
@@ -38,14 +40,20 @@ def test_run_search_study(redis_url):
# Load corpus data
corpus = utils.load_json(f"{TEST_DIR}/search_data/corpus.json")
corpus_data = eval_beir.process_corpus(corpus, emb_model)
+ indexing_start_time = time.time()
index.load(corpus_data)
# Wait for indexing to complete
while float(index.info()["percent_indexed"]) < 1:
- import time
-
time.sleep(1)
+ total_indexing_time = time.time() - indexing_start_time
+ # Sanity check: indexing time should be positive for a small test corpus.
+ assert total_indexing_time > 0.0
+
+ # Persist the measured indexing time so search_study can reuse it.
+ utils.set_last_indexing_time(redis_url, total_indexing_time)
+
# Run search study
metrics = run_search_study(
config_path=search_config_path,
@@ -57,6 +65,13 @@ def test_run_search_study(redis_url):
assert metrics.shape[0] == expected_trials
+ # total_indexing_time should be present and match the value we measured.
+ assert "total_indexing_time" in metrics.columns
+
+ unique_indexing_times = metrics["total_indexing_time"].unique()
+ assert len(unique_indexing_times) == 1
+ assert unique_indexing_times[0] == pytest.approx(total_indexing_time)
+
for score in metrics["f1"].tolist():
assert score > 0.0
@@ -67,13 +82,12 @@ def test_run_search_study(redis_url):
assert method in unique_methods
# Clean up
+ index.client.json().delete("ret-opt:last_indexing_time")
index.delete(drop=True)
def test_search_study_requires_embedding_model(redis_url):
"""Test that search study requires embedding_model in config."""
- import pytest
-
# Create a config without embedding_model
config_path = f"{TEST_DIR}/search_data/test_search_study_config_no_embedding.yaml"