From 9965a124200a965e85568a69de88f0e1309950b0 Mon Sep 17 00:00:00 2001 From: Dima Blochman Date: Wed, 21 Feb 2024 21:57:37 +0200 Subject: [PATCH 1/7] Added dotenv package for environment variable management and refactored Pinecone initialization in rag_engine.py --- rag_engine.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rag_engine.py b/rag_engine.py index 488d3ba..fcab70b 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -1,7 +1,7 @@ import os, tempfile import pinecone from pathlib import Path - +from dotenv import load_dotenv from langchain.chains import RetrievalQA, ConversationalRetrievalChain from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Chroma @@ -13,9 +13,12 @@ from langchain.embeddings.openai import OpenAIEmbeddings from langchain.memory import ConversationBufferMemory from langchain.memory.chat_message_histories import StreamlitChatMessageHistory +from pinecone import Pinecone, ServerlessSpec + import streamlit as st +load_dotenv() TMP_DIR = Path(__file__).resolve().parent.joinpath('data', 'tmp') LOCAL_VECTOR_STORE_DIR = Path(__file__).resolve().parent.joinpath('data', 'vector_store') @@ -42,9 +45,15 @@ def embeddings_on_local_vectordb(texts): return retriever def embeddings_on_pinecone(texts): - pinecone.init(api_key=st.session_state.pinecone_api_key, environment=st.session_state.pinecone_env) + + pc = Pinecone( + api_key=st.session_state.pinecone_api_key, + environment=st.session_state.pinecone_env + ) + + # pc.init(api_key=st.session_state.pinecone_api_key, environment=st.session_state.pinecone_env) embeddings = OpenAIEmbeddings(openai_api_key=st.session_state.openai_api_key) - vectordb = Pinecone.from_documents(texts, embeddings, index_name=st.session_state.pinecone_index) + vectordb = pc.from_documents(texts, embeddings, index_name=st.session_state.pinecone_index) retriever = vectordb.as_retriever() return retriever @@ -70,7 +79,7 @@ def input_fields(): # if "pinecone_api_key" in st.secrets: st.session_state.pinecone_api_key = st.secrets.pinecone_api_key - else: + else: st.session_state.pinecone_api_key = st.text_input("Pinecone API key", type="password") # if "pinecone_env" in st.secrets: @@ -125,7 +134,7 @@ def boot(): # for message in st.session_state.messages: st.chat_message('human').write(message[0]) - st.chat_message('ai').write(message[1]) + st.chat_message('ai').write(message[1]) # if query := st.chat_input(): st.chat_message("human").write(query) @@ -135,4 +144,3 @@ def boot(): if __name__ == '__main__': # boot() - \ No newline at end of file From d60353621fa873fb42e6dfcecacf9c00b5972c77 Mon Sep 17 00:00:00 2001 From: Dima Blochman Date: Thu, 22 Feb 2024 12:42:05 +0200 Subject: [PATCH 2/7] implemented new store and query for pinecone --- rag_engine.py | 42 ++++++++++++++++++++++++++++++------------ requirements.txt | 14 ++++++++++---- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/rag_engine.py b/rag_engine.py index fcab70b..225184e 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -13,8 +13,6 @@ from langchain.embeddings.openai import OpenAIEmbeddings from langchain.memory import ConversationBufferMemory from langchain.memory.chat_message_histories import StreamlitChatMessageHistory -from pinecone import Pinecone, ServerlessSpec - import streamlit as st @@ -45,18 +43,21 @@ def embeddings_on_local_vectordb(texts): return retriever def embeddings_on_pinecone(texts): - - pc = Pinecone( - api_key=st.session_state.pinecone_api_key, - environment=st.session_state.pinecone_env - ) - - # pc.init(api_key=st.session_state.pinecone_api_key, environment=st.session_state.pinecone_env) + pinecone.init(api_key=st.session_state.pinecone_api_key, environment=st.session_state.pinecone_env) embeddings = OpenAIEmbeddings(openai_api_key=st.session_state.openai_api_key) - vectordb = pc.from_documents(texts, embeddings, index_name=st.session_state.pinecone_index) + vectordb = Pinecone.from_documents(texts, embeddings, index_name=st.session_state.pinecone_index) retriever = vectordb.as_retriever() return retriever +def embedding_on_pinecone_new(texts): + from langchain_openai import OpenAIEmbeddings + embeddings = OpenAIEmbeddings(openai_api_key=st.session_state.openai_api_key) + from langchain_pinecone import Pinecone + + index_name = "quickstart" + docsearch = Pinecone.from_documents(texts , embeddings, index_name=index_name) + return docsearch.as_retriever() + def query_llm(retriever, query): qa_chain = ConversationalRetrievalChain.from_llm( llm=OpenAIChat(openai_api_key=st.session_state.openai_api_key), @@ -68,6 +69,23 @@ def query_llm(retriever, query): st.session_state.messages.append((query, result)) return result +def query_llm_new(retriever, query): + from langchain.chat_models import ChatOpenAI + from langchain.chains import RetrievalQA + llm = ChatOpenAI( + openai_api_key=st.session_state.openai_api_key, + model_name='gpt-3.5-turbo', + temperature=0.0 + ) + qa = RetrievalQA.from_chain_type( + llm=llm, + chain_type="stuff", + retriever=retriever + ) + result = qa.run(query) + st.session_state.messages.append((query, result)) + return result + def input_fields(): # with st.sidebar: @@ -119,7 +137,7 @@ def process_documents(): if not st.session_state.pinecone_db: st.session_state.retriever = embeddings_on_local_vectordb(texts) else: - st.session_state.retriever = embeddings_on_pinecone(texts) + st.session_state.retriever = embedding_on_pinecone_new(texts) except Exception as e: st.error(f"An error occurred: {e}") @@ -138,7 +156,7 @@ def boot(): # if query := st.chat_input(): st.chat_message("human").write(query) - response = query_llm(st.session_state.retriever, query) + response = query_llm_new(st.session_state.retriever, query) st.chat_message("ai").write(response) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index b7e9e69..75a3b6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,14 @@ -langchain==0.0.279 -pinecone_client==2.2.2 +pinecone_client==3.0.3 streamlit==1.26.0 unstructured unstructured[pdf] -openai +openai==1.12.0 chromadb -tiktoken \ No newline at end of file +tiktoken +langchain==0.1.8 +langchain-community==0.0.21 +langchain-core==0.1.25 +langchain-openai==0.0.6 +langchain-pinecone==0.0.2 +langdetect==1.0.9 +langsmith==0.1.5 From bb0047fe34b2132e6a87c5eb4ee8f64fd9ec251d Mon Sep 17 00:00:00 2001 From: Dima Blochman Date: Thu, 22 Feb 2024 12:46:29 +0200 Subject: [PATCH 3/7] added sources --- rag_engine.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rag_engine.py b/rag_engine.py index 225184e..83a4017 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -71,7 +71,10 @@ def query_llm(retriever, query): def query_llm_new(retriever, query): from langchain.chat_models import ChatOpenAI - from langchain.chains import RetrievalQA + from langchain.chains import RetrievalQA, RetrievalQAWithSourcesChain + + with_source = True + llm = ChatOpenAI( openai_api_key=st.session_state.openai_api_key, model_name='gpt-3.5-turbo', @@ -82,7 +85,12 @@ def query_llm_new(retriever, query): chain_type="stuff", retriever=retriever ) - result = qa.run(query) + qa_with_sources = RetrievalQAWithSourcesChain.from_chain_type( + llm=llm, + chain_type="stuff", + retriever=retriever + ) + result = qa_with_sources(query) if with_source else qa(query) st.session_state.messages.append((query, result)) return result From 4f697fa513d1992d69cb1fd0e06b8e7933cabf82 Mon Sep 17 00:00:00 2001 From: Dima Blochman Date: Thu, 22 Feb 2024 13:06:19 +0200 Subject: [PATCH 4/7] fix depends --- rag_engine.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/rag_engine.py b/rag_engine.py index 83a4017..032a2d7 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -3,16 +3,17 @@ from pathlib import Path from dotenv import load_dotenv from langchain.chains import RetrievalQA, ConversationalRetrievalChain -from langchain.embeddings import OpenAIEmbeddings -from langchain.vectorstores import Chroma -from langchain import OpenAI -from langchain.llms.openai import OpenAIChat -from langchain.document_loaders import DirectoryLoader +# from langchain_community.embeddings import OpenAIEmbeddings +# from langchain_community.vectorstores import Chroma +# from langchain import OpenAI +from langchain_community.llms.openai import OpenAIChat +from langchain_community.document_loaders import DirectoryLoader from langchain.text_splitter import CharacterTextSplitter -from langchain.vectorstores import Chroma, Pinecone -from langchain.embeddings.openai import OpenAIEmbeddings +from langchain_community.vectorstores import Chroma, Pinecone +# from langchain_community.embeddings.openai import OpenAIEmbeddings +from langchain_openai import OpenAIEmbeddings from langchain.memory import ConversationBufferMemory -from langchain.memory.chat_message_histories import StreamlitChatMessageHistory +# from langchain_community.chat_message_historie import StreamlitChatMessageHistory import streamlit as st @@ -50,7 +51,7 @@ def embeddings_on_pinecone(texts): return retriever def embedding_on_pinecone_new(texts): - from langchain_openai import OpenAIEmbeddings + # from langchain_community.embeddings.openai import OpenAIEmbeddings embeddings = OpenAIEmbeddings(openai_api_key=st.session_state.openai_api_key) from langchain_pinecone import Pinecone @@ -70,7 +71,7 @@ def query_llm(retriever, query): return result def query_llm_new(retriever, query): - from langchain.chat_models import ChatOpenAI + from langchain_openai import ChatOpenAI from langchain.chains import RetrievalQA, RetrievalQAWithSourcesChain with_source = True From 62528ac563fa2a3ad8647e3160bba60730c0ef26 Mon Sep 17 00:00:00 2001 From: Dima Blochman Date: Thu, 22 Feb 2024 13:07:49 +0200 Subject: [PATCH 5/7] added env vars file --- .gitignore | 5 ++++- .streamlit/example.secrets.toml | 6 ++++++ example.env | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .streamlit/example.secrets.toml create mode 100644 example.env diff --git a/.gitignore b/.gitignore index b3b3ed2..6a8d70c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -secrets.toml \ No newline at end of file +secrets.toml +.env +.streamlit/secrets.toml + diff --git a/.streamlit/example.secrets.toml b/.streamlit/example.secrets.toml new file mode 100644 index 0000000..fe48c43 --- /dev/null +++ b/.streamlit/example.secrets.toml @@ -0,0 +1,6 @@ +openai_api_key = "" +pinecone_api_key = "" +pinecone_index = "" +pinecone_env = "" + + diff --git a/example.env b/example.env new file mode 100644 index 0000000..f8adcb0 --- /dev/null +++ b/example.env @@ -0,0 +1,3 @@ +PINECONE_API_KEY= +PINECONE_INDEX_NAME= +OPENAI_API_KEY= From 2b0d43d55b99d46aa35e497119213efc459f9f48 Mon Sep 17 00:00:00 2001 From: Dima Blochman Date: Thu, 22 Feb 2024 13:12:32 +0200 Subject: [PATCH 6/7] cleanup --- rag_engine.py | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/rag_engine.py b/rag_engine.py index 032a2d7..962c3ef 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -1,19 +1,16 @@ import os, tempfile -import pinecone from pathlib import Path from dotenv import load_dotenv from langchain.chains import RetrievalQA, ConversationalRetrievalChain -# from langchain_community.embeddings import OpenAIEmbeddings -# from langchain_community.vectorstores import Chroma -# from langchain import OpenAI from langchain_community.llms.openai import OpenAIChat from langchain_community.document_loaders import DirectoryLoader from langchain.text_splitter import CharacterTextSplitter -from langchain_community.vectorstores import Chroma, Pinecone -# from langchain_community.embeddings.openai import OpenAIEmbeddings +from langchain_pinecone import Pinecone +from langchain_community.vectorstores import Chroma from langchain_openai import OpenAIEmbeddings from langchain.memory import ConversationBufferMemory -# from langchain_community.chat_message_historie import StreamlitChatMessageHistory +from langchain_openai import ChatOpenAI +from langchain.chains import RetrievalQA, RetrievalQAWithSourcesChain import streamlit as st @@ -43,37 +40,18 @@ def embeddings_on_local_vectordb(texts): retriever = vectordb.as_retriever(search_kwargs={'k': 7}) return retriever -def embeddings_on_pinecone(texts): - pinecone.init(api_key=st.session_state.pinecone_api_key, environment=st.session_state.pinecone_env) +def embedding_on_pinecone(texts): embeddings = OpenAIEmbeddings(openai_api_key=st.session_state.openai_api_key) - vectordb = Pinecone.from_documents(texts, embeddings, index_name=st.session_state.pinecone_index) - retriever = vectordb.as_retriever() - return retriever -def embedding_on_pinecone_new(texts): - # from langchain_community.embeddings.openai import OpenAIEmbeddings - embeddings = OpenAIEmbeddings(openai_api_key=st.session_state.openai_api_key) - from langchain_pinecone import Pinecone - index_name = "quickstart" + index_name = st.session_state.pinecone_index docsearch = Pinecone.from_documents(texts , embeddings, index_name=index_name) return docsearch.as_retriever() def query_llm(retriever, query): - qa_chain = ConversationalRetrievalChain.from_llm( - llm=OpenAIChat(openai_api_key=st.session_state.openai_api_key), - retriever=retriever, - return_source_documents=True, - ) - result = qa_chain({'question': query, 'chat_history': st.session_state.messages}) - result = result['answer'] - st.session_state.messages.append((query, result)) - return result -def query_llm_new(retriever, query): - from langchain_openai import ChatOpenAI - from langchain.chains import RetrievalQA, RetrievalQAWithSourcesChain + # TODO: export to sessions state with_source = True llm = ChatOpenAI( @@ -86,6 +64,7 @@ def query_llm_new(retriever, query): chain_type="stuff", retriever=retriever ) + qa_with_sources = RetrievalQAWithSourcesChain.from_chain_type( llm=llm, chain_type="stuff", @@ -146,7 +125,7 @@ def process_documents(): if not st.session_state.pinecone_db: st.session_state.retriever = embeddings_on_local_vectordb(texts) else: - st.session_state.retriever = embedding_on_pinecone_new(texts) + st.session_state.retriever = embedding_on_pinecone(texts) except Exception as e: st.error(f"An error occurred: {e}") @@ -165,7 +144,7 @@ def boot(): # if query := st.chat_input(): st.chat_message("human").write(query) - response = query_llm_new(st.session_state.retriever, query) + response = query_llm(st.session_state.retriever, query) st.chat_message("ai").write(response) if __name__ == '__main__': From c3fd64f7260eda71750de53ac4e1f6788f2829bd Mon Sep 17 00:00:00 2001 From: Dima Blochman Date: Thu, 22 Feb 2024 13:19:28 +0200 Subject: [PATCH 7/7] minor fixes --- rag_engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rag_engine.py b/rag_engine.py index 962c3ef..1c42d31 100644 --- a/rag_engine.py +++ b/rag_engine.py @@ -40,7 +40,7 @@ def embeddings_on_local_vectordb(texts): retriever = vectordb.as_retriever(search_kwargs={'k': 7}) return retriever -def embedding_on_pinecone(texts): +def embeddings_on_pinecone(texts): embeddings = OpenAIEmbeddings(openai_api_key=st.session_state.openai_api_key) @@ -52,7 +52,7 @@ def query_llm(retriever, query): # TODO: export to sessions state - with_source = True + with_source = False llm = ChatOpenAI( openai_api_key=st.session_state.openai_api_key, @@ -125,7 +125,7 @@ def process_documents(): if not st.session_state.pinecone_db: st.session_state.retriever = embeddings_on_local_vectordb(texts) else: - st.session_state.retriever = embedding_on_pinecone(texts) + st.session_state.retriever = embeddings_on_pinecone(texts) except Exception as e: st.error(f"An error occurred: {e}")