{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#used Libs\n# !pip install langchain\n# !pip install pinecone-client\n# !pip install InstructorEmbedding\n# !pip install sentence_transformers\n# !pip install faiss-cpu\n\n\n#specific to OPENAI Embeddings\n# !pip install openai\n# !pip install tiktoken","metadata":{"execution":{"iopub.status.busy":"2023-10-23T02:27:50.132810Z","iopub.execute_input":"2023-10-23T02:27:50.133543Z","iopub.status.idle":"2023-10-23T02:27:50.138049Z","shell.execute_reply.started":"2023-10-23T02:27:50.133505Z","shell.execute_reply":"2023-10-23T02:27:50.137119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport glob\nimport pickle\nimport pinecone\nimport langchain\nfrom tqdm.autonotebook import tqdm\nfrom langchain.document_loaders import PyPDFLoader, DirectoryLoader\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter\n\n\n#Instructor Open Source Embeddins Model\nfrom langchain.embeddings import HuggingFaceInstructEmbeddings\n\n\n# Library for Pinecone , FAISS Vectorstore\nfrom langchain.vectorstores import Pinecone\nfrom langchain.vectorstores import FAISS\n\n# Libs for Question-Answering \nfrom langchain.llms import HuggingFaceHub\nfrom langchain.chains.question_answering import load_qa_chain\n\n#Set the OpenAI API key and you can extract to use it \nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"OPENAI embeddings\")\nsecret_value_1 = user_secrets.get_secret(\"Pinecone_ENV\")\nsecret_value_2 = user_secrets.get_secret(\"Pinecone_api_key\")\nsecret_value_3 = user_secrets.get_secret(\"HUGGINGFACEHUB_API_TOKEN\")\n\n\nprint(sys.version)\nprint(\"langchain version\",langchain.__version__)\nprint(\"pinecone version\", pinecone.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:22:30.304625Z","iopub.execute_input":"2023-10-23T03:22:30.305579Z","iopub.status.idle":"2023-10-23T03:22:31.119168Z","shell.execute_reply.started":"2023-10-23T03:22:30.305545Z","shell.execute_reply":"2023-10-23T03:22:31.118280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configguration\nclass CFG:\n    \n    # LLMs\n    model_name = \"HuggingFaceH4/zephyr-7b-alpha\"  #\"llama2-13b\"\n    temperature = 1.0  # (strictly +ve for  HuggingFaceH4/zephyr-7b-alpha)\n    \n    \n    #split config\n    split_chunk_size = 200\n    split_overlap = 0\n    \n    # Text embedding config\n    embeddings_model_name = \"all-MiniLM-L6-v2\"\n    instruct_model_name =\"hkunlp/instructor-xl\"\n    \n    \n    \n    #Inputs Path \n    PDFs_path = \"/kaggle/input/pdf-data/\" \n    \n    \n\n","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:13:27.533217Z","iopub.execute_input":"2023-10-23T03:13:27.533565Z","iopub.status.idle":"2023-10-23T03:13:27.538740Z","shell.execute_reply.started":"2023-10-23T03:13:27.533536Z","shell.execute_reply":"2023-10-23T03:13:27.537803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loader = DirectoryLoader(\n        CFG.PDFs_path,\n#         glob = './*.pdf'  # Multiple pdfs can be loaded into single\n        loader_cls = PyPDFLoader,\n        show_progress=True,\n        use_multithreading=True\n        \n)\ndocumnets = loader.load()","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:13:28.605895Z","iopub.execute_input":"2023-10-23T03:13:28.606770Z","iopub.status.idle":"2023-10-23T03:13:33.646859Z","shell.execute_reply.started":"2023-10-23T03:13:28.606730Z","shell.execute_reply":"2023-10-23T03:13:33.645973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Splitting the Text into small chunks","metadata":{}},{"cell_type":"code","source":"text_splitter = RecursiveCharacterTextSplitter(chunk_size = CFG.split_chunk_size, chunk_overlap = CFG.split_overlap)\ntexts = text_splitter.split_documents(documnets)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:13:35.159652Z","iopub.execute_input":"2023-10-23T03:13:35.160012Z","iopub.status.idle":"2023-10-23T03:13:35.199295Z","shell.execute_reply.started":"2023-10-23T03:13:35.159983Z","shell.execute_reply":"2023-10-23T03:13:35.198400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Embeddings , Storing, Retrieving and Natural Response","metadata":{}},{"cell_type":"markdown","source":"## Create Embediings using Open Source LLMs","metadata":{}},{"cell_type":"code","source":"instructor_embeddings = HuggingFaceInstructEmbeddings(model_name=CFG.instruct_model_name, \n                                                      model_kwargs={\"device\": \"cuda\"})\n","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:13:38.437464Z","iopub.execute_input":"2023-10-23T03:13:38.437849Z","iopub.status.idle":"2023-10-23T03:13:52.309235Z","shell.execute_reply.started":"2023-10-23T03:13:38.437818Z","shell.execute_reply":"2023-10-23T03:13:52.308236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving the embeddings to Pinecone","metadata":{}},{"cell_type":"code","source":"# intiating the index in Pinecone\npinecone.init(api_key=secret_value_2,environment=secret_value_1)\nindex_name = 'langchaininstructembeddings'\ndocsearch = Pinecone.from_texts([t.page_content for t in texts[0:10]],instructor_embeddings,index_name=index_name)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:13:58.911935Z","iopub.execute_input":"2023-10-23T03:13:58.912284Z","iopub.status.idle":"2023-10-23T03:13:58.916727Z","shell.execute_reply.started":"2023-10-23T03:13:58.912258Z","shell.execute_reply":"2023-10-23T03:13:58.915746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retriving similar Response from Pinecone","metadata":{}},{"cell_type":"code","source":"query = \"what is datascience\"\ndocs = docsearch.similarity_search(query)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:14:06.318740Z","iopub.execute_input":"2023-10-23T03:14:06.319572Z","iopub.status.idle":"2023-10-23T03:14:06.501646Z","shell.execute_reply.started":"2023-10-23T03:14:06.319540Z","shell.execute_reply":"2023-10-23T03:14:06.500876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docs","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:14:09.230525Z","iopub.execute_input":"2023-10-23T03:14:09.231243Z","iopub.status.idle":"2023-10-23T03:14:09.237147Z","shell.execute_reply.started":"2023-10-23T03:14:09.231213Z","shell.execute_reply":"2023-10-23T03:14:09.236125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get the Response of the documents with an Natural Response using Open Source LLM","metadata":{}},{"cell_type":"code","source":"llm=HuggingFaceHub(repo_id=CFG.model_name,model_kwargs={\"temperature\":1}, huggingfacehub_api_token=secret_value_3)\nchain = load_qa_chain(llm,chain_type=\"stuff\")","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:14:11.950665Z","iopub.execute_input":"2023-10-23T03:14:11.951057Z","iopub.status.idle":"2023-10-23T03:14:12.321244Z","shell.execute_reply.started":"2023-10-23T03:14:11.951027Z","shell.execute_reply":"2023-10-23T03:14:12.320434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chain.run(input_documents=docs,question=query)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:14:15.015635Z","iopub.execute_input":"2023-10-23T03:14:15.016000Z","iopub.status.idle":"2023-10-23T03:14:15.101035Z","shell.execute_reply.started":"2023-10-23T03:14:15.015971Z","shell.execute_reply":"2023-10-23T03:14:15.100100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Use FAISS to Store the Embeddings in Pickle File","metadata":{}},{"cell_type":"code","source":"def store_embeddings(docs, embeddings, store_name, path): \n    if not os.path.exists(path):\n        os.makedirs(path,exist_ok=True)\n        \n    vectorStore = FAISS.from_documents(docs, embeddings)\n    with open(f\"{path}/faiss_{store_name}.pkl\", \"wb\") as f:\n        pickle.dump(vectorStore, f)\n    print(\"vector store created\")\n        \ndef load_embeddings(store_name, path):\n    with open(f\"{path}/faiss_{store_name}.pkl\", \"rb\") as f:\n        VectorStore = pickle.load(f)\n    return VectorStore","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:14:19.514356Z","iopub.execute_input":"2023-10-23T03:14:19.515167Z","iopub.status.idle":"2023-10-23T03:14:19.521312Z","shell.execute_reply.started":"2023-10-23T03:14:19.515137Z","shell.execute_reply":"2023-10-23T03:14:19.520416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_name = f\"/kaggle/working/Embedding_store\"\n#storing embeddings\nstore_embeddings(texts,instructor_embeddings,store_name='instructEmbeddings',path=path_name)\n\n#loading Embeddings\ndb_instructEmbed = load_embeddings(store_name=\"instructEmbeddings\",path=path_name)\n\n# Anothe Way\n\ndb_instructEmbed = FAISS.from_documents(texts,instructor_embeddings)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:14:20.279297Z","iopub.execute_input":"2023-10-23T03:14:20.279668Z","iopub.status.idle":"2023-10-23T03:15:27.510535Z","shell.execute_reply.started":"2023-10-23T03:14:20.279636Z","shell.execute_reply":"2023-10-23T03:15:27.509666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retriving the Similar Doucments from FAISS Vector store","metadata":{}},{"cell_type":"code","source":"retriever_obj = db_instructEmbed.as_retriever(search_kwargs={\"k\": 3})\nretriever_obj.search_type","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:15:27.512202Z","iopub.execute_input":"2023-10-23T03:15:27.512489Z","iopub.status.idle":"2023-10-23T03:15:27.518638Z","shell.execute_reply.started":"2023-10-23T03:15:27.512464Z","shell.execute_reply":"2023-10-23T03:15:27.517783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"query_2 =\"what are the examples of good data science teams\"\ndocs = retriever_obj.get_relevant_documents(query)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:17:39.376495Z","iopub.execute_input":"2023-10-23T03:17:39.376873Z","iopub.status.idle":"2023-10-23T03:17:39.421246Z","shell.execute_reply.started":"2023-10-23T03:17:39.376843Z","shell.execute_reply":"2023-10-23T03:17:39.420370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"docs","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:17:41.441809Z","iopub.execute_input":"2023-10-23T03:17:41.442265Z","iopub.status.idle":"2023-10-23T03:17:41.449521Z","shell.execute_reply.started":"2023-10-23T03:17:41.442231Z","shell.execute_reply":"2023-10-23T03:17:41.448489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chain.run(input_documents=docs,question=query_2)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T03:17:46.098285Z","iopub.execute_input":"2023-10-23T03:17:46.098640Z","iopub.status.idle":"2023-10-23T03:17:46.875172Z","shell.execute_reply.started":"2023-10-23T03:17:46.098613Z","shell.execute_reply":"2023-10-23T03:17:46.874263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}