diff --git a/Evaluations/AI4EIC_sample_dataset.csv b/Evaluations/AI4EIC_sample_dataset.csv
new file mode 100644
index 0000000..bf9a38c
--- /dev/null
+++ b/Evaluations/AI4EIC_sample_dataset.csv
@@ -0,0 +1,5 @@
+input_arxiv_id,input_category,input_nclaims,input_question,input_run_trace,output_claims,output_complete_response,output_individual_response,output_nclaims
+2204.08645v1,physics.ins-det,3, What are the main detectors proposed for the Electron-Ion Collider (EIC)? What are the optical processes involved in Cherenkov detectors? How is AI being utilized for imaging Cherenkov detectors at EIC?,https://smith.langchain.com/public/8a7701bd-64b6-4144-967f-35749a27ebf8/r,"[""Main detectors proposed for the Electron-Ion Collider (EIC)"", ""Optical processes involved in Cherenkov detectors"", ""Utilization of AI for imaging Cherenkov detectors at EIC""]"," All detector designs proposed for EIC have a dual radiator ring-imaging Cherenkov detector (dRICH) in the hadron direction, detection of internally reflected Cherenkov light (DIRC) in the barrel, and a modular-aerogel RICH (mRICH) in the electron direction. Cherenkov detectors involve optical processes with many photons that need to be tracked through complex surfaces, making these detectors relatively slow to simulate. AI is being utilized for optimizing the design of Cherenkov detectors and for reducing the computing budget necessary to explore a large number of design points.,
+ ","[""dual radiator ring-imaging Cherenkov detector (dRICH), detection of internally reflected Cherenkov light (DIRC), and a modular-aerogel RICH (mRICH)"", ""Optical processes with many photons that need to be tracked through complex surfaces"", ""AI is being utilized for optimizing the design of Cherenkov detectors and for reducing the computing budget necessary to explore a large number of design points""]",3
+2309.00818v1,physics.ins-det,2, What is the purpose of the high-granularity calorimeter insert for the Electron-Ion Collider (EIC)? What innovative features are used in its design?,https://smith.langchain.com/public/1a32c87e-6944-4241-a37e-a0f512e2db96/r,"[""Purpose of the high-granularity calorimeter insert for the Electron-Ion Collider (EIC)"", ""Innovative features used in the design of the high-granularity calorimeter insert""]"," The purpose of the high-granularity calorimeter insert for the Electron-Ion Collider (EIC) is to cover the range 3.2< 𝜂 < 4.0 and to offer high granularity to enhance performance in measuring jets, and to manage radiation damage and beam-gas interactions effectively. Among its innovative features are an ASIC-away-of-SiPM strategy for reducing cooling requirements and minimizing space use, along with employing 3D-printed frames to reduce optical crosstalk and dead areas.,
+ ","[""To cover the range 3.2< 𝜂 < 4.0 and to offer high granularity"", ""ASIC-away-of-SiPM strategy for reducing cooling requirements and minimizing space use, employing 3D-printed frames to reduce optical crosstalk and dead areas""]",2
\ No newline at end of file
diff --git a/Evaluations/RunRAGAS.ipynb b/Evaluations/RunRAGAS.ipynb
index df91267..b3d0c90 100644
--- a/Evaluations/RunRAGAS.ipynb
+++ b/Evaluations/RunRAGAS.ipynb
@@ -1,41 +1,37 @@
{
"cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Evaluating RAGAS for test dataset"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
+ "id": "359635bb",
"metadata": {},
"outputs": [],
"source": [
+ "\n",
"import toml, os, sys\n",
"\n",
"sys.path.append(os.path.realpath(\"../\"))\n",
"\n",
- "with open(\"../.streamlit/secrets.toml\") as f:\n",
- " secrets = toml.load(f)\n",
- " \n",
- "os.environ[\"OPENAI_API_KEY\"] = secrets[\"OPENAI_API_KEY\"]\n",
- "if secrets.get(\"LANGCHAIN_API_KEY\"):\n",
- " os.environ[\"LANGCHAIN_API_KEY\"] = secrets[\"LANGCHAIN_API_KEY\"]\n",
- " os.environ[\"LANGCHAIN_TRACING_V2\"] = \"false\"\n",
- " os.environ[\"LANGCHAIN_ENDPOINT\"] = secrets[\"LANGCHAIN_ENDPOINT\"]\n",
"\n",
- "os.environ[\"PINECONE_API_KEY\"] = secrets[\"PINECONE_API_KEY\"]"
+ "# set the API key as environment variable\n",
+ "langchain_api_key = os.getenv(\"LANGCHAIN_API_KEY\")\n",
+ "print(langchain_api_key)\n",
+ "os.environ[\"LANGCHAIN_API_KEY\"] = langchain_api_key\n",
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "# os.environ[\"LANGCHAIN_ENDPOINT\"] = secrets[\"LANGCHAIN_ENDPOINT\"]\n",
+ "print(f\"langchain_api_key :\" , {langchain_api_key})\n",
+ "\n",
+ "pinecone_api_key = os.getenv(\"PINECONE_API_KEY\")\n",
+ "os.environ[\"PINECONE_API_KEY\"] = os.getenv(\"PINECONE_API_KEY\") #secrets[\"PINECONE_API_KEY\"]"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
+ "id": "d1c1475a",
"metadata": {},
"outputs": [],
"source": [
- "# attach to the existing event loop when using jupyter notebooks\n",
"import nest_asyncio\n",
"\n",
"nest_asyncio.apply()\n",
@@ -49,39 +45,48 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 10,
+ "id": "b0c13672",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
- "df = pd.read_csv(\"AI4EIC2023_DATASETS.csv\", sep = \",\")"
+ "df = pd.read_csv(\"AI4EIC_sample_dataset.csv\", sep = \",\")"
]
},
{
"cell_type": "code",
"execution_count": null,
+ "id": "55a5bf42",
"metadata": {},
"outputs": [],
"source": [
- "from langchain_openai import OpenAIEmbeddings\n",
- "from langchain_openai import ChatOpenAI\n",
+ "from langchain_ollama import OllamaEmbeddings, ChatOllama\n",
+ "from langchain_chroma import Chroma\n",
"from streamlit_app.app_utilities import *\n",
"from streamlit_app.LangChainUtils.LLMChains import *\n",
"from langchain import callbacks\n",
"from langsmith import Client\n",
"from langchain_core.tracers.context import tracing_v2_enabled\n",
"from langchain.callbacks.tracers import LangChainTracer\n",
+ "from ragas.run_config import RunConfig\n",
+ "\n",
"\n",
- "def RunQuery(input_question, max_k, sim_score):\n",
+ "# def RunQuery(input_question, max_k, sim_score):\n",
+ "def RunQuery(input_question, max_k, sim_score,\n",
+ " collection_name=None, db_name=None, table_name=None):\n",
" \n",
- " llm = ChatOpenAI(model_name=\"gpt-3.5-turbo-1106\", temperature=0,\n",
- " max_tokens = 4096\n",
- " )\n",
- " embeddings = OpenAIEmbeddings()\n",
+ " #for generating output form prompt\n",
+ " llm = ChatOllama(model=\"llama3.2:latest\", temperature=0, num_predict=4096)\n",
+ "\n",
+ " # embeddings = OpenAIEmbeddings()\n",
+ " embeddings = OllamaEmbeddings(model=\"mxbai-embed-large:latest\")\n",
+ "\n",
" # Defining some props of DB\n",
" SimilarityDict = {\"Cosine similarity\" : \"similarity\", \"MMR\" : \"mmr\"}\n",
"\n",
- " DBProp = {\"PINECONE\" : {\"vector_config\" : {\"db_api_key\" : secrets[\"PINECONE_API_KEY\"], \n",
+ " # DBProp = {\"PINECONE\" : {\"vector_config\" : {\"db_api_key\" : secrets[\"PINECONE_API_KEY\"], \n",
+ " DBProp = {\"PINECONE\" : {\"vector_config\" : {\"db_api_key\" :pinecone_api_key, \n",
" \"index_name\" : \"llm-project\", \n",
" \"embedding_function\" : embeddings\n",
" },\n",
@@ -90,23 +95,47 @@
" },\n",
" \"available_metrics\" : [\"Cosine similarity\", \"MMR\"]\n",
" },\n",
+ " \"CHROMA\" : {\"vector_config\" : {\"db_name\" : db_name, \n",
+ " \"embedding_function\" : embeddings, \n",
+ " \"collection_name\": collection_name\n",
+ " },\n",
+ " \"search_config\" : {\"metric\" : sim_score, \n",
+ " \"search_kwargs\" : {\"k\" : max_k}\n",
+ " },\n",
+ " \"available_metrics\" : [\"Cosine similarity\", \"MMR\"]\n",
+ " },\n",
+ " \"LANCE\" : {\"vector_config\" : {\"db_name\" : db_name, \n",
+ " \"table_name\": table_name\n",
+ " },\n",
+ " \"search_config\" : {\"metric\" : sim_score, \n",
+ " \"search_kwargs\" : {\"k\" : max_k}\n",
+ " },\n",
+ " \"available_metrics\" : [\"Cosine similarity\", \"MMR\"]\n",
+ " },\n",
" }\n",
- " retriever = GetRetriever(\"PINECONE\", DBProp[\"PINECONE\"][\"vector_config\"], DBProp[\"PINECONE\"][\"search_config\"])\n",
- " project_name = f\"RAG-CHAT-ksuresh\"\n",
+ " # retriever = GetRetriever(\"PINECONE\", DBProp[\"PINECONE\"][\"vector_config\"], DBProp[\"PINECONE\"][\"search_config\"])\n",
+ " retriever = GetRetriever(\"CHROMA\", DBProp[\"CHROMA\"][\"vector_config\"], DBProp[\"CHROMA\"][\"search_config\"])\n",
+ " print(\"output of Getretriever\")\n",
+ " # project name for tracing in Langsmith\n",
+ " project_name = f\"RAG-CHAT-tapasi\"\n",
+ "\n",
" tracer = LangChainTracer(project_name = project_name)\n",
+ " print(\"out of LangChainTracer\")\n",
" run_name = \"Evaluation-testings\"\n",
- " trace_metadata = {\"DBType\": \"PINECONE\", \n",
+ " trace_metadata = {\"DBType\": \"CHROMA\", \n",
" \"similarity_score\": sim_score, \n",
" \"max_k\": max_k\n",
" }\n",
- " RUNCHAIN = RunChatBot(llm, retriever, \"/mnt/d/LLM-Project/EIC-RAG-Project/Templates\"\n",
+ " RUNCHAIN = RunChatBot(llm, retriever, \"../Templates\"\n",
" ).with_config({\"callbacks\": [tracer], \n",
" \"run_name\": run_name,\n",
" \"metadata\": trace_metadata}\n",
" )\n",
+ " print(\"out of RunCHatBot\")\n",
" trace_id = \"\"\n",
" response = \"\"\n",
" runid = \"\"\n",
+ " # run the chain with tracing enabled when os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
" with tracing_v2_enabled(project_name) as cb:\n",
" with callbacks.collect_runs() as ccb:\n",
" output = RUNCHAIN.invoke(input_question)\n",
@@ -123,12 +152,14 @@
" trace_id = run.trace_id\n",
" return response, trace_id, client.share_run(runid)\n",
"\n",
- "def RunLLM(input_question, GPTMODEL = 3):\n",
- " model_name = f\"gpt-3.5-turbo-1106\" if GPTMODEL == 3 else \"gpt-4-0125-preview\"\n",
- " print (input_question)\n",
- " llm = ChatOpenAI(model_name=model_name, temperature=0,\n",
- " max_tokens = 4096\n",
- " )\n",
+ "# use this function to generate answers when llm is not used through Chain \n",
+ "def RunLLM(input_question, MODEL = \"llama3.2:latest\"):\n",
+ " # model_name = f\"gpt-3.5-turbo-1106\" if GPTMODEL == 3 else \"gpt-4-0125-preview\"\n",
+ " print (f\"input_question, {input_question}\")\n",
+ " # llm = ChatOpenAI(model_name=model_name, temperature=0,\n",
+ " # max_tokens = 4096\n",
+ " # )\n",
+ " llm = ChatOllama(model_name=MODEL, temperature=0, num_predict=4096)\n",
" output = llm.invoke(input_question).content\n",
" return output"
]
@@ -136,13 +167,17 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "c1e414e3",
"metadata": {},
"outputs": [],
"source": [
"import pickle\n",
"from datasets import Dataset\n",
"\n",
- "from langchain_openai import OpenAIEmbeddings\n",
+ "from langchain_ollama import OllamaEmbeddings, ChatOllama\n",
+ "from ragas.llms import LangchainLLMWrapper\n",
+ "# from ragas.embeddings import LangchainEmbeddingsWrapper\n",
+ "from langchain_openai import OpenAIEmbeddings, ChatOpenAI\n",
"from ragas.metrics import (\n",
" faithfulness,\n",
" answer_relevancy,\n",
@@ -151,39 +186,78 @@
" answer_correctness\n",
")\n",
"\n",
+ "\n",
"import ragas\n",
- "embeddings = OpenAIEmbeddings()\n",
+ "evaluator_embeddings = OpenAIEmbeddings()\n",
+ "# wrapper is used due to a different name of the model instead of the original name \"gpt-4o-mini\"\n",
+ "evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o-mini-2024-07-18\"))\n",
+ "# embeddings = OllamaEmbeddings(model = \"mxbai-embed-large:latest\")\n",
+ "\n",
+ "'''RAGAS metrics uses openai models by default. So we explicitly define llm and embedding model of ollama and use to\n",
+ "compute evaluation metrics'''\n",
+ "\n",
+ "# ollama_embeddings = OllamaEmbeddings(model=\"mxbai-embed-large:latest\")\n",
+ "# ollama_llm = ChatOllama(model=\"llama3.2:latest\", temperature=0)\n",
+ "\n",
+ "# # wrap the above defined llm and embedding model\n",
+ "# wrapped_llm = LangchainLLMWrapper(ollama_llm)\n",
+ "# wrapped_embeddings = LangchainEmbeddingsWrapper(ollama_embeddings)\n",
+ "\n",
"ANSWER_CORRECTNESS = ragas.metrics.AnswerCorrectness(name = \"ANSWER_CORRECTNESS\",\n",
- " weights = [0.90, 0.10]\n",
+ " weights = [0.90, 0.10],\n",
+ " # llm = wrapped_llm,\n",
+ " # embeddings = wrapped_embeddings\n",
" )\n",
"ANSWER_RELEVANCY = ragas.metrics.AnswerRelevancy(name = \"ANSWER_RELEVANCY\",\n",
- " strictness = 5\n",
+ " strictness = 5,\n",
+ " # llm = wrapped_llm,\n",
+ " # embeddings = wrapped_embeddings\n",
" )\n",
- "CONTEXT_ENTITY_RECALL = ragas.metrics.ContextEntityRecall(name = \"CONTEXT_ENTITY_RECALL\"\n",
+ "CONTEXT_ENTITY_RECALL = ragas.metrics.ContextEntityRecall(name = \"CONTEXT_ENTITY_RECALL\",\n",
+ " # llm = wrapped_llm\n",
" )\n",
- "CONTEXT_PRECISION = ragas.metrics.ContextPrecision(name = \"CONTEXT_PRECISION\"\n",
+ "CONTEXT_PRECISION = ragas.metrics.ContextPrecision(name = \"CONTEXT_PRECISION\",\n",
+ " # llm = wrapped_llm\n",
" )\n",
- "CONTEXT_RECALL = ragas.metrics.ContextRecall(name = \"CONTEXT_RECALL\"\n",
- " )\n",
- "CONTEXT_RELEVANCY = ragas.metrics.ContextRelevancy(name = \"CONTEXT_RELEVANCY\"\n",
- " )\n",
- "FAITHFULNESS = ragas.metrics.Faithfulness(name = \"FAITHFULNESS\")\n",
+ "CONTEXT_RECALL = ragas.metrics.ContextRecall(name = \"CONTEXT_RECALL\",\n",
+ " # llm = wrapped_llm\n",
+ " )\n",
+ "## new RAGAS doesn't have this evaluation metric \n",
+ "# CONTEXT_RELEVANCY = ragas.metrics.ContextRelevancy(name = \"CONTEXT_RELEVANCY\")\n",
+ " \n",
+ "FAITHFULNESS = ragas.metrics.Faithfulness(name = \"FAITHFULNESS\",\n",
+ " )\n",
"\n",
"import pandas as pd\n",
- "df = pd.read_csv(\"AI4EIC2023_DATASETS.csv\", sep = \",\")\n",
+ "\n",
+ "# Read the benchmark Q&A dataframe\n",
+ "df = pd.read_csv(\"AI4EIC_sample_dataset.csv\", sep = \",\")\n",
+ "\n",
+ "\n",
"from ragas import evaluate\n",
+ "#output file to store answers from RAG system and will be used for RAGAS evaluation\n",
"dataset = {\"question\": [], \"answer\": [], \"contexts\": [], \"ground_truth\": [], \"arxiv_id\": [], \"input_arxiv_id\": [], \"trace_links\": []}\n",
"max_k = 3\n",
"sim_score = \"mmr\"\n",
+ "db_name=\"../ingestion/myChromaDB\"\n",
+ "collection_name = \"EIC_archive\"\n",
+ "table_name = \"arxiv_table\"\n",
"if (os.path.exists(f\"results_k_{max_k}_sim_{sim_score}.csv\")):\n",
" os.system(f\"rm -f results_k_{max_k}_sim_{sim_score}.csv\")\n",
+ "\n",
+ "# Iterate through the benchmark Q&A dataframe and run the query for each question\n",
"for index, row in df.iterrows():\n",
" question = row[\"input_question\"]\n",
- " answer, trace_id, trace_link = RunQuery(question, max_k, sim_score)\n",
"\n",
- " project_name = f\"RAG-CHAT-ksuresh\"\n",
+ " # Run the query, generate answer through llm\n",
+ " answer, trace_id, trace_link = RunQuery(question, max_k, sim_score, db_name=db_name, collection_name=collection_name)\n",
+ " print(f\" RunQuery : answer : {answer}, trace_id : {trace_id}, trace_link : {trace_link}\")\n",
+ "\n",
+ " project_name = f\"RAG-CHAT-tapasi\"\n",
" run_name = \"Evaluation-testings\"\n",
+ " # print(f\"before langsmith is called\")\n",
" runs = client.list_runs(project_name = project_name, trace_id = trace_id)\n",
+ " # print(f\"after langsmith client is called : , {runs}\")\n",
" contexts = []\n",
" cite_arxiv_ids = []\n",
" for run in runs:\n",
@@ -214,16 +288,19 @@
" tmpdataset[key] = [value[-1]]\n",
" DATASET = Dataset.from_dict(tmpdataset)\n",
" \n",
+ "\n",
" result = evaluate(DATASET,\n",
" metrics = [\n",
" FAITHFULNESS,\n",
- " CONTEXT_RELEVANCY,\n",
+ " # CONTEXT_RELEVANCY,\n",
" CONTEXT_ENTITY_RECALL,\n",
" CONTEXT_PRECISION,\n",
" CONTEXT_RECALL,\n",
" ANSWER_RELEVANCY,\n",
" ANSWER_CORRECTNESS\n",
- " ]\n",
+ " ],\n",
+ " llm = evaluator_llm,\n",
+ " embeddings = evaluator_embeddings,\n",
" )\n",
" result_df = result.to_pandas()\n",
" if (os.path.exists(f\"results_k_{max_k}_sim_{sim_score}.csv\")):\n",
@@ -235,210 +312,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import pickle\n",
- "from datasets import Dataset\n",
- "from langchain_openai import OpenAIEmbeddings\n",
- "from ragas.metrics import (\n",
- " faithfulness,\n",
- " answer_relevancy,\n",
- " context_precision,\n",
- " context_recall,\n",
- " answer_correctness\n",
- ")\n",
- "\n",
- "\n",
- "from ragas import evaluate\n",
- "dataset3 = {\"question\": [], \"answer\": [], \"contexts\": [], \"ground_truth\": [], \"arxiv_id\": [], \"input_arxiv_id\": [], \"trace_links\": []}\n",
- "dataset4 = {\"question\": [], \"answer\": [], \"contexts\": [], \"ground_truth\": [], \"arxiv_id\": [], \"input_arxiv_id\": [], \"trace_links\": []}\n",
- "import pandas as pd\n",
- "df = pd.read_csv(\"AI4EIC2023_DATASETS.csv\", sep = \",\")\n",
- "\n",
- "if (os.path.exists(f\"results_gpt3.csv\")):\n",
- " os.system(f\"rm -f results_gpt3.csv\")\n",
- "if (os.path.exists(f\"results_gpt4.csv\")):\n",
- " os.system(f\"rm -f results_gpt4.csv\")\n",
- "for index, row in df.iterrows():\n",
- " question = row[\"input_question\"]\n",
- " output3 = RunLLM(question, GPTMODEL = 3)\n",
- " output4 = RunLLM(question, GPTMODEL = 4)\n",
- " dataset3[\"question\"].append(question)\n",
- " dataset3[\"answer\"].append(output3)\n",
- " dataset3[\"contexts\"].append([row[\"output_complete_response\"]])\n",
- " dataset3[\"ground_truth\"].append(row[\"output_complete_response\"])\n",
- " dataset3[\"input_arxiv_id\"].append(row[\"input_arxiv_id\"])\n",
- " dataset3[\"arxiv_id\"].append([\"\"])\n",
- " dataset3[\"trace_links\"].append(\"\")\n",
- " \n",
- " dataset4[\"question\"].append(question)\n",
- " dataset4[\"answer\"].append(output4)\n",
- " dataset4[\"contexts\"].append([row[\"output_complete_response\"]])\n",
- " dataset4[\"ground_truth\"].append(row[\"output_complete_response\"])\n",
- " dataset4[\"input_arxiv_id\"].append(row[\"input_arxiv_id\"])\n",
- " dataset4[\"arxiv_id\"].append([\"\"])\n",
- " dataset4[\"trace_links\"].append(\"\")\n",
- " \n",
- " tmpdataset3 = {}\n",
- " for key, value in dataset3.items():\n",
- " tmpdataset3[key] = [value[-1]]\n",
- " DATASET3 = Dataset.from_dict(tmpdataset3)\n",
- " embeddings = OpenAIEmbeddings()\n",
- " result3 = evaluate(DATASET3,\n",
- " metrics = [\n",
- " FAITHFULNESS,\n",
- " CONTEXT_RELEVANCY,\n",
- " CONTEXT_ENTITY_RECALL,\n",
- " CONTEXT_PRECISION,\n",
- " CONTEXT_RECALL,\n",
- " ANSWER_RELEVANCY,\n",
- " ANSWER_CORRECTNESS\n",
- " ],\n",
- " embeddings = embeddings\n",
- " )\n",
- " result_df3 = result3.to_pandas()\n",
- " if (os.path.exists(f\"results_gpt3.csv\")):\n",
- " df = pd.read_csv(f\"results_gpt3.csv\", sep = \",\")\n",
- " result_df3 = pd.concat([df, result_df3])\n",
- " result_df3.to_csv(f\"results_gpt3.csv\", index = False)\n",
- " \n",
- " tmpdataset4 = {}\n",
- " for key, value in dataset4.items():\n",
- " tmpdataset4[key] = [value[-1]]\n",
- " DATASET4 = Dataset.from_dict(tmpdataset4)\n",
- " \n",
- " result4 = evaluate(DATASET4,\n",
- " metrics = [\n",
- " FAITHFULNESS,\n",
- " CONTEXT_RELEVANCY,\n",
- " CONTEXT_ENTITY_RECALL,\n",
- " CONTEXT_PRECISION,\n",
- " CONTEXT_RECALL,\n",
- " ANSWER_RELEVANCY,\n",
- " ANSWER_CORRECTNESS\n",
- " ],\n",
- " embeddings = embeddings\n",
- " )\n",
- " result_df4 = result4.to_pandas()\n",
- " if (os.path.exists(f\"results_gpt4.csv\")):\n",
- " df = pd.read_csv(f\"results_gpt4.csv\", sep = \",\")\n",
- " result_df4 = pd.concat([df, result_df4])\n",
- " result_df4.to_csv(f\"results_gpt4.csv\", index = False)\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "import matplotlib.pyplot as plt\n",
- "import seaborn as sns\n",
- "import pandas as pd\n",
- "max_k = 3\n",
- "sim_score = \"mmr\"\n",
- "%matplotlib inline\n",
- "def analysis(df_list, labels = [\"RAG4EIC\"]):\n",
- " sns.set_style(\"whitegrid\")\n",
- " df = df_list[0]\n",
- " fig, axs = plt.subplots(df.columns.size, 1, figsize=(5, 5*df.columns.size))\n",
- " colors = [\"orange\", \"blue\", \"green\"]\n",
- " for i,col in enumerate(df.columns):\n",
- " #sns.kdeplot(data=[df[col].values],legend=False,ax=axs[i],fill=True)\n",
- " sns.histplot(data=[df[col].values for df in df_list], kde=True, ax=axs[i], color = colors, alpha = 0.5)\n",
- " axs[i].set_title(f'{col} scores distribution')\n",
- " axs[i].legend(labels=labels)\n",
- " axs[i].set_xlabel(\"Score\")\n",
- " axs[i].set_xlim(0., 1.1)\n",
- " plt.tight_layout()\n",
- " plt.show()\n",
- "\n",
- "#columns = ['faithfulness', 'answer_relevancy', 'context_precision', 'context_recall', 'answer_correctness']\n",
- "#FAITHFULNESS,CONTEXT_RELEVANCY,CONTEXT_ENTITY_RECALL,CONTEXT_PRECISION,CONTEXT_RECALL,ANSWER_RELEVANCY,ANSWER_CORRECTNESS\n",
- "columns = [\"FAITHFULNESS\", \"CONTEXT_RELEVANCY\", \"CONTEXT_ENTITY_RECALL\", \"CONTEXT_PRECISION\", \"CONTEXT_RECALL\", \"ANSWER_RELEVANCY\", \"ANSWER_CORRECTNESS\"]\n",
- "result_df = pd.read_csv(f\"results_k_{max_k}_sim_{sim_score}.csv\", sep = \",\")[columns]\n",
- "result_df.fillna(0, inplace = True)\n",
- "gpt3_df = pd.read_csv(f\"results_gpt3.csv\", sep = \",\")[columns]\n",
- "gpt3_df.fillna(0, inplace = True)\n",
- "#gpt4_df = pd.read_csv(f\"results_gpt3.csv\", sep = \",\")[columns]\n",
- "analysis(\n",
- " [result_df, gpt3_df],\n",
- " [\"RAG4EIC\", \"GPT3\"]\n",
- ") \n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "result_df.plot(kind = \"box\", figsize = (20, 5), title = \"RAG4EIC\")\n",
- "gpt3_df.plot(kind = \"box\", figsize = (20, 5), title = \"GPT3\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "result_df.describe()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "gpt3_df.describe()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "gpt4_df.describe()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "gpt3_df = pd.read_csv(f\"results_gpt3.csv\", sep = \",\")[[\"answer\", \"ground_truth\"]]\n",
- "print (gpt3_df.loc[0].answer)\n",
- "print (gpt3_df.loc[0].ground_truth)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "gpt3_df[[\"answer\", \"ground_truth\"]]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "gpt3_df.columns"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
+ "id": "0dfa339e",
"metadata": {},
"outputs": [],
"source": []
@@ -446,7 +320,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "env_RAG-EIC",
+ "display_name": "venv_RAG-EIC",
"language": "python",
"name": "python3"
},
@@ -460,9 +334,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.10.13"
+ "version": "3.10.14"
}
},
"nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 5
}
diff --git a/Evaluations/test_RunRagas.py b/Evaluations/test_RunRagas.py
new file mode 100644
index 0000000..a69e24a
--- /dev/null
+++ b/Evaluations/test_RunRagas.py
@@ -0,0 +1,340 @@
+
+import toml, os, sys
+import time
+import torch # Added for GPU availability check
+
+# Check if GPU is available
+if not torch.cuda.is_available():
+ print("Warning: GPU not available, falling back to CPU")
+else:
+ print(f"GPU available: {torch.cuda.get_device_name(0)}")
+
+# -------------------------------
+sys.path.append(os.path.realpath("../"))
+
+# with open("../../.streamlit/secrets.toml") as f:
+# secrets = toml.load(f)
+
+# os.environ["OPENAI_API_KEY"] = secrets["OPENAI_API_KEY"]
+# if secrets.get("LANGCHAIN_API_KEY"):
+langchain_api_key = os.getenv("LANGCHAIN_API_KEY")
+print(langchain_api_key)
+os.environ["LANGCHAIN_API_KEY"] = langchain_api_key
+os.environ["LANGCHAIN_TRACING_V2"] = "true"
+# os.environ["LANGCHAIN_ENDPOINT"] = secrets["LANGCHAIN_ENDPOINT"]
+print(f"langchain_api_key :" , {langchain_api_key})
+
+pinecone_api_key = os.getenv("PINECONE_API_KEY")
+os.environ["PINECONE_API_KEY"] = os.getenv("PINECONE_API_KEY") #secrets["PINECONE_API_KEY"]
+
+#--------------------------
+import nest_asyncio
+
+nest_asyncio.apply()
+
+# Get the dataset
+from langsmith import Client
+from langsmith.utils import LangSmithError
+
+client = Client()
+
+#-----------------------
+import pandas as pd
+df = pd.read_csv("AI4EIC_sample_dataset.csv", sep = ",")
+
+#---------------------
+# from langchain_openai import OpenAIEmbeddings
+# from langchain_openai import ChatOpenAI
+from langchain_ollama import OllamaEmbeddings, ChatOllama
+from langchain_chroma import Chroma
+from streamlit_app.app_utilities import *
+from streamlit_app.LangChainUtils.LLMChains import *
+from langchain import callbacks
+from langsmith import Client
+from langchain_core.tracers.context import tracing_v2_enabled
+from langchain.callbacks.tracers import LangChainTracer
+from ragas.run_config import RunConfig
+import json
+
+
+# def RunQuery(input_question, max_k, sim_score):
+def RunQuery(input_question, max_k, sim_score,
+ collection_name=None, db_name=None, table_name=None):
+
+ #for generating output form prompt
+ llm = ChatOllama(model="llama3.2:latest", temperature=0, num_predict=4096)
+
+ ## Configure LLM with GPU-optimized parameters
+ # llm = ChatOllama(
+ # model="llama3.2:latest",
+ # temperature=0,
+ # num_predict=4096,
+ # num_gpu=999, # Use maximum available GPU layers
+ # num_threads=8 # Adjust based on your GPU/CPU setup
+ # )
+
+ # embeddings = OpenAIEmbeddings()
+ embeddings = OllamaEmbeddings(model="mxbai-embed-large:latest")
+
+ # Defining some props of DB
+ SimilarityDict = {"Cosine similarity" : "similarity", "MMR" : "mmr"}
+
+ #create an instance of the DB
+ DBProp = {"PINECONE" : {
+ "vector_config" : {"db_api_key" :pinecone_api_key, "index_name" : "llm-project", "embedding_function" : embeddings},
+ "search_config" : {"metric" : sim_score, "search_kwargs" : {"k" : max_k}},
+ "available_metrics" : ["Cosine similarity", "MMR"]
+ },
+ "CHROMA": {
+ "vector_config": {"db_name": db_name, "embedding_function": embeddings, "collection_name": collection_name},
+ "search_config": {"metric": sim_score, "search_kwargs": {"k": max_k}},
+ "available_metrics": ["Cosine similarity", "MMR"],
+ },
+ "LANCE": {
+ "vector_config": {"db_name": db_name, "table_name": table_name},
+ "search_config": {"metric": sim_score, "search_kwargs": {"k": max_k}},
+ "available_metrics": ["Cosine similarity", "MMR"],
+ },
+ }
+
+ #create an instance of the VectorDB
+ retriever = GetRetriever("CHROMA", DBProp["CHROMA"]["vector_config"], DBProp["CHROMA"]["search_config"])
+ print("output of Getretriever")
+
+ # project name for tracing in Langsmith
+ project_name = f"RAG-CHAT-tapasi"
+
+ # Create a LangChain tracer for tracing the run
+ tracer = LangChainTracer(project_name = project_name)
+ print("out of LangChainTracer")
+
+ run_name = "Evaluation-testings"
+ trace_metadata = {"DBType": "CHROMA",
+ "similarity_score": sim_score,
+ "max_k": max_k
+ }
+ RUNCHAIN = RunChatBot(llm, retriever, "../Templates"
+ ).with_config({"callbacks": [tracer],
+ "run_name": run_name,
+ "metadata": trace_metadata}
+ )
+ print("out of RunCHatBot")
+ trace_id = ""
+ response = ""
+ runid = ""
+ with tracing_v2_enabled(project_name) as cb:
+ with callbacks.collect_runs() as ccb:
+ output = RUNCHAIN.invoke(input_question)
+
+ ## modify to ensure json format
+ # response = output["answer"]
+ # Ensure output is a JSON-compatible dictionary
+ if isinstance(output, dict) and "answer" in output:
+ response = json.dumps({"answer": output["answer"]})
+ else:
+ response = json.dumps({"answer": str(output)})
+
+ print (output)
+ print (len(ccb.traced_runs))
+ for run in ccb.traced_runs:
+ runid = run.id
+ print (run.name)
+ print (run.id)
+ print (run.inputs)
+ print (run.outputs)
+ print (run.trace_id)
+ trace_id = run.trace_id
+ return response, trace_id, client.share_run(runid)
+
+def RunLLM(input_question, MODEL = "llama3.2:latest"):
+ # model_name = f"gpt-3.5-turbo-1106" if GPTMODEL == 3 else "gpt-4-0125-preview"
+ print (f"input_question, {input_question}")
+ # llm = ChatOpenAI(model_name=model_name, temperature=0,
+ # max_tokens = 4096
+ # )
+ llm = ChatOllama(model_name=MODEL, temperature=0, num_predict=4096)
+ # # For GPU
+ # llm = ChatOllama(
+ # model_name=MODEL,
+ # temperature=0,
+ # num_predict=4096,
+ # num_gpu=999 # Maximize GPU usage
+ # )
+ output = llm.invoke(input_question).content
+ ## for json format
+ # return output
+
+ print(f"output of llm : {output}")
+
+
+#---------------------------------------------
+import pickle
+from datasets import Dataset
+
+from langchain_ollama import OllamaEmbeddings, ChatOllama
+from ragas.llms import LangchainLLMWrapper
+from ragas.embeddings import LangchainEmbeddingsWrapper
+# from langchain_openai import OpenAIEmbeddings
+from ragas.metrics import (
+ faithfulness,
+ answer_relevancy,
+ context_precision,
+ context_recall,
+ answer_correctness
+)
+
+import ragas
+'''RAGAS metrics uses openai models by default. So we explicitly define llm and embedding model of ollama and use to
+compute evaluation metrics'''
+
+ANSWER_CORRECTNESS = ragas.metrics.AnswerCorrectness(name = "ANSWER_CORRECTNESS",
+ weights = [0.90, 0.10],
+ )
+ANSWER_RELEVANCY = ragas.metrics.AnswerRelevancy(name = "ANSWER_RELEVANCY",
+ strictness = 5,
+ )
+CONTEXT_ENTITY_RECALL = ragas.metrics.ContextEntityRecall(name = "CONTEXT_ENTITY_RECALL",
+ )
+CONTEXT_PRECISION = ragas.metrics.ContextPrecision(name = "CONTEXT_PRECISION"
+ )
+CONTEXT_RECALL = ragas.metrics.ContextRecall(name = "CONTEXT_RECALL")
+## new RAGAS doesn't have this evaluation metric
+# CONTEXT_RELEVANCY = ragas.metrics.ContextRelevancy(name = "CONTEXT_RELEVANCY")
+
+FAITHFULNESS = ragas.metrics.Faithfulness(name = "FAITHFULNESS")
+
+# the benchmark Q&A dataset
+import pandas as pd
+df = pd.read_csv("AI4EIC2023_DATASETS.csv", sep = ",")
+
+from ragas import evaluate
+dataset = {"question": [],
+ "answer": [],
+ "contexts": [],
+ "ground_truth": [],
+ "arxiv_id": [],
+ "input_arxiv_id": [],
+ "trace_links": []
+ }
+
+# no of chunks to be retrieved
+max_k = 20
+sim_score = "mmr"
+db_name="../ingestion/myChromaDB"
+collection_name = "EIC_archive"
+table_name = "arxiv_table"
+
+if (os.path.exists(f"results_k_{max_k}_sim_{sim_score}.csv")):
+ os.system(f"rm -f results_k_{max_k}_sim_{sim_score}.csv")
+
+for index, row in df.iterrows():
+ question = row["input_question"]
+ answer, trace_id, trace_link = RunQuery(question, max_k, sim_score, db_name=db_name, collection_name=collection_name)
+ print(f" RunQuery : answer : {answer}, trace_id : {trace_id}, trace_link : {trace_link}")
+
+ project_name = f"RAG-CHAT-tapasi"
+ run_name = "Evaluation-testings"
+
+ # if verbose==1:
+ # print(f"before langsmith is called")
+
+ runs = client.list_runs(project_name = project_name, trace_id = trace_id)
+ print(f"after langsmith client is called : , {runs}")
+ contexts = []
+ cite_arxiv_ids = []
+ for run in runs:
+ if (run.run_type.lower() == "retriever"):
+ print (run.name)
+ print (run.id)
+ print (run.inputs)
+ print (run.outputs)
+ for i in run.outputs['documents']:
+ contexts.append(i["page_content"])
+ cite_arxiv_ids.append(i["metadata"]["arxiv_id"].split("/")[-1].strip())
+ print (run.trace_id)
+ print ("-----")
+
+ dataset["question"].append(question)
+ print (answer.split("http://")[0].strip("\n"))
+ dataset["answer"].append(answer.split("http://")[0].strip("\n"))
+
+ dataset["contexts"].append(contexts)
+ dataset["ground_truth"].append(row["output_complete_response"])
+ dataset["input_arxiv_id"].append(row["input_arxiv_id"])
+ dataset["arxiv_id"].append(cite_arxiv_ids)
+ dataset["trace_links"].append(trace_link)
+
+ with open(f"dataset_k_{max_k}_sim_{sim_score}.pkl", "wb") as f:
+ pickle.dump(dataset, f)
+
+ tmpdataset = {}
+ for key, value in dataset.items():
+ tmpdataset[key] = [value[-1]]
+ DATASET = Dataset.from_dict(tmpdataset)
+
+ # Start time
+ start_time = time.time()
+ print(start_time)
+
+ # Configure run_config with custom timeout
+
+ run_config = RunConfig(
+ timeout=600, # Set timeout to 10 minutes (600 seconds)
+ max_workers=1 # Sequential processing to avoid Ollama overload
+ )
+
+ result = evaluate(DATASET,
+ metrics = [
+ # FAITHFULNESS,
+ # # CONTEXT_RELEVANCY,
+ CONTEXT_ENTITY_RECALL
+ # CONTEXT_PRECISION,
+ # CONTEXT_RECALL,
+ # ANSWER_RELEVANCY,
+ # ANSWER_CORRECTNESS
+ ],
+ run_config = run_config
+ )
+ result_df = result.to_pandas()
+ if (os.path.exists(f"results_k_{max_k}_sim_{sim_score}.csv")):
+ df = pd.read_csv(f"results_k_{max_k}_sim_{sim_score}.csv", sep = ",")
+ result_df = pd.concat([df, result_df])
+ result_df.to_csv(f"results_k_{max_k}_sim_{sim_score}.csv", index = False)
+
+ # End time
+ end_time = time.time()
+ delta_time = end_time - start_time
+ print(f"time taken : {delta_time}")
+
+
+# -------------------------------------------------------
+
+ # import asyncio
+ # from ragas import evaluate
+
+ # async def run_evaluation(dataset, metrics):
+ # return evaluate(dataset, metrics=metrics)
+
+ # try:
+ # result = asyncio.run(asyncio.wait_for(
+ # run_evaluation(
+ # DATASET,
+ # [
+ # FAITHFULNESS,
+ # CONTEXT_ENTITY_RECALL,
+ # CONTEXT_PRECISION,
+ # CONTEXT_RECALL,
+ # ANSWER_RELEVANCY,
+ # ANSWER_CORRECTNESS
+ # ]
+ # ),
+ # timeout=600 # 300 seconds
+ # ))
+ # result_df = result.to_pandas()
+ # if os.path.exists(f"results_k_{max_k}_sim_{sim_score}.csv"):
+ # df = pd.read_csv(f"results_k_{max_k}_sim_{sim_score}.csv", sep=",")
+ # result_df = pd.concat([df, result_df])
+ # result_df.to_csv(f"results_k_{max_k}_sim_{sim_score}.csv", index=False)
+ # except asyncio.TimeoutError:
+ # print(f"Evaluation timed out after 300 seconds")
+ # continue
\ No newline at end of file
diff --git a/Templates/reponse_01.template b/Templates/reponse_01.template
index 3924a13..df54424 100644
--- a/Templates/reponse_01.template
+++ b/Templates/reponse_01.template
@@ -27,6 +27,7 @@ Here is the response template that you need strictly follow:
- End with a closing remark and a list of sources with their respective URLs as a bullet list explicitly with full links which are enclosed in the tag and respectively.
- Your references have to strictly follow the `Example response` as template.
- Strictly use the styling of response based on the `Example response`.
+
---
Here is how an response would look like. Reproduce the same format for your response:
@@ -72,5 +73,13 @@ Make sure these citations have to be relavant and strictly do not repeat the con
REMEMBER: If there is no relevant information within the context, just say "Hmm, I'm \
not sure." or greet back. Don't try to make up an answer. Anything between the preceding 'context' \
html blocks is retrieved from a knowledge bank, not part of the conversation with the \
-user.\
+user.
+
+----
+Strictly note the points below
+- You must always return valid JSON fenced by a markdown code block. Do not return any additional text.
+- Do not add any text outside the JSON code block.
+- Escape special characters (e.g., quotes, backslashes) to ensure valid JSON.
+\
+
Question: {question}
\ No newline at end of file
diff --git a/ingestion/README.md b/ingestion/README.md
index 6e73d64..b135886 100644
--- a/ingestion/README.md
+++ b/ingestion/README.md
@@ -21,3 +21,8 @@ When approaching this within naive RAG framework. One can find intelligent ways
## Modular RAG System
+### To generate sample vectorDB after downloading sample arxive file
+- `python download_arxiv.py -i test_arxiv_sources.info -o downloaded_files`
+
+### to run the `ingest.py` file
+
diff --git a/ingestion/ingest.py b/ingestion/ingest.py
index 98379c0..3409524 100644
--- a/ingestion/ingest.py
+++ b/ingestion/ingest.py
@@ -1,76 +1,108 @@
+'''1. read the arxive document from the src_dir
+ 2. Define the metadata for the document
+ 3. Use Recursive chunking
+ 4. Use ollama model '''
+
import os, glob, arxiv, argparse
from langchain_community.document_loaders import PyPDFLoader
-from langchain_openai.embeddings import OpenAIEmbeddings
-from langchain_community.vectorstores import Chroma
+from langchain_ollama import OllamaEmbeddings
+from langchain_chroma import Chroma
from langchain_community.vectorstores import Pinecone as LangPinecone
from langchain.text_splitter import RecursiveCharacterTextSplitter
-from pinecone import Pinecone, ServerlessSpec
def SaveFromPDF(args):
- DB_NAME = args.db_name
- SRC_DIR = args.src_dir
- COLLECTION_NAME = args.table_name
- OPENAI_API_KEY = args.openai_api_key
- CHUNKING_TYPE = args.chunking
- if(args.db_api_key):
- os.environ["PINECONE_API_KEY"] = args.db_api_key
- pc = Pinecone(api_key = os.environ["PINECONE_API_KEY"])
- if COLLECTION_NAME not in pc.list_indexes().names():
- pc.create_index(name = COLLECTION_NAME,
- metric = "cosine",
- dimension = 1536,
- spec = ServerlessSpec(cloud = 'gcp-starter',
- region = 'us-central1'
- )
- )
- text_splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=20)
- os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
- embeddings = OpenAIEmbeddings()
- AllPDFs = [os.path.join(f, f.split("/")[-1] + ".pdf") for f in glob.glob(SRC_DIR + "/*")]
- DO = False
- for pdf in AllPDFs:
- if not DO:
- if pdf == "/mnt/d/LLM-Project/FinalRAG-Retrieval/ARXIV_SOURCES/2305.15593v1/2305.15593v1.pdf":
- DO = True
- continue
- print ("Processing: " + pdf)
-
- loader = PyPDFLoader(pdf)
- data = loader.load_and_split()
- arxiv_id = pdf.split("/")[-1].split(".pdf")[0]
- search = arxiv.Search(id_list=[arxiv_id])
- paper = next(arxiv.Client().results(search))
- meta_data = {"arxiv_id": paper.entry_id,
- "title": paper.title,
- "categories" : '\n'.join([f'{i+1}. {cat}' for i, cat in enumerate(paper.categories)]),
- "primary_category": paper.primary_category,
- "published": str(paper.published),
- "authors": '\n'.join([f'{i+1}. {auth.name}' for i, auth in enumerate(paper.authors)])
- }
- print ("Title is :" + """{}""".format(meta_data["title"]))
- for page in data:
- print ("Processing page " + str(page.metadata['page']))
- texts = text_splitter.create_documents([page.page_content], metadatas = [meta_data])
- if (args.db_type.lower() == "chroma"):
- Chroma.from_documents(texts, embeddings, persist_directory=DB_NAME, collection_name=COLLECTION_NAME)
- elif (args.db_type.lower() == "pinecone"):
- LangPinecone.from_documents(texts, embeddings, index_name=COLLECTION_NAME)
- del data, loader, page
+ DB_NAME = args.persistent_directory
+ SRC_DIR = args.src_dir
+ COLLECTION_NAME = args.table_name
+ CHUNKING_TYPE = args.chunking
+
+ ## recursive chunking
+ text_splitter = RecursiveCharacterTextSplitter(chunk_size=180, chunk_overlap=20)
+
+ ## Ollama embedding model
+ embeddings = OllamaEmbeddings(model="mxbai-embed-large:latest")
+
+ ## not able to extract the pdf id
+ # AllPDFs = [os.path.join(f, f.split("/")[-1] + ".pdf") for f in glob.glob(SRC_DIR + "/*")] #may not work in windows
+ AllPDFs = [os.path.join(f, os.path.basename(f) + ".pdf") for f in glob.glob(os.path.join(SRC_DIR, "*"))]
+
+ #check if the Vector DB persistent directory exists or not before looping through the pdfs
+ if (args.db_type.lower() == "chroma"):
+ if os.path.exists(DB_NAME):
+ #the code is same for load and create while loading for a path exist or not
+ # Load exising database
+ db = Chroma(persist_directory=DB_NAME, embedding_function=embeddings, collection_name=COLLECTION_NAME)
+ print("Chroma DB loaded successfully")
+ else:
+ # Create a new database
+ db = Chroma(persist_directory=DB_NAME, embedding_function=embeddings, collection_name=COLLECTION_NAME)
+ print("Chroma DB created successfully")
+
+ #for Pinecone and other vector DBs
+ elif (args.db_type.lower() == "pinecone"):
+ LangPinecone.from_documents(texts, embeddings, index_name=COLLECTION_NAME)
+
+ #
+ for pdf in AllPDFs:
+ #remove the .pdf extension to get the arxiv_id
+ arxiv_id = os.path.basename(pdf).replace(".pdf", "")
+
+ # Check whether the arxiv_id exists in db or not
+ existing_docs = db.get(where={"arxiv_id": arxiv_id}, include=["metadatas"])
+
+ #if the doc does not exists in the DB
+ if not existing_docs['ids']:
+ #load and split the document into pages
+ loader = PyPDFLoader(pdf)
+
+ #pages of the arxiv file
+ data = loader.load_and_split()
+
+ #
+ search = arxiv.Search(id_list=[arxiv_id])
+
+ #
+ paper = next(arxiv.Client().results(search))
+
+ #define the metadata for that document
+ meta_data = {"arxiv_id": os.path.basename(paper.entry_id),
+ "title": paper.title,
+ "categories" : '\n'.join([f'{i+1}. {cat}' for i, cat in enumerate(paper.categories)]),
+ "primary_category": paper.primary_category,
+ "published": str(paper.published),
+ "authors": '\n'.join([f'{i+1}. {auth.name}' for i, auth in enumerate(paper.authors)])
+ }
+ print ("Title is :" + """{}""".format(meta_data["title"]))
+
+ #loop through each splited page for a each arvix file
+ for page in data:
+ print ("Processing page " + str(page.metadata['page']))
+ ##creat chunk and add metadatas to each chunk
+ texts = text_splitter.create_documents([page.page_content], metadatas = [meta_data])
+ ## create the embeddings of all the chunks in that page and add to DB
+ db.add_documents(texts)
+ print("new data is added")
+ del data, loader, page
+ else:
+ print("Data already exists")
+ continue
+
+
if __name__ == "__main__":
argparser = argparse.ArgumentParser(description="Create Vector DB and perform ingestion from source files")
argparser.add_argument('-s', '--src_dir', type=str, required=True, help = "Source directory where arxiv sources are stored")
- argparser.add_argument('-db_type', '--db_type', type=str, required=True, help = "Type of VectorDB to be created")
- argparser.add_argument('-db', '--db_name', type=str, required=True, help = "Name of the database to be created")
+ argparser.add_argument('-db_type', '--db_type', type=str, required=True, help = "Type of VectorDB to be created", default="chroma")
+ argparser.add_argument('-db', '--db_name', type=str, required=False, help = "Name of the database to be created")
argparser.add_argument('-t', '--table_name', type=str, required=False, help = "Name of the table to be created", default = "EIC_archive")
- argparser.add_argument('-openai_key', '--openai_api_key', type=str, required=True, help = "OpenAI API key")
+ # argparser.add_argument('-openai_key', '--openai_api_key', type=str, required=True, help = "OpenAI API key")
argparser.add_argument('-c', '--chunking', type = str, required=False, help = "Type of Chunking PDF or LATEX", default = "PDF")
argparser.add_argument('-n', '--nthreads', type=int, default=-1)
argparser.add_argument('-db_api_key', '--db_api_key', type=str, required=False, help = "VectorDB API key")
-
+ argparser.add_argument('-persistent_dir', '--persistent_directory', type=str, required=True, help="Directory to store persistent ChromaDB", default="./chroma_db")
args = argparser.parse_args()
SaveFromPDF(args)
-
\ No newline at end of file
+
diff --git a/ingestion/requirements.txt b/ingestion/requirements.txt
new file mode 100644
index 0000000..125fddf
--- /dev/null
+++ b/ingestion/requirements.txt
@@ -0,0 +1,259 @@
+aiofiles==24.1.0
+aiohappyeyeballs==2.5.0
+aiohttp==3.11.13
+aiosignal==1.3.2
+annotated-types==0.7.0
+anyio==4.8.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+arxiv==2.1.3
+asgiref==3.8.1
+asttokens==3.0.0
+async-lru==2.0.4
+attrs==25.1.0
+babel==2.17.0
+backoff==2.2.1
+bcrypt==4.3.0
+beautifulsoup4==4.13.3
+bleach==6.2.0
+build==1.2.2.post1
+cachetools==5.5.2
+certifi==2025.1.31
+cffi==1.17.1
+chardet==5.2.0
+charset-normalizer==3.4.1
+chroma-hnswlib==0.7.6
+chromadb==0.6.3
+click==8.1.8
+coloredlogs==15.0.1
+comm==0.2.2
+contourpy==1.3.1
+cryptography==44.0.2
+cycler==0.12.1
+dataclasses-json==0.6.7
+debugpy==1.8.13
+decorator==5.2.1
+defusedxml==0.7.1
+Deprecated==1.2.18
+distro==1.9.0
+durationpy==0.9
+emoji==2.14.1
+eval_type_backport==0.2.2
+executing==2.2.0
+fastapi==0.115.11
+fastjsonschema==2.21.1
+feedparser==6.0.11
+filelock==3.17.0
+filetype==1.2.0
+flatbuffers==25.2.10
+fonttools==4.56.0
+fqdn==1.5.1
+frozenlist==1.5.0
+fsspec==2025.3.0
+google-auth==2.38.0
+googleapis-common-protos==1.69.2
+greenlet==3.1.1
+grpcio==1.71.0
+h11==0.14.0
+html5lib==1.1
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+httpx-sse==0.4.0
+huggingface-hub==0.29.2
+humanfriendly==10.0
+idna==3.10
+importlib_metadata==8.6.1
+importlib_resources==6.5.2
+ipykernel==6.29.5
+ipython==9.0.1
+ipython_pygments_lexers==1.1.1
+ipywidgets==8.1.5
+isoduration==20.11.0
+jedi==0.19.2
+Jinja2==3.1.6
+joblib==1.4.2
+json5==0.10.0
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter==1.1.1
+jupyter-console==6.6.3
+jupyter-events==0.12.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.15.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.3.5
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.13
+kiwisolver==1.4.8
+kubernetes==32.0.1
+langchain==0.3.20
+langchain-chroma==0.2.2
+langchain-community==0.3.19
+langchain-core==0.3.47
+langchain-ollama==0.3.0
+langchain-text-splitters==0.3.6
+langdetect==1.0.9
+langsmith==0.3.12
+lxml==5.3.1
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+marshmallow==3.26.1
+matplotlib==3.10.1
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mistune==3.1.2
+mmh3==5.1.0
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.1.0
+mypy-extensions==1.0.0
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.4.2
+nltk==3.9.1
+notebook==7.3.2
+notebook_shim==0.2.4
+numpy==1.26.4
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cusparselt-cu12==0.6.2
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127
+oauthlib==3.2.2
+olefile==0.47
+ollama==0.4.7
+onnx==1.17.0
+onnxruntime==1.21.0
+opencv-python==4.11.0.86
+opentelemetry-api==1.31.1
+opentelemetry-exporter-otlp-proto-common==1.31.1
+opentelemetry-exporter-otlp-proto-grpc==1.31.1
+opentelemetry-instrumentation==0.52b1
+opentelemetry-instrumentation-asgi==0.52b1
+opentelemetry-instrumentation-fastapi==0.52b1
+opentelemetry-proto==1.31.1
+opentelemetry-sdk==1.31.1
+opentelemetry-semantic-conventions==0.52b1
+opentelemetry-util-http==0.52b1
+orjson==3.10.15
+overrides==7.7.0
+packaging==24.2
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+pdf2image==1.17.0
+pdfminer.six==20231228
+pdfplumber==0.11.5
+pexpect==4.9.0
+pi_heif==0.21.0
+pillow==11.1.0
+platformdirs==4.3.6
+posthog==3.21.0
+prometheus_client==0.21.1
+prompt_toolkit==3.0.50
+propcache==0.3.0
+protobuf==5.29.2
+psutil==7.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pycparser==2.22
+pydantic==2.10.6
+pydantic-settings==2.8.1
+pydantic_core==2.27.2
+Pygments==2.19.1
+pyparsing==3.2.1
+pypdf==5.3.1
+pypdfium2==4.30.1
+PyPika==0.48.9
+pyproject_hooks==1.2.0
+pytesseract==0.3.13
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-iso639==2025.2.18
+python-json-logger==3.3.0
+python-magic==0.4.27
+python-multipart==0.0.20
+python-oxmsg==0.0.2
+pytz==2025.1
+PyYAML==6.0.2
+pyzmq==26.2.1
+RapidFuzz==3.12.2
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.9.4
+rpds-py==0.23.1
+rsa==4.9
+safetensors==0.5.3
+scipy==1.15.2
+Send2Trash==1.8.3
+setuptools==75.8.2
+sgmllib3k==1.0.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.6
+SQLAlchemy==2.0.38
+stack-data==0.6.3
+starlette==0.46.1
+sympy==1.13.1
+tenacity==9.0.0
+terminado==0.18.1
+timm==1.0.15
+tinycss2==1.4.0
+tokenizers==0.21.0
+torch==2.6.0
+torchvision==0.21.0
+tornado==6.4.2
+tqdm==4.67.1
+traitlets==5.14.3
+transformers==4.49.0
+triton==3.2.0
+typer==0.15.2
+types-python-dateutil==2.9.0.20241206
+typing-inspect==0.9.0
+typing-inspection==0.4.0
+typing_extensions==4.12.2
+tzdata==2025.1
+unstructured==0.16.25
+unstructured-client==0.31.1
+unstructured-inference==0.8.9
+unstructured.pytesseract==0.3.15
+uri-template==1.3.0
+urllib3==2.3.0
+uvicorn==0.34.0
+uvloop==0.21.0
+watchfiles==1.0.4
+wcwidth==0.2.13
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+websockets==15.0.1
+widgetsnbextension==4.0.13
+wrapt==1.17.2
+yarl==1.18.3
+zipp==3.21.0
+zstandard==0.23.0
diff --git a/requirements.txt b/requirements.txt
index 04f0d19..3525a74 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,235 +1,318 @@
-aiohttp==3.9.1
-aiosignal==1.3.1
+aiofiles==24.1.0
+aiohappyeyeballs==2.5.0
+aiohttp==3.11.13
+aiosignal==1.3.2
altair==5.2.0
-annotated-types==0.6.0
-anyio==4.2.0
+annotated-types==0.7.0
+anyio==4.8.0
appdirs==1.4.4
-arxiv==2.1.0
-asgiref==3.7.2
-asttokens==2.4.1
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+arxiv==2.1.3
+asgiref==3.8.1
+asttokens==3.0.0
+async-lru==2.0.4
async-timeout==4.0.3
-attrs==23.1.0
+attrs==25.1.0
+babel==2.17.0
backoff==2.2.1
-bcrypt==4.1.2
+bcrypt==4.3.0
+beautifulsoup4==4.13.3
+bleach==6.2.0
blinker==1.7.0
Bootstrap-Flask==2.3.2
-build==1.0.3
-cachetools==5.3.2
-certifi==2023.11.17
+bs4==0.0.2
+build==1.2.2.post1
+cachetools==5.5.2
+certifi==2025.1.31
+cffi==1.17.1
chardet==5.2.0
-charset-normalizer==3.3.2
-chroma-hnswlib==0.7.3
-chromadb==0.4.22
-click==8.1.7
+charset-normalizer==3.4.1
+chroma-hnswlib==0.7.6
+chromadb==0.6.3
+click==8.1.8
coloredlogs==15.0.1
-comm==0.2.1
-contourpy==1.2.0
+comm==0.2.2
+contourpy==1.3.1
+cryptography==44.0.2
cssselect==1.2.0
cssutils==2.9.0
cycler==0.12.1
-dataclasses-json==0.6.3
-datasets==2.18.0
-debugpy==1.8.0
-decorator==5.1.1
-Deprecated==1.2.14
+dataclasses-json==0.6.7
+datasets==3.4.1
+debugpy==1.8.13
+decorator==5.2.1
+defusedxml==0.7.1
+Deprecated==1.2.18
deprecation==2.1.0
dill==0.3.8
+diskcache==5.6.3
distro==1.9.0
dnspython==2.5.0
+durationpy==0.9
emails==0.6
+emoji==2.14.1
+eval_type_backport==0.2.2
exceptiongroup==1.2.0
-executing==2.0.1
-fastapi==0.109.0
-feedparser==6.0.10
-filelock==3.13.1
+executing==2.2.0
+fastapi==0.115.11
+fastjsonschema==2.21.1
+feedparser==6.0.11
+filelock==3.17.0
+filetype==1.2.0
Flask==3.0.0
Flask-SQLAlchemy==3.1.1
Flask-WTF==1.2.1
-flatbuffers==23.5.26
-fonttools==4.47.2
-frozenlist==1.4.1
-fsspec==2023.12.2
+flatbuffers==25.2.10
+fonttools==4.56.0
+fqdn==1.5.1
+frozenlist==1.5.0
+fsspec==2024.12.0
gitdb==4.0.11
GitPython==3.1.40
-google-auth==2.26.2
-googleapis-common-protos==1.62.0
-greenlet==3.0.1
-grpcio==1.60.0
+google-auth==2.38.0
+googleapis-common-protos==1.69.2
+greenlet==3.1.1
+grpcio==1.71.0
h11==0.14.0
-httpcore==1.0.2
-httptools==0.6.1
-httpx==0.26.0
-huggingface-hub==0.20.3
+html5lib==1.1
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+httpx-sse==0.4.0
+huggingface-hub==0.29.2
humanfriendly==10.0
-idna==3.6
-importlib-metadata==6.11.0
-importlib-resources==6.1.1
-ipykernel==6.29.0
-ipython==8.20.0
+idna==3.10
+importlib_metadata==8.6.1
+importlib_resources==6.5.2
+ipykernel==6.29.5
+ipython==8.34.0
+ipython_pygments_lexers==1.1.1
+ipywidgets==8.1.5
+isoduration==20.11.0
itsdangerous==2.1.2
-jedi==0.19.1
-Jinja2==3.1.2
-joblib==1.3.2
+jedi==0.19.2
+Jinja2==3.1.6
+jiter==0.9.0
+joblib==1.4.2
+json5==0.10.0
jsonpatch==1.33
-jsonpointer==2.4
-jsonschema==4.20.0
-jsonschema-specifications==2023.11.1
-jupyter_client==8.6.0
-jupyter_core==5.7.1
-kiwisolver==1.4.5
-kubernetes==29.0.0
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter==1.1.1
+jupyter-console==6.6.3
+jupyter-events==0.12.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.15.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.3.5
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.13
+kiwisolver==1.4.8
+kubernetes==32.0.1
lancedb==0.5.0
-langchain==0.1.12
-langchain-community==0.0.28
-langchain-core==0.1.32
-langchain-openai==0.0.5
-langchain-text-splitters==0.0.1
-langsmith==0.1.29
+langchain==0.3.20
+langchain-chroma==0.2.2
+langchain-community==0.3.19
+langchain-core==0.3.49
+langchain-ollama==0.3.0
+langchain-openai==0.3.11
+langchain-text-splitters==0.3.6
+langdetect==1.0.9
+langsmith==0.3.12
loguru==0.7.2
-lxml==5.1.0
+lxml==5.3.1
markdown-it-py==3.0.0
-MarkupSafe==2.1.3
-marshmallow==3.20.2
-matplotlib==3.8.2
-matplotlib-inline==0.1.6
+MarkupSafe==3.0.2
+marshmallow==3.26.1
+matplotlib==3.10.1
+matplotlib-inline==0.1.7
mdurl==0.1.2
-mmh3==4.1.0
+mistune==3.1.2
+mmh3==5.1.0
monotonic==1.6
mpmath==1.3.0
-multidict==6.0.4
+multidict==6.1.0
multiprocess==0.70.16
mypy-extensions==1.0.0
-nest_asyncio==1.6.0
-networkx==3.2.1
-numpy==1.26.2
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.19.3
-nvidia-nvjitlink-cu12==12.4.99
-nvidia-nvtx-cu12==12.1.105
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+networkx==3.4.2
+nltk==3.9.1
+notebook==7.3.2
+notebook_shim==0.2.4
+numpy==1.26.4
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cusparselt-cu12==0.6.2
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127
oauthlib==3.2.2
-onnxruntime==1.16.3
-openai==1.10.0
-opentelemetry-api==1.22.0
-opentelemetry-exporter-otlp-proto-common==1.22.0
-opentelemetry-exporter-otlp-proto-grpc==1.22.0
-opentelemetry-instrumentation==0.43b0
-opentelemetry-instrumentation-asgi==0.43b0
-opentelemetry-instrumentation-fastapi==0.43b0
-opentelemetry-proto==1.22.0
-opentelemetry-sdk==1.22.0
-opentelemetry-semantic-conventions==0.43b0
-opentelemetry-util-http==0.43b0
-orjson==3.9.15
-overrides==7.6.0
-packaging==23.2
-pandas==2.1.3
-parso==0.8.3
-pexpect==4.8.0
+olefile==0.47
+ollama==0.4.7
+onnx==1.17.0
+onnxruntime==1.21.0
+openai==1.68.2
+opencv-python==4.11.0.86
+opentelemetry-api==1.31.1
+opentelemetry-exporter-otlp-proto-common==1.31.1
+opentelemetry-exporter-otlp-proto-grpc==1.31.1
+opentelemetry-instrumentation==0.52b1
+opentelemetry-instrumentation-asgi==0.52b1
+opentelemetry-instrumentation-fastapi==0.52b1
+opentelemetry-proto==1.31.1
+opentelemetry-sdk==1.31.1
+opentelemetry-semantic-conventions==0.52b1
+opentelemetry-util-http==0.52b1
+orjson==3.10.15
+overrides==7.7.0
+packaging==24.2
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+pdf2image==1.17.0
+pdfminer.six==20231228
+pdfplumber==0.11.5
+pexpect==4.9.0
+pi_heif==0.21.0
pickleshare==0.7.5
-Pillow==10.1.0
+pillow==11.1.0
+pinecone==6.0.2
pinecone-client==3.0.2
-pip==24.0
-platformdirs==4.1.0
+pinecone-plugin-interface==0.0.7
+platformdirs==4.3.6
plotly==5.18.0
-posthog==3.3.2
+posthog==3.21.0
premailer==3.10.0
-prompt-toolkit==3.0.42
-protobuf==4.25.1
-psutil==5.9.8
+prometheus_client==0.21.1
+prompt_toolkit==3.0.50
+propcache==0.3.0
+protobuf==5.29.2
+psutil==7.0.0
ptyprocess==0.7.0
pulsar-client==3.4.0
-pure-eval==0.2.2
+pure_eval==0.2.3
py==1.11.0
-pyarrow==14.0.1
-pyarrow-hotfix==0.6
-pyasn1==0.5.1
-pyasn1-modules==0.3.0
-pydantic==2.5.3
-pydantic_core==2.14.6
+pyarrow==19.0.1
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pycparser==2.22
+pydantic==2.10.6
+pydantic-settings==2.8.1
+pydantic_core==2.27.2
pydeck==0.8.1b0
-Pygments==2.17.2
+Pygments==2.19.1
pylance==0.9.6
pymongo==4.6.1
-pyparsing==3.1.1
-pypdf==4.0.0
+pyparsing==3.2.1
+pypdf==5.3.1
+pypdfium2==4.30.1
pyphen==0.14.0
PyPika==0.48.9
-pyproject_hooks==1.0.0
-pysbd==0.3.4
-python-dateutil==2.8.2
+pyproject_hooks==1.2.0
+pytesseract==0.3.13
+python-dateutil==2.9.0.post0
python-dotenv==1.0.1
-pytz==2023.3.post1
-PyYAML==6.0.1
-pyzmq==25.1.2
-ragas==0.1.5
+python-iso639==2025.2.18
+python-json-logger==3.3.0
+python-magic==0.4.27
+python-multipart==0.0.20
+python-oxmsg==0.0.2
+pytz==2025.1
+PyYAML==6.0.2
+pyzmq==26.2.1
+ragas==0.2.14
+RapidFuzz==3.12.2
ratelimiter==1.2.0.post0
-referencing==0.31.0
-regex==2023.12.25
-requests==2.31.0
-requests-oauthlib==1.3.1
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
retry==0.9.2
-rich==13.7.0
-rpds-py==0.13.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.9.4
+rpds-py==0.23.1
rsa==4.9
-safetensors==0.4.2
-scikit-learn==1.4.1.post1
-scipy==1.12.0
-seaborn==0.13.2
+safetensors==0.5.3
+scipy==1.15.2
semver==3.0.2
-sentence-transformers==2.5.1
-setuptools==69.0.3
+Send2Trash==1.8.3
sgmllib3k==1.0.0
-six==1.16.0
+shellingham==1.5.4
+six==1.17.0
smmap==5.0.1
-sniffio==1.3.0
-SQLAlchemy==2.0.23
-stack-data==0.6.2
-starlette==0.35.1
+sniffio==1.3.1
+soupsieve==2.6
+SQLAlchemy==2.0.38
+stack-data==0.6.3
+starlette==0.46.1
streamlit==1.32.2
streamlit-feedback==0.1.3
streamlit-react-flow==0.0.3
-sympy==1.12
-tenacity==8.2.3
+sympy==1.13.1
+tenacity==9.0.0
+terminado==0.18.1
textstat==0.7.3
-threadpoolctl==3.4.0
-tiktoken==0.5.2
-tokenizers==0.15.1
+tiktoken==0.9.0
+timm==1.0.15
+tinycss2==1.4.0
+tokenizers==0.21.0
toml==0.10.2
tomli==2.0.1
toolz==0.12.0
-torch==2.2.1
-tornado==6.3.3
-tqdm==4.66.1
-traitlets==5.14.1
-transformers==4.39.0
-triton==2.2.0
+torch==2.6.0
+torchvision==0.21.0
+tornado==6.4.2
+tqdm==4.67.1
+traitlets==5.14.3
+transformers==4.49.0
+triton==3.2.0
trubrics==1.6.2
-typer==0.9.0
-typing_extensions==4.9.0
+typer==0.15.2
+types-python-dateutil==2.9.0.20241206
typing-inspect==0.9.0
-tzdata==2023.3
+typing-inspection==0.4.0
+typing_extensions==4.12.2
+tzdata==2025.1
tzlocal==5.2
-urllib3==2.1.0
-uvicorn==0.27.0
-uvloop==0.19.0
+unstructured==0.16.25
+unstructured-client==0.31.1
+unstructured-inference==0.8.9
+unstructured.pytesseract==0.3.15
+uri-template==1.3.0
+urllib3==2.3.0
+uvicorn==0.34.0
+uvloop==0.21.0
validators==0.22.0
watchdog==3.0.0
-watchfiles==0.21.0
+watchfiles==1.0.4
wcwidth==0.2.13
-websocket-client==1.7.0
-websockets==12.0
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+websockets==15.0.1
Werkzeug==3.0.1
-wheel==0.42.0
-wrapt==1.16.0
+widgetsnbextension==4.0.13
+wrapt==1.17.2
WTForms==3.1.1
-xxhash==3.4.1
-yarl==1.9.4
-zipp==3.17.0
+xxhash==3.5.0
+yarl==1.18.3
+zipp==3.21.0
+zstandard==0.23.0
diff --git a/streamlit_app/LangChainUtils/LLMChains.py b/streamlit_app/LangChainUtils/LLMChains.py
index f06b331..6a90eb8 100644
--- a/streamlit_app/LangChainUtils/LLMChains.py
+++ b/streamlit_app/LangChainUtils/LLMChains.py
@@ -72,6 +72,7 @@ def GeneralChain(llm):
) | llm | {"answer" : StrOutputParser()}
)
return general_chain
+#------------------------------------
def RunChatBot(llm, retriever, template_dir = None):
def format_docs(docs):
unique_arxiv = list(set(doc.metadata['arxiv_id'] for doc in docs))
@@ -100,7 +101,11 @@ def format_docs(docs):
- The content may have latex commands as well. Edit them to make it compatible within Github flavoured markdown by adding $ before and after the latex command.
- Make sure the citations are superscripted and has to be displayed properly when presented in a chat window.
- Do not include the and tags in your answer.
- - Strictly do no modify the reference URL nor its text. Strictly have only Footnotes with reference links in style of GithubFlavoured markdown.
+ - Strictly do not modify the reference URL nor its text. Strictly have only Footnotes with reference links in style of GithubFlavoured markdown.
+ Strictly note the points below
+ - You must always return valid JSON fenced by a markdown code block. Do not return any additional text.
+ - Do not add any text outside the JSON code block.
+ - Escape special characters (e.g., quotes, backslashes) to ensure valid JSON.
{markdown_response}
diff --git a/test.txt b/test.txt
new file mode 100644
index 0000000..7a7a206
--- /dev/null
+++ b/test.txt
@@ -0,0 +1 @@
+Testing first commit