HayStack docstore and hybrid context in Pufendorf bot.

app.py

@@ -91,6 +91,10 @@ except Exception as e:  # chromadb.errors.InvalidCollectionException:
     print("ERROR, no db")
     collection = None
 
+doc_store = InMemoryDocumentStore().load_from_disk("pufendorfdocs.store")
+print(f"Number of documents: {doc_store.count_documents()}.")
+hybrid_retrieval = create_hybrid_retriever(doc_store)
+
 
 # Contact OpenAI "moderator".
 def moderator(message):
@@ -129,6 +133,12 @@ def get_context(message):
     return data
 
 
+# Hybrid retriever from hybrid, uses pufendorfstore.
+def get_hybrid_context(message):
+    documents = retrieve(hybrid_retrieval, message, top_k=3, scale=True)
+    return documents
+
+
 def extract_persons(a_text) -> str:
     print(a_text)
     system_prompt = (
@@ -324,6 +334,16 @@ with gr.Blocks(theme=theme) as demo_blocks:
             return
 
         context = get_context(user_message)
+        hybrid_context = get_hybrid_context(user_message)
+        for hc in hybrid_context:
+            DBG(
+                str(hc.meta["file_path"])
+                + " "
+                + str(hc.meta["page_number"])
+                + "/"
+                + str(hc.content)
+            )
+
         DBG("FULL CONTEXT")
         for x in context:
             DBG(x)
@@ -340,8 +360,21 @@ with gr.Blocks(theme=theme) as demo_blocks:
             context_str = "Context:\n"
             for i, x in enumerate(context):  # note different after reranking
                 DBG(x)
-                context_str += "### " + str(i) + "\n" + x + "\n"
+                context_str += x + "\n\n"
+        # The hc is the new haystack contents.
+        hybridkeep = os.getenv("HYBRIDKEEP")
+        if not hybridkeep:
+            hybridkeep = 3
+        else:
+            hybridkeep = int(hybridkeep)
+        DBG("hybrid context keep: " + str(hybridkeep))
+        if hybridkeep > 0:
+            hybrid_context = hybrid_context[0:hybridkeep]
+            for i, x in enumerate(hybrid_context):
+                DBG(x)
+                context_str += x.content + "\n\n"
             ctx_text = context_str
+        if ctxkeep > 0 or hybridkeep > 0:
             prompt = f"Context: {context_str}\nQuestion:{user_message}\n"
         else:
             ctx_text = "(no retrieved context used)"
@@ -375,6 +408,8 @@ with gr.Blocks(theme=theme) as demo_blocks:
         messages += history[:-1]  # because the prompt has the context.
         ## Truncate the messages when too many?
         messages.append({"role": "user", "content": prompt})  ## should be ChatMessage
+        # ctx_text = str(messages)
+        # DBG(prompt)
         # format_history(messages)
         # print("=" * 40)
         # print(messages)
@@ -420,6 +455,4 @@ with gr.Blocks(theme=theme) as demo_blocks:
 # demo.launch(share=True)
 if __name__ == "__main__":
     print("Starting")
-    doc_store = InMemoryDocumentStore().load_from_disk("pufendorfdocs.store")
-    print(f"Number of documents: {doc_store.count_documents()}.")
     demo_blocks.launch()

hybrid.py

@@ -31,8 +31,7 @@ python hybrid.py -c newstore.store
 python hybrid.py -r newstore.store -q "who is pufendorf"
 """
 
-# embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
-embedding_model = "sentence-transformers/all-MiniLM-L12-v2"
+embedding_model = "sentence-transformers/all-MiniLM-L6-v2"
 
 # see https://huggingface.co/BAAI/bge-m3
 reranker_model = "BAAI/bge-reranker-base"

vector3_db/a1b2bf9f-4f30-46a6-a6c2-b6ca99effce9/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8146ecc3e4c3a36ea9b3edc3778630c452f483990ec942d38e8006f4661e430
+oid sha256:a2f33a640a6a1c9930a46cf5a6b6ebc0e07c52d85a754892208b3725ec6d7964
 size 16760000

vector3_db/a1b2bf9f-4f30-46a6-a6c2-b6ca99effce9/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc238a7a80a8cb9db9df824df6a3252ba0dd6f473223db345f2c4727a127151f
+oid sha256:174a07871bf6956d282f718bf1af45ecb44ff58d1120450e65572884e2655044
 size 40000

vector3_db/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:861003a15d8d7e1a50386e60541f0f36e1a5a431a3c76c10abbe6ac42cf8c560
+oid sha256:2a4d40287f44cd70d89a2167703709b18734676b57e399607ebef5145003eda0
 size 11452416