clean code

app.py

@@ -2,12 +2,9 @@ import streamlit as st
 import os
 from dotenv import load_dotenv
 from langsmith import traceable
-from datetime import datetime
-from typing import List, Dict, Optional
-
 from app.chat import initialize_session_state, display_chat_history
-from app.data_loader import get_data, list_all_files, load_docs
-from app.document_processor import process_documents, save_vector_store_to_supabase, load_vector_store_from_supabase
+from app.data_loader import get_data, load_docs
+from app.document_processor import process_documents, save_vector_store_to_supabase, load_vector_store_from_supabase, get_latest_data_timestamp_from_files,get_supabase_vector_store_timestamp, vector_store_is_outdated
 from app.prompts import sahabat_prompt
 from app.db import supabase
 from langchain_community.llms import Replicate
@@ -17,58 +14,9 @@ from langchain_community.document_transformers import LongContextReorder
 
 load_dotenv()
 
-# ---------------------------------------------------------
-# ⚡️ CONFIG
-# ---------------------------------------------------------
 BUCKET_NAME = "pnp-bot-storage-archive"
 VECTOR_STORE_PREFIX = "vector_store"
 
-# ---------------------------------------------------------
-# ⚡️ UTILITY
-# ---------------------------------------------------------
-def get_latest_data_timestamp_from_files(bucket_name: str) -> float:
-    """Get the latest timestamp from files in a Supabase storage bucket."""
-    files = list_all_files(bucket_name)
-    latest_time = 0.0
-    for file in files:
-        iso_time = file.get("updated_at") or file.get("created_at")
-        if iso_time:
-            try:
-                timestamp = datetime.fromisoformat(iso_time.replace('Z', '+00:00')).timestamp()
-                latest_time = max(latest_time, timestamp)
-            except Exception as e:
-                print(f"Gagal parsing waktu dari {file.get('name')}: {e}")
-    return latest_time
-
-
-def get_supabase_vector_store_timestamp() -> Optional[str]:
-    """Get the latest timestamp of vector store files in the Supabase storage."""
-    try:
-        response = supabase.storage.from_(BUCKET_NAME).list()
-        timestamps = []
-        for file in response:
-            if file["name"].startswith(VECTOR_STORE_PREFIX) and (
-                file["name"].endswith(".faiss") or file["name"].endswith(".pkl")
-            ):
-                timestamps.append(file["updated_at"])
-        if len(timestamps) >= 2:
-            return max(timestamps)
-        return None
-    except Exception as e:
-        print(f"Error getting Supabase timestamp: {e}")
-        return None
-
-
-def vector_store_is_outdated() -> bool:
-    """Check if vector store needs to be updated based on files in Supabase storage."""
-    supabase_timestamp = get_supabase_vector_store_timestamp()
-    if supabase_timestamp is None:
-        return True
-    supabase_time = datetime.fromisoformat(supabase_timestamp.replace("Z", "+00:00")).timestamp()
-    data_time = get_latest_data_timestamp_from_files("pnp-bot-storage")
-
-    return data_time > supabase_time
-
 
 def reorder_embedding(docs):
     """Reorder documents for long context retrieval."""
@@ -76,9 +24,6 @@ def reorder_embedding(docs):
     return reordering.transform_documents(docs)
 
 
-# ---------------------------------------------------------
-# ⚡️ RAG CHAIN
-# ---------------------------------------------------------
 @traceable(name="Create RAG Conversational Chain")
 def create_conversational_chain(vector_store):
     """Create a Conversational Retrieval Chain for RAG."""
@@ -101,10 +46,6 @@ def get_rag_chain(vector_store):
     """Return a Conversational Retrieval Chain for external use."""
     return create_conversational_chain(vector_store)
 
-
-# ---------------------------------------------------------
-# ⚡️ MAIN FUNCTION
-# ---------------------------------------------------------
 @traceable(name="Main Chatbot RAG App")
 def main():
     initialize_session_state()

app/document_processor.py

@@ -4,7 +4,8 @@ from langchain_community.vectorstores import FAISS
 import os
 import tempfile
 import streamlit as st
-from langchain.schema import Document
+from datetime import datetime
+from data_loader import list_all_files
 
 
 def save_vector_store_to_supabase(vector_store, supabase, bucket_name, file_prefix="vector_store"):
@@ -112,4 +113,46 @@ def process_documents(docs):
     text_chunks = text_splitter.split_documents(docs)
     vector_store = FAISS.from_documents(text_chunks, embeddings)
     
-    return vector_store
+    return vector_store
+
+def get_latest_data_timestamp_from_files(bucket_name: str) -> float:
+    """Get the latest timestamp from files in a Supabase storage bucket."""
+    files = list_all_files(bucket_name)
+    latest_time = 0.0
+    for file in files:
+        iso_time = file.get("updated_at") or file.get("created_at")
+        if iso_time:
+            try:
+                timestamp = datetime.fromisoformat(iso_time.replace('Z', '+00:00')).timestamp()
+                latest_time = max(latest_time, timestamp)
+            except Exception as e:
+                print(f"Gagal parsing waktu dari {file.get('name')}: {e}")
+    return latest_time
+
+
+def get_supabase_vector_store_timestamp() -> Optional[str]:
+    """Get the latest timestamp of vector store files in the Supabase storage."""
+    try:
+        response = supabase.storage.from_(BUCKET_NAME).list()
+        timestamps = []
+        for file in response:
+            if file["name"].startswith(VECTOR_STORE_PREFIX) and (
+                file["name"].endswith(".faiss") or file["name"].endswith(".pkl")
+            ):
+                timestamps.append(file["updated_at"])
+        if len(timestamps) >= 2:
+            return max(timestamps)
+        return None
+    except Exception as e:
+        print(f"Error getting Supabase timestamp: {e}")
+        return None
+
+def vector_store_is_outdated() -> bool:
+    """Check if vector store needs to be updated based on files in Supabase storage."""
+    supabase_timestamp = get_supabase_vector_store_timestamp()
+    if supabase_timestamp is None:
+        return True
+    supabase_time = datetime.fromisoformat(supabase_timestamp.replace("Z", "+00:00")).timestamp()
+    data_time = get_latest_data_timestamp_from_files("pnp-bot-storage")
+
+    return data_time > supabase_time