updated models, added multi-threaded generation

LLMHelper.py

@@ -13,15 +13,10 @@ AVAILABLE_MODELS_GGUF = {
         "model_file": "mistral-7b-instruct-v0.2.Q4_K_M.gguf",
         "model_type": "mistral"
     },
-    "TheBloke/zephyr-7B-beta-GGUF": {
-        "model_file": "zephyr-7b-beta.Q4_K_M.gguf",
-        "model_type": "zephyr"
-    },
-    "TheBloke/una-cybertron-7B-v2-GGUF": {
-        "model_file": "una-cybertron-7b-v2-bf16.Q4_K_M.gguf",
-        "model_type": "cybertron"
-    },
-
+    "TheBloke/LeoScorpius-7B-GGUF": {
+        "model_file": "leoscorpius-7b.Q4_K_M.gguf",
+        "model_type": "leoscorpius"
+    }
 }
 
 AVAILABLE_MODELS_OPENAI = [
@@ -55,7 +50,8 @@ def generate_cover_letter_open_source(job_description, resume, selected_model, c
                                                reset=True,
                                                stream=True,
                                                # top_k=2,
-                                               temperature=0.5
+                                               temperature=0.5,
+                                               threads=(os.cpu_count()//2)
                                                )
 
     llm_response = llm(prompt)

app.py

@@ -44,14 +44,18 @@ def generate_response(cover_letter_gen, start_time):
                         generated_text_placeholder.write(st.session_state.cover_letter_stream)
                     st.write(f"generated words: {len(st.session_state.cover_letter_stream.split())}")
                     st.write(f"generation time: {round(time.time() - start_time, 2)} seconds")
+                    st.write(
+                        f"tokens per second: {round(len(st.session_state.cover_letter_stream.split())/(round(time.time() - start_time, 2)))} seconds")
 
 
 if 'running' not in st.session_state:
     st.session_state.running = False
 
 st.session_state.cover_letter_stream = ""
-st.set_page_config(page_title='Cover Letter Generator', layout="wide")
-st.markdown("## Cover Letter Generator")
+st.set_page_config(page_title='LLM Cover Letter Generator', layout="wide")
+st.markdown("## Cover Letter Generator using Large Language Models (LLM)")
+st.info("Please be patient with the open source LLM models, as they are running on a CPU on the server.\n "
+        "Average generation time around 5 mins.")
 info = st.expander("Information")
 info.write(f"This project aims to:\n"
            f"- Explore various open-source Large Language Models (LLMs).\n"