primary upload of the app

LLMHelper.py

@@ -0,0 +1,88 @@
+import os
+
+from ctransformers import AutoModelForCausalLM
+from langchain.llms import OpenAI
+from langchain.prompts import PromptTemplate
+
+AVAILABLE_MODELS_GGUF = {
+    "TheBloke/Marcoroni-7B-v3-GGUF": {
+        "model_file": "marcoroni-7b-v3.Q4_K_M.gguf",
+        "model_type": "marcoroni"
+    },
+    "TheBloke/Mistral-7B-Instruct-v0.2-GGUF": {
+        "model_file": "mistral-7b-instruct-v0.2.Q5_K_M.gguf",
+        "model_type": "mistral"
+    },
+    "TheBloke/zephyr-7B-beta-GGUF": {
+        "model_file": "zephyr-7b-beta.Q5_K_M.gguf",
+        "model_type": "zephyr"
+    },
+    "TheBloke/una-cybertron-7B-v2-GGUF": {
+        "model_file": "una-cybertron-7b-v2-bf16.Q5_K_M.gguf",
+        "model_type": "cybertron"
+    },
+
+}
+
+AVAILABLE_MODELS_OPENAI = [
+    "gpt-4-1106-preview", "gpt-4-32k", "gpt-3.5-turbo-1106",
+]
+
+
+
+
+def generate_cover_letter_open_source(job_description, resume, selected_model, context_length=8000):
+    print(f"selected_model: {selected_model}, "
+          f"{AVAILABLE_MODELS_GGUF[selected_model]['model_file']}, {AVAILABLE_MODELS_GGUF[selected_model]['model_type']}")
+    print(f"context_length: {context_length}")
+
+    prompt = (f"Do the following steps: "
+              f"1. Read the following job description,"
+              f"2. Read the following resume, "
+              f"3. Write a formal cover letter to the hiring manager for the job description based on the given resume, "
+              # f"4. The cover letter MUST BE within {output_size_range[0]} to {output_size_range[1]} words. "
+              # f"4. The cover letter MUST BE within 100 words. "
+              f"4. Return ONLY the cover letter ONCE, nothing else. "
+              f"Job Description: '{job_description}'. Resume: '{resume}'")
+
+    # prompt = "What is an LLM"
+
+    llm = AutoModelForCausalLM.from_pretrained(selected_model,
+                                               model_file=AVAILABLE_MODELS_GGUF[selected_model]['model_file'],
+                                               model_type=AVAILABLE_MODELS_GGUF[selected_model]['model_type'],
+                                               context_length=context_length,
+                                               max_new_tokens=1000,
+                                               reset=True,
+                                               stream=True,
+                                               # top_k=2,
+                                               temperature=0.5
+                                               )
+
+    llm_response = llm(prompt)
+
+    return llm_response
+
+
+def generate_cover_letter_openai(job_description, resume, selected_model, openai_key=None):
+    os.environ["OPENAI_API_KEY"] = openai_key
+    temp = "Do the following steps: " \
+           "1. Read the following job description," \
+           "2. Read the following resume, " \
+           "3. Write a formal cover letter to the hiring manager for the job description based on the given resume, " \
+           "4. Return ONLY the cover letter ONCE, nothing else. " \
+           "Job Description: '{job_description}'. Resume: '{resume}'"
+
+    prompt = PromptTemplate(
+        template=temp,
+        input_variables=["job_description", "resume"]
+    )
+
+    # model = OpenAI(openai_api_key=openai_key, max_tokens=-1)
+    print(f'openai key: {openai_key}')
+    # model = OpenAI(model_name=selected_model, openai_api_key="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")
+    model = OpenAI(model_name=selected_model)
+
+    _input = prompt.format(job_description=job_description, resume=resume)
+    output = model.stream(_input)
+
+    return output

Readme.md

@@ -0,0 +1,13 @@
+---
+title: Llm Cover Letter Generator
+emoji: 📝
+colorFrom: blue
+colorTo: green
+sdk: streamlit
+sdk_version: 1.29.0
+app_file: app.py
+pinned: false
+---
+
+1. pip install -r requirements.txt
+2. streamlit run app.py

app.py

@@ -0,0 +1,98 @@
+import time
+import streamlit as st
+import LLMHelper
+
+
+def generate_open_source():
+    with output_col:
+        st.session_state.running = True
+        start_time = time.time()
+        context_length = (len(st.session_state.get('resume', '').split()) +
+                          len(st.session_state.get('jd', '').split()) +
+                          2000)
+        cover_letter_generator = LLMHelper.generate_cover_letter_open_source(
+            job_description=st.session_state['jd'], resume=st.session_state['resume'],
+            selected_model=selected_model, context_length=context_length
+        )
+        print(f'generated text: {cover_letter_generator}')
+        generate_response(cover_letter_generator, start_time)
+        st.session_state.running = False
+
+
+def generate_openai():
+    with output_col:
+        start_time = time.time()
+        try:
+            cover_letter_generator = LLMHelper.generate_cover_letter_openai(
+                job_description=st.session_state['jd'], resume=st.session_state['resume'],
+                selected_model=selected_model, openai_key=open_ai_key
+            )
+            generate_response(cover_letter_generator, start_time)
+        except ValueError as e:
+            st.error("Please provide a valid Open AI API key")
+        st.session_state.running = False
+
+
+def generate_response(cover_letter_gen, start_time):
+    with output_col:
+        if cover_letter_gen is not None:
+            with st.container(border=True):
+                with st.spinner("Generating text..."):
+                    generated_text_placeholder = st.empty()
+                    for chunk in cover_letter_gen:
+                        st.session_state.cover_letter_stream += chunk
+                        generated_text_placeholder.write(st.session_state.cover_letter_stream)
+                    st.write(f"generated words: {len(st.session_state.cover_letter_stream.split())}")
+                    st.write(f"generation time: {round(time.time() - start_time, 2)} seconds")
+
+
+if 'running' not in st.session_state:
+    st.session_state.running = False
+
+st.session_state.cover_letter_stream = ""
+st.set_page_config(page_title='Cover Letter Generator', layout="wide")
+st.markdown("## Cover Letter Generator")
+info = st.expander("Information")
+info.write(f"This project aims to:\n"
+           f"- Explore various open-source Large Language Models (LLMs).\n"
+           f"- Compare them to OpenAI models for performance. \n"
+           f"- Highlight benefits of open-source LLMs: \n"
+           f"  - Faster generation on OpenAI models due to non-local execution.\n"
+           f"  - Run open-source models on CPU with 10GB RAM (Around 5-min generation time).\n"
+           f"  - Significantly faster generation on GPUs. \n"
+           f"  - Free of cost and user-data ownership.\n\n"
+           f"Checkout my profile: https://zayedupal.github.io")
+
+input_col, output_col = st.columns(2)
+
+with input_col:
+    st.session_state['jd'] = st.text_area("Job Description",
+                                          placeholder="Paste the job description here",
+                                          disabled=st.session_state.running)
+    st.write(f"{len(st.session_state.get('jd', '').split())} words")
+
+    st.session_state['resume'] = st.text_area("Resume Information",
+                                              placeholder="Paste the resume content here",
+                                              disabled=st.session_state.running)
+
+    st.write(f"{len(st.session_state.get('resume', '').split())} words")
+
+    with output_col:
+        llm_tab = st.radio("LLM type", ["Open Source LLMs", "Open AI LLMs"], horizontal=True)
+        if llm_tab == "Open Source LLMs":
+            cover_letter_generator = None
+            st.session_state.cover_letter_stream = ""
+            selected_model = st.selectbox("Select LLM Model", options=LLMHelper.AVAILABLE_MODELS_GGUF.keys(),
+                                          disabled=st.session_state.running)
+
+            st.button("Generate Cover Letter", key='open_source_gen_key', on_click=generate_open_source,
+                      disabled=st.session_state.running)
+
+        elif llm_tab == "Open AI Models":
+            cover_letter_generator = None
+            st.session_state.cover_letter_stream = ""
+            selected_model = st.selectbox("Select Open AI Model", options=LLMHelper.AVAILABLE_MODELS_OPENAI,
+                                          disabled=st.session_state.running)
+            open_ai_key = st.text_input("Enter your open ai API key")
+            st.button("Generate Cover Letter", key='open_ai_gen_key', disabled=st.session_state.running,
+                      on_click=generate_openai)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+huggingface-hub
+ctransformers
+langchain
+openai==0.28.1