making the submits + removing internal / external

app.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import subprocess
@@ -18,6 +19,7 @@ from src.display.utils import (
 )
 from src.envs import (
     API,
+    DATA_PATH,
     H4_TOKEN,
     HF_HOME,
     HF_TOKEN_PRIVATE,
@@ -26,8 +28,9 @@ from src.envs import (
     PERSISTENT_FILE_CHECK_PATH,
     REPO_ID,
     RESET_JUDGEMENT_ENV,
+    SUBMITS_META_FILE,
 )
-from src.leaderboard.build_leaderboard import build_leadearboard_df, download_openbench
+from src.leaderboard.build_leaderboard import build_leadearboard_df, download_meta
 
 os.environ["GRADIO_ANALYTICS_ENABLED"] = "false"
 
@@ -37,7 +40,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(
 # Start ephemeral Spaces on PRs (see config in README.md)
 enable_space_ci()
 
-download_openbench()
+download_meta()
 
 
 def build_demo():
@@ -75,15 +78,45 @@ def build_demo():
 
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
-                    submitter_username = gr.Textbox(label="Username")
+                    submitter_username = gr.Textbox(label="Username")  # can we get this info from hf??
 
                     def upload_file(file):
-                        file_path = file.name.split("/")[-1] if "/" in file.name else file.name
-                        logging.info("New submition: file saved to %s", file_path)
+                        file_name = file.name.split("/")[-1] if "/" in file.name else file.name
+
+                        with open(f"{DATA_PATH}/{SUBMITS_META_FILE}", "r", encoding="utf-8") as submit_meta_file:
+                            current_info = json.loads(submit_meta_file)
+
+                            # for now just do not save same name model
+                            if model_name_textbox in current_info:
+                                return False
+
+                            submit_info = {
+                                "username": submitter_username,
+                                "file_name": file_name,
+                            }
+
+                            current_info[model_name_textbox] = submit_info
+
+                        with open(f"{DATA_PATH}/{SUBMITS_META_FILE}", "w", encoding="utf-8") as submit_meta_file:
+                            submit_meta_file.write(json.dumps(current_info))
+
+                        logging.info(
+                            "New submition: file from %s saved to %s with model %s",
+                            submitter_username,
+                            file_name,
+                            model_name_textbox,
+                        )
                         API.upload_file(
                             path_or_fileobj=file.name,
-                            path_in_repo="model_answers/external/" + file_path,
-                            repo_id="Vikhrmodels/openbench-eval",
+                            path_in_repo="arena-hard-v0.1/model_answers/" + file_name,
+                            repo_id=METAINFO_DATASET,
+                            repo_type="dataset",
+                            token=HF_TOKEN_PRIVATE,
+                        )
+                        API.upload_file(
+                            path_or_fileobj=SUBMITS_META_FILE,
+                            path_in_repo=SUBMITS_META_FILE,
+                            repo_id=METAINFO_DATASET,
                             repo_type="dataset",
                             token=HF_TOKEN_PRIVATE,
                         )

src/envs.py

@@ -35,6 +35,8 @@ RESET_JUDGEMENT_ENV = "RESET_JUDGEMENT"
 
 API = HfApi(token=H4_TOKEN)
 
+# if any more info about service pls make this file just a json
+SUBMITS_META_FILE = "submits_info.json"
 PERSISTENT_FILE_CHECK = "persistent_file_check"
 PERSISTENT_FILE_CHECK_PATH = f"{DATA_PATH}/{PERSISTENT_FILE_CHECK}"
 

src/gen/gen_answer.py

@@ -3,6 +3,7 @@
 Usage:
 python gen_api_answer --parallel 32
 """
+
 import argparse
 import concurrent.futures
 import json
@@ -138,7 +139,7 @@ if __name__ == "__main__":
     settings = make_config(args.setting_file)
     endpoint_list = make_config(args.endpoint_file)
 
-    existing_answer = load_model_answers(os.path.join("data", settings["bench_name"], "model_answers", "internal"))
+    existing_answer = load_model_answers(os.path.join("data", settings["bench_name"], "model_answers"))
 
     print(settings)
 

src/gen/gen_judgment.py

@@ -116,12 +116,12 @@ def judgment(**args):
         result = {"user_prompt": conv[1]["content"], "judgment": judgment, "score": score}
         output["games"].append(result)
 
-    with open(output_file, "a") as f:
+    with open(output_file, "a", encoding="utf-8") as f:
         f.write(json.dumps(output, ensure_ascii=False) + "\n")
     huggingface_hub.HfApi().upload_file(
         output_file,
         path_in_repo=f'model_judgment/{configs["judge_model"]}/{output_file.split("/")[-1]}',
-        repo_id="Vikhrmodels/openbench-eval",
+        repo_id="Vikhrmodels/-eval",
         repo_type="dataset",
     )
 
@@ -145,21 +145,16 @@ if __name__ == "__main__":
         pattern = re.compile(configs["regex_pattern"])
 
     question_file = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "question.jsonl")
-    internal_dir = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "model_answers/internal")
-    external_dir = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "model_answers/external")
+    answers_dir = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "model_answers")
     ref_answer_dir = os.path.join(f"{HF_HOME}/data", configs["bench_name"], "reference_answer")
 
     questions = load_questions(question_file)
-    model_answers_external = load_model_answers(external_dir)
-    model_answers_internal = load_model_answers(internal_dir)
-
-    # internal has priority
-    model_answers = {**model_answers_external, **model_answers_internal}
+    model_answers = load_model_answers(answers_dir)
 
     # if user choose a set of models, only judge those models
     models = [
         model.split("/")[-1].split(".")[0]
-        for model in glob.glob(f"{HF_HOME}/data/arena-hard-v0.1/model_answers/external/*.jsonl")
+        for model in glob.glob(f"{HF_HOME}/data/arena-hard-v0.1/model_answers/*.jsonl")
     ]
 
     ref_answers = None

src/gen/show_result.py

@@ -198,7 +198,7 @@ if __name__ == "__main__":
         args.load_battles and args.load_bootstrap
     ), "If loading prexisting bootstrapping data, you must also load preexisting battles."
 
-    answer_dir = os.path.join("data", args.bench_name, "model_answers/external")
+    answer_dir = os.path.join("data", args.bench_name, "model_answers")
     model_answers = load_model_answers(answer_dir)
 
     if args.load_battles:
@@ -265,7 +265,7 @@ if __name__ == "__main__":
         huggingface_hub.HfApi().upload_file(
             path_or_fileobj=json_file_name,
             path_in_repo="leaderboard.json",
-            repo_id="Vikhrmodels/arena-leaderboard-metainfo",
+            repo_id=METAINFO_DATASET,
             repo_type="dataset",
             token=HF_TOKEN_PRIVATE,
         )

src/leaderboard/build_leaderboard.py

@@ -6,7 +6,7 @@ import time
 import pandas as pd
 from huggingface_hub import snapshot_download
 
-from src.envs import DATA_ARENA_PATH, DATA_PATH, HF_TOKEN_PRIVATE, METAINFO_DATASET
+from src.envs import DATA_PATH, HF_TOKEN_PRIVATE, METAINFO_DATASET, SUBMITS_META_FILE
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -51,16 +51,17 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, ba
     logging.error("Failed to download %s after %s attempts", repo_id, max_attempts)
 
 
-def download_openbench():
-    # download prev autogenerated leaderboard files
+def download_meta():
+    # download all metainfo about submits
+    # also all the submits questions
+    # also all other stuff
     download_dataset(METAINFO_DATASET, DATA_PATH)
 
-    # download answers of different models that we trust
-    download_dataset("Vikhrmodels/openbench-eval", DATA_ARENA_PATH)
-
 
 def build_leadearboard_df():
     # Retrieve the leaderboard DataFrame
     with open(f"{DATA_PATH}/leaderboard.json", "r", encoding="utf-8") as eval_file:
-        leaderboard_df = pd.DataFrame.from_records(json.load(eval_file))
-    return leaderboard_df.copy()
+        battle_info = pd.DataFrame.from_records(json.load(eval_file))
+    with open(f"{DATA_PATH}/{SUBMITS_META_FILE}", "r", encoding="utf-8") as submit_meta_file:
+        submit_info = pd.DataFrame.from_records(json.load(submit_meta_file))
+    return battle_info.copy()