Upload app.py

app.py

@@ -19,18 +19,18 @@ from content import format_error, format_warning, format_log, TITLE, INTRODUCTIO
 TOKEN = os.environ.get("TOKEN", None)
 
 OWNER="Blanca"
-DATA_DATASET = f"gaia-benchmark/GAIA"
-INTERNAL_DATA_DATASET = f"gaia-benchmark/GAIA"
-SUBMISSION_DATASET = f"{DATA_DATASET}/test.jsonl"
-SUBMISSION_DATASET_PUBLIC = f"gaia-benchmark/submissions_public"
-#CONTACT_DATASET = f"{OWNER}/contact_info"
-RESULTS_DATASET = f"gaia-benchmark/results_public"
-LEADERBOARD_PATH = f"{OWNER}/Critical_Questions_leaderboard"
+DATA_DATASET = f"{OWNER}/critical_questions_generation"
+INTERNAL_DATA_DATASET = f"{OWNER}/critical_questions_generation"
+SUBMISSION_DATASET = f"{OWNER}/submissions_internal"
+SUBMISSION_DATASET_PUBLIC = f"{OWNER}/submissions_public"
+CONTACT_DATASET = f"{OWNER}/contact_info"
+RESULTS_DATASET = f"{OWNER}/results_public"
+LEADERBOARD_PATH = f"HiTZ/Critical_Questions_Leaderboard"
 api = HfApi()
 
-YEAR_VERSION = "2023"
-ref_scores_len = {"test": 301}
-ref_level_len = {"test": {1: 93, 2: 159, 3: 49}}
+YEAR_VERSION = "2025"
+ref_scores_len = {"test": 34}
+#ref_level_len = {"validation": {1: 53, 2: 86, 3: 26}, "test": {1: 93, 2: 159, 3: 49}}
 
 os.makedirs("scored", exist_ok=True)
 
@@ -39,33 +39,29 @@ LOCAL_DEBUG = False #not (os.environ.get("system") == "spaces")
 
 # Display the results
 eval_results = load_dataset(RESULTS_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True)
-#contact_infos = load_dataset(CONTACT_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True)
+contact_infos = load_dataset(CONTACT_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True)
 def get_dataframe_from_results(eval_results, split):
     local_df = eval_results[split]
     local_df = local_df.map(lambda row: {"model": model_hyperlink(row["url"], row["model"])})
     local_df = local_df.remove_columns(["system_prompt", "url"])
     local_df = local_df.rename_column("model", "Agent name")
     local_df = local_df.rename_column("model_family", "Model family")
-    local_df = local_df.rename_column("score", "Average score (%)")
-    for i in [1, 2, 3]:
-        local_df = local_df.rename_column(f"score_level{i}", f"Level {i} score (%)")
+    local_df = local_df.rename_column("score", "Score (%)")
     local_df = local_df.rename_column("date", "Submission date")
     df = pd.DataFrame(local_df)
-    df = df.sort_values(by=["Average score (%)"], ascending=False)
+    df = df.sort_values(by=["Score (%)"], ascending=False)
 
-    numeric_cols = [c for c in local_df.column_names if "score" in c]
-    df[numeric_cols] = df[numeric_cols].multiply(100).round(decimals=2)
-    #df = df.style.format("{:.2%}", subset=numeric_cols)
+    df["Score (%)"] = df["Score (%)"].multiply(100).round(2)
 
     return df
 
-eval_dataframe_val = get_dataframe_from_results(eval_results=eval_results, split="validation")
+
 eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")
 
 # Gold answers
 gold_results = {}
-gold_dataset = load_dataset(INTERNAL_DATA_DATASET, f"{YEAR_VERSION}_all", token=TOKEN, trust_remote_code=True)
-gold_results = {split: {row["task_id"]: row for row in gold_dataset[split]} for split in ["test"]}
+gold_dataset = load_dataset(INTERNAL_DATA_DATASET, token=TOKEN, trust_remote_code=True)
+gold_results = {"test": {row["intervention_id"]: row for row in gold_dataset["test"]}}
 
 
 def restart_space():
@@ -88,16 +84,16 @@ def add_new_eval(
     user_data = requests.get(f"https://huggingface.co/api/users/{profile.username}/overview")
     creation_date = json.loads(user_data.content)["createdAt"]
     if datetime.datetime.now() - datetime.datetime.strptime(creation_date, '%Y-%m-%dT%H:%M:%S.%fZ') < datetime.timedelta(days=60):
-        return format_error("This account is not authorized to submit on GAIA.")
+        return format_error("This account is not authorized to submit on this leaderboard.")
         
 
-    #contact_infos = load_dataset(CONTACT_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True)
-    #user_submission_dates = sorted(row["date"] for row in contact_infos[val_or_test] if row["username"] == profile.username)
-    #if len(user_submission_dates) > 0 and user_submission_dates[-1] == datetime.datetime.today().strftime('%Y-%m-%d'):
-    #    return format_error("You already submitted once today, please try again tomorrow.")
+    contact_infos = load_dataset(CONTACT_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", verification_mode=VerificationMode.NO_CHECKS, trust_remote_code=True)
+    user_submission_dates = sorted(row["date"] for row in contact_infos[val_or_test] if row["username"] == profile.username)
+    if len(user_submission_dates) > 0 and user_submission_dates[-1] == datetime.datetime.today().strftime('%Y-%m-%d'):
+        return format_error("You already submitted once today, please try again tomorrow.")
 
 
-    #is_validation = val_or_test == "validation"
+    is_validation = val_or_test == "validation"
     # Very basic email parsing
     _, parsed_mail = parseaddr(mail)
     if not "@" in parsed_mail:
@@ -134,17 +130,18 @@ def add_new_eval(
         "mail": mail,
         "date": datetime.datetime.today().strftime('%Y-%m-%d')
     }
-    #contact_infos[val_or_test]= contact_infos[val_or_test].add_item(contact_info)
+    contact_infos[val_or_test]= contact_infos[val_or_test].add_item(contact_info)
     if LOCAL_DEBUG:
         print("mock uploaded contact info")
-    #else:
-    #    contact_infos.push_to_hub(CONTACT_DATASET, config_name = YEAR_VERSION, token=TOKEN)
+    else:
+        contact_infos.push_to_hub(CONTACT_DATASET, config_name = YEAR_VERSION, token=TOKEN)
 
     # SCORE SUBMISSION
     file_path = path_to_file.name        
-    scores = {"all": 0, 1: 0, 2: 0, 3: 0}
-    num_questions = {"all": 0, 1: 0, 2: 0, 3: 0}
+    scores = 0
+    num_questions = 0
     task_ids = []
+
     with open(f"scored/{organisation}_{model}.jsonl", "w") as scored_file:
         with open(file_path, 'r') as f:
             for ix, line in enumerate(f):
@@ -152,39 +149,35 @@ def add_new_eval(
                     task = json.loads(line)
                 except Exception:
                     return format_error(f"Line {ix} is incorrectly formatted. Please fix it and resubmit your file.")
-
                 if "model_answer" not in task:
-                    return format_error(f"Line {ix} contains no model_answer key. Please fix it and resubmit your file.")
+                    return format_error(f"Line {ix} missing 'model_answer'.")
                 answer = task["model_answer"]
                 task_id = task["task_id"]
-                try:
-                    level = int(gold_results[val_or_test][task_id]["Level"])
-                except KeyError:
-                    return format_error(f"{task_id} not found in split {val_or_test}. Are you sure you submitted the correct file?")
 
-                score = question_scorer(task['model_answer'], gold_results[val_or_test][task_id]["Final answer"])
-                
+                if task_id not in gold_results[val_or_test]:
+                    return format_error(f"{task_id} not found in gold set.")
+
+                score = question_scorer(answer, gold_results[val_or_test][task_id]["Final answer"])
+
                 scored_file.write(
                     json.dumps({
                         "id": task_id,
                         "model_answer": answer,
-                        "score": score,
-                        "level": level
+                        "score": score
                     }) + "\n"
                 )
+
                 task_ids.append(task_id)
+                scores += score
+                num_questions += 1
 
-                scores["all"] += score
-                scores[level] += score
-                num_questions["all"] += 1
-                num_questions[level] += 1
 
     # Check if there's any duplicate in the submission
     if len(task_ids) != len(set(task_ids)):
         return format_error("There are duplicates in your submission. Please check your file and resubmit it.")
 
-    if any([num_questions[level] != ref_level_len[val_or_test][level] for level in [1, 2, 3]]):
-        return format_error(f"Your submission has {num_questions[1]} questions for level 1, {num_questions[2]} for level 2, and {num_questions[3]} for level 3, but it should have {ref_level_len[val_or_test][1]}, {ref_level_len[val_or_test][2]}, and {ref_level_len[val_or_test][3]} respectively. Please check your submission.")
+    #if any([num_questions[level] != ref_level_len[val_or_test][level] for level in [1, 2, 3]]):
+    #    return format_error(f"Your submission has {num_questions[1]} questions for level 1, {num_questions[2]} for level 2, and {num_questions[3]} for level 3, but it should have {ref_level_len[val_or_test][1]}, {ref_level_len[val_or_test][2]}, and {ref_level_len[val_or_test][3]} respectively. Please check your submission.")
 
     # SAVE SCORED SUBMISSION
     if LOCAL_DEBUG:
@@ -215,14 +208,14 @@ def add_new_eval(
         "system_prompt": system_prompt,
         "url": url,
         "organisation": organisation,
-        "score": scores["all"]/ref_scores_len[val_or_test],
-        "score_level1": scores[1]/num_questions[1],
-        "score_level2": scores[2]/num_questions[2],
-        "score_level3": scores[3]/num_questions[3],
+        "score": scores / ref_scores_len,#[val_or_test],
+        #"score_level1": scores[1]/num_questions[1],
+        #"score_level2": scores[2]/num_questions[2],
+        #"score_level3": scores[3]/num_questions[3],
         "date": datetime.datetime.today().strftime('%Y-%m-%d')
     }
     if num_questions[1] + num_questions[2] + num_questions[3] != ref_scores_len[val_or_test]:
-        return format_error(f"Your submission has {len(scores['all'])} questions for the test set, but it should have {ref_scores_len['test']}. Please check your submission.")
+        return format_error(f"Your submission has {len(scores['all'])} questions for the {val_or_test} set, but it should have {ref_scores_len[val_or_test]}. Please check your submission.")
     # Catching spam submissions of 100%
     if all((eval_entry[k] == 1 for k in ["score_level1", "score_level2", "score_level3"])):
         return format_error(f"There was a problem with your submission. Please open a discussion.")
@@ -246,10 +239,10 @@ def add_new_eval(
 
 def refresh():
     eval_results = load_dataset(RESULTS_DATASET, YEAR_VERSION, token=TOKEN, download_mode="force_redownload", verification_mode=VerificationMode.NO_CHECKS,trust_remote_code=True)
-    #eval_dataframe_val = get_dataframe_from_results(eval_results=eval_results, split="validation")
     eval_dataframe_test = get_dataframe_from_results(eval_results=eval_results, split="test")
     return eval_dataframe_test
 
+
 def upload_file(files):
     file_paths = [file.name for file in files]
     return file_paths
@@ -273,21 +266,15 @@ with demo:
             value=eval_dataframe_test, datatype=TYPES, interactive=False,
             column_widths=["20%"] 
         )
-    with gr.Tab("Results: Validation"):
-        leaderboard_table_val = gr.components.Dataframe(
-            value=eval_dataframe_val, datatype=TYPES, interactive=False,
-            column_widths=["20%"] 
-        )
+    #with gr.Tab("Results: Validation"):
+    #    leaderboard_table_val = gr.components.Dataframe(
+    #        value=eval_dataframe_val, datatype=TYPES, interactive=False,
+    #        column_widths=["20%"] 
+    #    )
 
     refresh_button = gr.Button("Refresh")
-    refresh_button.click(
-        refresh,
-        inputs=[],
-        outputs=[
-            leaderboard_table_val,
-            leaderboard_table_test,
-        ],
-    )
+    refresh_button.click(refresh, inputs=[], outputs=[leaderboard_table_test])
+
     with gr.Accordion("Submit a new model for evaluation"):
         with gr.Row():
             gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text")
@@ -326,4 +313,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
-demo.launch(debug=True)
+demo.launch(debug=True)