Update space

css_html.py

@@ -0,0 +1,64 @@
+custom_css = """
+#changelog-text {
+    font-size: 16px !important;
+}
+#changelog-text h2 {
+    font-size: 18px !important;
+}
+.markdown-text {
+    font-size: 16px !important;
+}
+#models-to-add-text {
+    font-size: 18px !important;
+}
+#citation-button span {
+    font-size: 16px !important;
+}
+#citation-button textarea {
+    font-size: 16px !important;
+}
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+#leaderboard-table {
+    margin-top: 15px
+}
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+.tab-buttons button {
+    font-size: 20px;
+}
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+#scale-logo .download {
+    display: none;
+}
+"""

results.json

@@ -0,0 +1,1106 @@
+{
+    "gpt-4o": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 70.1,
+            "cpp_input": 64.6,
+            "d_input": 71.6,
+            "go_input": 75.4,
+            "java_input": 69.8,
+            "js_input": 73.2,
+            "jl_input": 67.0,
+            "lua_input": 73.0,
+            "pl_input": 70.1,
+            "php_input": 74.8,
+            "py_input": 70.6,
+            "r_input": 74.4,
+            "rkt_input": 67.4,
+            "rb_input": 72.0,
+            "rs_input": 73.6,
+            "scala_input": 65.4,
+            "sh_input": 70.6,
+            "swift_input": 74.2,
+            "ts_input": 74.0,
+            "cs_output": 75.0,
+            "cpp_output": 74.8,
+            "d_output": 71.3,
+            "go_output": 77.0,
+            "java_output": 73.2,
+            "js_output": 77.6,
+            "jl_output": 73.6,
+            "lua_output": 74.8,
+            "pl_output": 74.0,
+            "php_output": 75.4,
+            "py_output": 75.4,
+            "r_output": 72.0,
+            "rkt_output": 70.8,
+            "rb_output": 74.0,
+            "rs_output": 74.4,
+            "scala_output": 71.8,
+            "sh_output": 71.6,
+            "swift_output": 76.0,
+            "ts_output": 76.4
+        },
+        "prompted": false,
+        "size": null
+    },
+    "gpt-4o-mini": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 58.8,
+            "cpp_input": 52.2,
+            "d_input": 60.6,
+            "go_input": 62.0,
+            "java_input": 57.2,
+            "js_input": 59.6,
+            "jl_input": 56.2,
+            "lua_input": 63.4,
+            "pl_input": 57.4,
+            "php_input": 61.0,
+            "py_input": 59.6,
+            "r_input": 60.4,
+            "rkt_input": 51.2,
+            "rb_input": 61.6,
+            "rs_input": 61.2,
+            "scala_input": 52.6,
+            "sh_input": 57.2,
+            "swift_input": 63.4,
+            "ts_input": 61.2,
+            "cs_output": 63.0,
+            "cpp_output": 63.0,
+            "d_output": 61.4,
+            "go_output": 63.4,
+            "java_output": 54.0,
+            "js_output": 61.8,
+            "jl_output": 57.8,
+            "lua_output": 60.0,
+            "pl_output": 57.4,
+            "php_output": 64.2,
+            "py_output": 61.6,
+            "r_output": 59.6,
+            "rkt_output": 56.6,
+            "rb_output": 61.2,
+            "rs_output": 61.8,
+            "scala_output": 61.2,
+            "sh_output": 56.2,
+            "swift_output": 63.0,
+            "ts_output": 61.2
+        },
+        "prompted": true,
+        "size": null
+    },
+    "gpt-3.5-turbo": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 52.2,
+            "cpp_input": 39.2,
+            "d_input": 50.2,
+            "go_input": 53.4,
+            "java_input": 55.4,
+            "js_input": 50.0,
+            "jl_input": 47.0,
+            "lua_input": 53.2,
+            "pl_input": 47.6,
+            "php_input": 52.2,
+            "py_input": 51.6,
+            "r_input": 48.6,
+            "rkt_input": 45.4,
+            "rb_input": 49.6,
+            "rs_input": 53.0,
+            "scala_input": 54.2,
+            "sh_input": 47.6,
+            "swift_input": 58.2,
+            "ts_input": 48.4,
+            "cs_output": 54.2,
+            "cpp_output": 43.2,
+            "d_output": 56.0,
+            "go_output": 53.2,
+            "java_output": 43.6,
+            "js_output": 56.2,
+            "jl_output": 54.2,
+            "lua_output": 54.6,
+            "pl_output": 51.8,
+            "php_output": 55.2,
+            "py_output": 57.2,
+            "r_output": 49.4,
+            "rkt_output": 48.0,
+            "rb_output": 56.4,
+            "rs_output": 54.6,
+            "scala_output": 56.4,
+            "sh_output": 51.0,
+            "swift_output": 57.8,
+            "ts_output": 53.6
+        },
+        "prompted": true,
+        "size": null
+    },
+    "DeepSeek-Coder-V2-0724": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 63.8,
+            "cpp_input": 57.0,
+            "d_input": 66.6,
+            "go_input": 64.0,
+            "java_input": 64.8,
+            "js_input": 67.0,
+            "jl_input": 58.4,
+            "lua_input": 62.0,
+            "pl_input": 61.4,
+            "php_input": 64.2,
+            "py_input": 64.0,
+            "r_input": 65.8,
+            "rkt_input": 58.0,
+            "rb_input": 63.2,
+            "rs_input": 63.6,
+            "scala_input": 58.2,
+            "sh_input": 62.4,
+            "swift_input": 62.6,
+            "ts_input": 66.6,
+            "cs_output": 66.6,
+            "cpp_output": 66.2,
+            "d_output": 63.4,
+            "go_output": 68.0,
+            "java_output": 67.6,
+            "js_output": 65.4,
+            "jl_output": 64.8,
+            "lua_output": 63.6,
+            "pl_output": 63.0,
+            "php_output": 67.4,
+            "py_output": 66.8,
+            "r_output": 63.0,
+            "rkt_output": 62.2,
+            "rb_output": 65.2,
+            "rs_output": 65.8,
+            "scala_output": 63.2,
+            "sh_output": 58.8,
+            "swift_output": 67.8,
+            "ts_output": 66.4
+        },
+        "prompted": true,
+        "size": 23
+    },
+    "Qwen2-72B-Instruct-GPTQ-Int4": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 52.0,
+            "cpp_input": 54.2,
+            "d_input": 49.6,
+            "go_input": 55.4,
+            "java_input": 50.0,
+            "js_input": 51.6,
+            "jl_input": 51.0,
+            "lua_input": 51.2,
+            "pl_input": 47.8,
+            "php_input": 55.2,
+            "py_input": 52.4,
+            "r_input": 53.2,
+            "rkt_input": 47.8,
+            "rb_input": 54.4,
+            "rs_input": 57.2,
+            "scala_input": 50.6,
+            "sh_input": 52.4,
+            "swift_input": 51.6,
+            "ts_input": 52.0,
+            "cs_output": 51.2,
+            "cpp_output": 50.2,
+            "d_output": 51.6,
+            "go_output": 53.6,
+            "java_output": 38.2,
+            "js_output": 52.0,
+            "jl_output": 51.0,
+            "lua_output": 49.0,
+            "pl_output": 45.8,
+            "php_output": 50.8,
+            "py_output": 51.2,
+            "r_output": 45.0,
+            "rkt_output": 46.8,
+            "rb_output": 50.8,
+            "rs_output": 51.0,
+            "scala_output": 51.0,
+            "sh_output": 45.6,
+            "swift_output": 50.4,
+            "ts_output": 53.2
+        },
+        "prompted": true,
+        "size": 72
+    },
+    "CodeLlama-34b-Python-hf": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 38.8,
+            "cpp_input": 40.0,
+            "d_input": 39.2,
+            "go_input": 39.0,
+            "java_input": 41.4,
+            "js_input": 45.8,
+            "jl_input": 44.8,
+            "lua_input": 45.0,
+            "pl_input": 43.2,
+            "php_input": 48.0,
+            "py_input": 46.8,
+            "r_input": 42.2,
+            "rkt_input": 38.8,
+            "rb_input": 44.0,
+            "rs_input": 44.2,
+            "scala_input": 43.0,
+            "sh_input": 44.6,
+            "swift_input": 45.0,
+            "ts_input": 44.0,
+            "cs_output": 41.4,
+            "cpp_output": 44.8,
+            "d_output": 45.6,
+            "go_output": 41.8,
+            "java_output": 41.4,
+            "js_output": 45.4,
+            "jl_output": 45.2,
+            "lua_output": 42.8,
+            "pl_output": 43.6,
+            "php_output": 43.8,
+            "py_output": 43.8,
+            "r_output": 42.4,
+            "rkt_output": 38.6,
+            "rb_output": 42.8,
+            "rs_output": 46.6,
+            "scala_output": 43.8,
+            "sh_output": 42.0,
+            "swift_output": 44.4,
+            "ts_output": 44.8
+        },
+        "prompted": true,
+        "size": 34
+    },
+    "CodeLlama-34b-Instruct-hf": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 44.6,
+            "cpp_input": 48.4,
+            "d_input": 43.8,
+            "go_input": 46.0,
+            "java_input": 44.4,
+            "js_input": 52.6,
+            "jl_input": 50.4,
+            "lua_input": 49.4,
+            "pl_input": 46.0,
+            "php_input": 52.0,
+            "py_input": 51.2,
+            "r_input": 48.4,
+            "rkt_input": 42.4,
+            "rb_input": 48.2,
+            "rs_input": 48.6,
+            "scala_input": 48.0,
+            "sh_input": 46.2,
+            "swift_input": 49.4,
+            "ts_input": 53.2,
+            "cs_output": 44.4,
+            "cpp_output": 46.2,
+            "d_output": 45.8,
+            "go_output": 46.8,
+            "java_output": 40.6,
+            "js_output": 47.4,
+            "jl_output": 45.6,
+            "lua_output": 42.8,
+            "pl_output": 44.0,
+            "php_output": 44.8,
+            "py_output": 44.0,
+            "r_output": 40.2,
+            "rkt_output": 38.2,
+            "rb_output": 44.2,
+            "rs_output": 46.4,
+            "scala_output": 43.8,
+            "sh_output": 40.6,
+            "swift_output": 45.2,
+            "ts_output": 45.0
+        },
+        "prompted": true,
+        "size": 34
+    },
+    "CodeLlama-34b-hf": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 40.4,
+            "cpp_input": 44.6,
+            "d_input": 45.6,
+            "go_input": 41.2,
+            "java_input": 39.0,
+            "js_input": 50.0,
+            "jl_input": 49.0,
+            "lua_input": 47.0,
+            "pl_input": 46.6,
+            "php_input": 48.8,
+            "py_input": 49.8,
+            "r_input": 47.6,
+            "rkt_input": 39.8,
+            "rb_input": 46.6,
+            "rs_input": 46.8,
+            "scala_input": 44.6,
+            "sh_input": 44.4,
+            "swift_input": 50.0,
+            "ts_input": 48.6,
+            "cs_output": 44.6,
+            "cpp_output": 47.8,
+            "d_output": 44.2,
+            "go_output": 45.2,
+            "java_output": 38.4,
+            "js_output": 47.0,
+            "jl_output": 45.8,
+            "lua_output": 42.8,
+            "pl_output": 43.8,
+            "php_output": 46.4,
+            "py_output": 46.4,
+            "r_output": 38.8,
+            "rkt_output": 38.4,
+            "rb_output": 45.4,
+            "rs_output": 47.2,
+            "scala_output": 47.4,
+            "sh_output": 43.8,
+            "swift_output": 47.6,
+            "ts_output": 47.4
+        },
+        "prompted": true,
+        "size": 34
+    },
+    "WizardCoder-33B-V1.1": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 44.8,
+            "cpp_input": 25.4,
+            "d_input": 46.4,
+            "go_input": 47.6,
+            "java_input": 48.4,
+            "js_input": 45.6,
+            "jl_input": 49.2,
+            "lua_input": 48.8,
+            "pl_input": 44.6,
+            "php_input": 50.0,
+            "py_input": 50.0,
+            "r_input": 45.0,
+            "rkt_input": 42.4,
+            "rb_input": 49.2,
+            "rs_input": 48.2,
+            "scala_input": 48.2,
+            "sh_input": 45.4,
+            "swift_input": 51.0,
+            "ts_input": 46.4,
+            "cs_output": 47.0,
+            "cpp_output": 46.8,
+            "d_output": 45.8,
+            "go_output": 44.2,
+            "java_output": 50.8,
+            "js_output": 50.0,
+            "jl_output": 47.0,
+            "lua_output": 46.0,
+            "pl_output": 45.2,
+            "php_output": 51.4,
+            "py_output": 49.6,
+            "r_output": 44.0,
+            "rkt_output": 42.4,
+            "rb_output": 48.2,
+            "rs_output": 47.8,
+            "scala_output": 45.0,
+            "sh_output": 44.4,
+            "swift_output": 48.0,
+            "ts_output": 49.8
+        },
+        "prompted": true,
+        "size": 33
+    },
+    "deepseek-coder-33b-instruct": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 46.0,
+            "cpp_input": 43.6,
+            "d_input": 49.8,
+            "go_input": 49.0,
+            "java_input": 46.8,
+            "js_input": 48.8,
+            "jl_input": 47.0,
+            "lua_input": 50.0,
+            "pl_input": 46.8,
+            "php_input": 52.0,
+            "py_input": 51.8,
+            "r_input": 48.2,
+            "rkt_input": 41.6,
+            "rb_input": 52.0,
+            "rs_input": 48.4,
+            "scala_input": 47.0,
+            "sh_input": 48.2,
+            "swift_input": 52.2,
+            "ts_input": 49.6,
+            "cs_output": 52.0,
+            "cpp_output": 51.4,
+            "d_output": 49.0,
+            "go_output": 48.8,
+            "java_output": 53.2,
+            "js_output": 55.0,
+            "jl_output": 50.4,
+            "lua_output": 50.4,
+            "pl_output": 50.0,
+            "php_output": 53.0,
+            "py_output": 52.2,
+            "r_output": 48.2,
+            "rkt_output": 46.6,
+            "rb_output": 52.8,
+            "rs_output": 50.6,
+            "scala_output": 48.0,
+            "sh_output": 49.4,
+            "swift_output": 52.8,
+            "ts_output": 53.6
+        },
+        "prompted": true,
+        "size": 33
+    },
+    "deepseek-coder-33b-base": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 41.2,
+            "cpp_input": 42.8,
+            "d_input": 43.2,
+            "go_input": 45.6,
+            "java_input": 43.8,
+            "js_input": 46.0,
+            "jl_input": 47.6,
+            "lua_input": 47.4,
+            "pl_input": 47.2,
+            "php_input": 48.6,
+            "py_input": 49.2,
+            "r_input": 50.6,
+            "rkt_input": 42.8,
+            "rb_input": 47.4,
+            "rs_input": 46.8,
+            "scala_input": 44.0,
+            "sh_input": 46.4,
+            "swift_input": 48.2,
+            "ts_input": 45.0,
+            "cs_output": 48.2,
+            "cpp_output": 50.0,
+            "d_output": 46.0,
+            "go_output": 48.6,
+            "java_output": 49.2,
+            "js_output": 51.4,
+            "jl_output": 46.8,
+            "lua_output": 48.0,
+            "pl_output": 48.4,
+            "php_output": 52.0,
+            "py_output": 49.8,
+            "r_output": 45.2,
+            "rkt_output": 46.4,
+            "rb_output": 49.0,
+            "rs_output": 46.2,
+            "scala_output": 47.6,
+            "sh_output": 46.0,
+            "swift_output": 49.2,
+            "ts_output": 51.2
+        },
+        "prompted": true,
+        "size": 33
+    },
+    "starcoder2-15b": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 41.4,
+            "cpp_input": 43.8,
+            "d_input": 51.6,
+            "go_input": 45.2,
+            "java_input": 42.6,
+            "js_input": 44.0,
+            "jl_input": 48.2,
+            "lua_input": 44.6,
+            "pl_input": 44.8,
+            "php_input": 49.8,
+            "py_input": 46.6,
+            "r_input": 45.8,
+            "rkt_input": 45.0,
+            "rb_input": 49.0,
+            "rs_input": 46.6,
+            "scala_input": 37.0,
+            "sh_input": 47.4,
+            "swift_input": 52.2,
+            "ts_input": 46.2,
+            "cs_output": 46.0,
+            "cpp_output": 47.4,
+            "d_output": 47.2,
+            "go_output": 49.0,
+            "java_output": 48.4,
+            "js_output": 50.0,
+            "jl_output": 49.2,
+            "lua_output": 44.8,
+            "pl_output": 49.4,
+            "php_output": 48.4,
+            "py_output": 48.4,
+            "r_output": 47.2,
+            "rkt_output": 45.0,
+            "rb_output": 51.0,
+            "rs_output": 48.8,
+            "scala_output": 45.2,
+            "sh_output": 45.8,
+            "swift_output": 49.6,
+            "ts_output": 48.6
+        },
+        "prompted": true,
+        "size": 15
+    },
+    "WizardCoder-15B-V1.0": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 29.2,
+            "cpp_input": 30.0,
+            "d_input": 30.6,
+            "go_input": 28.6,
+            "java_input": 29.6,
+            "js_input": 33.0,
+            "jl_input": 34.8,
+            "lua_input": 33.6,
+            "pl_input": 36.2,
+            "php_input": 36.8,
+            "py_input": 33.2,
+            "r_input": 33.4,
+            "rkt_input": 36.4,
+            "rb_input": 33.6,
+            "rs_input": 33.0,
+            "scala_input": 29.0,
+            "sh_input": 35.0,
+            "swift_input": 34.0,
+            "ts_input": 32.4,
+            "cs_output": 25.2,
+            "cpp_output": 30.0,
+            "d_output": 30.6,
+            "go_output": 33.2,
+            "java_output": 26.8,
+            "js_output": 33.6,
+            "jl_output": 30.2,
+            "lua_output": 30.2,
+            "pl_output": 31.0,
+            "php_output": 33.0,
+            "py_output": 34.0,
+            "r_output": 31.6,
+            "rkt_output": 29.6,
+            "rb_output": 32.8,
+            "rs_output": 31.2,
+            "scala_output": 31.2,
+            "sh_output": 29.8,
+            "swift_output": 34.2,
+            "ts_output": 34.0
+        },
+        "prompted": true,
+        "size": 15
+    },
+    "starcoder": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 28.2,
+            "cpp_input": 30.0,
+            "d_input": 33.0,
+            "go_input": 33.2,
+            "java_input": 33.4,
+            "js_input": 35.2,
+            "jl_input": 34.4,
+            "lua_input": 31.6,
+            "pl_input": 34.0,
+            "php_input": 36.4,
+            "py_input": 34.8,
+            "r_input": 33.4,
+            "rkt_input": 36.6,
+            "rb_input": 35.0,
+            "rs_input": 34.8,
+            "scala_input": 27.4,
+            "sh_input": 37.0,
+            "swift_input": 30.8,
+            "ts_input": 33.2,
+            "cs_output": 20.4,
+            "cpp_output": 31.6,
+            "d_output": 31.8,
+            "go_output": 31.0,
+            "java_output": 18.4,
+            "js_output": 33.4,
+            "jl_output": 32.2,
+            "lua_output": 31.8,
+            "pl_output": 29.8,
+            "php_output": 32.6,
+            "py_output": 32.6,
+            "r_output": 30.0,
+            "rkt_output": 29.2,
+            "rb_output": 33.4,
+            "rs_output": 32.6,
+            "scala_output": 30.0,
+            "sh_output": 30.2,
+            "swift_output": 33.0,
+            "ts_output": 33.0
+        },
+        "prompted": true,
+        "size": 15
+    },
+    "Phi-3-medium-4k-instruct": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 31.8,
+            "cpp_input": 26.0,
+            "d_input": 38.8,
+            "go_input": 36.4,
+            "java_input": 37.2,
+            "js_input": 42.4,
+            "jl_input": 36.2,
+            "lua_input": 37.2,
+            "pl_input": 35.6,
+            "php_input": 41.2,
+            "py_input": 43.4,
+            "r_input": 39.2,
+            "rkt_input": 24.4,
+            "rb_input": 36.0,
+            "rs_input": 36.8,
+            "scala_input": 38.0,
+            "sh_input": 33.6,
+            "swift_input": 41.2,
+            "ts_input": 42.8,
+            "cs_output": 34.2,
+            "cpp_output": 37.6,
+            "d_output": 39.0,
+            "go_output": 31.0,
+            "java_output": 34.2,
+            "js_output": 41.6,
+            "jl_output": 41.2,
+            "lua_output": 34.4,
+            "pl_output": 35.8,
+            "php_output": 37.8,
+            "py_output": 42.4,
+            "r_output": 36.6,
+            "rkt_output": 24.6,
+            "rb_output": 42.2,
+            "rs_output": 37.4,
+            "scala_output": 36.2,
+            "sh_output": 37.2,
+            "swift_output": 41.4,
+            "ts_output": 43.0
+        },
+        "prompted": true,
+        "size": 14
+    },
+    "Meta-Llama-3-8B-Instruct-hf": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 37.0,
+            "cpp_input": 36.4,
+            "d_input": 35.0,
+            "go_input": 38.6,
+            "java_input": 36.2,
+            "js_input": 38.4,
+            "jl_input": 39.6,
+            "lua_input": 40.0,
+            "pl_input": 36.2,
+            "php_input": 36.6,
+            "py_input": 38.4,
+            "r_input": 42.2,
+            "rkt_input": 24.2,
+            "rb_input": 35.8,
+            "rs_input": 37.6,
+            "scala_input": 38.0,
+            "sh_input": 31.6,
+            "swift_input": 42.2,
+            "ts_input": 38.2,
+            "cs_output": 32.0,
+            "cpp_output": 30.8,
+            "d_output": 31.2,
+            "go_output": 31.4,
+            "java_output": 25.0,
+            "js_output": 35.0,
+            "jl_output": 31.4,
+            "lua_output": 34.0,
+            "pl_output": 29.6,
+            "php_output": 27.0,
+            "py_output": 33.6,
+            "r_output": 27.2,
+            "rkt_output": 28.0,
+            "rb_output": 31.8,
+            "rs_output": 34.4,
+            "scala_output": 33.8,
+            "sh_output": 32.0,
+            "swift_output": 36.4,
+            "ts_output": 33.8
+        },
+        "prompted": true,
+        "size": 8
+    },
+    "CodeQwen1.5-7B-Chat": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 42.8,
+            "cpp_input": 42.0,
+            "d_input": 43.0,
+            "go_input": 46.4,
+            "java_input": 44.6,
+            "js_input": 43.8,
+            "jl_input": 42.2,
+            "lua_input": 42.8,
+            "pl_input": 41.6,
+            "php_input": 44.8,
+            "py_input": 43.0,
+            "r_input": 43.4,
+            "rkt_input": 38.2,
+            "rb_input": 43.6,
+            "rs_input": 42.0,
+            "scala_input": 39.4,
+            "sh_input": 46.6,
+            "swift_input": 45.8,
+            "ts_input": 43.6,
+            "cs_output": 37.8,
+            "cpp_output": 40.2,
+            "d_output": 40.2,
+            "go_output": 40.6,
+            "java_output": 35.4,
+            "js_output": 43.6,
+            "jl_output": 42.6,
+            "lua_output": 40.4,
+            "pl_output": 39.6,
+            "php_output": 43.0,
+            "py_output": 41.4,
+            "r_output": 38.2,
+            "rkt_output": 39.0,
+            "rb_output": 44.6,
+            "rs_output": 42.0,
+            "scala_output": 35.0,
+            "sh_output": 38.2,
+            "swift_output": 43.8,
+            "ts_output": 42.2
+        },
+        "prompted": true,
+        "size": 7
+    },
+    "CodeLlama-7b-Instruct-hf": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 38.6,
+            "cpp_input": 36.0,
+            "d_input": 38.4,
+            "go_input": 38.4,
+            "java_input": 38.2,
+            "js_input": 39.6,
+            "jl_input": 42.2,
+            "lua_input": 43.4,
+            "pl_input": 36.4,
+            "php_input": 40.4,
+            "py_input": 41.0,
+            "r_input": 41.0,
+            "rkt_input": 38.8,
+            "rb_input": 41.6,
+            "rs_input": 37.6,
+            "scala_input": 42.6,
+            "sh_input": 39.6,
+            "swift_input": 40.2,
+            "ts_input": 41.0,
+            "cs_output": 32.2,
+            "cpp_output": 35.6,
+            "d_output": 34.4,
+            "go_output": 35.0,
+            "java_output": 24.4,
+            "js_output": 38.2,
+            "jl_output": 35.2,
+            "lua_output": 32.2,
+            "pl_output": 34.2,
+            "php_output": 36.0,
+            "py_output": 35.4,
+            "r_output": 32.0,
+            "rkt_output": 29.6,
+            "rb_output": 37.0,
+            "rs_output": 37.4,
+            "scala_output": 33.0,
+            "sh_output": 33.0,
+            "swift_output": 34.6,
+            "ts_output": 38.8
+        },
+        "prompted": true,
+        "size": 7
+    },
+    "CodeLlama-7b-hf": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 36.4,
+            "cpp_input": 36.2,
+            "d_input": 36.8,
+            "go_input": 34.6,
+            "java_input": 36.4,
+            "js_input": 36.6,
+            "jl_input": 40.2,
+            "lua_input": 39.6,
+            "pl_input": 36.0,
+            "php_input": 39.4,
+            "py_input": 40.2,
+            "r_input": 40.0,
+            "rkt_input": 36.6,
+            "rb_input": 39.2,
+            "rs_input": 35.4,
+            "scala_input": 37.8,
+            "sh_input": 36.8,
+            "swift_input": 39.2,
+            "ts_input": 38.8,
+            "cs_output": 32.6,
+            "cpp_output": 34.4,
+            "d_output": 33.8,
+            "go_output": 33.4,
+            "java_output": 28.4,
+            "js_output": 38.0,
+            "jl_output": 35.2,
+            "lua_output": 34.4,
+            "pl_output": 35.2,
+            "php_output": 38.0,
+            "py_output": 34.4,
+            "r_output": 32.6,
+            "rkt_output": 30.8,
+            "rb_output": 34.8,
+            "rs_output": 36.8,
+            "scala_output": 33.4,
+            "sh_output": 31.0,
+            "swift_output": 35.0,
+            "ts_output": 38.2
+        },
+        "prompted": true,
+        "size": 7
+    },
+    "deepseek-coder-6.7b-instruct": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 35.0,
+            "cpp_input": 37.0,
+            "d_input": 35.6,
+            "go_input": 40.4,
+            "java_input": 35.0,
+            "js_input": 36.6,
+            "jl_input": 39.2,
+            "lua_input": 38.8,
+            "pl_input": 39.4,
+            "php_input": 42.2,
+            "py_input": 38.2,
+            "r_input": 42.0,
+            "rkt_input": 37.2,
+            "rb_input": 40.2,
+            "rs_input": 37.4,
+            "scala_input": 36.8,
+            "sh_input": 42.8,
+            "swift_input": 40.8,
+            "ts_input": 34.2,
+            "cs_output": 34.8,
+            "cpp_output": 41.8,
+            "d_output": 40.4,
+            "go_output": 39.4,
+            "java_output": 32.8,
+            "js_output": 47.6,
+            "jl_output": 42.6,
+            "lua_output": 38.8,
+            "pl_output": 42.0,
+            "php_output": 43.8,
+            "py_output": 43.6,
+            "r_output": 40.8,
+            "rkt_output": 39.2,
+            "rb_output": 43.2,
+            "rs_output": 41.8,
+            "scala_output": 40.6,
+            "sh_output": 37.8,
+            "swift_output": 43.2,
+            "ts_output": 44.0
+        },
+        "prompted": true,
+        "size": 6.7
+    },
+    "deepseek-coder-6.7b-base": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 38.8,
+            "cpp_input": 42.4,
+            "d_input": 41.2,
+            "go_input": 43.2,
+            "java_input": 40.4,
+            "js_input": 43.6,
+            "jl_input": 42.6,
+            "lua_input": 42.8,
+            "pl_input": 41.6,
+            "php_input": 46.4,
+            "py_input": 41.4,
+            "r_input": 46.2,
+            "rkt_input": 43.0,
+            "rb_input": 44.6,
+            "rs_input": 41.6,
+            "scala_input": 40.8,
+            "sh_input": 44.8,
+            "swift_input": 43.4,
+            "ts_input": 41.8,
+            "cs_output": 41.2,
+            "cpp_output": 46.2,
+            "d_output": 43.2,
+            "go_output": 42.8,
+            "java_output": 42.6,
+            "js_output": 44.8,
+            "jl_output": 46.0,
+            "lua_output": 41.0,
+            "pl_output": 40.4,
+            "php_output": 41.8,
+            "py_output": 44.8,
+            "r_output": 42.8,
+            "rkt_output": 43.0,
+            "rb_output": 42.6,
+            "rs_output": 42.0,
+            "scala_output": 43.2,
+            "sh_output": 40.6,
+            "swift_output": 47.6,
+            "ts_output": 45.4
+        },
+        "prompted": true,
+        "size": 6.7
+    },
+    "codegen-6B-multi": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 28.8,
+            "cpp_input": 25.4,
+            "d_input": 6.2,
+            "go_input": 25.6,
+            "java_input": 36.2,
+            "js_input": 25.2,
+            "jl_input": 17.4,
+            "lua_input": 24.4,
+            "pl_input": 38.4,
+            "php_input": 22.8,
+            "py_input": 22.6,
+            "r_input": 27.2,
+            "rkt_input": 16.2,
+            "rb_input": 6.4,
+            "rs_input": 18.8,
+            "scala_input": 31.0,
+            "sh_input": 48.6,
+            "swift_input": 32.4,
+            "ts_input": 25.2,
+            "cs_output": 21.4,
+            "cpp_output": 23.6,
+            "d_output": 25.0,
+            "go_output": 26.4,
+            "java_output": 21.6,
+            "js_output": 22.8,
+            "jl_output": 22.8,
+            "lua_output": 23.8,
+            "pl_output": 20.4,
+            "php_output": 25.2,
+            "py_output": 24.8,
+            "r_output": 23.4,
+            "rkt_output": 17.8,
+            "rb_output": 24.0,
+            "rs_output": 25.2,
+            "scala_output": 22.0,
+            "sh_output": 22.2,
+            "swift_output": 25.0,
+            "ts_output": 21.4
+        },
+        "prompted": true,
+        "size": 6
+    },
+    "phi-1_5": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 29.2,
+            "cpp_input": 16.0,
+            "d_input": 13.2,
+            "go_input": 25.8,
+            "java_input": 26.8,
+            "js_input": 9.8,
+            "jl_input": 30.4,
+            "lua_input": 26.6,
+            "pl_input": 17.8,
+            "php_input": 26.6,
+            "py_input": 25.8,
+            "r_input": 8.4,
+            "rkt_input": 6.6,
+            "rb_input": 1.4,
+            "rs_input": 25.2,
+            "scala_input": 30.4,
+            "sh_input": 34.4,
+            "swift_input": 26.6,
+            "ts_input": 30.8,
+            "cs_output": 16.0,
+            "cpp_output": 26.0,
+            "d_output": 24.8,
+            "go_output": 22.6,
+            "java_output": 15.8,
+            "js_output": 23.0,
+            "jl_output": 23.6,
+            "lua_output": 21.2,
+            "pl_output": 22.0,
+            "php_output": 22.2,
+            "py_output": 25.6,
+            "r_output": 21.8,
+            "rkt_output": 16.8,
+            "rb_output": 19.6,
+            "rs_output": 22.0,
+            "scala_output": 21.6,
+            "sh_output": 17.6,
+            "swift_output": 25.6,
+            "ts_output": 25.2
+        },
+        "prompted": true,
+        "size": 1.3
+    },
+    "phi-1": {
+        "link": "",
+        "open-data": "NONE",
+        "pass@1": {
+            "cs_input": 0.2,
+            "cpp_input": 7.0,
+            "d_input": 9.6,
+            "go_input": 3.6,
+            "java_input": 2.8,
+            "js_input": 17.0,
+            "jl_input": 19.0,
+            "lua_input": 17.4,
+            "pl_input": 23.6,
+            "php_input": 9.2,
+            "py_input": 11.8,
+            "r_input": 9.4,
+            "rkt_input": 11.2,
+            "rb_input": 6.8,
+            "rs_input": 5.4,
+            "scala_input": 1.8,
+            "sh_input": 19.8,
+            "swift_input": 14.0,
+            "ts_input": 14.0,
+            "cs_output": 5.8,
+            "cpp_output": 9.0,
+            "d_output": 13.2,
+            "go_output": 14.8,
+            "java_output": 4.6,
+            "js_output": 20.8,
+            "jl_output": 19.2,
+            "lua_output": 15.8,
+            "pl_output": 15.6,
+            "php_output": 18.6,
+            "py_output": 22.4,
+            "r_output": 17.6,
+            "rkt_output": 10.4,
+            "rb_output": 18.0,
+            "rs_output": 16.4,
+            "scala_output": 11.0,
+            "sh_output": 16.4,
+            "swift_output": 19.2,
+            "ts_output": 19.0
+        },
+        "prompted": true,
+        "size": 1.3
+    }
+  }

text_content.py

@@ -0,0 +1,45 @@
+HEAD_TEXT = """
+Based on the CRUXEVAL-X benchmark, we evaluated the executing and reasoning ability of different LLMs in 19 different programing languages.
+            
+More details about how to evalute the LLM are available in the [CRUXEVAL-X GitHub repository](https://github.com/CRUXEVAL-X/cruxeval-x). For a complete description of CRUXEVAL-X benchmark and related experimental analysis, please refer to the paper: [CRUXEval-X: A Benchmark for Multilingual Code Reasoning, Understanding and Execution](https://arxiv.org/abs/2408.13001). [![](https://img.shields.io/badge/arXiv-2408.13001-b31b1b.svg)](https://arxiv.org/abs/2408.13001)
+**_Latest News_** 🔥
+- [24/08/26] We release our CRUXEVAL-X benchmark, leaderboard and paper.
+"""
+
+ABOUT_TEXT = """# What is CRUXEVAL-X benchmark?
+CRUXEVAL-X is a multilingual code reasoning, understanding and execution benchmark that focuses on code reasoning ability in different languages. 
+Its goal is to evaluate LLM's code reasoning (given input, reasoning output; and given output, reasoning input) ability. 
+# How to evaluate?
+To facilitate evaluation on the CRUXEVAL-X benchmark, we provide the evaluation data and easy-to-use evaluation scripts in our [CRUXEVAL-X GitHub repository](https://github.com/CRUXEVAL-X/cruxeval-x). 
+Additionally, factors involving execution-based evaluation are conducted in a virtual environment to ensure evaluation security.
+# Contact
+If you have any questions, feel free to reach out to us at [xuruiyang2022@iscas.ac.cn](mailto:xuruiyang2022@iscas.ac.cn).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+
+CITATION_BUTTON_TEXT = r"""
+@misc{xu2024cruxevalxbenchmarkmultilingualcode,
+      title={CRUXEval-X: A Benchmark for Multilingual Code Reasoning, Understanding and Execution}, 
+      author={Ruiyang Xu and Jialun Cao and Yaojie Lu and Hongyu Lin and Xianpei Han and Ben He and Shing-Chi Cheung and Le Sun},
+      year={2024},
+      eprint={2408.13001},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI},
+      url={https://arxiv.org/abs/2408.13001}, 
+}
+"""
+
+ACKNOWLEDGEMENT_TEXT = """
+Inspired from the [🤗 Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard).
+"""
+
+
+NOTES_TEXT = """
+**Notes:**
+- Evaluate using pass@1 as the evaluation metric.
+- `Average` denotes the average results of 19 different languages in a specific task.
+- you can choose differt tasks in `⏬ Tasks`, `input reasoning` denotes given output, reasoning input, `output reasoning` denotes given input, reasoning output.
+- `⏬ Languages` can choose languages you want to show in the leaderboard.
+- For more explanation check the 📝 About section.
+"""

utils.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+
+lang_map = {
+    "java": "Java",
+    "cpp": "C++",
+    "go": "Go",
+    "cs": "C#",
+    "d": "D",
+    "jl": "Julia",
+    "js": "JavaScript",
+    "php": "PHP",
+    "pl": "Perl",
+    "py": "Python",
+    "r": "R",
+    "lua": "Lua",
+    "rb": "Ruby",
+    "rkt": "Racket",
+    "rs": "Rust",
+    "scala": "Scala",
+    "sh": "Shell",
+    "swift": "Swift",
+    "ts": "TypeScript"
+}
+
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+
+
+def fields(raw_class):
+    return [
+        v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
+    ]
+
+
+@dataclass(frozen=True)
+class AutoEvalColumn:  # Auto evals column
+    model = ColumnContent("Model", "markdown", True)
+    size = ColumnContent("Size", "number", True)
+    average = ColumnContent("Average", "number", True)
+    l_0 = ColumnContent("Java", "number", True)
+    l_1 = ColumnContent("C++", "number", True)
+    l_2 = ColumnContent("C#", "number", True)
+    l_3 = ColumnContent("D", "number", True)
+    l_4 = ColumnContent("Go", "number", True)
+    l_5 = ColumnContent("Julia", "number", True)
+    l_6 = ColumnContent("JavaScript", "number", True)
+    l_7 = ColumnContent("Lua", "number", True)
+    l_8 = ColumnContent("PHP", "number", True)
+    l_9 = ColumnContent("Perl", "number", True)
+    l_10 = ColumnContent("Python", "number", True)
+    l_11 = ColumnContent("R", "number", True)
+    l_12 = ColumnContent("Ruby", "number", True)
+    l_13 = ColumnContent("Racket", "number", True)
+    l_14 = ColumnContent("Rust", "number", True)
+    l_15 = ColumnContent("Scala", "number", True)
+    l_16 = ColumnContent("Shell", "number", True)
+    l_17 = ColumnContent("Swift", "number", True)
+    l_18 = ColumnContent("TypeScript", "number", True)