add
Browse files
app.py
CHANGED
|
@@ -29,7 +29,6 @@ UPDATE_INTERVAL = 86400 # 24 hours in seconds
|
|
| 29 |
LEADERBOARD_COLUMNS = [
|
| 30 |
("Agent Name", "string"),
|
| 31 |
("Organization", "string"),
|
| 32 |
-
("GitHub Name", "string"),
|
| 33 |
("Total PRs", "number"),
|
| 34 |
("Merged PRs", "number"),
|
| 35 |
("Acceptance Rate (%)", "number"),
|
|
@@ -70,8 +69,8 @@ def save_jsonl(filename, data):
|
|
| 70 |
|
| 71 |
|
| 72 |
def cache_to_dict(cache_list):
|
| 73 |
-
"""Convert list of cache entries to dictionary
|
| 74 |
-
return {entry['
|
| 75 |
|
| 76 |
|
| 77 |
def dict_to_cache(cache_dict):
|
|
@@ -189,12 +188,12 @@ def get_github_token():
|
|
| 189 |
return token
|
| 190 |
|
| 191 |
|
| 192 |
-
def validate_github_username(
|
| 193 |
"""Verify that a GitHub username exists with backoff-aware requests."""
|
| 194 |
try:
|
| 195 |
token = get_github_token()
|
| 196 |
headers = {'Authorization': f'token {token}'} if token else {}
|
| 197 |
-
url = f'https://api.github.com/users/{
|
| 198 |
response = request_with_backoff('GET', url, headers=headers, max_retries=6)
|
| 199 |
if response is None:
|
| 200 |
return False, "Validation error: network/rate limit exhausted"
|
|
@@ -208,7 +207,7 @@ def validate_github_username(github_name):
|
|
| 208 |
return False, f"Validation error: {str(e)}"
|
| 209 |
|
| 210 |
|
| 211 |
-
def fetch_all_prs(
|
| 212 |
"""
|
| 213 |
Fetch all pull requests authored by a GitHub user.
|
| 214 |
Uses pagination to retrieve all results.
|
|
@@ -221,7 +220,7 @@ def fetch_all_prs(github_name, token=None):
|
|
| 221 |
while True:
|
| 222 |
url = f'https://api.github.com/search/issues'
|
| 223 |
params = {
|
| 224 |
-
'q': f'is:pr author:{
|
| 225 |
'per_page': per_page,
|
| 226 |
'page': page
|
| 227 |
}
|
|
@@ -229,11 +228,11 @@ def fetch_all_prs(github_name, token=None):
|
|
| 229 |
try:
|
| 230 |
response = request_with_backoff('GET', url, headers=headers, params=params, max_retries=6)
|
| 231 |
if response is None:
|
| 232 |
-
print(f"Error fetching PRs for {
|
| 233 |
break
|
| 234 |
|
| 235 |
if response.status_code != 200:
|
| 236 |
-
print(f"Error fetching PRs for {
|
| 237 |
break
|
| 238 |
|
| 239 |
data = response.json()
|
|
@@ -252,7 +251,7 @@ def fetch_all_prs(github_name, token=None):
|
|
| 252 |
time.sleep(0.5) # Courtesy delay between pages
|
| 253 |
|
| 254 |
except Exception as e:
|
| 255 |
-
print(f"Error fetching PRs for {
|
| 256 |
break
|
| 257 |
|
| 258 |
return all_prs
|
|
@@ -305,15 +304,15 @@ def calculate_pr_stats(prs):
|
|
| 305 |
}
|
| 306 |
|
| 307 |
|
| 308 |
-
def fetch_agent_stats(
|
| 309 |
"""
|
| 310 |
Fetch and calculate PR statistics for a single agent.
|
| 311 |
Returns dictionary with all stats and metadata.
|
| 312 |
"""
|
| 313 |
-
print(f"Fetching data for {
|
| 314 |
-
prs = fetch_all_prs(
|
| 315 |
stats = calculate_pr_stats(prs)
|
| 316 |
-
stats['
|
| 317 |
stats['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
| 318 |
return stats
|
| 319 |
|
|
@@ -330,7 +329,7 @@ def load_submissions_dataset():
|
|
| 330 |
for row in dataset:
|
| 331 |
submissions.append({
|
| 332 |
'agent_name': row.get('agent_name', 'Unknown'),
|
| 333 |
-
'
|
| 334 |
'organization': row.get('organization', 'Unknown'),
|
| 335 |
'description': row.get('description', ''),
|
| 336 |
})
|
|
@@ -377,12 +376,12 @@ def save_submission_to_hf(data):
|
|
| 377 |
# Create timestamped filename
|
| 378 |
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 379 |
year = datetime.now().year
|
| 380 |
-
|
| 381 |
-
filename = f"{year}/{timestamp}_{
|
| 382 |
|
| 383 |
# Save locally first
|
| 384 |
os.makedirs(str(year), exist_ok=True)
|
| 385 |
-
filepath = f"{year}/{timestamp}_{
|
| 386 |
|
| 387 |
with open(filepath, 'w') as f:
|
| 388 |
json.dump(data, f, indent=2)
|
|
@@ -468,19 +467,19 @@ def update_all_agents():
|
|
| 468 |
|
| 469 |
# Update each agent
|
| 470 |
for agent in agents:
|
| 471 |
-
|
| 472 |
-
if not
|
| 473 |
-
print(f"Warning: Skipping agent without
|
| 474 |
continue
|
| 475 |
|
| 476 |
try:
|
| 477 |
# Fetch fresh PR statistics
|
| 478 |
-
stats = fetch_agent_stats(
|
| 479 |
|
| 480 |
# Merge metadata with stats
|
| 481 |
-
cache_dict[
|
| 482 |
'agent_name': agent.get('agent_name', 'Unknown'),
|
| 483 |
-
'
|
| 484 |
'organization': agent.get('organization', 'Unknown'),
|
| 485 |
'description': agent.get('description', ''),
|
| 486 |
**stats
|
|
@@ -488,10 +487,10 @@ def update_all_agents():
|
|
| 488 |
|
| 489 |
# Progressive save
|
| 490 |
save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
|
| 491 |
-
print(f"✓ Updated {
|
| 492 |
|
| 493 |
except Exception as e:
|
| 494 |
-
print(f"✗ Error updating {
|
| 495 |
continue
|
| 496 |
|
| 497 |
return cache_dict
|
|
@@ -549,13 +548,13 @@ def get_leaderboard_dataframe():
|
|
| 549 |
cache_dict = cache_to_dict(cache_list)
|
| 550 |
|
| 551 |
rows = []
|
| 552 |
-
for
|
| 553 |
# Only include display-relevant fields
|
| 554 |
# Normalize date formats for consistent display
|
| 555 |
rows.append([
|
| 556 |
data.get('agent_name', 'Unknown'),
|
| 557 |
data.get('organization', 'Unknown'),
|
| 558 |
-
|
| 559 |
data.get('total_prs', 0),
|
| 560 |
data.get('merged', 0),
|
| 561 |
data.get('acceptance_rate', 0.0),
|
|
@@ -594,13 +593,13 @@ def refresh_leaderboard():
|
|
| 594 |
return error_msg, get_leaderboard_dataframe()
|
| 595 |
|
| 596 |
|
| 597 |
-
def submit_agent(
|
| 598 |
"""
|
| 599 |
Submit a new agent to the leaderboard.
|
| 600 |
Validates input, saves submission, and fetches PR data.
|
| 601 |
"""
|
| 602 |
# Validate required fields
|
| 603 |
-
if not
|
| 604 |
return "❌ GitHub username is required", get_leaderboard_dataframe()
|
| 605 |
if not agent_name or not agent_name.strip():
|
| 606 |
return "❌ Agent name is required", get_leaderboard_dataframe()
|
|
@@ -608,27 +607,27 @@ def submit_agent(github_name, agent_name, organization, description):
|
|
| 608 |
return "❌ Organization name is required", get_leaderboard_dataframe()
|
| 609 |
|
| 610 |
# Clean inputs
|
| 611 |
-
|
| 612 |
agent_name = agent_name.strip()
|
| 613 |
organization = organization.strip()
|
| 614 |
description = description.strip() if description else ""
|
| 615 |
|
| 616 |
# Validate GitHub username
|
| 617 |
-
is_valid, message = validate_github_username(
|
| 618 |
if not is_valid:
|
| 619 |
return f"❌ {message}", get_leaderboard_dataframe()
|
| 620 |
|
| 621 |
# Check for duplicates
|
| 622 |
agents = load_jsonl(AGENTS_FILE)
|
| 623 |
-
existing_names = {agent['
|
| 624 |
|
| 625 |
-
if
|
| 626 |
-
return f"⚠️ Agent with
|
| 627 |
|
| 628 |
# Create submission
|
| 629 |
submission = {
|
| 630 |
'agent_name': agent_name,
|
| 631 |
-
'
|
| 632 |
'organization': organization,
|
| 633 |
'description': description,
|
| 634 |
}
|
|
@@ -644,12 +643,12 @@ def submit_agent(github_name, agent_name, organization, description):
|
|
| 644 |
# Fetch PR data immediately
|
| 645 |
token = get_github_token()
|
| 646 |
try:
|
| 647 |
-
stats = fetch_agent_stats(
|
| 648 |
|
| 649 |
# Update cache
|
| 650 |
cache_list = load_jsonl(CACHE_FILE)
|
| 651 |
cache_dict = cache_to_dict(cache_list)
|
| 652 |
-
cache_dict[
|
| 653 |
save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
|
| 654 |
|
| 655 |
# Save to HuggingFace
|
|
@@ -714,7 +713,7 @@ with gr.Blocks(title="SWE Agent PR Leaderboard", theme=gr.themes.Soft()) as app:
|
|
| 714 |
leaderboard_table = Leaderboard(
|
| 715 |
value=get_leaderboard_dataframe(),
|
| 716 |
datatype=LEADERBOARD_COLUMNS,
|
| 717 |
-
search_columns=["Agent Name", "Organization"
|
| 718 |
filter_columns=["Total PRs", "Acceptance Rate (%)"]
|
| 719 |
)
|
| 720 |
|
|
|
|
| 29 |
LEADERBOARD_COLUMNS = [
|
| 30 |
("Agent Name", "string"),
|
| 31 |
("Organization", "string"),
|
|
|
|
| 32 |
("Total PRs", "number"),
|
| 33 |
("Merged PRs", "number"),
|
| 34 |
("Acceptance Rate (%)", "number"),
|
|
|
|
| 69 |
|
| 70 |
|
| 71 |
def cache_to_dict(cache_list):
|
| 72 |
+
"""Convert list of cache entries to dictionary by identifier."""
|
| 73 |
+
return {entry['identifier']: entry for entry in cache_list}
|
| 74 |
|
| 75 |
|
| 76 |
def dict_to_cache(cache_dict):
|
|
|
|
| 188 |
return token
|
| 189 |
|
| 190 |
|
| 191 |
+
def validate_github_username(identifier):
|
| 192 |
"""Verify that a GitHub username exists with backoff-aware requests."""
|
| 193 |
try:
|
| 194 |
token = get_github_token()
|
| 195 |
headers = {'Authorization': f'token {token}'} if token else {}
|
| 196 |
+
url = f'https://api.github.com/users/{identifier}'
|
| 197 |
response = request_with_backoff('GET', url, headers=headers, max_retries=6)
|
| 198 |
if response is None:
|
| 199 |
return False, "Validation error: network/rate limit exhausted"
|
|
|
|
| 207 |
return False, f"Validation error: {str(e)}"
|
| 208 |
|
| 209 |
|
| 210 |
+
def fetch_all_prs(identifier, token=None):
|
| 211 |
"""
|
| 212 |
Fetch all pull requests authored by a GitHub user.
|
| 213 |
Uses pagination to retrieve all results.
|
|
|
|
| 220 |
while True:
|
| 221 |
url = f'https://api.github.com/search/issues'
|
| 222 |
params = {
|
| 223 |
+
'q': f'is:pr author:{identifier}',
|
| 224 |
'per_page': per_page,
|
| 225 |
'page': page
|
| 226 |
}
|
|
|
|
| 228 |
try:
|
| 229 |
response = request_with_backoff('GET', url, headers=headers, params=params, max_retries=6)
|
| 230 |
if response is None:
|
| 231 |
+
print(f"Error fetching PRs for {identifier}: retries exhausted")
|
| 232 |
break
|
| 233 |
|
| 234 |
if response.status_code != 200:
|
| 235 |
+
print(f"Error fetching PRs for {identifier}: HTTP {response.status_code}")
|
| 236 |
break
|
| 237 |
|
| 238 |
data = response.json()
|
|
|
|
| 251 |
time.sleep(0.5) # Courtesy delay between pages
|
| 252 |
|
| 253 |
except Exception as e:
|
| 254 |
+
print(f"Error fetching PRs for {identifier}: {str(e)}")
|
| 255 |
break
|
| 256 |
|
| 257 |
return all_prs
|
|
|
|
| 304 |
}
|
| 305 |
|
| 306 |
|
| 307 |
+
def fetch_agent_stats(identifier, token=None):
|
| 308 |
"""
|
| 309 |
Fetch and calculate PR statistics for a single agent.
|
| 310 |
Returns dictionary with all stats and metadata.
|
| 311 |
"""
|
| 312 |
+
print(f"Fetching data for {identifier}...")
|
| 313 |
+
prs = fetch_all_prs(identifier, token)
|
| 314 |
stats = calculate_pr_stats(prs)
|
| 315 |
+
stats['identifier'] = identifier
|
| 316 |
stats['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
| 317 |
return stats
|
| 318 |
|
|
|
|
| 329 |
for row in dataset:
|
| 330 |
submissions.append({
|
| 331 |
'agent_name': row.get('agent_name', 'Unknown'),
|
| 332 |
+
'identifier': row.get('identifier'),
|
| 333 |
'organization': row.get('organization', 'Unknown'),
|
| 334 |
'description': row.get('description', ''),
|
| 335 |
})
|
|
|
|
| 376 |
# Create timestamped filename
|
| 377 |
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 378 |
year = datetime.now().year
|
| 379 |
+
identifier = data['identifier']
|
| 380 |
+
filename = f"{year}/{timestamp}_{identifier}.json"
|
| 381 |
|
| 382 |
# Save locally first
|
| 383 |
os.makedirs(str(year), exist_ok=True)
|
| 384 |
+
filepath = f"{year}/{timestamp}_{identifier}.json"
|
| 385 |
|
| 386 |
with open(filepath, 'w') as f:
|
| 387 |
json.dump(data, f, indent=2)
|
|
|
|
| 467 |
|
| 468 |
# Update each agent
|
| 469 |
for agent in agents:
|
| 470 |
+
identifier = agent.get('identifier')
|
| 471 |
+
if not identifier:
|
| 472 |
+
print(f"Warning: Skipping agent without identifier: {agent}")
|
| 473 |
continue
|
| 474 |
|
| 475 |
try:
|
| 476 |
# Fetch fresh PR statistics
|
| 477 |
+
stats = fetch_agent_stats(identifier, token)
|
| 478 |
|
| 479 |
# Merge metadata with stats
|
| 480 |
+
cache_dict[identifier] = {
|
| 481 |
'agent_name': agent.get('agent_name', 'Unknown'),
|
| 482 |
+
'identifier': identifier,
|
| 483 |
'organization': agent.get('organization', 'Unknown'),
|
| 484 |
'description': agent.get('description', ''),
|
| 485 |
**stats
|
|
|
|
| 487 |
|
| 488 |
# Progressive save
|
| 489 |
save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
|
| 490 |
+
print(f"✓ Updated {identifier}")
|
| 491 |
|
| 492 |
except Exception as e:
|
| 493 |
+
print(f"✗ Error updating {identifier}: {str(e)}")
|
| 494 |
continue
|
| 495 |
|
| 496 |
return cache_dict
|
|
|
|
| 548 |
cache_dict = cache_to_dict(cache_list)
|
| 549 |
|
| 550 |
rows = []
|
| 551 |
+
for identifier, data in cache_dict.items():
|
| 552 |
# Only include display-relevant fields
|
| 553 |
# Normalize date formats for consistent display
|
| 554 |
rows.append([
|
| 555 |
data.get('agent_name', 'Unknown'),
|
| 556 |
data.get('organization', 'Unknown'),
|
| 557 |
+
identifier,
|
| 558 |
data.get('total_prs', 0),
|
| 559 |
data.get('merged', 0),
|
| 560 |
data.get('acceptance_rate', 0.0),
|
|
|
|
| 593 |
return error_msg, get_leaderboard_dataframe()
|
| 594 |
|
| 595 |
|
| 596 |
+
def submit_agent(identifier, agent_name, organization, description):
|
| 597 |
"""
|
| 598 |
Submit a new agent to the leaderboard.
|
| 599 |
Validates input, saves submission, and fetches PR data.
|
| 600 |
"""
|
| 601 |
# Validate required fields
|
| 602 |
+
if not identifier or not identifier.strip():
|
| 603 |
return "❌ GitHub username is required", get_leaderboard_dataframe()
|
| 604 |
if not agent_name or not agent_name.strip():
|
| 605 |
return "❌ Agent name is required", get_leaderboard_dataframe()
|
|
|
|
| 607 |
return "❌ Organization name is required", get_leaderboard_dataframe()
|
| 608 |
|
| 609 |
# Clean inputs
|
| 610 |
+
identifier = identifier.strip()
|
| 611 |
agent_name = agent_name.strip()
|
| 612 |
organization = organization.strip()
|
| 613 |
description = description.strip() if description else ""
|
| 614 |
|
| 615 |
# Validate GitHub username
|
| 616 |
+
is_valid, message = validate_github_username(identifier)
|
| 617 |
if not is_valid:
|
| 618 |
return f"❌ {message}", get_leaderboard_dataframe()
|
| 619 |
|
| 620 |
# Check for duplicates
|
| 621 |
agents = load_jsonl(AGENTS_FILE)
|
| 622 |
+
existing_names = {agent['identifier'] for agent in agents}
|
| 623 |
|
| 624 |
+
if identifier in existing_names:
|
| 625 |
+
return f"⚠️ Agent with identifier '{identifier}' already exists", get_leaderboard_dataframe()
|
| 626 |
|
| 627 |
# Create submission
|
| 628 |
submission = {
|
| 629 |
'agent_name': agent_name,
|
| 630 |
+
'identifier': identifier,
|
| 631 |
'organization': organization,
|
| 632 |
'description': description,
|
| 633 |
}
|
|
|
|
| 643 |
# Fetch PR data immediately
|
| 644 |
token = get_github_token()
|
| 645 |
try:
|
| 646 |
+
stats = fetch_agent_stats(identifier, token)
|
| 647 |
|
| 648 |
# Update cache
|
| 649 |
cache_list = load_jsonl(CACHE_FILE)
|
| 650 |
cache_dict = cache_to_dict(cache_list)
|
| 651 |
+
cache_dict[identifier] = {**submission, **stats}
|
| 652 |
save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
|
| 653 |
|
| 654 |
# Save to HuggingFace
|
|
|
|
| 713 |
leaderboard_table = Leaderboard(
|
| 714 |
value=get_leaderboard_dataframe(),
|
| 715 |
datatype=LEADERBOARD_COLUMNS,
|
| 716 |
+
search_columns=["Agent Name", "Organization"],
|
| 717 |
filter_columns=["Total PRs", "Acceptance Rate (%)"]
|
| 718 |
)
|
| 719 |
|