zhiminy commited on
Commit
d376fd8
·
1 Parent(s): f1724e4
Files changed (1) hide show
  1. app.py +38 -39
app.py CHANGED
@@ -29,7 +29,6 @@ UPDATE_INTERVAL = 86400 # 24 hours in seconds
29
  LEADERBOARD_COLUMNS = [
30
  ("Agent Name", "string"),
31
  ("Organization", "string"),
32
- ("GitHub Name", "string"),
33
  ("Total PRs", "number"),
34
  ("Merged PRs", "number"),
35
  ("Acceptance Rate (%)", "number"),
@@ -70,8 +69,8 @@ def save_jsonl(filename, data):
70
 
71
 
72
  def cache_to_dict(cache_list):
73
- """Convert list of cache entries to dictionary keyed by github_name."""
74
- return {entry['github_name']: entry for entry in cache_list}
75
 
76
 
77
  def dict_to_cache(cache_dict):
@@ -189,12 +188,12 @@ def get_github_token():
189
  return token
190
 
191
 
192
- def validate_github_username(github_name):
193
  """Verify that a GitHub username exists with backoff-aware requests."""
194
  try:
195
  token = get_github_token()
196
  headers = {'Authorization': f'token {token}'} if token else {}
197
- url = f'https://api.github.com/users/{github_name}'
198
  response = request_with_backoff('GET', url, headers=headers, max_retries=6)
199
  if response is None:
200
  return False, "Validation error: network/rate limit exhausted"
@@ -208,7 +207,7 @@ def validate_github_username(github_name):
208
  return False, f"Validation error: {str(e)}"
209
 
210
 
211
- def fetch_all_prs(github_name, token=None):
212
  """
213
  Fetch all pull requests authored by a GitHub user.
214
  Uses pagination to retrieve all results.
@@ -221,7 +220,7 @@ def fetch_all_prs(github_name, token=None):
221
  while True:
222
  url = f'https://api.github.com/search/issues'
223
  params = {
224
- 'q': f'is:pr author:{github_name}',
225
  'per_page': per_page,
226
  'page': page
227
  }
@@ -229,11 +228,11 @@ def fetch_all_prs(github_name, token=None):
229
  try:
230
  response = request_with_backoff('GET', url, headers=headers, params=params, max_retries=6)
231
  if response is None:
232
- print(f"Error fetching PRs for {github_name}: retries exhausted")
233
  break
234
 
235
  if response.status_code != 200:
236
- print(f"Error fetching PRs for {github_name}: HTTP {response.status_code}")
237
  break
238
 
239
  data = response.json()
@@ -252,7 +251,7 @@ def fetch_all_prs(github_name, token=None):
252
  time.sleep(0.5) # Courtesy delay between pages
253
 
254
  except Exception as e:
255
- print(f"Error fetching PRs for {github_name}: {str(e)}")
256
  break
257
 
258
  return all_prs
@@ -305,15 +304,15 @@ def calculate_pr_stats(prs):
305
  }
306
 
307
 
308
- def fetch_agent_stats(github_name, token=None):
309
  """
310
  Fetch and calculate PR statistics for a single agent.
311
  Returns dictionary with all stats and metadata.
312
  """
313
- print(f"Fetching data for {github_name}...")
314
- prs = fetch_all_prs(github_name, token)
315
  stats = calculate_pr_stats(prs)
316
- stats['github_name'] = github_name
317
  stats['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
318
  return stats
319
 
@@ -330,7 +329,7 @@ def load_submissions_dataset():
330
  for row in dataset:
331
  submissions.append({
332
  'agent_name': row.get('agent_name', 'Unknown'),
333
- 'github_name': row.get('github_name'),
334
  'organization': row.get('organization', 'Unknown'),
335
  'description': row.get('description', ''),
336
  })
@@ -377,12 +376,12 @@ def save_submission_to_hf(data):
377
  # Create timestamped filename
378
  timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
379
  year = datetime.now().year
380
- github_name = data['github_name']
381
- filename = f"{year}/{timestamp}_{github_name}.json"
382
 
383
  # Save locally first
384
  os.makedirs(str(year), exist_ok=True)
385
- filepath = f"{year}/{timestamp}_{github_name}.json"
386
 
387
  with open(filepath, 'w') as f:
388
  json.dump(data, f, indent=2)
@@ -468,19 +467,19 @@ def update_all_agents():
468
 
469
  # Update each agent
470
  for agent in agents:
471
- github_name = agent.get('github_name')
472
- if not github_name:
473
- print(f"Warning: Skipping agent without github_name: {agent}")
474
  continue
475
 
476
  try:
477
  # Fetch fresh PR statistics
478
- stats = fetch_agent_stats(github_name, token)
479
 
480
  # Merge metadata with stats
481
- cache_dict[github_name] = {
482
  'agent_name': agent.get('agent_name', 'Unknown'),
483
- 'github_name': github_name,
484
  'organization': agent.get('organization', 'Unknown'),
485
  'description': agent.get('description', ''),
486
  **stats
@@ -488,10 +487,10 @@ def update_all_agents():
488
 
489
  # Progressive save
490
  save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
491
- print(f"✓ Updated {github_name}")
492
 
493
  except Exception as e:
494
- print(f"✗ Error updating {github_name}: {str(e)}")
495
  continue
496
 
497
  return cache_dict
@@ -549,13 +548,13 @@ def get_leaderboard_dataframe():
549
  cache_dict = cache_to_dict(cache_list)
550
 
551
  rows = []
552
- for github_name, data in cache_dict.items():
553
  # Only include display-relevant fields
554
  # Normalize date formats for consistent display
555
  rows.append([
556
  data.get('agent_name', 'Unknown'),
557
  data.get('organization', 'Unknown'),
558
- github_name,
559
  data.get('total_prs', 0),
560
  data.get('merged', 0),
561
  data.get('acceptance_rate', 0.0),
@@ -594,13 +593,13 @@ def refresh_leaderboard():
594
  return error_msg, get_leaderboard_dataframe()
595
 
596
 
597
- def submit_agent(github_name, agent_name, organization, description):
598
  """
599
  Submit a new agent to the leaderboard.
600
  Validates input, saves submission, and fetches PR data.
601
  """
602
  # Validate required fields
603
- if not github_name or not github_name.strip():
604
  return "❌ GitHub username is required", get_leaderboard_dataframe()
605
  if not agent_name or not agent_name.strip():
606
  return "❌ Agent name is required", get_leaderboard_dataframe()
@@ -608,27 +607,27 @@ def submit_agent(github_name, agent_name, organization, description):
608
  return "❌ Organization name is required", get_leaderboard_dataframe()
609
 
610
  # Clean inputs
611
- github_name = github_name.strip()
612
  agent_name = agent_name.strip()
613
  organization = organization.strip()
614
  description = description.strip() if description else ""
615
 
616
  # Validate GitHub username
617
- is_valid, message = validate_github_username(github_name)
618
  if not is_valid:
619
  return f"❌ {message}", get_leaderboard_dataframe()
620
 
621
  # Check for duplicates
622
  agents = load_jsonl(AGENTS_FILE)
623
- existing_names = {agent['github_name'] for agent in agents}
624
 
625
- if github_name in existing_names:
626
- return f"⚠️ Agent with GitHub name '{github_name}' already exists", get_leaderboard_dataframe()
627
 
628
  # Create submission
629
  submission = {
630
  'agent_name': agent_name,
631
- 'github_name': github_name,
632
  'organization': organization,
633
  'description': description,
634
  }
@@ -644,12 +643,12 @@ def submit_agent(github_name, agent_name, organization, description):
644
  # Fetch PR data immediately
645
  token = get_github_token()
646
  try:
647
- stats = fetch_agent_stats(github_name, token)
648
 
649
  # Update cache
650
  cache_list = load_jsonl(CACHE_FILE)
651
  cache_dict = cache_to_dict(cache_list)
652
- cache_dict[github_name] = {**submission, **stats}
653
  save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
654
 
655
  # Save to HuggingFace
@@ -714,7 +713,7 @@ with gr.Blocks(title="SWE Agent PR Leaderboard", theme=gr.themes.Soft()) as app:
714
  leaderboard_table = Leaderboard(
715
  value=get_leaderboard_dataframe(),
716
  datatype=LEADERBOARD_COLUMNS,
717
- search_columns=["Agent Name", "Organization", "GitHub Name"],
718
  filter_columns=["Total PRs", "Acceptance Rate (%)"]
719
  )
720
 
 
29
  LEADERBOARD_COLUMNS = [
30
  ("Agent Name", "string"),
31
  ("Organization", "string"),
 
32
  ("Total PRs", "number"),
33
  ("Merged PRs", "number"),
34
  ("Acceptance Rate (%)", "number"),
 
69
 
70
 
71
  def cache_to_dict(cache_list):
72
+ """Convert list of cache entries to dictionary by identifier."""
73
+ return {entry['identifier']: entry for entry in cache_list}
74
 
75
 
76
  def dict_to_cache(cache_dict):
 
188
  return token
189
 
190
 
191
+ def validate_github_username(identifier):
192
  """Verify that a GitHub username exists with backoff-aware requests."""
193
  try:
194
  token = get_github_token()
195
  headers = {'Authorization': f'token {token}'} if token else {}
196
+ url = f'https://api.github.com/users/{identifier}'
197
  response = request_with_backoff('GET', url, headers=headers, max_retries=6)
198
  if response is None:
199
  return False, "Validation error: network/rate limit exhausted"
 
207
  return False, f"Validation error: {str(e)}"
208
 
209
 
210
+ def fetch_all_prs(identifier, token=None):
211
  """
212
  Fetch all pull requests authored by a GitHub user.
213
  Uses pagination to retrieve all results.
 
220
  while True:
221
  url = f'https://api.github.com/search/issues'
222
  params = {
223
+ 'q': f'is:pr author:{identifier}',
224
  'per_page': per_page,
225
  'page': page
226
  }
 
228
  try:
229
  response = request_with_backoff('GET', url, headers=headers, params=params, max_retries=6)
230
  if response is None:
231
+ print(f"Error fetching PRs for {identifier}: retries exhausted")
232
  break
233
 
234
  if response.status_code != 200:
235
+ print(f"Error fetching PRs for {identifier}: HTTP {response.status_code}")
236
  break
237
 
238
  data = response.json()
 
251
  time.sleep(0.5) # Courtesy delay between pages
252
 
253
  except Exception as e:
254
+ print(f"Error fetching PRs for {identifier}: {str(e)}")
255
  break
256
 
257
  return all_prs
 
304
  }
305
 
306
 
307
+ def fetch_agent_stats(identifier, token=None):
308
  """
309
  Fetch and calculate PR statistics for a single agent.
310
  Returns dictionary with all stats and metadata.
311
  """
312
+ print(f"Fetching data for {identifier}...")
313
+ prs = fetch_all_prs(identifier, token)
314
  stats = calculate_pr_stats(prs)
315
+ stats['identifier'] = identifier
316
  stats['last_updated'] = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
317
  return stats
318
 
 
329
  for row in dataset:
330
  submissions.append({
331
  'agent_name': row.get('agent_name', 'Unknown'),
332
+ 'identifier': row.get('identifier'),
333
  'organization': row.get('organization', 'Unknown'),
334
  'description': row.get('description', ''),
335
  })
 
376
  # Create timestamped filename
377
  timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
378
  year = datetime.now().year
379
+ identifier = data['identifier']
380
+ filename = f"{year}/{timestamp}_{identifier}.json"
381
 
382
  # Save locally first
383
  os.makedirs(str(year), exist_ok=True)
384
+ filepath = f"{year}/{timestamp}_{identifier}.json"
385
 
386
  with open(filepath, 'w') as f:
387
  json.dump(data, f, indent=2)
 
467
 
468
  # Update each agent
469
  for agent in agents:
470
+ identifier = agent.get('identifier')
471
+ if not identifier:
472
+ print(f"Warning: Skipping agent without identifier: {agent}")
473
  continue
474
 
475
  try:
476
  # Fetch fresh PR statistics
477
+ stats = fetch_agent_stats(identifier, token)
478
 
479
  # Merge metadata with stats
480
+ cache_dict[identifier] = {
481
  'agent_name': agent.get('agent_name', 'Unknown'),
482
+ 'identifier': identifier,
483
  'organization': agent.get('organization', 'Unknown'),
484
  'description': agent.get('description', ''),
485
  **stats
 
487
 
488
  # Progressive save
489
  save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
490
+ print(f"✓ Updated {identifier}")
491
 
492
  except Exception as e:
493
+ print(f"✗ Error updating {identifier}: {str(e)}")
494
  continue
495
 
496
  return cache_dict
 
548
  cache_dict = cache_to_dict(cache_list)
549
 
550
  rows = []
551
+ for identifier, data in cache_dict.items():
552
  # Only include display-relevant fields
553
  # Normalize date formats for consistent display
554
  rows.append([
555
  data.get('agent_name', 'Unknown'),
556
  data.get('organization', 'Unknown'),
557
+ identifier,
558
  data.get('total_prs', 0),
559
  data.get('merged', 0),
560
  data.get('acceptance_rate', 0.0),
 
593
  return error_msg, get_leaderboard_dataframe()
594
 
595
 
596
+ def submit_agent(identifier, agent_name, organization, description):
597
  """
598
  Submit a new agent to the leaderboard.
599
  Validates input, saves submission, and fetches PR data.
600
  """
601
  # Validate required fields
602
+ if not identifier or not identifier.strip():
603
  return "❌ GitHub username is required", get_leaderboard_dataframe()
604
  if not agent_name or not agent_name.strip():
605
  return "❌ Agent name is required", get_leaderboard_dataframe()
 
607
  return "❌ Organization name is required", get_leaderboard_dataframe()
608
 
609
  # Clean inputs
610
+ identifier = identifier.strip()
611
  agent_name = agent_name.strip()
612
  organization = organization.strip()
613
  description = description.strip() if description else ""
614
 
615
  # Validate GitHub username
616
+ is_valid, message = validate_github_username(identifier)
617
  if not is_valid:
618
  return f"❌ {message}", get_leaderboard_dataframe()
619
 
620
  # Check for duplicates
621
  agents = load_jsonl(AGENTS_FILE)
622
+ existing_names = {agent['identifier'] for agent in agents}
623
 
624
+ if identifier in existing_names:
625
+ return f"⚠️ Agent with identifier '{identifier}' already exists", get_leaderboard_dataframe()
626
 
627
  # Create submission
628
  submission = {
629
  'agent_name': agent_name,
630
+ 'identifier': identifier,
631
  'organization': organization,
632
  'description': description,
633
  }
 
643
  # Fetch PR data immediately
644
  token = get_github_token()
645
  try:
646
+ stats = fetch_agent_stats(identifier, token)
647
 
648
  # Update cache
649
  cache_list = load_jsonl(CACHE_FILE)
650
  cache_dict = cache_to_dict(cache_list)
651
+ cache_dict[identifier] = {**submission, **stats}
652
  save_jsonl(CACHE_FILE, dict_to_cache(cache_dict))
653
 
654
  # Save to HuggingFace
 
713
  leaderboard_table = Leaderboard(
714
  value=get_leaderboard_dataframe(),
715
  datatype=LEADERBOARD_COLUMNS,
716
+ search_columns=["Agent Name", "Organization"],
717
  filter_columns=["Total PRs", "Acceptance Rate (%)"]
718
  )
719