Update app.py

app.py

@@ -1,7 +1,8 @@
 import json
 import os
+import re
 from functools import lru_cache
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Dict
 
 import gradio as gr
 from huggingface_hub import HfApi, hf_hub_download
@@ -161,10 +162,130 @@ def _filter_records(repo: str, pr: str, sha: str) -> List[dict]:
     records.sort(key=_sort_key, reverse=True)
     print(f"DEBUG: Returning {len(records)} records after filtering")
     return records[:MAX_ROWS]
-    return records[:MAX_ROWS]
 
 
-def query(repo: str, pr: str, sha: str) -> Tuple[List[List[str]], str, str]:
+def _generate_summary_tables(record: dict) -> Tuple[List[List[str]], List[List[str]]]:
+    """Generate by_test and by_model summary tables from a record."""
+    
+    # By test table
+    by_test_rows = []
+    by_test_data = record.get("by_test", {})
+    for test_name, test_info in by_test_data.items():
+        count = test_info.get("count", 0)
+        errors = test_info.get("errors", {})
+        # Format errors as "Nx error_message" for each unique error
+        error_list = [f"{cnt}× {err}" for err, cnt in errors.items()]
+        error_str = "; ".join(error_list)
+        by_test_rows.append([test_name, str(count), error_str])
+    
+    # By model table
+    by_model_rows = []
+    by_model_data = record.get("by_model", {})
+    for model_name, model_info in by_model_data.items():
+        count = model_info.get("count", 0)
+        errors = model_info.get("errors", {})
+        # Format errors as "Nx error_message" for each unique error
+        error_list = [f"{cnt}× {err}" for err, cnt in errors.items()]
+        error_str = "; ".join(error_list)
+        by_model_rows.append([model_name, str(count), error_str])
+    
+    return by_test_rows, by_model_rows
+
+
+def _generate_markdown_summary(record: dict) -> str:
+    """Generate markdown summary for copy-paste to GitHub."""
+    
+    md = "# Failure summary\n\n"
+    
+    # By test section
+    md += "## By test\n\n"
+    md += "| Test | Failures | Full error(s) |\n"
+    md += "| --- | --- | --- |\n"
+    
+    by_test_data = record.get("by_test", {})
+    for test_name, test_info in by_test_data.items():
+        count = test_info.get("count", 0)
+        errors = test_info.get("errors", {})
+        error_list = [f"{cnt}× {err}" for err, cnt in errors.items()]
+        error_str = "; ".join(error_list)
+        md += f"| {test_name} | {count} | {error_str} |\n"
+    
+    # By model section
+    md += "\n## By model\n\n"
+    md += "| Model | Failures | Full error(s) |\n"
+    md += "| --- | --- | --- |\n"
+    
+    by_model_data = record.get("by_model", {})
+    for model_name, model_info in by_model_data.items():
+        count = model_info.get("count", 0)
+        errors = model_info.get("errors", {})
+        error_list = [f"{cnt}× {err}" for err, cnt in errors.items()]
+        error_str = "; ".join(error_list)
+        md += f"| {model_name} | {count} | {error_str} |\n"
+    
+    return md
+
+
+def _generate_pytest_commands(record: dict) -> str:
+    """Generate helpful pytest commands based on the failures."""
+    
+    commands = []
+    
+    by_test_data = record.get("by_test", {})
+    by_model_data = record.get("by_model", {})
+    
+    # Add header
+    commands.append("# Helpful pytest commands\n")
+    
+    # Commands by test name pattern
+    if by_test_data:
+        commands.append("## Run specific test patterns:")
+        # Extract unique test name patterns (without the variants)
+        test_patterns = {}  # Use dict to preserve one example per pattern
+        
+        for test_name in by_test_data.keys():
+            if "::" in test_name:
+                parts = test_name.split("::")
+                if len(parts) >= 3:
+                    # Extract method name without variant suffix
+                    method = parts[2]
+                    # Remove _XX_ variant suffixes (like _00_fp16_pad_left_sdpa_kernels)
+                    method_base = re.sub(r'_\d+_.*$', '', method)
+                    
+                    # Store the pattern with the original test as example
+                    if method_base not in test_patterns:
+                        test_patterns[method_base] = test_name
+        
+        # Generate commands
+        for method_base in sorted(test_patterns.keys())[:5]:  # Limit to 5 examples
+            commands.append(f"```bash\npytest -k {method_base}\n```")
+        
+        # Add a note if there are more patterns
+        if len(test_patterns) > 5:
+            commands.append(f"\n*...and {len(test_patterns) - 5} more test patterns*")
+    
+    # Commands by model
+    if by_model_data:
+        commands.append("\n## Run tests for specific models:")
+        for model_name in sorted(by_model_data.keys())[:5]:  # Limit to 5 examples
+            commands.append(f"```bash\npytest tests/models/{model_name}/\n```")
+        
+        # Add a note if there are more models
+        if len(by_model_data) > 5:
+            commands.append(f"\n*...and {len(by_model_data) - 5} more models*")
+    
+    return "\n".join(commands)
+
+
+def query(repo: str, pr: str, sha: str) -> Tuple[
+    str,  # metadata_info
+    List[List[str]],  # by_test_table
+    List[List[str]],  # by_model_table
+    str,  # markdown_summary
+    str,  # pytest_commands
+    str,  # raw_json
+    str,  # status
+]:
     repo = repo.strip()
     pr = pr.strip()
     sha = sha.strip()
@@ -172,75 +293,180 @@ def query(repo: str, pr: str, sha: str) -> Tuple[List[List[str]], str, str]:
     print(f"DEBUG: Query called with repo='{repo}', pr='{pr}', sha='{sha}'")
 
     if not pr:
-        return [], json.dumps({"error": "PR number is required."}, indent=2), "Provide a PR number to search."
+        return (
+            "**Error:** PR number is required.",
+            [],
+            [],
+            "",
+            "",
+            json.dumps({"error": "PR number is required."}, indent=2),
+            "❌ Provide a PR number to search."
+        )
 
     records = _filter_records(repo, pr, sha)
     print(f"DEBUG: _filter_records returned {len(records)} records")
     
     if not records:
-        return [], json.dumps({"error": "No records found."}, indent=2), f"No records found for PR {pr}."
-
-    table_rows = []
-    for record in records:
-        metadata = record.get("metadata") or {}
-        table_rows.append(
-            [
-                metadata.get("collected_at", ""),
-                metadata.get("repository", ""),
-                metadata.get("branch", ""),
-                metadata.get("pull_request_number", ""),
-                (metadata.get("commit_sha") or "")[:12],
-                metadata.get("workflow_id", ""),
-                str(len(record.get("failures", []))),
-            ]
+        return (
+            f"**No records found** for PR {pr}.",
+            [],
+            [],
+            "",
+            "",
+            json.dumps({"error": "No records found."}, indent=2),
+            f"❌ No records found for PR {pr}."
         )
 
-    latest_payload = json.dumps(records[0], indent=2)
-    status = f"Showing {len(records)} record(s) for PR {pr}."
-    print(f"DEBUG: Returning {len(table_rows)} table rows")
-    return table_rows, latest_payload, status
+    # Use the latest record
+    latest_record = records[0]
+    metadata = latest_record.get("metadata", {})
+    
+    # Generate metadata info
+    metadata_lines = [
+        f"**Repository:** {metadata.get('repository', 'N/A')}",
+        f"**Branch:** {metadata.get('branch', 'N/A')}",
+        f"**PR:** #{metadata.get('pull_request_number', 'N/A')}",
+        f"**Commit:** `{metadata.get('commit_sha', 'N/A')[:12]}`",
+        f"**Workflow ID:** {metadata.get('workflow_id', 'N/A')}",
+        f"**Collected at:** {metadata.get('collected_at', 'N/A')}",
+        f"**Total failures:** {len(latest_record.get('failures', []))}",
+    ]
+    metadata_info = "\n\n".join(metadata_lines)
+    
+    # Generate tables
+    by_test_rows, by_model_rows = _generate_summary_tables(latest_record)
+    
+    # Generate markdown summary
+    markdown_summary = _generate_markdown_summary(latest_record)
+    
+    # Generate pytest commands
+    pytest_commands = _generate_pytest_commands(latest_record)
+    
+    # Raw JSON
+    raw_json = json.dumps(latest_record, indent=2)
+    
+    status = f"✅ Showing latest result from {len(records)} record(s) for PR {pr}."
+    
+    return (
+        metadata_info,
+        by_test_rows,
+        by_model_rows,
+        markdown_summary,
+        pytest_commands,
+        raw_json,
+        status
+    )
 
 
 def refresh_dataset() -> str:
     _list_collection_files.cache_clear()
-    return "Cleared cached manifest. Data will be reloaded on next search."
-
-
-with gr.Blocks() as demo:
+    return "✅ Cleared cached manifest. Data will be reloaded on next search."
+
+
+# Custom CSS for better styling
+custom_css = """
+.metadata-box {
+    background-color: #f6f8fa;
+    border: 1px solid #d0d7de;
+    border-radius: 6px;
+    padding: 16px;
+    margin: 8px 0;
+}
+.dataframe-container {
+    max-height: 500px;
+    overflow-y: auto;
+}
+"""
+
+with gr.Blocks(css=custom_css, title="CircleCI Test Results Viewer") as demo:
     gr.Markdown(
         """
-        # CircleCI Test Collection Helper
-
-        Use the filters below to inspect CircleCI test aggregation records for the Transformers repository (or any
-        repository that uploads data to the `transformers-community/circleci-test-results` dataset).
+        # 🔍 CircleCI Test Results Viewer
         
-        Files are named `failure_summary.json` and organized as `pr-{PR}/sha-{COMMIT}/failure_summary.json`.
+        Explore test failure summaries from the Transformers repository CI runs.
+        
+        **Quick start:** Enter a PR number and click Search to see the latest test failures.
         """
     )
 
     with gr.Row():
-        repo_box = gr.Textbox(label="Repository", placeholder="huggingface/transformers")
-        pr_box = gr.Textbox(label="PR number (required)")
-        sha_box = gr.Textbox(label="Commit SHA (prefix accepted)")
+        with gr.Column(scale=1):
+            repo_box = gr.Textbox(
+                label="Repository", 
+                placeholder="huggingface/transformers",
+                info="Optional: filter by repository name"
+            )
+        with gr.Column(scale=1):
+            pr_box = gr.Textbox(
+                label="PR Number", 
+                placeholder="42240",
+                info="Required: PR number to search"
+            )
+        with gr.Column(scale=1):
+            sha_box = gr.Textbox(
+                label="Commit SHA", 
+                placeholder="50947fc",
+                info="Optional: commit SHA prefix"
+            )
 
     with gr.Row():
-        search_btn = gr.Button("Search")
-        refresh_btn = gr.Button("Clear cache")
-
-    table = gr.Dataframe(
-        headers=[
-            "Collected at",
-            "Repository",
-            "Branch",
-            "PR",
-            "Commit",
-            "Workflow ID",
-            "Failures",
-        ],
-        wrap=True,
-    )
-    json_view = gr.Code(label="Latest entry details", language="json")
-    status = gr.Markdown("")
+        search_btn = gr.Button("🔎 Search", variant="primary", scale=2)
+        refresh_btn = gr.Button("🔄 Clear Cache", scale=1)
+
+    status_md = gr.Markdown("")
+    
+    with gr.Tabs() as tabs:
+        with gr.Tab("📊 Summary"):
+            metadata_box = gr.Markdown(label="Metadata", elem_classes=["metadata-box"])
+            
+            gr.Markdown("### 📝 By Test")
+            by_test_table = gr.Dataframe(
+                headers=["Test", "Failures", "Full error(s)"],
+                wrap=True,
+                interactive=False,
+                elem_classes=["dataframe-container"],
+            )
+            
+            gr.Markdown("### 🏷️ By Model")
+            by_model_table = gr.Dataframe(
+                headers=["Model", "Failures", "Full error(s)"],
+                wrap=True,
+                interactive=False,
+                elem_classes=["dataframe-container"],
+            )
+        
+        with gr.Tab("📋 Copy for GitHub"):
+            gr.Markdown(
+                """
+                Copy the markdown below to paste directly into a GitHub comment.
+                """
+            )
+            markdown_output = gr.Textbox(
+                label="Markdown Summary",
+                lines=20,
+                max_lines=30,
+                show_copy_button=True,
+            )
+        
+        with gr.Tab("🧪 Pytest Commands"):
+            gr.Markdown(
+                """
+                Helpful pytest commands to run specific failing tests locally.
+                """
+            )
+            pytest_output = gr.Markdown()
+        
+        with gr.Tab("🔧 Raw JSON"):
+            gr.Markdown(
+                """
+                Full JSON data for debugging or custom processing.
+                """
+            )
+            json_view = gr.Code(
+                label="Latest entry details", 
+                language="json",
+                lines=20,
+            )
 
     def get_url_params(request: gr.Request):
         """Get URL parameters from the request"""
@@ -261,16 +487,60 @@ with gr.Blocks() as demo:
             print(f"DEBUG: Auto-triggering search with repo={repo}, pr={pr}, sha={sha}")
             return query(repo, pr, sha)
         else:
-            return [], "", "No PR provided - enter parameters and click Search"
+            return (
+                "Enter a PR number and click Search",
+                [],
+                [],
+                "",
+                "",
+                "",
+                "💡 Enter a PR number above to get started"
+            )
+    
+    # Connect the search button
+    search_btn.click(
+        query, 
+        inputs=[repo_box, pr_box, sha_box], 
+        outputs=[
+            metadata_box,
+            by_test_table,
+            by_model_table,
+            markdown_output,
+            pytest_output,
+            json_view,
+            status_md
+        ]
+    )
     
-    search_btn.click(query, inputs=[repo_box, pr_box, sha_box], outputs=[table, json_view, status])
-    refresh_btn.click(refresh_dataset, outputs=status)
+    # Connect the refresh button
+    refresh_btn.click(refresh_dataset, outputs=status_md)
     
     # Load URL parameters when page loads, then auto-search if PR is present
-    demo.load(get_url_params, outputs=[repo_box, pr_box, sha_box]).then(
+    demo.load(
+        get_url_params, 
+        outputs=[repo_box, pr_box, sha_box]
+    ).then(
         auto_search_if_params, 
         inputs=[repo_box, pr_box, sha_box], 
-        outputs=[table, json_view, status]
+        outputs=[
+            metadata_box,
+            by_test_table,
+            by_model_table,
+            markdown_output,
+            pytest_output,
+            json_view,
+            status_md
+        ]
+    )
+
+    gr.Markdown(
+        """
+        ---
+        
+        **Data source:** [transformers-community/circleci-test-results](https://huggingface.co/datasets/transformers-community/circleci-test-results)
+        
+        Files are organized as `pr-{PR}/sha-{COMMIT}/failure_summary.json`
+        """
     )
 
 if __name__ == "__main__":