refine

msr.py

@@ -397,38 +397,53 @@ def fetch_all_pr_metadata_streaming(conn, identifiers, start_date, end_date):
         file_patterns_sql = '[' + ', '.join([f"'{fp}'" for fp in file_patterns]) + ']'
         
         # Query for this batch
+        # Note: GitHub Archive schema varies - older data has full pull_request objects,
+        # newer data (Oct 2025+) has stripped-down objects. We use TRY() to handle both.
         query = f"""
-        WITH pr_events AS (
+        WITH raw_events AS (
+            SELECT * FROM read_json(
+                {file_patterns_sql},
+                union_by_name=true,
+                filename=true,
+                compression='gzip',
+                format='newline_delimited',
+                ignore_errors=true,
+                maximum_object_size=2147483648
+            )
+            WHERE type = 'PullRequestEvent'
+                AND payload.action IN ('opened', 'closed')
+        ),
+        pr_events AS (
             SELECT
                 CONCAT(
                     REPLACE(repo.url, 'api.github.com/repos/', 'github.com/'),
                     '/pull/',
                     CAST(payload.pull_request.number AS VARCHAR)
                 ) as url,
-                payload.pull_request.user.login as pr_author, -- CHANGED
-                created_at as event_time,
                 payload.action as event_action,
-                payload.pull_request.merged_at as merged_at -- CHANGED
-            FROM read_json({file_patterns_sql}, union_by_name=true, filename=true, compression='gzip', format='newline_delimited', ignore_errors=true, maximum_object_size=2147483648)
-            WHERE
-                type = 'PullRequestEvent'
-                AND payload.action IN ('opened', 'closed')
-                AND payload.pull_request.number IS NOT NULL
-                AND payload.pull_request.user.login IN ({identifier_list}) -- CHANGED
+                CASE
+                    WHEN payload.action = 'opened' THEN actor.login
+                    ELSE NULL
+                END as pr_author,
+                created_at as event_time,
+                TRY_CAST(json_extract_string(to_json(payload), '$.pull_request.merged_at') AS VARCHAR) as merged_at
+            FROM raw_events
+            WHERE payload.pull_request.number IS NOT NULL
         ),
         pr_timeline AS (
             SELECT
                 url,
-                pr_author,
+                MAX(pr_author) as pr_author,
                 MIN(CASE WHEN event_action = 'opened' THEN event_time END) as created_at,
                 MAX(CASE WHEN event_action = 'closed' THEN event_time END) as closed_at,
                 MAX(merged_at) as merged_at
             FROM pr_events
-            GROUP BY url, pr_author
+            GROUP BY url
         )
         SELECT url, pr_author, created_at, merged_at, closed_at
         FROM pr_timeline
         WHERE created_at IS NOT NULL
+            AND pr_author IN ({identifier_list})
         """
         
         try: