refine

msr.py

@@ -691,82 +691,6 @@ def load_agents_from_hf():
         return []
 
 
-def load_review_metadata():
-    """
-    Load all review metadata from HuggingFace dataset within LEADERBOARD_TIME_FRAME_DAYS.
-
-    Returns:
-        List of dictionaries with 'agent_identifier' added to each review metadata.
-    """
-    # Calculate cutoff date
-    current_time = datetime.now(timezone.utc)
-    cutoff_date = current_time - timedelta(days=LEADERBOARD_TIME_FRAME_DAYS)
-
-    try:
-        api = HfApi()
-        token = get_hf_token()
-
-        # List all files in the repository
-        files = list_repo_files_with_backoff(api=api, repo_id=REVIEW_METADATA_REPO, repo_type="dataset")
-
-        # Filter for JSONL files matching pattern: [agent_identifier]/YYYY.MM.DD.jsonl
-        time_frame_files = []
-        for f in files:
-            if f.endswith('.jsonl'):
-                parts = f.split('/')
-                if len(parts) == 2:
-                    filename = parts[1]
-                    # Parse date from filename: YYYY.MM.DD.jsonl
-                    try:
-                        date_part = filename.replace('.jsonl', '')
-                        date_components = date_part.split('.')
-                        if len(date_components) == 3:
-                            file_year, file_month, file_day = map(int, date_components)
-                            file_date = datetime(file_year, file_month, file_day, tzinfo=timezone.utc)
-
-                            # Only include files within time frame
-                            if file_date >= cutoff_date:
-                                time_frame_files.append(f)
-                    except Exception:
-                        continue
-
-        print(f"Loading review metadata from last {LEADERBOARD_TIME_FRAME_DAYS} days ({len(time_frame_files)} daily files)...")
-
-        all_metadata = []
-
-        for filename in time_frame_files:
-            try:
-                # Extract agent_identifier from path
-                parts = filename.split('/')
-                if len(parts) != 2:
-                    continue
-
-                agent_identifier = parts[0]
-
-                file_path = hf_hub_download_with_backoff(
-                    repo_id=REVIEW_METADATA_REPO,
-                    filename=filename,
-                    repo_type="dataset",
-                    token=token
-                )
-                day_metadata = load_jsonl(file_path)
-
-                # Add agent_identifier to each review
-                for review_meta in day_metadata:
-                    review_meta['agent_identifier'] = agent_identifier
-                    all_metadata.append(review_meta)
-
-            except Exception as e:
-                print(f"   Warning: Could not load {filename}: {str(e)}")
-
-        print(f"Loaded {len(all_metadata)} total reviews from last {LEADERBOARD_TIME_FRAME_DAYS} days")
-        return all_metadata
-
-    except Exception as e:
-        print(f"Error loading review metadata: {str(e)}")
-        return []
-
-
 def get_pr_status_from_metadata(review_meta):
     """
     Derive PR status from merged_at and closed_at fields.
@@ -818,10 +742,14 @@ def calculate_review_stats_from_metadata(metadata_list):
     }
 
 
-def calculate_monthly_metrics_by_agent():
+def calculate_monthly_metrics_by_agent(all_metadata_dict, agents):
     """
     Calculate monthly metrics for all agents for visualization.
 
+    Args:
+        all_metadata_dict: Dictionary mapping agent identifier to list of PR metadata
+        agents: List of agent dictionaries with metadata
+
     Returns:
         dict: {
             'agents': list of agent names,
@@ -835,38 +763,33 @@ def calculate_monthly_metrics_by_agent():
             }
         }
     """
-    # Load agents
-    agents = load_agents_from_hf()
-
     # Create mapping from agent_identifier to agent_name
     identifier_to_name = {agent.get('github_identifier'): agent.get('name') for agent in agents if agent.get('github_identifier')}
 
-    # Load all review metadata
-    all_metadata = load_review_metadata()
-
-    if not all_metadata:
+    if not all_metadata_dict:
         return {'agents': [], 'months': [], 'data': {}}
 
     # Group by agent and month
     agent_month_data = defaultdict(lambda: defaultdict(list))
 
-    for review_meta in all_metadata:
-        agent_identifier = review_meta.get('agent_identifier')
-        reviewed_at = review_meta.get('reviewed_at')
+    # Flatten the dict of lists into a single list with agent_identifier added
+    for agent_identifier, metadata_list in all_metadata_dict.items():
+        for review_meta in metadata_list:
+            reviewed_at = review_meta.get('reviewed_at')
 
-        if not agent_identifier or not reviewed_at:
-            continue
+            if not reviewed_at:
+                continue
 
-        # Get agent_name from identifier
-        agent_name = identifier_to_name.get(agent_identifier, agent_identifier)
+            # Get agent_name from identifier
+            agent_name = identifier_to_name.get(agent_identifier, agent_identifier)
 
-        try:
-            dt = datetime.fromisoformat(reviewed_at.replace('Z', '+00:00'))
-            month_key = f"{dt.year}-{dt.month:02d}"
-            agent_month_data[agent_name][month_key].append(review_meta)
-        except Exception as e:
-            print(f"Warning: Could not parse date '{reviewed_at}': {e}")
-            continue
+            try:
+                dt = datetime.fromisoformat(reviewed_at.replace('Z', '+00:00'))
+                month_key = f"{dt.year}-{dt.month:02d}"
+                agent_month_data[agent_name][month_key].append(review_meta)
+            except Exception as e:
+                print(f"Warning: Could not parse date '{reviewed_at}': {e}")
+                continue
 
     # Get all unique months and sort them
     all_months = set()
@@ -918,26 +841,24 @@ def calculate_monthly_metrics_by_agent():
     }
 
 
-def construct_leaderboard_from_metadata():
+def construct_leaderboard_from_metadata(all_metadata_dict, agents):
     """
-    Construct leaderboard from stored review metadata.
+    Construct leaderboard from in-memory review metadata.
+
+    Args:
+        all_metadata_dict: Dictionary mapping agent identifier to list of PR metadata
+        agents: List of agent dictionaries with metadata
 
     Returns:
         Dictionary of agent stats.
     """
     print("Constructing leaderboard from review metadata...")
 
-    # Load agents
-    agents = load_agents_from_hf()
     if not agents:
         print("No agents found")
         return {}
 
-    print(f"Loaded {len(agents)} agents")
-
-    # Load all review metadata
-    all_metadata = load_review_metadata()
-    print(f"Loaded {len(all_metadata)} review metadata entries")
+    print(f"Processing {len(agents)} agents")
 
     cache_dict = {}
 
@@ -945,8 +866,8 @@ def construct_leaderboard_from_metadata():
         identifier = agent.get('github_identifier')
         agent_name = agent.get('name', 'Unknown')
 
-        # Filter metadata for this agent
-        bot_metadata = [review for review in all_metadata if review.get("agent_identifier") == identifier]
+        # Get metadata for this agent from the dictionary
+        bot_metadata = all_metadata_dict.get(identifier, [])
 
         # Calculate stats
         stats = calculate_review_stats_from_metadata(bot_metadata)
@@ -1095,12 +1016,12 @@ def mine_all_agents():
     print(f"{'='*80}\n")
 
     try:
-        # Construct leaderboard
-        leaderboard_dict = construct_leaderboard_from_metadata()
+        # Construct leaderboard from in-memory data
+        leaderboard_dict = construct_leaderboard_from_metadata(all_metadata, agents)
 
-        # Calculate monthly metrics
+        # Calculate monthly metrics from in-memory data
         print(f"Calculating monthly metrics...")
-        monthly_metrics = calculate_monthly_metrics_by_agent()
+        monthly_metrics = calculate_monthly_metrics_by_agent(all_metadata, agents)
 
         # Save to HuggingFace
         print(f"Saving leaderboard data to HuggingFace...")