refin

Dockerfile

@@ -9,16 +9,9 @@ RUN apt-get update && apt-get install -y \
     g++ \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy requirements first to leverage Docker cache
-COPY requirements.txt .
-
 # Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy application code
-COPY msr.py .
-COPY .env .env
-
 # Set environment variables
 ENV PYTHONUNBUFFERED=1
 

docker-compose.yml

@@ -10,12 +10,10 @@ services:
     env_file:
       - .env
     volumes:
+      # Mount entire workspace for live code updates
+      - .:/app
       # Mount gharchive data directory
       - ../gharchive/data:/gharchive/data:ro
-      # Persist DuckDB cache
-      - ./gharchive_cache.duckdb:/app/gharchive_cache.duckdb
-      # Persist logs
-      - ./logs:/app/logs
     environment:
       - PYTHONUNBUFFERED=1
     logging:

msr.py

@@ -137,7 +137,6 @@ def download_file(url):
 
     # Skip if json.gz already exists
     if os.path.exists(filepath):
-        print(f"   ✓ {filename} (already exists)")
         return True
 
     # Download with retry logic
@@ -147,7 +146,6 @@ def download_file(url):
             response.raise_for_status()
             with open(filepath, "wb") as f:
                 f.write(response.content)
-            print(f"   ✓ {filename} (downloaded)")
             return True
 
         except requests.exceptions.HTTPError as e:
@@ -194,10 +192,6 @@ def download_all_gharchive_data():
     Returns:
         bool: True if all downloads completed (some may have failed), False if critical error
     """
-    print(f"\n{'='*80}")
-    print(f"DOWNLOADING GHARCHIVE DATA")
-    print(f"{'='*80}")
-
     # Create data directory if it doesn't exist
     os.makedirs(GHARCHIVE_DATA_DIR, exist_ok=True)
 
@@ -215,11 +209,6 @@ def download_all_gharchive_data():
             urls.append(url)
         current_date += timedelta(days=1)
 
-    print(f"Downloading {len(urls)} files ({len(urls)//24} days × 24 hours)")
-    print(f"Workers: {DOWNLOAD_WORKERS}")
-    print(f"Target directory: {GHARCHIVE_DATA_DIR}")
-    print(f"{'='*80}\n")
-
     downloads_processed = 0
 
     try:
@@ -230,18 +219,12 @@ def download_all_gharchive_data():
             # Wait for downloads to complete
             for future in as_completed(futures):
                 downloads_processed += 1
-                if downloads_processed % 100 == 0:
-                    print(f"   Progress: {downloads_processed}/{len(urls)} files processed ({downloads_processed*100//len(urls)}%)")
 
-        print(f"\n{'='*80}")
-        print(f"Download complete: {downloads_processed}/{len(urls)} files processed")
-        print(f"{'='*80}\n")
+        print(f"Download complete: {downloads_processed} files")
         return True
 
     except Exception as e:
-        print(f"\n{'='*80}")
         print(f"Error during download: {str(e)}")
-        print(f"{'='*80}\n")
         import traceback
         traceback.print_exc()
         return False
@@ -435,9 +418,6 @@ def fetch_all_pr_metadata_single_query(conn, identifiers, start_date, end_date):
             ...
         }
     """
-    print(f"Querying DuckDB for ALL {len(identifiers)} agents in ONE QUERY")
-    print(f"   Time range: {start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}")
-
     # Generate file path patterns for review period
     review_patterns = generate_file_path_patterns(start_date, end_date)
 
@@ -530,13 +510,6 @@ def fetch_all_pr_metadata_single_query(conn, identifiers, start_date, end_date):
     ORDER BY re.reviewer, re.reviewed_at DESC
     """
 
-    # Calculate number of days for reporting
-    review_days = (end_date - start_date).days
-    status_days = (end_date - status_start_date).days
-
-    print(f"   Querying {review_days} days for reviews, {status_days} days for PR status...")
-    print(f"   Agents: {', '.join(identifiers[:5])}{'...' if len(identifiers) > 5 else ''}")
-
     try:
         # Create cache table name based on date range
         cache_table_name = f"pr_cache_{start_date.strftime('%Y%m%d')}_{end_date.strftime('%Y%m%d')}"
@@ -548,14 +521,12 @@ def fetch_all_pr_metadata_single_query(conn, identifiers, start_date, end_date):
         """).fetchone()[0] > 0
 
         if cache_exists:
-            print(f"   Using cached results from table {cache_table_name}")
             results = conn.execute(f"""
                 SELECT reviewer, url, reviewed_at, merged_at, closed_at
                 FROM {cache_table_name}
                 WHERE reviewer IN ({identifier_list})
             """).fetchall()
         else:
-            print(f"   Cache miss - executing full query and caching to {cache_table_name}")
             # Execute query with parameters
             results = conn.execute(query, {'review_patterns': review_patterns, 'status_patterns': status_patterns}).fetchall()
 
@@ -575,9 +546,6 @@ def fetch_all_pr_metadata_single_query(conn, identifiers, start_date, end_date):
                     [r[3] for r in results],
                     [r[4] for r in results]
                 ])
-                print(f"   Cached {len(results)} results to {cache_table_name}")
-
-        print(f"   Found {len(results)} total PR review records across all agents")
 
         # Group results by agent
         metadata_by_agent = defaultdict(list)
@@ -596,22 +564,11 @@ def fetch_all_pr_metadata_single_query(conn, identifiers, start_date, end_date):
                 'closed_at': closed_at,
             })
 
-        # Print breakdown by agent
-        print(f"Results breakdown by agent:")
-        for identifier in identifiers:
-            count = len(metadata_by_agent.get(identifier, []))
-            if count > 0:
-                metadata = metadata_by_agent[identifier]
-                merged_count = sum(1 for m in metadata if m['merged_at'] is not None)
-                closed_count = sum(1 for m in metadata if m['closed_at'] is not None and m['merged_at'] is None)
-                open_count = count - merged_count - closed_count
-                print(f"      {identifier}: {count} PRs ({merged_count} merged, {closed_count} closed, {open_count} open)")
-
         # Convert defaultdict to regular dict
         return dict(metadata_by_agent)
 
     except Exception as e:
-        print(f"   DuckDB error: {str(e)}")
+        print(f"DuckDB error: {str(e)}")
         import traceback
         traceback.print_exc()
         return {}
@@ -710,23 +667,17 @@ def batch_upload_review_metadata(all_metadata):
                 grouped = group_metadata_by_date(metadata_list)
                 total_files += len(grouped)
 
-        print(f"\n{'='*80}")
-        print(f"Starting batch upload: {len(all_metadata)} agents, {total_files} total files")
-        print(f"Upload delay: {UPLOAD_DELAY_SECONDS}s between files")
-        print(f"{'='*80}\n")
+        print(f"Uploading {total_files} files for {len(all_metadata)} agents...")
 
         file_count = 0
 
         for agent_idx, (agent_identifier, metadata_list) in enumerate(all_metadata.items(), 1):
             if not metadata_list:
-                print(f"[{agent_idx}/{len(all_metadata)}] Skipping {agent_identifier} (no data)")
                 continue
 
             # Group by date
             grouped = group_metadata_by_date(metadata_list)
 
-            print(f"[{agent_idx}/{len(all_metadata)}] Uploading {len(grouped)} files for {agent_identifier}...")
-
             # Create temporary files for this agent
             agent_temp_dir = tempfile.mkdtemp()
 
@@ -752,8 +703,6 @@ def batch_upload_review_metadata(all_metadata):
                 for file_idx, (local_path, repo_path, review_count) in enumerate(local_files, 1):
                     file_count += 1
 
-                    print(f"   [{file_count}/{total_files}] Uploading {repo_path} ({review_count} reviews)...", end='')
-
                     if upload_single_file_with_retry(
                         api=api,
                         local_path=local_path,
@@ -773,20 +722,16 @@ def batch_upload_review_metadata(all_metadata):
                     if file_idx < len(local_files):
                         time.sleep(UPLOAD_DELAY_SECONDS)
 
-                print(f"   Agent {agent_identifier}: {agent_success} uploaded, {agent_error} errors\n")
-
             finally:
                 # Clean up temp directory
                 if os.path.exists(agent_temp_dir):
                     import shutil
                     shutil.rmtree(agent_temp_dir)
 
-        print(f"\n{'='*80}")
-        print(f"Batch upload complete!")
-        print(f"   Total files: {total_files}")
-        print(f"   Successful: {success_count}")
-        print(f"   Errors: {error_count}")
-        print(f"{'='*80}\n")
+        if error_count > 0:
+            print(f"Upload complete: {success_count}/{total_files} succeeded, {error_count} errors")
+        else:
+            print(f"Upload complete: {success_count}/{total_files} files")
 
         return success_count, error_count
 
@@ -813,8 +758,6 @@ def load_agents_from_hf():
         # Filter for JSON files only
         json_files = [f for f in files if f.endswith('.json')]
 
-        print(f"Found {len(json_files)} agent files in {AGENTS_REPO}")
-
         # Download and parse each JSON file
         for json_file in json_files:
             try:
@@ -838,10 +781,9 @@ def load_agents_from_hf():
                     agents.append(agent_data)
 
             except Exception as e:
-                print(f"Warning: Could not load {json_file}: {str(e)}")
+                print(f"Error loading {json_file}: {str(e)}")
                 continue
 
-        print(f"Loaded {len(agents)} agents from HuggingFace")
         return agents
 
     except Exception as e:
@@ -1010,14 +952,10 @@ def construct_leaderboard_from_metadata(all_metadata_dict, agents):
     Returns:
         Dictionary of agent stats.
     """
-    print("Constructing leaderboard from review metadata...")
-
     if not agents:
-        print("No agents found")
+        print("Error: No agents found")
         return {}
 
-    print(f"Processing {len(agents)} agents")
-
     cache_dict = {}
 
     for agent in agents:
@@ -1037,8 +975,6 @@ def construct_leaderboard_from_metadata(all_metadata_dict, agents):
             **stats
         }
 
-    print(f"Constructed cache with {len(cache_dict)} agent entries")
-
     return cache_dict
 
 
@@ -1077,7 +1013,6 @@ def save_leaderboard_data_to_hf(leaderboard_dict, monthly_metrics):
 
         try:
             # Upload to HuggingFace with retry logic
-            print(f"Uploading leaderboard data...", end='')
             upload_file_with_backoff(
                 api=api,
                 path_or_fileobj=filename,
@@ -1085,7 +1020,6 @@ def save_leaderboard_data_to_hf(leaderboard_dict, monthly_metrics):
                 repo_id=LEADERBOARD_REPO,
                 repo_type="dataset"
             )
-            print(f"Saved leaderboard data to HuggingFace: {filename}")
             return True
         finally:
             # Always clean up local file
@@ -1109,35 +1043,26 @@ def mine_all_agents():
     Downloads GHArchive data first, then uses ONE DuckDB query for ALL agents, then batch uploads with time gaps.
     """
     # Step 1: Download GHArchive data
-    print(f"\n{'='*80}")
-    print(f"STEP 1: DOWNLOADING GHARCHIVE DATA")
-    print(f"{'='*80}\n")
+    print(f"\n[1/5] Downloading GHArchive data...")
 
     if not download_all_gharchive_data():
-        print("Warning: Download had errors, but continuing with available data...")
+        print("Warning: Download had errors, continuing with available data...")
 
     # Step 2: Load agent metadata from HuggingFace
-    print(f"\n{'='*80}")
-    print(f"STEP 2: LOADING AGENT METADATA")
-    print(f"{'='*80}\n")
+    print(f"\n[2/5] Loading agent metadata...")
 
     agents = load_agents_from_hf()
     if not agents:
-        print("No agents found in HuggingFace dataset")
+        print("Error: No agents found")
         return
 
     # Extract all identifiers
     identifiers = [agent['github_identifier'] for agent in agents if agent.get('github_identifier')]
     if not identifiers:
-        print("No valid agent identifiers found")
+        print("Error: No valid agent identifiers found")
         return
 
-    print(f"\n{'='*80}")
-    print(f"STEP 3: MINING REVIEW METADATA")
-    print(f"{'='*80}")
-    print(f"Agents: {len(identifiers)}")
-    print(f"Time frame: Last {LEADERBOARD_TIME_FRAME_DAYS} days")
-    print(f"{'='*80}")
+    print(f"\n[3/5] Mining review metadata ({len(identifiers)} agents, {LEADERBOARD_TIME_FRAME_DAYS} days)...")
 
     # Initialize DuckDB connection
     try:
@@ -1161,12 +1086,7 @@ def mine_all_agents():
         total_prs = sum(len(metadata_list) for metadata_list in all_metadata.values())
         agents_with_data = sum(1 for metadata_list in all_metadata.values() if metadata_list)
 
-        print(f"\n{'='*80}")
-        print(f"DuckDB query complete!")
-        print(f"   Total agents: {len(agents)}")
-        print(f"   Agents with data: {agents_with_data}")
-        print(f"   Total PRs found: {total_prs}")
-        print(f"{'='*80}")
+        print(f"Query complete: {total_prs} PRs found for {agents_with_data}/{len(agents)} agents")
 
     except Exception as e:
         print(f"Error during DuckDB fetch: {str(e)}")
@@ -1178,39 +1098,27 @@ def mine_all_agents():
         conn.close()
 
     # Step 4: Batch upload review metadata with time gaps
-    print(f"\n{'='*80}")
-    print(f"STEP 4: UPLOADING REVIEW METADATA")
-    print(f"{'='*80}\n")
+    print(f"\n[4/5] Uploading review metadata...")
 
     success_count, error_count = batch_upload_review_metadata(all_metadata)
 
     # Step 5: Construct and save leaderboard data
-    print(f"\n{'='*80}")
-    print(f"STEP 5: CONSTRUCTING AND SAVING LEADERBOARD")
-    print(f"{'='*80}\n")
+    print(f"\n[5/5] Saving leaderboard...")
 
     try:
         # Construct leaderboard from in-memory data
         leaderboard_dict = construct_leaderboard_from_metadata(all_metadata, agents)
 
         # Calculate monthly metrics from in-memory data
-        print(f"Calculating monthly metrics...")
         monthly_metrics = calculate_monthly_metrics_by_agent(all_metadata, agents)
 
         # Save to HuggingFace
-        print(f"Saving leaderboard data to HuggingFace...")
         save_leaderboard_data_to_hf(leaderboard_dict, monthly_metrics)
 
-        print(f"\n{'='*80}")
-        print(f"ALL TASKS COMPLETE!")
-        print(f"   Review metadata: {success_count} files uploaded, {error_count} errors")
-        print(f"   Leaderboard entries: {len(leaderboard_dict)}")
-        print(f"   Monthly data points: {len(monthly_metrics.get('months', []))} months")
-        print(f"   Saved to: {LEADERBOARD_REPO}/swe-review.json")
-        print(f"{'='*80}")
+        print(f"\nCOMPLETE: {success_count} files uploaded" + (f", {error_count} errors" if error_count > 0 else ""))
 
     except Exception as e:
-        print(f"Failed to construct/save leaderboard data: {str(e)}")
+        print(f"Error saving leaderboard: {str(e)}")
         import traceback
         traceback.print_exc()
 
@@ -1258,21 +1166,17 @@ def setup_scheduler():
     )
 
     # Print schedule information
-    print(f"{'='*80}")
-    print(f"SCHEDULER CONFIGURED")
-    print(f"{'='*80}")
-    print(f"Schedule: Monthly on day {SCHEDULE_DAY_OF_MONTH} at {SCHEDULE_HOUR:02d}:{SCHEDULE_MINUTE:02d} {SCHEDULE_TIMEZONE}")
-    print(f"Next run: {scheduler.get_jobs()[0].next_run_time}")
-    print(f"{'='*80}\n")
+    from datetime import datetime
+    next_run = trigger.get_next_fire_time(None, datetime.now(trigger.timezone))
+    print(f"Scheduler: Monthly on day {SCHEDULE_DAY_OF_MONTH} at {SCHEDULE_HOUR:02d}:{SCHEDULE_MINUTE:02d} {SCHEDULE_TIMEZONE}")
+    print(f"Next run: {next_run}\n")
 
     # Run immediately on startup
-    print("Running initial mining job on startup...")
+    print("Running initial mining job...")
     mine_all_agents()
 
     # Start scheduler (blocking call)
-    print(f"\n{'='*80}")
-    print("Starting scheduler... (Press Ctrl+C to exit)")
-    print(f"{'='*80}\n")
+    print(f"\nScheduler started (Press Ctrl+C to exit)")
 
     try:
         scheduler.start()