Spaces:
Sleeping
Sleeping
refine
Browse files
README.md
CHANGED
|
@@ -65,7 +65,7 @@ For each PR reviewed by an agent, we determine its status:
|
|
| 65 |
The leaderboard refreshes automatically every day at 12:00 AM UTC.
|
| 66 |
|
| 67 |
**Community Submissions**
|
| 68 |
-
Anyone can submit a coding agent to track via the leaderboard. We store agent metadata in Hugging Face datasets (`SWE-Arena/
|
| 69 |
|
| 70 |
## Using the Leaderboard
|
| 71 |
|
|
|
|
| 65 |
The leaderboard refreshes automatically every day at 12:00 AM UTC.
|
| 66 |
|
| 67 |
**Community Submissions**
|
| 68 |
+
Anyone can submit a coding agent to track via the leaderboard. We store agent metadata in Hugging Face datasets (`SWE-Arena/bot_metadata`) and review metadata in (`SWE-Arena/review_metadata`). The leaderboard is dynamically constructed from the review metadata. All submissions are automatically validated through GitHub's API to ensure the account exists and has public activity.
|
| 69 |
|
| 70 |
## Using the Leaderboard
|
| 71 |
|
app.py
CHANGED
|
@@ -28,7 +28,7 @@ load_dotenv()
|
|
| 28 |
# CONFIGURATION
|
| 29 |
# =============================================================================
|
| 30 |
|
| 31 |
-
AGENTS_REPO = "SWE-Arena/
|
| 32 |
REVIEW_METADATA_REPO = "SWE-Arena/review_metadata" # HuggingFace dataset for review metadata
|
| 33 |
LEADERBOARD_REPO = "SWE-Arena/leaderboard_metadata" # HuggingFace dataset for leaderboard data
|
| 34 |
LEADERBOARD_TIME_FRAME_DAYS = 180 # Time frame for constructing leaderboard
|
|
@@ -2176,10 +2176,10 @@ def construct_leaderboard_from_metadata():
|
|
| 2176 |
agent_name = agent.get('name', 'Unknown')
|
| 2177 |
|
| 2178 |
# Filter metadata for this agent
|
| 2179 |
-
|
| 2180 |
|
| 2181 |
# Calculate stats
|
| 2182 |
-
stats = calculate_review_stats_from_metadata(
|
| 2183 |
|
| 2184 |
cache_dict[identifier] = {
|
| 2185 |
'name': agent_name,
|
|
|
|
| 28 |
# CONFIGURATION
|
| 29 |
# =============================================================================
|
| 30 |
|
| 31 |
+
AGENTS_REPO = "SWE-Arena/bot_metadata" # HuggingFace dataset for agent metadata
|
| 32 |
REVIEW_METADATA_REPO = "SWE-Arena/review_metadata" # HuggingFace dataset for review metadata
|
| 33 |
LEADERBOARD_REPO = "SWE-Arena/leaderboard_metadata" # HuggingFace dataset for leaderboard data
|
| 34 |
LEADERBOARD_TIME_FRAME_DAYS = 180 # Time frame for constructing leaderboard
|
|
|
|
| 2176 |
agent_name = agent.get('name', 'Unknown')
|
| 2177 |
|
| 2178 |
# Filter metadata for this agent
|
| 2179 |
+
bot_metadata = [review for review in all_metadata if review.get("agent_identifier") == identifier]
|
| 2180 |
|
| 2181 |
# Calculate stats
|
| 2182 |
+
stats = calculate_review_stats_from_metadata(bot_metadata)
|
| 2183 |
|
| 2184 |
cache_dict[identifier] = {
|
| 2185 |
'name': agent_name,
|
msr.py
CHANGED
|
@@ -21,7 +21,7 @@ load_dotenv()
|
|
| 21 |
# CONFIGURATION
|
| 22 |
# =============================================================================
|
| 23 |
|
| 24 |
-
AGENTS_REPO = "SWE-Arena/
|
| 25 |
REVIEW_METADATA_REPO = "SWE-Arena/review_metadata"
|
| 26 |
LEADERBOARD_REPO = "SWE-Arena/leaderboard_metadata" # HuggingFace dataset for leaderboard data
|
| 27 |
LEADERBOARD_TIME_FRAME_DAYS = 180 # Time frame for leaderboard
|
|
@@ -919,10 +919,10 @@ def construct_leaderboard_from_metadata():
|
|
| 919 |
agent_name = agent.get('name', 'Unknown')
|
| 920 |
|
| 921 |
# Filter metadata for this agent
|
| 922 |
-
|
| 923 |
|
| 924 |
# Calculate stats
|
| 925 |
-
stats = calculate_review_stats_from_metadata(
|
| 926 |
|
| 927 |
cache_dict[identifier] = {
|
| 928 |
'name': agent_name,
|
|
|
|
| 21 |
# CONFIGURATION
|
| 22 |
# =============================================================================
|
| 23 |
|
| 24 |
+
AGENTS_REPO = "SWE-Arena/bot_metadata"
|
| 25 |
REVIEW_METADATA_REPO = "SWE-Arena/review_metadata"
|
| 26 |
LEADERBOARD_REPO = "SWE-Arena/leaderboard_metadata" # HuggingFace dataset for leaderboard data
|
| 27 |
LEADERBOARD_TIME_FRAME_DAYS = 180 # Time frame for leaderboard
|
|
|
|
| 919 |
agent_name = agent.get('name', 'Unknown')
|
| 920 |
|
| 921 |
# Filter metadata for this agent
|
| 922 |
+
bot_metadata = [review for review in all_metadata if review.get("agent_identifier") == identifier]
|
| 923 |
|
| 924 |
# Calculate stats
|
| 925 |
+
stats = calculate_review_stats_from_metadata(bot_metadata)
|
| 926 |
|
| 927 |
cache_dict[identifier] = {
|
| 928 |
'name': agent_name,
|