support private submissions

app.py

@@ -288,6 +288,7 @@ def add_new_eval(
     mail: str,
     training_set: str,
     additional_info: str,
+    submission_visibility: str,
     profile: gr.OAuthProfile,
 ) -> str:
     """Add a new evaluation to the leaderboard."""
@@ -296,7 +297,7 @@ def add_new_eval(
         # Validate email address
         _, parsed_mail = parseaddr(mail)
         if "@" not in parsed_mail:
-            yield "⚠️ Please provide a valid email address."
+            yield "⚠️  Please provide a valid email address."
             return
 
         # Check monthly submission limit (5 submissions per month)
@@ -313,20 +314,20 @@ def add_new_eval(
         ]
 
         if len(current_month_submissions) >= 5:
-            yield f"⚠️ You have reached the monthly submission limit of 5 submissions. Please try again next month."
+            yield f"⚠️  You have reached the monthly submission limit of 5 submissions. Please try again next month."
             return
 
         # Validate file submission
         if path_to_file is None:
-            yield "⚠️ Please upload a file."
+            yield "⚠️  Please upload a file."
             return
 
         if not (path_to_file.endswith(".npz") or path_to_file.endswith(".json")):
-            yield "⚠️ Please submit a valid npz or json file"
+            yield "⚠️  Please submit a valid npz or json file"
             return
 
         # Evaluate the submission
-        yield "⚙️ Evaluating your submission...(do not close/refresh this page!)"
+        yield "⚙️  Evaluating your submission...(do not close/refresh this page!)"
         metrics = evaluate(
             leaderboard_data.target_paths[eval_type],
             path_to_file,
@@ -335,18 +336,7 @@ def add_new_eval(
 
         submission_time = datetime.today().strftime("%Y-%m-%d-%H:%M")
 
-        # Upload submission file
-        yield "☁️ Uploading submission file..."
-        api.upload_file(
-            repo_id=SUBMISSION_DATASET,
-            path_or_fileobj=path_to_file,
-            path_in_repo=f"{organization}/{model}/submissions/{training_set}/{contact_key}_{submission_time}_{os.path.basename(path_to_file)}",
-            repo_type="dataset",
-            token=TOKEN,
-        )
-
         # Update leaderboard data
-        yield "📋 Updating leaderboard data..."
         eval_results, _ = leaderboard_data.load_eval_data()
         eval_entry = {
             "Model": model,
@@ -367,20 +357,40 @@ def add_new_eval(
         else:
             eval_results[eval_type] = eval_results[eval_type].add_item(eval_entry)
 
-        data_file_name = leaderboard_data.result_paths[eval_type]
-
-        # Upload results
-        yield "💾 Saving results to database..."
-        with tempfile.NamedTemporaryFile(suffix=".parquet") as tmp_file:
-            eval_results[eval_type].to_parquet(tmp_file.name)
+        if submission_visibility == "Public":
+            # Upload submission file
+            yield "☁️  Uploading submission file..."
             api.upload_file(
-                repo_id=RESULTS_DATASET,
-                path_or_fileobj=tmp_file.name,
-                path_in_repo=f"data/{data_file_name}",
+                repo_id=SUBMISSION_DATASET,
+                path_or_fileobj=path_to_file,
+                path_in_repo=f"{organization}/{model}/submissions/{training_set}/{contact_key}_{submission_time}_{os.path.basename(path_to_file)}",
                 repo_type="dataset",
                 token=TOKEN,
             )
 
+            data_file_name = leaderboard_data.result_paths[eval_type]
+
+            # Upload results
+            yield "💾 Saving results to database..."
+            with tempfile.NamedTemporaryFile(suffix=".parquet") as tmp_file:
+                eval_results[eval_type].to_parquet(tmp_file.name)
+                api.upload_file(
+                    repo_id=RESULTS_DATASET,
+                    path_or_fileobj=tmp_file.name,
+                    path_in_repo=f"data/{data_file_name}",
+                    repo_type="dataset",
+                    token=TOKEN,
+                )
+        else:
+            eval_df = Dataset.from_dict({k: [v] for k, v in eval_entry.items()})
+            private_df = leaderboard_data._get_eval_df_from_results(
+                {eval_type: eval_df}, eval_type
+            )
+            private_df["Model"] = model
+            private_df.columns = [col.replace("\n", " ") for col in private_df.columns]
+            private_df.set_index("Model")
+            yield f"✅ Private evaluation complete. Please consider submitting publicly once you are comfortable with the results.\n{private_df}"
+
         # Save contact information
         contact_info = {
             "model": model,
@@ -401,8 +411,9 @@ def add_new_eval(
 
         leaderboard_data.contact_infos.push_to_hub(CONTACT_DATASET, token=TOKEN)
 
-        success_str = f"✅ Model {model} is successfully evaluated and stored in our database.\nPlease wait an hour and refresh the leaderboard to see your results displayed."
-        yield success_str
+        if submission_visibility == "Public":
+            success_str = f"✅ Model {model} is successfully evaluated and stored in our database.\nPlease wait an hour and refresh the leaderboard to see your results displayed."
+            yield success_str
     except Exception as e:
         print(f"Error during submission: {e}")
         yield (
@@ -599,11 +610,16 @@ def create_submission_interface() -> Tuple[gr.components.Component, ...]:
                         label="Eval Type",
                         interactive=True,
                     )
+                    submission_visibility = gr.Dropdown(
+                        label="Submission visibility", choices=["Public", "Private"]
+                    )
                     with gr.Column():
                         gr.LoginButton()
                     with gr.Column():
                         submit_button = gr.Button("Submit Eval")
-                submission_result = gr.Textbox(label="Status")
+                submission_result = gr.Textbox(
+                    label="Status", autoscroll=False, lines=10
+                )
 
     return (
         submit_button,
@@ -618,6 +634,7 @@ def create_submission_interface() -> Tuple[gr.components.Component, ...]:
         dataset,
         additional_info,
         submission_result,
+        submission_visibility,
     )
 
 
@@ -669,6 +686,7 @@ def create_interface() -> gr.Blocks:
             dataset,
             additional_info,
             submission_result,
+            submission_visibility,
         ) = create_submission_interface()
 
         submit_button.click(
@@ -684,6 +702,7 @@ def create_interface() -> gr.Blocks:
                 mail,
                 dataset,
                 additional_info,
+                submission_visibility,
             ],
             submission_result,
         )