Request too large ... TPM 6000, Requested 6841 Reduce max_tokens for the judge

ragbench_eval/judge.py

@@ -41,7 +41,8 @@ class RAGJudge:
             },
             {"role": "user", "content": prompt},
         ]
-        raw = self.client.chat(messages, max_tokens=2048)
+        #raw = self.client.chat(messages, max_tokens=2048)
+        raw = self.client.chat(messages, max_tokens=512)
 
         try:
             data = json.loads(raw)

ragbench_eval/metrics.py

@@ -72,6 +72,7 @@ def compute_rmse_auc(
             roc_auc_score(y_true_adh, y_pred_adh)
         )
     else:
-        metrics["auroc_adherence"] = float("nan")
+        #metrics["auroc_adherence"] = float("nan")
+        metrics["auroc_adherence"] = 0.5  # or None, but not float("nan")
 
     return metrics