Upload folder using huggingface_hub

.gitattributes

@@ -60,3 +60,4 @@ llmtf_eval_k5_bs8/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval_k5_bs8/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval_k5_bs8/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval_k5_bs8/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text
+llmtf_eval_k5_bs1/daru_treewayabstractive.jsonl filter=lfs diff=lfs merge=lfs -text

llmtf_eval_k5_bs1/daru_treewayabstractive_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "openchat/openchat-3.5-0106",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 1,
+            "do_sample": true,
+            "eos_token_id": [
+                32000
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 512,
+            "pad_token_id": 32000,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "{role} {content}<|end_of_turn|>",
+            "user_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template_incomplete": "{role} {content}",
+            "user_role": "GPT4 Correct User:",
+            "bot_role": "GPT4 Correct Assistant:",
+            "system_role": "GPT4 Correct System:",
+            "global_prefix": "<s>",
+            "suffix": "GPT4 Correct Assistant:",
+            "add_special_tokens": false,
+            "eos_token": "<|end_of_turn|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 28705,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 1,
+        "max_sample_per_dataset": 500,
+        "method": "generate"
+    }
+}

llmtf_eval_k5_bs1/daru_treewayabstractive_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "daru/treewayabstractive",
+    "results": {
+        "rouge1": 0.357438599714093,
+        "rouge2": 0.13372912507444903
+    },
+    "leaderboard_result": 0.245583862394271
+}

llmtf_eval_k5_bs1/darumeru_cp_para_en_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "openchat/openchat-3.5-0106",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 1,
+            "do_sample": true,
+            "eos_token_id": [
+                32000
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1024,
+            "pad_token_id": 32000,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "{role} {content}<|end_of_turn|>",
+            "user_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template_incomplete": "{role} {content}",
+            "user_role": "GPT4 Correct User:",
+            "bot_role": "GPT4 Correct Assistant:",
+            "system_role": "GPT4 Correct System:",
+            "global_prefix": "<s>",
+            "suffix": "GPT4 Correct Assistant:",
+            "add_special_tokens": false,
+            "eos_token": "<|end_of_turn|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 28705,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k5_bs1/darumeru_cp_para_en_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_para_en",
+    "results": {
+        "symbol_per_token": 3.960763996832381,
+        "len": 0.9995281850843424,
+        "lcs": 0.9811766452032213
+    },
+    "leaderboard_result": 0.9811766452032213
+}

llmtf_eval_k5_bs1/darumeru_cp_para_ru_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "openchat/openchat-3.5-0106",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 1,
+            "do_sample": true,
+            "eos_token_id": [
+                32000
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1024,
+            "pad_token_id": 32000,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "{role} {content}<|end_of_turn|>",
+            "user_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template_incomplete": "{role} {content}",
+            "user_role": "GPT4 Correct User:",
+            "bot_role": "GPT4 Correct Assistant:",
+            "system_role": "GPT4 Correct System:",
+            "global_prefix": "<s>",
+            "suffix": "GPT4 Correct Assistant:",
+            "add_special_tokens": false,
+            "eos_token": "<|end_of_turn|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 28705,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

llmtf_eval_k5_bs1/darumeru_cp_para_ru_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_para_ru",
+    "results": {
+        "symbol_per_token": 2.4702341373518975,
+        "len": 0.9993717494721948,
+        "lcs": 0.958885193897962
+    },
+    "leaderboard_result": 0.958885193897962
+}

llmtf_eval_k5_bs1/evaluation_log.txt

@@ -251,3 +251,23 @@ INFO: 2024-07-13 16:31:58,534: llmtf.base.evaluator: Ended eval
 INFO: 2024-07-13 16:31:58,610: llmtf.base.evaluator: 
 mean	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
 0.627	0.404	0.533	0.830	0.507	0.608	0.122	1.000	0.999	0.487	0.754	0.540	0.876	0.659	0.528	0.553
+INFO: 2024-07-13 16:33:42,900: llmtf.base.daru/treewayabstractive: Processing Dataset: 2112.96s
+INFO: 2024-07-13 16:33:42,904: llmtf.base.daru/treewayabstractive: Results for daru/treewayabstractive:
+INFO: 2024-07-13 16:33:42,937: llmtf.base.daru/treewayabstractive: {'rouge1': 0.357438599714093, 'rouge2': 0.13372912507444903}
+INFO: 2024-07-13 16:33:42,941: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-13 16:33:42,951: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.603	0.246	0.404	0.533	0.830	0.507	0.608	0.122	1.000	0.999	0.487	0.754	0.540	0.876	0.659	0.528	0.553
+INFO: 2024-07-13 16:34:08,837: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 908.48s
+INFO: 2024-07-13 16:34:08,840: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
+INFO: 2024-07-13 16:34:08,857: llmtf.base.darumeru/cp_para_ru: {'symbol_per_token': 2.4702341373518975, 'len': 0.9993717494721948, 'lcs': 0.958885193897962}
+INFO: 2024-07-13 16:34:08,859: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [32000]
+INFO: 2024-07-13 16:34:08,859: llmtf.base.hfmodel: Updated generation_config.stop_strings: []
+INFO: 2024-07-13 16:34:11,992: llmtf.base.darumeru/cp_para_en: Loading Dataset: 3.13s
+INFO: 2024-07-13 16:45:47,059: llmtf.base.darumeru/cp_para_en: Processing Dataset: 695.07s
+INFO: 2024-07-13 16:45:47,066: llmtf.base.darumeru/cp_para_en: Results for darumeru/cp_para_en:
+INFO: 2024-07-13 16:45:47,099: llmtf.base.darumeru/cp_para_en: {'symbol_per_token': 3.960763996832381, 'len': 0.9995281850843424, 'lcs': 0.9811766452032213}
+INFO: 2024-07-13 16:45:47,100: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-13 16:45:47,126: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.644	0.246	0.404	0.533	0.830	0.507	0.608	0.122	0.981	0.959	1.000	0.999	0.487	0.754	0.540	0.876	0.659	0.528	0.553

llmtf_eval_k5_bs1/evaluation_results.txt

@@ -1,2 +1,2 @@
-mean	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
-0.627	0.404	0.533	0.830	0.507	0.608	0.122	1.000	0.999	0.487	0.754	0.540	0.876	0.659	0.528	0.553
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.644	0.246	0.404	0.533	0.830	0.507	0.608	0.122	0.981	0.959	1.000	0.999	0.487	0.754	0.540	0.876	0.659	0.528	0.553

llmtf_eval_k5_bs1/nlpcoreteam_ruMMLU_params.jsonl

@@ -0,0 +1,54 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "openchat/openchat-3.5-0106",
+        "generation_config": {
+            "_from_model_config": true,
+            "bos_token_id": 1,
+            "do_sample": true,
+            "eos_token_id": [
+                32000
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 32000,
+            "stop_strings": [],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2"
+        },
+        "conversation_template": {
+            "system_prompt": "",
+            "system_message_template": "{role} {content}<|end_of_turn|>",
+            "user_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template": "{role} {content}<|end_of_turn|>",
+            "bot_message_template_incomplete": "{role} {content}",
+            "user_role": "GPT4 Correct User:",
+            "bot_role": "GPT4 Correct Assistant:",
+            "system_role": "GPT4 Correct System:",
+            "global_prefix": "<s>",
+            "suffix": "GPT4 Correct Assistant:",
+            "add_special_tokens": false,
+            "eos_token": "<|end_of_turn|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 28705,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 1,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

llmtf_eval_k5_bs1/nlpcoreteam_ruMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "nlpcoreteam/ruMMLU",
+    "results": {
+        "acc": 0.5283401236619901
+    },
+    "leaderboard_result": 0.5283401236619901
+}