Upload TransformerForCausalLM

README.md

@@ -8,4 +8,4 @@ license: mit
 datasets:
 - cerebras/SlimPajama-627B
 library_name: fla
----
+---

config.json

@@ -5,9 +5,11 @@
   ],
   "attention_bias": false,
   "bos_token_id": 1,
+  "elementwise_affine": true,
   "eos_token_id": 2,
   "fuse_cross_entropy": true,
   "fuse_norm": true,
+  "fuse_swiglu": true,
   "hidden_act": "swish",
   "hidden_ratio": 4,
   "hidden_size": 2048,
@@ -15,12 +17,16 @@
   "intermediate_size": null,
   "max_position_embeddings": 2048,
   "model_type": "transformer",
+  "norm_eps": 1e-06,
   "num_heads": 32,
   "num_hidden_layers": 24,
+  "num_kv_heads": null,
   "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.39.2",
+  "transformers_version": "4.48.2",
   "use_cache": true,
-  "vocab_size": 32000
+  "vocab_size": 32000,
+  "window_size": null
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.39.2"
+  "transformers_version": "4.48.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc9d7ff91e3c9c0d3ff9fa51db36f79b375d7b0565e4ef1a4a8e3c80be015d5b
-size 2728617032
+oid sha256:780b0295e9a6bd686c004e281bcec94130dc5a3e1d111ba440f2cd5f601fe790
+size 2728619696