Add files using upload-large-folder tool

config.json

@@ -94,7 +94,7 @@
         },
         "output_activations": null,
         "targets": [
-          "re:.*mlp\\..*"
+          "re:.*mlp\\.(?!shared_experts).*"
         ],
         "weights": {
           "actorder": null,
@@ -110,6 +110,44 @@
           "type": "float",
           "zp_dtype": null
         }
+      },
+      "group_2": {
+        "format": "float-quantized",
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": true,
+          "group_size": 128,
+          "num_bits": 8,
+          "observer": null,
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "group",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        },
+        "output_activations": null,
+        "targets": [
+          "re:.*shared_experts.*"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": [
+            128,
+            128
+          ],
+          "dynamic": false,
+          "group_size": null,
+          "num_bits": 8,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": null,
+          "strategy": "block",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        }
       }
     },
     "format": "mixed-precision",
@@ -463,4 +501,4 @@
     "use_vision_norm": true,
     "vision_encoder_type": "hmlp"
   }
-}
+}

model-00033-of-00033.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f09547eecf7b6944dec2a2ec4bd2d56953fdd80c7f0ee492ef0282ea82120f20
+size 7248678304

model.safetensors.index.json

@@ -91,8 +91,6 @@
     "model.llm.layers.10.mlp.gate.bias": "model-00001-of-00032.safetensors",
     "model.llm.layers.10.mlp.gate.global_scale": "model-00001-of-00032.safetensors",
     "model.llm.layers.10.mlp.gate.weight": "model-00001-of-00032.safetensors",
-    "model.llm.layers.10.mlp.shared_experts.shared_w13_weight": "model-00001-of-00032.safetensors",
-    "model.llm.layers.10.mlp.shared_experts.shared_w2_weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.10.mlp_norm.weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.10.mlp_sconv.weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.11.attn.k_norm.weight": "model-00001-of-00032.safetensors",
@@ -123,8 +121,6 @@
     "model.llm.layers.11.mlp.gate.bias": "model-00001-of-00032.safetensors",
     "model.llm.layers.11.mlp.gate.global_scale": "model-00001-of-00032.safetensors",
     "model.llm.layers.11.mlp.gate.weight": "model-00001-of-00032.safetensors",
-    "model.llm.layers.11.mlp.shared_experts.shared_w13_weight": "model-00001-of-00032.safetensors",
-    "model.llm.layers.11.mlp.shared_experts.shared_w2_weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.11.mlp_norm.weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.11.mlp_sconv.weight": "model-00001-of-00032.safetensors",
     "model.llm.layers.12.attn.k_norm.weight": "model-00001-of-00032.safetensors",
@@ -155,8 +151,6 @@
     "model.llm.layers.12.mlp.gate.bias": "model-00002-of-00032.safetensors",
     "model.llm.layers.12.mlp.gate.global_scale": "model-00002-of-00032.safetensors",
     "model.llm.layers.12.mlp.gate.weight": "model-00002-of-00032.safetensors",
-    "model.llm.layers.12.mlp.shared_experts.shared_w13_weight": "model-00002-of-00032.safetensors",
-    "model.llm.layers.12.mlp.shared_experts.shared_w2_weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.12.mlp_norm.weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.12.mlp_sconv.weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.13.attn.k_norm.weight": "model-00002-of-00032.safetensors",
@@ -187,8 +181,6 @@
     "model.llm.layers.13.mlp.gate.bias": "model-00002-of-00032.safetensors",
     "model.llm.layers.13.mlp.gate.global_scale": "model-00002-of-00032.safetensors",
     "model.llm.layers.13.mlp.gate.weight": "model-00002-of-00032.safetensors",
-    "model.llm.layers.13.mlp.shared_experts.shared_w13_weight": "model-00002-of-00032.safetensors",
-    "model.llm.layers.13.mlp.shared_experts.shared_w2_weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.13.mlp_norm.weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.13.mlp_sconv.weight": "model-00002-of-00032.safetensors",
     "model.llm.layers.14.attn.k_norm.weight": "model-00002-of-00032.safetensors",
@@ -219,8 +211,6 @@
     "model.llm.layers.14.mlp.gate.bias": "model-00003-of-00032.safetensors",
     "model.llm.layers.14.mlp.gate.global_scale": "model-00003-of-00032.safetensors",
     "model.llm.layers.14.mlp.gate.weight": "model-00003-of-00032.safetensors",
-    "model.llm.layers.14.mlp.shared_experts.shared_w13_weight": "model-00003-of-00032.safetensors",
-    "model.llm.layers.14.mlp.shared_experts.shared_w2_weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.14.mlp_norm.weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.14.mlp_sconv.weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.15.attn.k_norm.weight": "model-00003-of-00032.safetensors",
@@ -251,8 +241,6 @@
     "model.llm.layers.15.mlp.gate.bias": "model-00003-of-00032.safetensors",
     "model.llm.layers.15.mlp.gate.global_scale": "model-00003-of-00032.safetensors",
     "model.llm.layers.15.mlp.gate.weight": "model-00003-of-00032.safetensors",
-    "model.llm.layers.15.mlp.shared_experts.shared_w13_weight": "model-00003-of-00032.safetensors",
-    "model.llm.layers.15.mlp.shared_experts.shared_w2_weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.15.mlp_norm.weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.15.mlp_sconv.weight": "model-00003-of-00032.safetensors",
     "model.llm.layers.16.attn.k_norm.weight": "model-00003-of-00032.safetensors",
@@ -283,8 +271,6 @@
     "model.llm.layers.16.mlp.gate.bias": "model-00004-of-00032.safetensors",
     "model.llm.layers.16.mlp.gate.global_scale": "model-00004-of-00032.safetensors",
     "model.llm.layers.16.mlp.gate.weight": "model-00004-of-00032.safetensors",
-    "model.llm.layers.16.mlp.shared_experts.shared_w13_weight": "model-00004-of-00032.safetensors",
-    "model.llm.layers.16.mlp.shared_experts.shared_w2_weight": "model-00004-of-00032.safetensors",
     "model.llm.layers.16.mlp_norm.weight": "model-00004-of-00032.safetensors",
     "model.llm.layers.16.mlp_sconv.weight": "model-00004-of-00032.safetensors",
     "model.llm.layers.17.attn.k_norm.weight": "model-00004-of-00032.safetensors",
@@ -315,8 +301,6 @@
     "model.llm.layers.17.mlp.gate.bias": "model-00004-of-00032.safetensors",
     "model.llm.layers.17.mlp.gate.global_scale": "model-00004-of-00032.safetensors",
     "model.llm.layers.17.mlp.gate.weight": "model-00004-of-00032.safetensors",
-    "model.llm.layers.17.mlp.shared_experts.shared_w13_weight": "model-00004-of-00032.safetensors",
-    "model.llm.layers.17.mlp.shared_experts.shared_w2_weight": "model-00004-of-00032.safetensors",
     "model.llm.layers.17.mlp_norm.weight": "model-00004-of-00032.safetensors",
     "model.llm.layers.17.mlp_sconv.weight": "model-00004-of-00032.safetensors",
     "model.llm.layers.18.attn.k_norm.weight": "model-00004-of-00032.safetensors",
@@ -347,8 +331,6 @@
     "model.llm.layers.18.mlp.gate.bias": "model-00005-of-00032.safetensors",
     "model.llm.layers.18.mlp.gate.global_scale": "model-00005-of-00032.safetensors",
     "model.llm.layers.18.mlp.gate.weight": "model-00005-of-00032.safetensors",
-    "model.llm.layers.18.mlp.shared_experts.shared_w13_weight": "model-00005-of-00032.safetensors",
-    "model.llm.layers.18.mlp.shared_experts.shared_w2_weight": "model-00005-of-00032.safetensors",
     "model.llm.layers.18.mlp_norm.weight": "model-00005-of-00032.safetensors",
     "model.llm.layers.18.mlp_sconv.weight": "model-00005-of-00032.safetensors",
     "model.llm.layers.19.attn.k_norm.weight": "model-00005-of-00032.safetensors",
@@ -379,8 +361,6 @@
     "model.llm.layers.19.mlp.gate.bias": "model-00005-of-00032.safetensors",
     "model.llm.layers.19.mlp.gate.global_scale": "model-00005-of-00032.safetensors",
     "model.llm.layers.19.mlp.gate.weight": "model-00005-of-00032.safetensors",
-    "model.llm.layers.19.mlp.shared_experts.shared_w13_weight": "model-00005-of-00032.safetensors",
-    "model.llm.layers.19.mlp.shared_experts.shared_w2_weight": "model-00005-of-00032.safetensors",
     "model.llm.layers.19.mlp_norm.weight": "model-00005-of-00032.safetensors",
     "model.llm.layers.19.mlp_sconv.weight": "model-00005-of-00032.safetensors",
     "model.llm.layers.2.attn.k_norm.weight": "model-00005-of-00032.safetensors",
@@ -411,8 +391,6 @@
     "model.llm.layers.2.mlp.gate.bias": "model-00006-of-00032.safetensors",
     "model.llm.layers.2.mlp.gate.global_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.2.mlp.gate.weight": "model-00006-of-00032.safetensors",
-    "model.llm.layers.2.mlp.shared_experts.shared_w13_weight": "model-00006-of-00032.safetensors",
-    "model.llm.layers.2.mlp.shared_experts.shared_w2_weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.2.mlp_norm.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.2.mlp_sconv.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.20.attn.k_norm.weight": "model-00006-of-00032.safetensors",
@@ -443,8 +421,6 @@
     "model.llm.layers.20.mlp.gate.bias": "model-00006-of-00032.safetensors",
     "model.llm.layers.20.mlp.gate.global_scale": "model-00006-of-00032.safetensors",
     "model.llm.layers.20.mlp.gate.weight": "model-00006-of-00032.safetensors",
-    "model.llm.layers.20.mlp.shared_experts.shared_w13_weight": "model-00006-of-00032.safetensors",
-    "model.llm.layers.20.mlp.shared_experts.shared_w2_weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.20.mlp_norm.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.20.mlp_sconv.weight": "model-00006-of-00032.safetensors",
     "model.llm.layers.21.attn.k_norm.weight": "model-00006-of-00032.safetensors",
@@ -475,8 +451,6 @@
     "model.llm.layers.21.mlp.gate.bias": "model-00007-of-00032.safetensors",
     "model.llm.layers.21.mlp.gate.global_scale": "model-00007-of-00032.safetensors",
     "model.llm.layers.21.mlp.gate.weight": "model-00007-of-00032.safetensors",
-    "model.llm.layers.21.mlp.shared_experts.shared_w13_weight": "model-00007-of-00032.safetensors",
-    "model.llm.layers.21.mlp.shared_experts.shared_w2_weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.21.mlp_norm.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.21.mlp_sconv.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.22.attn.k_norm.weight": "model-00007-of-00032.safetensors",
@@ -507,8 +481,6 @@
     "model.llm.layers.22.mlp.gate.bias": "model-00007-of-00032.safetensors",
     "model.llm.layers.22.mlp.gate.global_scale": "model-00007-of-00032.safetensors",
     "model.llm.layers.22.mlp.gate.weight": "model-00007-of-00032.safetensors",
-    "model.llm.layers.22.mlp.shared_experts.shared_w13_weight": "model-00007-of-00032.safetensors",
-    "model.llm.layers.22.mlp.shared_experts.shared_w2_weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.22.mlp_norm.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.22.mlp_sconv.weight": "model-00007-of-00032.safetensors",
     "model.llm.layers.23.attn.k_norm.weight": "model-00007-of-00032.safetensors",
@@ -539,8 +511,6 @@
     "model.llm.layers.23.mlp.gate.bias": "model-00008-of-00032.safetensors",
     "model.llm.layers.23.mlp.gate.global_scale": "model-00008-of-00032.safetensors",
     "model.llm.layers.23.mlp.gate.weight": "model-00008-of-00032.safetensors",
-    "model.llm.layers.23.mlp.shared_experts.shared_w13_weight": "model-00008-of-00032.safetensors",
-    "model.llm.layers.23.mlp.shared_experts.shared_w2_weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.23.mlp_norm.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.23.mlp_sconv.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.24.attn.k_norm.weight": "model-00008-of-00032.safetensors",
@@ -571,8 +541,6 @@
     "model.llm.layers.24.mlp.gate.bias": "model-00008-of-00032.safetensors",
     "model.llm.layers.24.mlp.gate.global_scale": "model-00008-of-00032.safetensors",
     "model.llm.layers.24.mlp.gate.weight": "model-00008-of-00032.safetensors",
-    "model.llm.layers.24.mlp.shared_experts.shared_w13_weight": "model-00008-of-00032.safetensors",
-    "model.llm.layers.24.mlp.shared_experts.shared_w2_weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.24.mlp_norm.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.24.mlp_sconv.weight": "model-00008-of-00032.safetensors",
     "model.llm.layers.25.attn.k_norm.weight": "model-00008-of-00032.safetensors",
@@ -603,8 +571,6 @@
     "model.llm.layers.25.mlp.gate.bias": "model-00009-of-00032.safetensors",
     "model.llm.layers.25.mlp.gate.global_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.25.mlp.gate.weight": "model-00009-of-00032.safetensors",
-    "model.llm.layers.25.mlp.shared_experts.shared_w13_weight": "model-00009-of-00032.safetensors",
-    "model.llm.layers.25.mlp.shared_experts.shared_w2_weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.25.mlp_norm.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.25.mlp_sconv.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.26.attn.k_norm.weight": "model-00009-of-00032.safetensors",
@@ -635,8 +601,6 @@
     "model.llm.layers.26.mlp.gate.bias": "model-00009-of-00032.safetensors",
     "model.llm.layers.26.mlp.gate.global_scale": "model-00009-of-00032.safetensors",
     "model.llm.layers.26.mlp.gate.weight": "model-00009-of-00032.safetensors",
-    "model.llm.layers.26.mlp.shared_experts.shared_w13_weight": "model-00009-of-00032.safetensors",
-    "model.llm.layers.26.mlp.shared_experts.shared_w2_weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.26.mlp_norm.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.26.mlp_sconv.weight": "model-00009-of-00032.safetensors",
     "model.llm.layers.27.attn.k_norm.weight": "model-00009-of-00032.safetensors",
@@ -667,8 +631,6 @@
     "model.llm.layers.27.mlp.gate.bias": "model-00010-of-00032.safetensors",
     "model.llm.layers.27.mlp.gate.global_scale": "model-00010-of-00032.safetensors",
     "model.llm.layers.27.mlp.gate.weight": "model-00010-of-00032.safetensors",
-    "model.llm.layers.27.mlp.shared_experts.shared_w13_weight": "model-00010-of-00032.safetensors",
-    "model.llm.layers.27.mlp.shared_experts.shared_w2_weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.27.mlp_norm.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.27.mlp_sconv.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.28.attn.k_norm.weight": "model-00010-of-00032.safetensors",
@@ -699,8 +661,6 @@
     "model.llm.layers.28.mlp.gate.bias": "model-00010-of-00032.safetensors",
     "model.llm.layers.28.mlp.gate.global_scale": "model-00010-of-00032.safetensors",
     "model.llm.layers.28.mlp.gate.weight": "model-00010-of-00032.safetensors",
-    "model.llm.layers.28.mlp.shared_experts.shared_w13_weight": "model-00010-of-00032.safetensors",
-    "model.llm.layers.28.mlp.shared_experts.shared_w2_weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.28.mlp_norm.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.28.mlp_sconv.weight": "model-00010-of-00032.safetensors",
     "model.llm.layers.29.attn.k_norm.weight": "model-00010-of-00032.safetensors",
@@ -731,8 +691,6 @@
     "model.llm.layers.29.mlp.gate.bias": "model-00011-of-00032.safetensors",
     "model.llm.layers.29.mlp.gate.global_scale": "model-00011-of-00032.safetensors",
     "model.llm.layers.29.mlp.gate.weight": "model-00011-of-00032.safetensors",
-    "model.llm.layers.29.mlp.shared_experts.shared_w13_weight": "model-00011-of-00032.safetensors",
-    "model.llm.layers.29.mlp.shared_experts.shared_w2_weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.29.mlp_norm.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.29.mlp_sconv.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.3.attn.k_norm.weight": "model-00011-of-00032.safetensors",
@@ -763,8 +721,6 @@
     "model.llm.layers.3.mlp.gate.bias": "model-00011-of-00032.safetensors",
     "model.llm.layers.3.mlp.gate.global_scale": "model-00011-of-00032.safetensors",
     "model.llm.layers.3.mlp.gate.weight": "model-00011-of-00032.safetensors",
-    "model.llm.layers.3.mlp.shared_experts.shared_w13_weight": "model-00011-of-00032.safetensors",
-    "model.llm.layers.3.mlp.shared_experts.shared_w2_weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.3.mlp_norm.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.3.mlp_sconv.weight": "model-00011-of-00032.safetensors",
     "model.llm.layers.30.attn.k_norm.weight": "model-00011-of-00032.safetensors",
@@ -795,8 +751,6 @@
     "model.llm.layers.30.mlp.gate.bias": "model-00012-of-00032.safetensors",
     "model.llm.layers.30.mlp.gate.global_scale": "model-00012-of-00032.safetensors",
     "model.llm.layers.30.mlp.gate.weight": "model-00012-of-00032.safetensors",
-    "model.llm.layers.30.mlp.shared_experts.shared_w13_weight": "model-00012-of-00032.safetensors",
-    "model.llm.layers.30.mlp.shared_experts.shared_w2_weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.30.mlp_norm.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.30.mlp_sconv.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.31.attn.k_norm.weight": "model-00012-of-00032.safetensors",
@@ -827,8 +781,6 @@
     "model.llm.layers.31.mlp.gate.bias": "model-00012-of-00032.safetensors",
     "model.llm.layers.31.mlp.gate.global_scale": "model-00012-of-00032.safetensors",
     "model.llm.layers.31.mlp.gate.weight": "model-00012-of-00032.safetensors",
-    "model.llm.layers.31.mlp.shared_experts.shared_w13_weight": "model-00012-of-00032.safetensors",
-    "model.llm.layers.31.mlp.shared_experts.shared_w2_weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.31.mlp_norm.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.31.mlp_sconv.weight": "model-00012-of-00032.safetensors",
     "model.llm.layers.32.attn.k_norm.weight": "model-00012-of-00032.safetensors",
@@ -859,8 +811,6 @@
     "model.llm.layers.32.mlp.gate.bias": "model-00013-of-00032.safetensors",
     "model.llm.layers.32.mlp.gate.global_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.32.mlp.gate.weight": "model-00013-of-00032.safetensors",
-    "model.llm.layers.32.mlp.shared_experts.shared_w13_weight": "model-00013-of-00032.safetensors",
-    "model.llm.layers.32.mlp.shared_experts.shared_w2_weight": "model-00013-of-00032.safetensors",
     "model.llm.layers.32.mlp_norm.weight": "model-00013-of-00032.safetensors",
     "model.llm.layers.32.mlp_sconv.weight": "model-00013-of-00032.safetensors",
     "model.llm.layers.33.attn.k_norm.weight": "model-00013-of-00032.safetensors",
@@ -891,8 +841,6 @@
     "model.llm.layers.33.mlp.gate.bias": "model-00013-of-00032.safetensors",
     "model.llm.layers.33.mlp.gate.global_scale": "model-00013-of-00032.safetensors",
     "model.llm.layers.33.mlp.gate.weight": "model-00013-of-00032.safetensors",
-    "model.llm.layers.33.mlp.shared_experts.shared_w13_weight": "model-00013-of-00032.safetensors",
-    "model.llm.layers.33.mlp.shared_experts.shared_w2_weight": "model-00013-of-00032.safetensors",
     "model.llm.layers.33.mlp_norm.weight": "model-00013-of-00032.safetensors",
     "model.llm.layers.33.mlp_sconv.weight": "model-00013-of-00032.safetensors",
     "model.llm.layers.34.attn.k_norm.weight": "model-00013-of-00032.safetensors",
@@ -923,8 +871,6 @@
     "model.llm.layers.34.mlp.gate.bias": "model-00014-of-00032.safetensors",
     "model.llm.layers.34.mlp.gate.global_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.34.mlp.gate.weight": "model-00014-of-00032.safetensors",
-    "model.llm.layers.34.mlp.shared_experts.shared_w13_weight": "model-00014-of-00032.safetensors",
-    "model.llm.layers.34.mlp.shared_experts.shared_w2_weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.34.mlp_norm.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.34.mlp_sconv.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.35.attn.k_norm.weight": "model-00014-of-00032.safetensors",
@@ -955,8 +901,6 @@
     "model.llm.layers.35.mlp.gate.bias": "model-00014-of-00032.safetensors",
     "model.llm.layers.35.mlp.gate.global_scale": "model-00014-of-00032.safetensors",
     "model.llm.layers.35.mlp.gate.weight": "model-00014-of-00032.safetensors",
-    "model.llm.layers.35.mlp.shared_experts.shared_w13_weight": "model-00014-of-00032.safetensors",
-    "model.llm.layers.35.mlp.shared_experts.shared_w2_weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.35.mlp_norm.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.35.mlp_sconv.weight": "model-00014-of-00032.safetensors",
     "model.llm.layers.36.attn.k_norm.weight": "model-00014-of-00032.safetensors",
@@ -987,8 +931,6 @@
     "model.llm.layers.36.mlp.gate.bias": "model-00015-of-00032.safetensors",
     "model.llm.layers.36.mlp.gate.global_scale": "model-00015-of-00032.safetensors",
     "model.llm.layers.36.mlp.gate.weight": "model-00015-of-00032.safetensors",
-    "model.llm.layers.36.mlp.shared_experts.shared_w13_weight": "model-00015-of-00032.safetensors",
-    "model.llm.layers.36.mlp.shared_experts.shared_w2_weight": "model-00015-of-00032.safetensors",
     "model.llm.layers.36.mlp_norm.weight": "model-00015-of-00032.safetensors",
     "model.llm.layers.36.mlp_sconv.weight": "model-00015-of-00032.safetensors",
     "model.llm.layers.37.attn.k_norm.weight": "model-00015-of-00032.safetensors",
@@ -1019,8 +961,6 @@
     "model.llm.layers.37.mlp.gate.bias": "model-00015-of-00032.safetensors",
     "model.llm.layers.37.mlp.gate.global_scale": "model-00015-of-00032.safetensors",
     "model.llm.layers.37.mlp.gate.weight": "model-00015-of-00032.safetensors",
-    "model.llm.layers.37.mlp.shared_experts.shared_w13_weight": "model-00015-of-00032.safetensors",
-    "model.llm.layers.37.mlp.shared_experts.shared_w2_weight": "model-00015-of-00032.safetensors",
     "model.llm.layers.37.mlp_norm.weight": "model-00015-of-00032.safetensors",
     "model.llm.layers.37.mlp_sconv.weight": "model-00015-of-00032.safetensors",
     "model.llm.layers.38.attn.k_norm.weight": "model-00015-of-00032.safetensors",
@@ -1051,8 +991,6 @@
     "model.llm.layers.38.mlp.gate.bias": "model-00016-of-00032.safetensors",
     "model.llm.layers.38.mlp.gate.global_scale": "model-00016-of-00032.safetensors",
     "model.llm.layers.38.mlp.gate.weight": "model-00016-of-00032.safetensors",
-    "model.llm.layers.38.mlp.shared_experts.shared_w13_weight": "model-00016-of-00032.safetensors",
-    "model.llm.layers.38.mlp.shared_experts.shared_w2_weight": "model-00016-of-00032.safetensors",
     "model.llm.layers.38.mlp_norm.weight": "model-00016-of-00032.safetensors",
     "model.llm.layers.38.mlp_sconv.weight": "model-00016-of-00032.safetensors",
     "model.llm.layers.39.attn.k_norm.weight": "model-00016-of-00032.safetensors",
@@ -1083,8 +1021,6 @@
     "model.llm.layers.39.mlp.gate.bias": "model-00016-of-00032.safetensors",
     "model.llm.layers.39.mlp.gate.global_scale": "model-00016-of-00032.safetensors",
     "model.llm.layers.39.mlp.gate.weight": "model-00016-of-00032.safetensors",
-    "model.llm.layers.39.mlp.shared_experts.shared_w13_weight": "model-00016-of-00032.safetensors",
-    "model.llm.layers.39.mlp.shared_experts.shared_w2_weight": "model-00016-of-00032.safetensors",
     "model.llm.layers.39.mlp_norm.weight": "model-00016-of-00032.safetensors",
     "model.llm.layers.39.mlp_sconv.weight": "model-00016-of-00032.safetensors",
     "model.llm.layers.4.attn.k_norm.weight": "model-00016-of-00032.safetensors",
@@ -1115,8 +1051,6 @@
     "model.llm.layers.4.mlp.gate.bias": "model-00017-of-00032.safetensors",
     "model.llm.layers.4.mlp.gate.global_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.4.mlp.gate.weight": "model-00017-of-00032.safetensors",
-    "model.llm.layers.4.mlp.shared_experts.shared_w13_weight": "model-00017-of-00032.safetensors",
-    "model.llm.layers.4.mlp.shared_experts.shared_w2_weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.4.mlp_norm.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.4.mlp_sconv.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.40.attn.k_norm.weight": "model-00017-of-00032.safetensors",
@@ -1147,8 +1081,6 @@
     "model.llm.layers.40.mlp.gate.bias": "model-00017-of-00032.safetensors",
     "model.llm.layers.40.mlp.gate.global_scale": "model-00017-of-00032.safetensors",
     "model.llm.layers.40.mlp.gate.weight": "model-00017-of-00032.safetensors",
-    "model.llm.layers.40.mlp.shared_experts.shared_w13_weight": "model-00017-of-00032.safetensors",
-    "model.llm.layers.40.mlp.shared_experts.shared_w2_weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.40.mlp_norm.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.40.mlp_sconv.weight": "model-00017-of-00032.safetensors",
     "model.llm.layers.41.attn.k_norm.weight": "model-00017-of-00032.safetensors",
@@ -1179,8 +1111,6 @@
     "model.llm.layers.41.mlp.gate.bias": "model-00018-of-00032.safetensors",
     "model.llm.layers.41.mlp.gate.global_scale": "model-00018-of-00032.safetensors",
     "model.llm.layers.41.mlp.gate.weight": "model-00018-of-00032.safetensors",
-    "model.llm.layers.41.mlp.shared_experts.shared_w13_weight": "model-00018-of-00032.safetensors",
-    "model.llm.layers.41.mlp.shared_experts.shared_w2_weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.41.mlp_norm.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.41.mlp_sconv.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.42.attn.k_norm.weight": "model-00018-of-00032.safetensors",
@@ -1211,8 +1141,6 @@
     "model.llm.layers.42.mlp.gate.bias": "model-00018-of-00032.safetensors",
     "model.llm.layers.42.mlp.gate.global_scale": "model-00018-of-00032.safetensors",
     "model.llm.layers.42.mlp.gate.weight": "model-00018-of-00032.safetensors",
-    "model.llm.layers.42.mlp.shared_experts.shared_w13_weight": "model-00018-of-00032.safetensors",
-    "model.llm.layers.42.mlp.shared_experts.shared_w2_weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.42.mlp_norm.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.42.mlp_sconv.weight": "model-00018-of-00032.safetensors",
     "model.llm.layers.43.attn.k_norm.weight": "model-00018-of-00032.safetensors",
@@ -1243,8 +1171,6 @@
     "model.llm.layers.43.mlp.gate.bias": "model-00019-of-00032.safetensors",
     "model.llm.layers.43.mlp.gate.global_scale": "model-00019-of-00032.safetensors",
     "model.llm.layers.43.mlp.gate.weight": "model-00019-of-00032.safetensors",
-    "model.llm.layers.43.mlp.shared_experts.shared_w13_weight": "model-00019-of-00032.safetensors",
-    "model.llm.layers.43.mlp.shared_experts.shared_w2_weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.43.mlp_norm.weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.43.mlp_sconv.weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.44.attn.k_norm.weight": "model-00019-of-00032.safetensors",
@@ -1275,8 +1201,6 @@
     "model.llm.layers.44.mlp.gate.bias": "model-00019-of-00032.safetensors",
     "model.llm.layers.44.mlp.gate.global_scale": "model-00019-of-00032.safetensors",
     "model.llm.layers.44.mlp.gate.weight": "model-00019-of-00032.safetensors",
-    "model.llm.layers.44.mlp.shared_experts.shared_w13_weight": "model-00019-of-00032.safetensors",
-    "model.llm.layers.44.mlp.shared_experts.shared_w2_weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.44.mlp_norm.weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.44.mlp_sconv.weight": "model-00019-of-00032.safetensors",
     "model.llm.layers.45.attn.k_norm.weight": "model-00019-of-00032.safetensors",
@@ -1307,8 +1231,6 @@
     "model.llm.layers.45.mlp.gate.bias": "model-00020-of-00032.safetensors",
     "model.llm.layers.45.mlp.gate.global_scale": "model-00020-of-00032.safetensors",
     "model.llm.layers.45.mlp.gate.weight": "model-00020-of-00032.safetensors",
-    "model.llm.layers.45.mlp.shared_experts.shared_w13_weight": "model-00020-of-00032.safetensors",
-    "model.llm.layers.45.mlp.shared_experts.shared_w2_weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.45.mlp_norm.weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.45.mlp_sconv.weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.46.attn.k_norm.weight": "model-00020-of-00032.safetensors",
@@ -1339,8 +1261,6 @@
     "model.llm.layers.46.mlp.gate.bias": "model-00020-of-00032.safetensors",
     "model.llm.layers.46.mlp.gate.global_scale": "model-00020-of-00032.safetensors",
     "model.llm.layers.46.mlp.gate.weight": "model-00020-of-00032.safetensors",
-    "model.llm.layers.46.mlp.shared_experts.shared_w13_weight": "model-00020-of-00032.safetensors",
-    "model.llm.layers.46.mlp.shared_experts.shared_w2_weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.46.mlp_norm.weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.46.mlp_sconv.weight": "model-00020-of-00032.safetensors",
     "model.llm.layers.47.attn.k_norm.weight": "model-00020-of-00032.safetensors",
@@ -1371,8 +1291,6 @@
     "model.llm.layers.47.mlp.gate.bias": "model-00021-of-00032.safetensors",
     "model.llm.layers.47.mlp.gate.global_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.47.mlp.gate.weight": "model-00021-of-00032.safetensors",
-    "model.llm.layers.47.mlp.shared_experts.shared_w13_weight": "model-00021-of-00032.safetensors",
-    "model.llm.layers.47.mlp.shared_experts.shared_w2_weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.47.mlp_norm.weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.47.mlp_sconv.weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.48.attn.k_norm.weight": "model-00021-of-00032.safetensors",
@@ -1403,8 +1321,6 @@
     "model.llm.layers.48.mlp.gate.bias": "model-00021-of-00032.safetensors",
     "model.llm.layers.48.mlp.gate.global_scale": "model-00021-of-00032.safetensors",
     "model.llm.layers.48.mlp.gate.weight": "model-00021-of-00032.safetensors",
-    "model.llm.layers.48.mlp.shared_experts.shared_w13_weight": "model-00021-of-00032.safetensors",
-    "model.llm.layers.48.mlp.shared_experts.shared_w2_weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.48.mlp_norm.weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.48.mlp_sconv.weight": "model-00021-of-00032.safetensors",
     "model.llm.layers.49.attn.k_norm.weight": "model-00021-of-00032.safetensors",
@@ -1435,8 +1351,6 @@
     "model.llm.layers.49.mlp.gate.bias": "model-00022-of-00032.safetensors",
     "model.llm.layers.49.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.49.mlp.gate.weight": "model-00022-of-00032.safetensors",
-    "model.llm.layers.49.mlp.shared_experts.shared_w13_weight": "model-00022-of-00032.safetensors",
-    "model.llm.layers.49.mlp.shared_experts.shared_w2_weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.49.mlp_norm.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.49.mlp_sconv.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.attn.k_norm.weight": "model-00022-of-00032.safetensors",
@@ -1467,8 +1381,6 @@
     "model.llm.layers.5.mlp.gate.bias": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.mlp.gate.global_scale": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.mlp.gate.weight": "model-00022-of-00032.safetensors",
-    "model.llm.layers.5.mlp.shared_experts.shared_w13_weight": "model-00022-of-00032.safetensors",
-    "model.llm.layers.5.mlp.shared_experts.shared_w2_weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.mlp_norm.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.5.mlp_sconv.weight": "model-00022-of-00032.safetensors",
     "model.llm.layers.50.attn.k_norm.weight": "model-00022-of-00032.safetensors",
@@ -1499,8 +1411,6 @@
     "model.llm.layers.50.mlp.gate.bias": "model-00023-of-00032.safetensors",
     "model.llm.layers.50.mlp.gate.global_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.50.mlp.gate.weight": "model-00023-of-00032.safetensors",
-    "model.llm.layers.50.mlp.shared_experts.shared_w13_weight": "model-00023-of-00032.safetensors",
-    "model.llm.layers.50.mlp.shared_experts.shared_w2_weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.50.mlp_norm.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.50.mlp_sconv.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.51.attn.k_norm.weight": "model-00023-of-00032.safetensors",
@@ -1531,8 +1441,6 @@
     "model.llm.layers.51.mlp.gate.bias": "model-00023-of-00032.safetensors",
     "model.llm.layers.51.mlp.gate.global_scale": "model-00023-of-00032.safetensors",
     "model.llm.layers.51.mlp.gate.weight": "model-00023-of-00032.safetensors",
-    "model.llm.layers.51.mlp.shared_experts.shared_w13_weight": "model-00023-of-00032.safetensors",
-    "model.llm.layers.51.mlp.shared_experts.shared_w2_weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.51.mlp_norm.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.51.mlp_sconv.weight": "model-00023-of-00032.safetensors",
     "model.llm.layers.52.attn.k_norm.weight": "model-00023-of-00032.safetensors",
@@ -1563,8 +1471,6 @@
     "model.llm.layers.52.mlp.gate.bias": "model-00024-of-00032.safetensors",
     "model.llm.layers.52.mlp.gate.global_scale": "model-00024-of-00032.safetensors",
     "model.llm.layers.52.mlp.gate.weight": "model-00024-of-00032.safetensors",
-    "model.llm.layers.52.mlp.shared_experts.shared_w13_weight": "model-00024-of-00032.safetensors",
-    "model.llm.layers.52.mlp.shared_experts.shared_w2_weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.52.mlp_norm.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.52.mlp_sconv.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.53.attn.k_norm.weight": "model-00024-of-00032.safetensors",
@@ -1595,8 +1501,6 @@
     "model.llm.layers.53.mlp.gate.bias": "model-00024-of-00032.safetensors",
     "model.llm.layers.53.mlp.gate.global_scale": "model-00024-of-00032.safetensors",
     "model.llm.layers.53.mlp.gate.weight": "model-00024-of-00032.safetensors",
-    "model.llm.layers.53.mlp.shared_experts.shared_w13_weight": "model-00024-of-00032.safetensors",
-    "model.llm.layers.53.mlp.shared_experts.shared_w2_weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.53.mlp_norm.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.53.mlp_sconv.weight": "model-00024-of-00032.safetensors",
     "model.llm.layers.54.attn.k_norm.weight": "model-00024-of-00032.safetensors",
@@ -1627,8 +1531,6 @@
     "model.llm.layers.54.mlp.gate.bias": "model-00025-of-00032.safetensors",
     "model.llm.layers.54.mlp.gate.global_scale": "model-00025-of-00032.safetensors",
     "model.llm.layers.54.mlp.gate.weight": "model-00025-of-00032.safetensors",
-    "model.llm.layers.54.mlp.shared_experts.shared_w13_weight": "model-00025-of-00032.safetensors",
-    "model.llm.layers.54.mlp.shared_experts.shared_w2_weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.54.mlp_norm.weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.54.mlp_sconv.weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.55.attn.k_norm.weight": "model-00025-of-00032.safetensors",
@@ -1659,8 +1561,6 @@
     "model.llm.layers.55.mlp.gate.bias": "model-00025-of-00032.safetensors",
     "model.llm.layers.55.mlp.gate.global_scale": "model-00025-of-00032.safetensors",
     "model.llm.layers.55.mlp.gate.weight": "model-00025-of-00032.safetensors",
-    "model.llm.layers.55.mlp.shared_experts.shared_w13_weight": "model-00025-of-00032.safetensors",
-    "model.llm.layers.55.mlp.shared_experts.shared_w2_weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.55.mlp_norm.weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.55.mlp_sconv.weight": "model-00025-of-00032.safetensors",
     "model.llm.layers.56.attn.k_norm.weight": "model-00025-of-00032.safetensors",
@@ -1691,8 +1591,6 @@
     "model.llm.layers.56.mlp.gate.bias": "model-00026-of-00032.safetensors",
     "model.llm.layers.56.mlp.gate.global_scale": "model-00026-of-00032.safetensors",
     "model.llm.layers.56.mlp.gate.weight": "model-00026-of-00032.safetensors",
-    "model.llm.layers.56.mlp.shared_experts.shared_w13_weight": "model-00026-of-00032.safetensors",
-    "model.llm.layers.56.mlp.shared_experts.shared_w2_weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.56.mlp_norm.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.56.mlp_sconv.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.57.attn.k_norm.weight": "model-00026-of-00032.safetensors",
@@ -1723,8 +1621,6 @@
     "model.llm.layers.57.mlp.gate.bias": "model-00026-of-00032.safetensors",
     "model.llm.layers.57.mlp.gate.global_scale": "model-00026-of-00032.safetensors",
     "model.llm.layers.57.mlp.gate.weight": "model-00026-of-00032.safetensors",
-    "model.llm.layers.57.mlp.shared_experts.shared_w13_weight": "model-00026-of-00032.safetensors",
-    "model.llm.layers.57.mlp.shared_experts.shared_w2_weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.57.mlp_norm.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.57.mlp_sconv.weight": "model-00026-of-00032.safetensors",
     "model.llm.layers.58.attn.k_norm.weight": "model-00026-of-00032.safetensors",
@@ -1755,8 +1651,6 @@
     "model.llm.layers.58.mlp.gate.bias": "model-00027-of-00032.safetensors",
     "model.llm.layers.58.mlp.gate.global_scale": "model-00027-of-00032.safetensors",
     "model.llm.layers.58.mlp.gate.weight": "model-00027-of-00032.safetensors",
-    "model.llm.layers.58.mlp.shared_experts.shared_w13_weight": "model-00027-of-00032.safetensors",
-    "model.llm.layers.58.mlp.shared_experts.shared_w2_weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.58.mlp_norm.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.58.mlp_sconv.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.59.attn.k_norm.weight": "model-00027-of-00032.safetensors",
@@ -1787,8 +1681,6 @@
     "model.llm.layers.59.mlp.gate.bias": "model-00027-of-00032.safetensors",
     "model.llm.layers.59.mlp.gate.global_scale": "model-00027-of-00032.safetensors",
     "model.llm.layers.59.mlp.gate.weight": "model-00027-of-00032.safetensors",
-    "model.llm.layers.59.mlp.shared_experts.shared_w13_weight": "model-00027-of-00032.safetensors",
-    "model.llm.layers.59.mlp.shared_experts.shared_w2_weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.59.mlp_norm.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.59.mlp_sconv.weight": "model-00027-of-00032.safetensors",
     "model.llm.layers.6.attn.k_norm.weight": "model-00027-of-00032.safetensors",
@@ -1819,8 +1711,6 @@
     "model.llm.layers.6.mlp.gate.bias": "model-00028-of-00032.safetensors",
     "model.llm.layers.6.mlp.gate.global_scale": "model-00028-of-00032.safetensors",
     "model.llm.layers.6.mlp.gate.weight": "model-00028-of-00032.safetensors",
-    "model.llm.layers.6.mlp.shared_experts.shared_w13_weight": "model-00028-of-00032.safetensors",
-    "model.llm.layers.6.mlp.shared_experts.shared_w2_weight": "model-00028-of-00032.safetensors",
     "model.llm.layers.6.mlp_norm.weight": "model-00028-of-00032.safetensors",
     "model.llm.layers.6.mlp_sconv.weight": "model-00028-of-00032.safetensors",
     "model.llm.layers.60.attn.k_norm.weight": "model-00028-of-00032.safetensors",
@@ -1851,8 +1741,6 @@
     "model.llm.layers.60.mlp.gate.bias": "model-00028-of-00032.safetensors",
     "model.llm.layers.60.mlp.gate.global_scale": "model-00028-of-00032.safetensors",
     "model.llm.layers.60.mlp.gate.weight": "model-00028-of-00032.safetensors",
-    "model.llm.layers.60.mlp.shared_experts.shared_w13_weight": "model-00028-of-00032.safetensors",
-    "model.llm.layers.60.mlp.shared_experts.shared_w2_weight": "model-00028-of-00032.safetensors",
     "model.llm.layers.60.mlp_norm.weight": "model-00028-of-00032.safetensors",
     "model.llm.layers.60.mlp_sconv.weight": "model-00028-of-00032.safetensors",
     "model.llm.layers.61.attn.k_norm.weight": "model-00028-of-00032.safetensors",
@@ -1883,8 +1771,6 @@
     "model.llm.layers.61.mlp.gate.bias": "model-00029-of-00032.safetensors",
     "model.llm.layers.61.mlp.gate.global_scale": "model-00029-of-00032.safetensors",
     "model.llm.layers.61.mlp.gate.weight": "model-00029-of-00032.safetensors",
-    "model.llm.layers.61.mlp.shared_experts.shared_w13_weight": "model-00029-of-00032.safetensors",
-    "model.llm.layers.61.mlp.shared_experts.shared_w2_weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.61.mlp_norm.weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.61.mlp_sconv.weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.62.attn.k_norm.weight": "model-00029-of-00032.safetensors",
@@ -1915,8 +1801,6 @@
     "model.llm.layers.62.mlp.gate.bias": "model-00029-of-00032.safetensors",
     "model.llm.layers.62.mlp.gate.global_scale": "model-00029-of-00032.safetensors",
     "model.llm.layers.62.mlp.gate.weight": "model-00029-of-00032.safetensors",
-    "model.llm.layers.62.mlp.shared_experts.shared_w13_weight": "model-00029-of-00032.safetensors",
-    "model.llm.layers.62.mlp.shared_experts.shared_w2_weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.62.mlp_norm.weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.62.mlp_sconv.weight": "model-00029-of-00032.safetensors",
     "model.llm.layers.63.attn.k_norm.weight": "model-00029-of-00032.safetensors",
@@ -1947,8 +1831,6 @@
     "model.llm.layers.63.mlp.gate.bias": "model-00030-of-00032.safetensors",
     "model.llm.layers.63.mlp.gate.global_scale": "model-00030-of-00032.safetensors",
     "model.llm.layers.63.mlp.gate.weight": "model-00030-of-00032.safetensors",
-    "model.llm.layers.63.mlp.shared_experts.shared_w13_weight": "model-00030-of-00032.safetensors",
-    "model.llm.layers.63.mlp.shared_experts.shared_w2_weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.63.mlp_norm.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.63.mlp_sconv.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.64.attn.k_norm.weight": "model-00030-of-00032.safetensors",
@@ -1979,8 +1861,6 @@
     "model.llm.layers.64.mlp.gate.bias": "model-00030-of-00032.safetensors",
     "model.llm.layers.64.mlp.gate.global_scale": "model-00030-of-00032.safetensors",
     "model.llm.layers.64.mlp.gate.weight": "model-00030-of-00032.safetensors",
-    "model.llm.layers.64.mlp.shared_experts.shared_w13_weight": "model-00030-of-00032.safetensors",
-    "model.llm.layers.64.mlp.shared_experts.shared_w2_weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.64.mlp_norm.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.64.mlp_sconv.weight": "model-00030-of-00032.safetensors",
     "model.llm.layers.65.attn.k_norm.weight": "model-00030-of-00032.safetensors",
@@ -2011,8 +1891,6 @@
     "model.llm.layers.65.mlp.gate.bias": "model-00031-of-00032.safetensors",
     "model.llm.layers.65.mlp.gate.global_scale": "model-00031-of-00032.safetensors",
     "model.llm.layers.65.mlp.gate.weight": "model-00031-of-00032.safetensors",
-    "model.llm.layers.65.mlp.shared_experts.shared_w13_weight": "model-00031-of-00032.safetensors",
-    "model.llm.layers.65.mlp.shared_experts.shared_w2_weight": "model-00031-of-00032.safetensors",
     "model.llm.layers.65.mlp_norm.weight": "model-00031-of-00032.safetensors",
     "model.llm.layers.65.mlp_sconv.weight": "model-00031-of-00032.safetensors",
     "model.llm.layers.7.attn.k_norm.weight": "model-00031-of-00032.safetensors",
@@ -2043,8 +1921,6 @@
     "model.llm.layers.7.mlp.gate.bias": "model-00031-of-00032.safetensors",
     "model.llm.layers.7.mlp.gate.global_scale": "model-00031-of-00032.safetensors",
     "model.llm.layers.7.mlp.gate.weight": "model-00031-of-00032.safetensors",
-    "model.llm.layers.7.mlp.shared_experts.shared_w13_weight": "model-00031-of-00032.safetensors",
-    "model.llm.layers.7.mlp.shared_experts.shared_w2_weight": "model-00031-of-00032.safetensors",
     "model.llm.layers.7.mlp_norm.weight": "model-00031-of-00032.safetensors",
     "model.llm.layers.7.mlp_sconv.weight": "model-00031-of-00032.safetensors",
     "model.llm.layers.8.attn.k_norm.weight": "model-00031-of-00032.safetensors",
@@ -2075,8 +1951,6 @@
     "model.llm.layers.8.mlp.gate.bias": "model-00032-of-00032.safetensors",
     "model.llm.layers.8.mlp.gate.global_scale": "model-00032-of-00032.safetensors",
     "model.llm.layers.8.mlp.gate.weight": "model-00032-of-00032.safetensors",
-    "model.llm.layers.8.mlp.shared_experts.shared_w13_weight": "model-00032-of-00032.safetensors",
-    "model.llm.layers.8.mlp.shared_experts.shared_w2_weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.8.mlp_norm.weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.8.mlp_sconv.weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.9.attn.k_norm.weight": "model-00032-of-00032.safetensors",
@@ -2107,8 +1981,6 @@
     "model.llm.layers.9.mlp.gate.bias": "model-00032-of-00032.safetensors",
     "model.llm.layers.9.mlp.gate.global_scale": "model-00032-of-00032.safetensors",
     "model.llm.layers.9.mlp.gate.weight": "model-00032-of-00032.safetensors",
-    "model.llm.layers.9.mlp.shared_experts.shared_w13_weight": "model-00032-of-00032.safetensors",
-    "model.llm.layers.9.mlp.shared_experts.shared_w2_weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.9.mlp_norm.weight": "model-00032-of-00032.safetensors",
     "model.llm.layers.9.mlp_sconv.weight": "model-00032-of-00032.safetensors",
     "model.llm.norm.weight": "model-00032-of-00032.safetensors",
@@ -2280,6 +2152,262 @@
     "model.visual.layers.linear_3.weight": "model-00032-of-00032.safetensors",
     "model.visual.layers.norm_0.weight": "model-00032-of-00032.safetensors",
     "model.visual.layers.norm_1.weight": "model-00032-of-00032.safetensors",
-    "model.visual.layers.norm_2.weight": "model-00032-of-00032.safetensors"
+    "model.visual.layers.norm_2.weight": "model-00032-of-00032.safetensors",
+    "model.llm.layers.10.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.10.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.10.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.10.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.11.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.11.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.11.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.11.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.12.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.12.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.12.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.12.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.13.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.13.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.13.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.13.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.14.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.14.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.14.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.14.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.15.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.15.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.15.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.15.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.16.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.16.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.16.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.16.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.17.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.17.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.17.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.17.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.18.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.18.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.18.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.18.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.19.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.19.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.19.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.19.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.2.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.2.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.2.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.2.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.20.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.20.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.20.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.20.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.21.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.21.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.21.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.21.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.22.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.22.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.22.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.22.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.23.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.23.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.23.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.23.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.24.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.24.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.24.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.24.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.25.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.25.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.25.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.25.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.26.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.26.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.26.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.26.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.27.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.27.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.27.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.27.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.28.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.28.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.28.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.28.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.29.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.29.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.29.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.29.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.3.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.3.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.3.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.3.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.30.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.30.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.30.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.30.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.31.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.31.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.31.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.31.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.32.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.32.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.32.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.32.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.33.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.33.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.33.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.33.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.34.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.34.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.34.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.34.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.35.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.35.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.35.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.35.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.36.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.36.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.36.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.36.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.37.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.37.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.37.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.37.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.38.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.38.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.38.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.38.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.39.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.39.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.39.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.39.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.4.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.4.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.4.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.4.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.40.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.40.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.40.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.40.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.41.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.41.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.41.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.41.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.42.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.42.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.42.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.42.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.43.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.43.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.43.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.43.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.44.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.44.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.44.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.44.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.45.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.45.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.45.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.45.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.46.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.46.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.46.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.46.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.47.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.47.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.47.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.47.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.48.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.48.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.48.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.48.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.49.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.49.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.49.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.49.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.5.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.5.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.5.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.5.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.50.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.50.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.50.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.50.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.51.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.51.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.51.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.51.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.52.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.52.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.52.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.52.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.53.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.53.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.53.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.53.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.54.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.54.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.54.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.54.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.55.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.55.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.55.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.55.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.56.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.56.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.56.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.56.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.57.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.57.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.57.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.57.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.58.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.58.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.58.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.58.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.59.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.59.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.59.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.59.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.6.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.6.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.6.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.6.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.60.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.60.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.60.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.60.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.61.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.61.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.61.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.61.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.62.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.62.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.62.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.62.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.63.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.63.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.63.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.63.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.64.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.64.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.64.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.64.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.65.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.65.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.65.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.65.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.7.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.7.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.7.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.7.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.8.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.8.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.8.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.8.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.9.mlp.shared_experts.shared_w13_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.9.mlp.shared_experts.shared_w13_weight.weight_scale": "model-00033-of-00033.safetensors",
+    "model.llm.layers.9.mlp.shared_experts.shared_w2_weight.weight": "model-00033-of-00033.safetensors",
+    "model.llm.layers.9.mlp.shared_experts.shared_w2_weight.weight_scale": "model-00033-of-00033.safetensors"
   }
-}
+}