{ "architectures": ["MistralLarge4ForConditionalGeneration"], "dtype": "bfloat16", "image_token_index": 10, "model_type": "mistral_large4", "multimodal_projector_bias": false, "projector_hidden_act": "gelu", "quantization_config": { "activation_scheme": "dynamic", "modules_to_not_convert": [ "model.vision_tower", "model.multi_modal_projector", "lm_head", "kv_b_proj", "indexer.wk", "indexer.weights_proj", "kv_a_proj_with_mqa" ], "quant_method": "fp8", "weight_block_size": [ 128, 128 ] }, "spatial_merge_size": 2, "text_config": { "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": 1, "eos_token_id": 2, "final_logit_softcapping": 30.0, "first_k_dense_replace": 1, "head_dim": 64, "hidden_act": "silu", "hidden_size": 6144, "index_head_dim": 128, "index_n_heads": 32, "index_topk": 2048, "indexer_types": [ "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full", "full", "shared", "shared", "full" ], "initializer_range": 0.02, "intermediate_size": 24576, "kv_lora_rank": 512, "kv_source_layers": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 35, 40, 41, 42, 35, 44, 45, 46, 35, 48, 49, 50, 35, 52, 53, 54, 35, 56, 57, 58, 35, 60, 61, 62, 35, 64, 65, 66, 35, 68, 69, 70, 35 ], "layer_types": [ "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention", "indexed_attention" ], "max_position_embeddings": 1048576, "mlp_layer_types": [ "dense", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse", "sparse" ], "model_type": "mistral_large4_text", "moe_intermediate_size": 6144, "n_group": 1, "n_routed_experts": 128, "n_shared_experts": 1, "no_rope_layer_interval": 4, "no_rope_layers": [ 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0 ], "num_attention_heads": 64, "num_experts_per_tok": 4, "num_hidden_layers": 72, "num_key_value_heads": 64, "output_router_logits": false, "pad_token_id": null, "qk_head_dim": 128, "qk_nope_head_dim": 64, "qk_rope_head_dim": 64, "rms_norm_eps": 1e-05, "rope_parameters": { "attention_factor": 1.0, "beta_fast": 32, "beta_slow": 1, "factor": 2048, "original_max_position_embeddings": 512, "rope_theta": 10000.0, "rope_type": "yarn" }, "routed_scaling_factor": 1.0, "tie_word_embeddings": false, "topk_group": 1, "use_cache": true, "v_head_dim": 128, "vocab_size": 204800 }, "tie_word_embeddings": false, "transformers_version": "5.21.0.dev0", "transformers_weights": "consolidated.safetensors.index.json", "vision_config": { "attention_dropout": 0.0, "head_dim": 128, "hidden_act": "silu", "hidden_size": 2048, "image_size": 1540, "initializer_range": 0.02, "intermediate_size": 6144, "model_type": "pixtral", "num_attention_heads": 16, "num_channels": 3, "num_hidden_layers": 30, "patch_size": 14, "rope_parameters": { "rope_theta": 10000.0, "rope_type": "axial" } }, "vision_feature_layer": -1 }