Update modeling_typhoon2audio.py

modeling_typhoon2audio.py

@@ -18,6 +18,7 @@ from transformers import (
     WhisperModel,
     PreTrainedModel,
     AutoTokenizer,
+    AutoConfig,
     AutoModelForCausalLM,
 )
 from transformers.cache_utils import Cache, StaticCache
@@ -63,6 +64,7 @@ from transformers.modeling_utils import (
     apply_chunking_to_forward,
     find_pruneable_heads_and_indices,
     prune_linear_layer,
+    no_init_weights
 )
 from transformers.models.bert.configuration_bert import BertConfig
 
@@ -841,9 +843,9 @@ class Typhoon2AudioForConditionalGeneration(PreTrainedModel, GenerationMixin):
         self.second_stride = config.second_stride
 
         # 2. LLM (e.g., Llama3)
-        self.llama_model = AutoModelForCausalLM.from_pretrained(
-            config.llama_base_model, attn_implementation=attn_implementation
-        )
+        with no_init_weights(_enable=True):
+            llm_config = AutoConfig.from_pretrained(config.llama_base_model)
+            self.llama_model = AutoModelForCausalLM.from_config(llm_config, attn_implementation=attn_implementation)
         # tokenizer
         self.llama_tokenizer = AutoTokenizer.from_pretrained(
             config.llama_base_model, use_fast=False