code: def prepare_inputs_labels_for_multimodal(
      self,
      input_ids,
      attention_mask,
      past_key_values,
      labels,
      images,
      prompts=None,
  ):
      if prompts is None and hasattr(self, "prompts"):
          prompts = self.prompts
  
      vision_tower = self.get_vision_tower()
      if (
          vision_tower is None
          or images is None
          or input_ids.shape[1] == 1
      ):
          if (
              past_key_values is not None
              and vision_tower is not None
              and images is not None
              and input_ids.shape[1] == 1
          ):
              attention_mask = torch.ones(
                  (
                      attention_mask.shape[0],
                      past_key_values[-1][-1].shape[-2] + 1,
                  ),
                  dtype=attention_mask.dtype,
                  device=attention_mask.device,
              )
          return (
              input_ids,
              attention_mask,
              past_key_values,
              None,
              labels,
          )
  
      # pre-process images for long video
      if images[0].shape[-1] > 1000:
          long_video = True
      else:
          long_video = False
  
      if type(images) is list or images.ndim == 5:
          # not reseshape for long video
          if not long_video:
              images = [
                  image if len(image.shape) == 4 else image.unsqueeze(0)
                  for image in images
              ]
          image_counts = [image.shape[0] for image in images]
          concat_images = torch.cat(images, dim=0)
          image_features = self.encode_images(
              concat_images,
              prompts,
              image_counts,
              long_video=long_video,
          )
      else:
          image_features = self.encode_images(
              images, prompts, long_video=long_video
          )
  
      new_input_embeds = []
      new_labels = [] if labels is not None else None
      cur_image_idx = 0
      for batch_idx, cur_input_ids in enumerate(input_ids):
          if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
              # multimodal LLM, but the current sample is not multimodal
              # FIXME: this is a hacky fix, for deepspeed zero3 to work
              half_len = cur_input_ids.shape[0] // 2
              if isinstance(image_features, list):
                  cur_image_features = image_features[cur_image_idx][0]
              else:
                  cur_image_features = image_features[cur_image_idx]
              cur_input_embeds_1 = self.get_model().embed_tokens(
                  cur_input_ids[:half_len]
              )
              cur_input_embeds_2 = self.get_model().embed_tokens(
                  cur_input_ids[half_len:]
              )
              cur_input_embeds = torch.cat(
                  [
                      cur_input_embeds_1,
                      cur_image_features[0:0],
                      cur_input_embeds_2,
                  ],
                  dim=0,
              )
              new_input_embeds.append(cur_input_embeds)
              if labels is not None:
                  new_labels.append(labels[batch_idx])
              cur_image_idx += 1
              continue
  
          image_token_indices = torch.where(
              cur_input_ids == IMAGE_TOKEN_INDEX
          )[0]
          cur_new_input_embeds = []
          if labels is not None:
              cur_labels = labels[batch_idx]
              cur_new_labels = []
              assert cur_labels.shape == cur_input_ids.shape
  
          if not long_video:
              token_idx = 0
              while image_token_indices.numel() > 0:
                  if isinstance(image_features, list):
                      cur_image_features = image_features[
                          cur_image_idx
                      ][token_idx]
                  else:
                      cur_image_features = image_features[cur_image_idx]
                  image_token_start = image_token_indices[0]
  
                  if getattr(
                      self.config, "tune_mm_mlp_adapter", False
                  ) and getattr(
                      self.config, "mm_use_im_start_end", False
                  ):
                      cur_new_input_embeds.append(
                          self.get_model()
                          .embed_tokens(
                              cur_input_ids[: image_token_start - 1]
                          )
                          .detach()
                      )
                      cur_new_input_embeds.append(
                          self.get_model().embed_tokens(
                              cur_input_ids[
                                  image_token_start
                                  - 1 : image_token_start
                              ]
                          )
                      )
                      cur_new_input_embeds.append(cur_image_features)
                      cur_new_input_embeds.append(
                          self.get_model().embed_tokens(
                              cur_input_ids[
                                  image_token_start
                                  + 1 : image_token_start
                                  + 2
                              ]
                          )
                      )
                      if labels is not None:
                          cur_new_labels.append(
                              cur_labels[:image_token_start]
                          )
                          cur_new_labels.append(
                              torch.full(
                                  (cur_image_features.shape[0],),
                                  IGNORE_INDEX,
                                  device=labels.device,
                                  dtype=labels.dtype,
                              )
                          )
                          cur_new_labels.append(
                              cur_labels[
                                  image_token_start : image_token_start
                                  + 1
                              ]
                          )
                          cur_labels = cur_labels[
                              image_token_start + 2 :
                          ]
                  else:
                      cur_new_input_embeds.append(
                          self.get_model().embed_tokens(
                              cur_input_ids[:image_token_start]
                          )
                      )
                      cur_new_input_embeds.append(cur_image_features)
                      if labels is not None:
                          cur_new_labels.append(
                              cur_labels[:image_token_start]
                          )
                          cur_new_labels.append(
                              torch.full(
                                  (cur_image_features.shape[0],),
                                  IGNORE_INDEX,
                                  device=labels.device,
                                  dtype=labels.dtype,
                              )
                          )
                          cur_labels = cur_labels[
                              image_token_start + 1 :
                          ]
                  if getattr(
                      self.config, "tune_mm_mlp_adapter", False
                  ) and getattr(
                      self.config, "mm_use_im_start_end", False
                  ):
                      cur_input_ids = cur_input_ids[
                          image_token_start + 2 :
                      ]
                  else:
                      cur_input_ids = cur_input_ids[
                          image_token_start + 1 :
                      ]
                  image_token_indices = torch.where(
                      cur_input_ids == IMAGE_TOKEN_INDEX
                  )[0]
                  token_idx += 1
  
              # changle image idx after processing one sample
              cur_image_idx += 1
              if cur_input_ids.numel() > 0:
                  if getattr(
                      self.config, "tune_mm_mlp_adapter", False
                  ) and getattr(
                      self.config, "mm_use_im_start_end", False
                  ):
                      cur_new_input_embeds.append(
                          self.get_model()
                          .embed_tokens(cur_input_ids)
                          .detach()
                      )
                  else:
                      cur_new_input_embeds.append(
                          self.get_model().embed_tokens(cur_input_ids)
                      )
                  if labels is not None:
                      cur_new_labels.append(cur_labels)
              cur_new_input_embeds = [
                  x.to(device=self.device) for x in cur_new_input_embeds
              ]
              cur_new_input_embeds = torch.cat(
                  cur_new_input_embeds, dim=0
              )
              new_input_embeds.append(cur_new_input_embeds)
              if labels is not None:
                  cur_new_labels = torch.cat(cur_new_labels, dim=0)
                  new_labels.append(cur_new_labels)
          else:
              cur_new_input_embeds = torch.Tensor(
                  len(cur_input_ids), self.config.hidden_size
              ).to(dtype=self.dtype, device=self.device)
              text_token_indices = torch.where(
                  cur_input_ids != IMAGE_TOKEN_INDEX
              )[0]
              if (
                  not self.training
                  and self.get_model().embed_tokens.weight.device
                  != cur_input_ids.device
              ):
                  model_device = (
                      self.get_model().embed_tokens.weight.device
                  )
                  data_device = cur_input_ids.device
                  cur_input_ids_text = cur_input_ids[
                      text_token_indices
                  ].to(device=model_device)
                  cur_new_input_embeds[text_token_indices] = (
                      self.get_model()
                      .embed_tokens(cur_input_ids_text)
                      .to(device=data_device)
                  )
              else:
                  cur_new_input_embeds[
                      text_token_indices
                  ] = self.get_model().embed_tokens(
                      cur_input_ids[text_token_indices]
                  )
              cur_image_features = image_features[cur_image_idx]
              cur_new_input_embeds[
                  image_token_indices
              ] = cur_image_features
              new_input_embeds.append(cur_new_input_embeds)
              if labels is not None:
                  new_labels.append(cur_labels)
              cur_image_idx += 1
  
      if any(
          x.shape != new_input_embeds[0].shape for x in new_input_embeds
      ):
          max_len = max(x.shape[0] for x in new_input_embeds)
  
          new_input_embeds_align = []
          for cur_new_embed in new_input_embeds:
              cur_new_embed = torch.cat(
                  (
                      cur_new_embed,
                      torch.zeros(
                          (
                              max_len - cur_new_embed.shape[0],
                              cur_new_embed.shape[1],
                          ),
                          dtype=cur_new_embed.dtype,
                          device=cur_new_embed.device,
                      ),
                  ),
                  dim=0,
              )
              new_input_embeds_align.append(cur_new_embed)
          new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
  
          if labels is not None:
              new_labels_align = []
              _new_labels = new_labels
              for cur_new_label in new_labels:
                  cur_new_label = torch.cat(
                      (
                          cur_new_label,
                          torch.full(
                              (max_len - cur_new_label.shape[0],),
                              IGNORE_INDEX,
                              dtype=cur_new_label.dtype,
                              device=cur_new_label.device,
                          ),
                      ),
                      dim=0,
                  )
                  new_labels_align.append(cur_new_label)
              new_labels = torch.stack(new_labels_align, dim=0)
  
          # only used for right padding in tokenlizer
          if attention_mask is not None:
              new_attention_mask = []
              for (
                  cur_attention_mask,
                  cur_new_labels,
                  cur_new_labels_align,
              ) in zip(attention_mask, _new_labels, new_labels):
                  new_attn_mask_pad_left = torch.full(
                      (cur_new_labels.shape[0] - labels.shape[1],),
                      True,
                      dtype=attention_mask.dtype,
                      device=attention_mask.device,
                  )
                  new_attn_mask_pad_right = torch.full(
                      (
                          cur_new_labels_align.shape[0]
                          - cur_new_labels.shape[0],
                      ),
                      False,
                      dtype=attention_mask.dtype,
                      device=attention_mask.device,
                  )
                  cur_new_attention_mask = torch.cat(
                      (
                          new_attn_mask_pad_left,
                          cur_attention_mask,
                          new_attn_mask_pad_right,
                      ),
                      dim=0,
                  )
                  new_attention_mask.append(cur_new_attention_mask)
              attention_mask = torch.stack(new_attention_mask, dim=0)
              assert attention_mask.shape == new_labels.shape
      else:
          new_input_embeds = torch.stack(new_input_embeds, dim=0)
          if labels is not None:
              new_labels = torch.stack(new_labels, dim=0)
  
          # only used for right padding in tokenlizer
          if attention_mask is not None:
              new_attn_mask_pad_left = torch.full(
                  (
                      attention_mask.shape[0],
                      new_input_embeds.shape[1] - input_ids.shape[1],
                  ),
                  True,
                  dtype=attention_mask.dtype,
                  device=attention_mask.device,
              )
              attention_mask = torch.cat(
                  (new_attn_mask_pad_left, attention_mask), dim=1
              )
              assert attention_mask.shape == new_input_embeds.shape[:2]
  
      return (
          None,
          attention_mask,
          past_key_values,
          new_input_embeds,
          new_labels,
      )
  
imports: from abc import ABC, abstractmethod
  import os
  import json
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from transformers import BertTokenizer
  from transformers.models.bert.modeling_bert import BertLMHeadModel
  from .qformer import BertConfig
  from .qformer import BertLMHeadModel
  from .multimodal_encoder.builder import build_vision_tower
  from .multimodal_projector.builder import build_vision_projector
  from llamavid.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
label_desc: Use image node_id 7 for calling the LLaMAVIDMetaForCausalLM obj's underlying member method code with example usage: obj.prepare_inputs_labels_for_multimodal(input_ids, attention_mask, past_key_values, labels, images, prompts) and returns: None,  attention_mask,  past_key_values,  new_input_embeds,  new_labels, input_ids,  attention_mask,  past_key_values,  None,  labels