code: def forward(
      self,
      input_ids: Optional[torch.Tensor] = None,
      past_key_values: Optional[List[torch.Tensor]] = None,
      attention_mask: Optional[torch.Tensor] = None,
      token_type_ids: Optional[torch.Tensor] = None,
      position_ids: Optional[torch.Tensor] = None,
      head_mask: Optional[torch.Tensor] = None,
      inputs_embeds: Optional[torch.Tensor] = None,
      encoder_hidden_states: Optional[torch.Tensor] = None,
      encoder_attention_mask: Optional[torch.Tensor] = None,
      use_cache: Optional[bool] = None,
      output_attentions: Optional[bool] = None,
      output_hidden_states: Optional[bool] = None,
      return_dict: Optional[bool] = None,
  ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
      output_attentions = (
          output_attentions
          if output_attentions is not None
          else self.config.output_attentions
      )
      output_hidden_states = (
          output_hidden_states
          if output_hidden_states is not None
          else self.config.output_hidden_states
      )
      use_cache = (
          use_cache if use_cache is not None else self.config.use_cache
      )
      return_dict = (
          return_dict
          if return_dict is not None
          else self.config.use_return_dict
      )
  
      if input_ids is not None and inputs_embeds is not None:
          raise ValueError(
              "You cannot specify both input_ids and inputs_embeds at the same time"
          )
      elif input_ids is not None:
          self.warn_if_padding_and_no_attention_mask(
              input_ids, attention_mask
          )
          input_shape = input_ids.size()
          input_ids = input_ids.view(-1, input_shape[-1])
          batch_size = input_ids.shape[0]
      elif inputs_embeds is not None:
          input_shape = inputs_embeds.size()[:-1]
          batch_size = inputs_embeds.shape[0]
      else:
          raise ValueError(
              "You have to specify either input_ids or inputs_embeds"
          )
  
      if batch_size <= 0:
          raise ValueError("batch_size has to be defined and > 0")
  
      device = (
          input_ids.device
          if input_ids is not None
          else inputs_embeds.device
      )
  
      if token_type_ids is not None:
          token_type_ids = token_type_ids.view(-1, input_shape[-1])
  
      if past_key_values is None:
          past_length = 0
          past_key_values = tuple([None] * len(self.h))
      else:
          past_length = past_key_values[0].size(-2)
  
      if (
          attention_mask is not None
          and len(attention_mask.shape) == 2
          and position_ids is None
      ):
          # create position_ids on the fly for batch generation
          position_ids = attention_mask.long().cumsum(-1) - 1
          position_ids.masked_fill_(attention_mask == 0, 1)
          if past_length > 0:
              position_ids = position_ids[
                  :, past_length : input_shape[-1] + past_length :
              ]
      elif position_ids is None:
          position_ids = torch.arange(
              past_length,
              input_shape[-1] + past_length,
              dtype=torch.long,
              device=device,
          )
          position_ids = position_ids.unsqueeze(0)
  
      # Self-attention mask.
      query_length = input_shape[-1]
      key_length = past_length + query_length
      self_attention_mask = self.bias[
          None, key_length - query_length : key_length, :key_length
      ]
  
      if self._use_flash_attention_2:
          # 2d mask is passed through the layers
          attention_mask = (
              attention_mask.bool()
              if (attention_mask is not None and 0 in attention_mask)
              else None
          )
          encoder_attention_mask = (
              encoder_attention_mask.bool()
              if (
                  encoder_attention_mask is not None
                  and 0 in encoder_attention_mask
              )
              else None
          )
      else:
          # 4d mask is passed through the layers
          if attention_mask is not None:
              self_attention_mask = (
                  self_attention_mask
                  * attention_mask.view(batch_size, 1, -1).to(
                      dtype=torch.bool,
                      device=self_attention_mask.device,
                  )
              )
  
          # MQA models: (batch_size, query_length, n_heads, key_length)
          # MHA models: (batch_size, n_heads, query_length, key_length)
          self_attention_mask = self_attention_mask.unsqueeze(
              2 if self.multi_query else 1
          )
  
          if (
              self._use_sdpa
              and head_mask is None
              and not output_attentions
          ):
              # output_attentions=True can not be supported when using SDPA, and we fall back on
              # the manual implementation that requires a 4D causal mask in all cases.
              if self.multi_query:
                  # gpt_bigcode using MQA has the bad taste to use a causal mask with shape
                  # [batch_size, target_length, 1, source_length], not compatible with SDPA, hence this transpose.
                  self_attention_mask = self_attention_mask.transpose(
                      1, 2
                  )
  
              if query_length > 1 and attention_mask is not None:
                  # From PyTorch 2.1 onwards, F.scaled_dot_product_attention with the memory-efficient attention backend
                  # produces nans if sequences are completely unattended in the attention mask. Details: https://github.com/pytorch/pytorch/issues/110213
                  self_attention_mask = (
                      AttentionMaskConverter._unmask_unattended(
                          self_attention_mask,
                          attention_mask,
                          unmasked_value=True,
                      )
                  )
  
              # SDPA with a custom mask is much faster in fp16/fp32 dtype rather than bool. Cast here to floating point instead of at every layer.
              dtype = self.wte.weight.dtype
              self_attention_mask = torch.where(
                  self_attention_mask,
                  torch.full(
                      [],
                      0.0,
                      dtype=dtype,
                      device=self_attention_mask.device,
                  ),
                  torch.full(
                      [],
                      torch.finfo(self.wte.weight.dtype).min,
                      dtype=dtype,
                      device=self_attention_mask.device,
                  ),
              )
  
          attention_mask = self_attention_mask
  
          # If a 2D or 3D attention mask is provided for the cross-attention
          # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
          if (
              self.config.add_cross_attention
              and encoder_hidden_states is not None
              and encoder_attention_mask is not None
          ):
              if encoder_attention_mask.dim() == 2:
                  encoder_attention_mask.unsqueeze(1)
              assert encoder_attention_mask.dim() == 3
              encoder_attention_mask = (
                  encoder_attention_mask.bool().unsqueeze(
                      2 if self.multi_query else 1
                  )
              )
          else:
              encoder_attention_mask = None
  
      # Prepare head mask if needed
      # 1.0 in head_mask indicate we keep the head
      # attention_probs has shape bsz x n_heads x N x N
      # head_mask has shape n_layer x batch x n_heads x N x N
      head_mask = self.get_head_mask(head_mask, self.config.n_layer)
  
      if inputs_embeds is None:
          inputs_embeds = self.wte(input_ids)
      position_embeds = self.wpe(position_ids)
      hidden_states = inputs_embeds + position_embeds
  
      if token_type_ids is not None:
          token_type_embeds = self.wte(token_type_ids)
          hidden_states = hidden_states + token_type_embeds
  
      hidden_states = self.drop(hidden_states)
  
      output_shape = input_shape + (hidden_states.size(-1),)
  
      presents = [] if use_cache else None
      all_self_attentions = () if output_attentions else None
      all_cross_attentions = (
          ()
          if output_attentions and self.config.add_cross_attention
          else None
      )
      all_hidden_states = () if output_hidden_states else None
      for i, (block, layer_past) in enumerate(
          zip(self.h, past_key_values)
      ):
          if output_hidden_states:
              all_hidden_states = all_hidden_states + (hidden_states,)
  
          if self.gradient_checkpointing and self.training:
              outputs = self._gradient_checkpointing_func(
                  block.__call__,
                  hidden_states,
                  None,
                  attention_mask,
                  head_mask[i],
                  encoder_hidden_states,
                  encoder_attention_mask,
                  use_cache,
                  output_attentions,
              )
          else:
              outputs = block(
                  hidden_states,
                  layer_past=layer_past,
                  attention_mask=attention_mask,
                  head_mask=head_mask[i],
                  encoder_hidden_states=encoder_hidden_states,
                  encoder_attention_mask=encoder_attention_mask,
                  use_cache=use_cache,
                  output_attentions=output_attentions,
              )
  
          hidden_states = outputs[0]
          if use_cache:
              presents.append(outputs[1])
  
          if output_attentions:
              all_self_attentions = all_self_attentions + (
                  outputs[2 if use_cache else 1],
              )
              if self.config.add_cross_attention:
                  all_cross_attentions = all_cross_attentions + (
                      outputs[3 if use_cache else 2],
                  )
  
      hidden_states = self.ln_f(hidden_states)
  
      hidden_states = hidden_states.view(output_shape)
      # Add last hidden state
      if output_hidden_states:
          all_hidden_states = all_hidden_states + (hidden_states,)
  
      if not return_dict:
          return tuple(
              v
              for v in [
                  hidden_states,
                  presents,
                  all_hidden_states,
                  all_self_attentions,
                  all_cross_attentions,
              ]
              if v is not None
          )
  
      return BaseModelOutputWithPastAndCrossAttentions(
          last_hidden_state=hidden_states,
          past_key_values=presents,
          hidden_states=all_hidden_states,
          attentions=all_self_attentions,
          cross_attentions=all_cross_attentions,
      )
  
imports: import math
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.nn.functional
  import torch.utils.checkpoint
  from torch import nn
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
  from ...activations import ACT2FN
  from ...modeling_attn_mask_utils import AttentionMaskConverter
  from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions, SequenceClassifierOutputWithPast, TokenClassifierOutput
  from ...modeling_utils import PreTrainedModel
  from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, is_flash_attn_2_available, is_flash_attn_greater_or_equal_2_10, logging
  from .configuration_gpt_bigcode import GPTBigCodeConfig
label_desc: Use image node_id 4 for calling the GPTBigCodeModel obj's underlying member method code with example usage: obj.forward(input_ids, past_key_values, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, use_cache, output_attentions, output_hidden_states, return_dict) and returns: BaseModelOutputWithPastAndCrossAttentions, tuple