docstr: # Args.
      input_embeds: Tensor, (batch * beam, max_input_length, hidden_dim),
          input hidden states.
      attention_mask: Tensor, (batch * beam, max_input_length, max_input_length),
          input attention mask.
      input_lengths: (batch * beam,), input sequence lengths.
      memory_length: int, the length of memory to keep key/cache values.
      compact_index: IntTensor, (compact_batch_size,)
          The index of input sequences of a compact batch. If None, the FT op
          doesn't apply the shared context feature and as result the inference
          time may increase.
      batch_to_compact_index: IntTensor, (batch * beam,)
          The index map from the original input batch to the compact batch.
          This must be provided if compact_index is not None.
      linear_bias_slopes: (num_heads,)
          The slope per head of linear attention bias - ALiBi. If None, a base
          self attention will be performed.
  # Returns
      hidden_states: Tensor, (batch * beam, max_input_length, hidden_dim),
          decoder outputs.
      key_cache: Tensor, (num_layers, batch * beam, local_num_heads, size_per_head / x, memory_length, x), # noqa: E501
          key cache of attention of inputs.
          x = 16 / sizeof(T), memory_length = max_input_length or max_input_length + gen_length # noqa: E501
      value_cache: Tensor, (num_layers, batch * beam, local_num_heads, memory_length, hidden_dim) # noqa: E501
          value cache of attention
      last_token_hidden_states: Tensor, (batch * beam, hidden_dim)
          hidden states of the last input token.
docstr_tok: ["#","Args",".","input_embeds",":","Tensor",",","(","batch","*","beam",",","max_input_length",",","hidden_dim",")",",","input","hidden","states",".","attention_mask",":","Tensor",",","(","batch","*","beam",",","max_input_length",",","max_input_length",")",",","input","attention","mask",".","input_lengths",":","(","batch","*","beam",",",")",",","input","sequence","lengths",".","memory_length",":","int",",","the","length","of","memory","to","keep","key\/cache","values",".","compact_index",":","IntTensor",",","(","compact_batch_size",",",")","The","index","of","input","sequences","of","a","compact","batch",".","If","None",",","the","FT","op","does","n't","apply","the","shared","context","feature","and","as","result","the","inference","time","may","increase",".","batch_to_compact_index",":","IntTensor",",","(","batch","*","beam",",",")","The","index","map","from","the","original","input","batch","to","the","compact","batch",".","This","must","be","provided","if","compact_index","is","not","None",".","linear_bias_slopes",":","(","num_heads",",",")","The","slope","per","head","of","linear","attention","bias","-","ALiBi",".","If","None",",","a","base","self","attention","will","be","performed",".","#","Returns","hidden_states",":","Tensor",",","(","batch","*","beam",",","max_input_length",",","hidden_dim",")",",","decoder","outputs",".","key_cache",":","Tensor",",","(","num_layers",",","batch","*","beam",",","local_num_heads",",","size_per_head","\/","x",",","memory_length",",","x",")",",","#","noqa",":","E501","key","cache","of","attention","of","inputs",".","x","=","16","\/","sizeof","(","T",")",",","memory_length","=","max_input_length","or","max_input_length","+","gen_length","#","noqa",":","E501","value_cache",":","Tensor",",","(","num_layers",",","batch","*","beam",",","local_num_heads",",","memory_length",",","hidden_dim",")","#","noqa",":","E501","value","cache","of","attention","last_token_hidden_states",":","Tensor",",","(","batch","*","beam",",","hidden_dim",")","hidden","states","of","the","last","input","token","."]
code: def forward(
      self,
      input_embeds: torch.Tensor,
      attention_mask: torch.Tensor,
      input_lengths: torch.IntTensor,
      memory_length: Optional[int] = None,
      compact_index: Optional[torch.IntTensor] = None,
      batch_to_compact_index: Optional[torch.IntTensor] = None,
      linear_bias_slopes: Optional[torch.Tensor] = None,
  ):
      """
  
      # Args.
          input_embeds: Tensor, (batch * beam, max_input_length, hidden_dim),
              input hidden states.
          attention_mask: Tensor, (batch * beam, max_input_length, max_input_length),
              input attention mask.
          input_lengths: (batch * beam,), input sequence lengths.
          memory_length: int, the length of memory to keep key/cache values.
          compact_index: IntTensor, (compact_batch_size,)
              The index of input sequences of a compact batch. If None, the FT op
              doesn't apply the shared context feature and as result the inference
              time may increase.
          batch_to_compact_index: IntTensor, (batch * beam,)
              The index map from the original input batch to the compact batch.
              This must be provided if compact_index is not None.
          linear_bias_slopes: (num_heads,)
              The slope per head of linear attention bias - ALiBi. If None, a base
              self attention will be performed.
      # Returns
          hidden_states: Tensor, (batch * beam, max_input_length, hidden_dim),
              decoder outputs.
          key_cache: Tensor, (num_layers, batch * beam, local_num_heads, size_per_head / x, memory_length, x), # noqa: E501
              key cache of attention of inputs.
              x = 16 / sizeof(T), memory_length = max_input_length or max_input_length + gen_length # noqa: E501
          value_cache: Tensor, (num_layers, batch * beam, local_num_heads, memory_length, hidden_dim) # noqa: E501
              value cache of attention
          last_token_hidden_states: Tensor, (batch * beam, hidden_dim)
              hidden states of the last input token.
      """
      self._initialize_model()
      # outputs: output hidden states
      (
          decoder_ouptut,
          key_cache,
          value_cache,
          last_token_hidden_states,
      ) = self.ft_op.forward(
          input_embeds,
          attention_mask,
          input_lengths,
          memory_length,
          compact_index,
          batch_to_compact_index,
          linear_bias_slopes,
      )
      return (
          decoder_ouptut,
          key_cache,
          value_cache,
          last_token_hidden_states,
      )
  
imports: import math
  from abc import abstractmethod
  from pathlib import Path
  from typing import List, Literal, Optional, Union
  import os
  import numpy
  from .None import comm
  from .None import profiler
  from .gpt import GptInitModelParameters
  from nebullvm.optional_modules.torch import torch
label_desc: Use image node_id 5 for calling the GptContextDecoder obj's underlying member method code with example usage: obj.forward(input_embeds, attention_mask, input_lengths, memory_length, compact_index, batch_to_compact_index, linear_bias_slopes) and returns: decoder_ouptut,  key_cache,  value_cache,  last_token_hidden_states