code: def __init__(
      self,
      num_heads: int,
      size_per_head: int,
      inter_size: int,
      num_layers: int,
      tensor_para_size: int = 1,
      pipeline_para_size: int = 1,
      layernorm_eps: float = 1e-6,
      layernorm_type: LayernormType = "pre_layernorm",
      activation_type: str = "gelu",
      has_adapters: bool = False,
      adapter_inter_size: int = 0,
      int8_mode: int = 0,
  ):
      super().__init__()
      self.num_heads = num_heads
      self.size_per_head = size_per_head
      self.hidden_size = self.num_heads * self.size_per_head
      self.inter_size = inter_size
      self.num_layers = num_layers
  
      self.tensor_para_size = tensor_para_size
      self.pipeline_para_size = pipeline_para_size
  
      self.layernorm_eps = layernorm_eps
      self.layernorm_type = layernorm_type
      self.activation_type = activation_type
      self.has_adapters = has_adapters
      self.adapter_inter_size = adapter_inter_size
  
      self.int8_mode = int8_mode
  
      self.ft_op = None
      self.weight = None
  
imports: import math
  from abc import abstractmethod
  from pathlib import Path
  from typing import List, Literal, Optional, Union
  import os
  import numpy
  from .None import comm
  from .None import profiler
  from .gpt import GptInitModelParameters
  from nebullvm.optional_modules.torch import torch
label_desc: Use image node_id 1 to create a new GptDecoder object from inherited base classes: FtModuleBase with example: obj = GptDecoder(num_heads, size_per_head, inter_size, num_layers, tensor_para_size, pipeline_para_size, layernorm_eps, layernorm_type, activation_type, has_adapters, adapter_inter_size, int8_mode)