code: def __init__(
      self,
      num_heads: int,
      size_per_head: int,
      num_layers: int,
      vocab_size: int,
      start_id: int,
      end_id: int,
      lib_path: PathLike,
      tensor_para_size: int = 1,
      pipeline_para_size: int = 1,
      remove_padding: bool = True,
      shared_contexts_ratio: float = 1.0,
      layernorm_eps: float = 1e-6,
      layernorm_type: LayernormType = "pre_layernorm",
      activation_type: str = "gelu",
      has_positional_encoding: bool = True,
      max_seq_len: int = 0,
      has_pre_decoder_layernorm: bool = False,
      has_post_decoder_layernorm: bool = True,
      has_adapters: bool = False,
      adapter_inter_size: int = 0,
      int8_mode: int = 0,
      inference_data_type: Optional[str] = None,
      weights_data_type: str = "fp32",
      use_fp32_to_compute_logit: bool = False,
      **kwargs,
  ):
      super().__init__()
  
      inference_data_type = inference_data_type or weights_data_type
  
      self.config = GptInitModelParameters(
          head_num=num_heads,
          size_per_head=size_per_head,
          layer_num=num_layers,
          max_seq_len=max_seq_len,
          tensor_para_size=tensor_para_size,
          vocab_size=vocab_size,
          start_id=start_id,
          end_id=end_id,
          pipeline_para_size=pipeline_para_size,
          data_type=inference_data_type,
          weights_data_type=weights_data_type,
          layernorm_eps=layernorm_eps,
          layernorm_type=layernorm_type,
          activation_type=activation_type,
          has_positional_encoding=has_positional_encoding,
          has_pre_decoder_layernorm=has_pre_decoder_layernorm,
          has_post_decoder_layernorm=has_post_decoder_layernorm,
          has_adapters=has_adapters,
          adapter_inter_size=adapter_inter_size,
          int8_mode=int8_mode,
          sparse=kwargs.get("sparse", False),
      )
      self.use_fp32_to_compute_logit = use_fp32_to_compute_logit
  
      self.weight = None
      self.shared_contexts_ratio = shared_contexts_ratio
  
      torch.classes.load_library(os.path.abspath(lib_path))
  
      # Embeddings to encode or decode tokens.
      hidden_dim = num_heads * size_per_head
  
      # Pad vocab size for FT.
      local_vocab_size = math.ceil(
          self.config.vocab_size / self.config.tensor_para_size
      )
      if self.config.data_type == "fp16":
          local_vocab_size = math.ceil(local_vocab_size / 8) * 8
      self.vocab_size_padded = (
          local_vocab_size * self.config.tensor_para_size
      )
      self.vocab_size = self.config.vocab_size
  
      self.decode_op = torch.classes.FasterTransformer.DynamicDecodeOp(
          self.vocab_size,
          self.vocab_size_padded,
          self.config.tensor_para_size,
          self.config.pipeline_para_size,
          torch.float,
      )
  
      self._parameters = {}
  
      def register_param(name, p):
          self._parameters[name] = p
          setattr(self, name, p)
  
      register_param(
          "context_decoder",
          GptContextDecoder.from_config(
              self.config,
              remove_padding=remove_padding,
              shared_contexts_ratio=shared_contexts_ratio,
              **kwargs,
          ),
      )
      register_param(
          "decoder", GptDecoder.from_config(self.config, **kwargs)
      )
  
      compute_dtype = to_torch_dtype(inference_data_type)
  
      if comm.is_pipeline_group_first():
          register_param(
              "word_embedding",
              torch.nn.Embedding(
                  self.vocab_size_padded,
                  hidden_dim,
                  dtype=compute_dtype,
              ),
          )
          self._mask_padded_vocab_weights(self.word_embedding.weight)
          if self.config.has_positional_encoding:
              register_param(
                  "position_encoding",
                  torch.nn.Embedding(
                      self.config.max_seq_len,
                      hidden_dim,
                      dtype=compute_dtype,
                  ),
              )
          else:
              self.position_encoding = None
          if self.config.has_pre_decoder_layernorm:
              register_param(
                  "pre_decoder_layernorm",
                  torch.nn.LayerNorm(
                      hidden_dim, eps=layernorm_eps, dtype=compute_dtype
                  ),
              )
          else:
              self.pre_decoder_layernorm = None
  
      if comm.is_pipeline_group_last():
          if has_post_decoder_layernorm:
              register_param(
                  "post_decoder_layernorm",
                  torch.nn.LayerNorm(
                      hidden_dim, eps=layernorm_eps, dtype=compute_dtype
                  ),
              )
          else:
              self.post_decoder_layernorm = None
          self.lm_head_ctype = (
              compute_dtype
              if not self.use_fp32_to_compute_logit
              else torch.float32
          )
          register_param(
              "lm_head",
              torch.nn.Linear(
                  hidden_dim,
                  self.vocab_size_padded,
                  bias=False,
                  dtype=self.lm_head_ctype,
              ),
          )
          self._mask_padded_vocab_weights(self.lm_head.weight)
  
imports: import math
  from abc import abstractmethod
  from pathlib import Path
  from typing import List, Literal, Optional, Union
  import os
  import numpy
  from .None import comm
  from .None import profiler
  from .gpt import GptInitModelParameters
  from nebullvm.optional_modules.torch import torch
label_desc: Use image node_id 1 to create a new Gpt object  with example: obj = Gpt(num_heads, size_per_head, num_layers, vocab_size, start_id, end_id, lib_path, tensor_para_size, pipeline_para_size, remove_padding, shared_contexts_ratio, layernorm_eps, layernorm_type, activation_type, has_positional_encoding, max_seq_len, has_pre_decoder_layernorm, has_post_decoder_layernorm, has_adapters, adapter_inter_size, int8_mode, inference_data_type, weights_data_type, use_fp32_to_compute_logit)