code: def __init__(
      self,
      num_heads: int,
      size_per_head: int,
      inter_size: int,
      num_layers: int,
      tensor_para_size: int = 1,
      pipeline_para_size: int = 1,
      has_adapters: bool = False,
      adapter_inter_size: int = 0,
      int8_mode: int = 0,
  ):
      assert num_heads % tensor_para_size == 0, (
          f"num_heads ({num_heads}) is not multiple of "
          "tensor para size ({tensor_para_size})"
      )
  
      self.num_heads = num_heads
      self.size_per_head = size_per_head
      self.hidden_units = num_heads * size_per_head
      self.num_layers = num_layers
  
      self.tensor_para_size = tensor_para_size
      self.tensor_para_rank = comm.get_tensor_para_rank()
      self.pipeline_para_size = pipeline_para_size
      self.pipeline_para_rank = comm.get_pipeline_para_rank()
  
      self.has_adapters = has_adapters
      self.adapter_inter_size = adapter_inter_size
  
      self.local_num_layers = num_layers // pipeline_para_size
      self.local_num_heads = num_heads // tensor_para_size
      self.local_hidden_units = self.local_num_heads * size_per_head
      self.local_inter_size = inter_size // tensor_para_size
      self.local_adapter_inter_size = (
          self.adapter_inter_size // tensor_para_size
      )
  
      self.weight_transpose_calibrate_quantize = None
      assert int8_mode in [
          0,
          1,
      ], "Invalid int8 mode for GPT. Must be 0 or 1"
      self.int8_mode = int8_mode
      if self.int8_mode == 1:
          quant = (
              torch.ops.fastertransformer.symmetric_quantize_last_axis_of_batched_matrix  # noqa: E501
          )
          self.weight_transpose_calibrate_quantize = lambda x: quant(
              x, torch.int8
          )
  
      self.weights = None
      self.int8_weights = None
      self.int8_scales = None
  
      self.expected_weight_shapes = list()
  
      # pylint:disable=line-too-long
      # Transformer blocks
      self.expected_weight_shapes.extend(
          [(self.hidden_units,)] * self.local_num_layers
      )  # input layernorm weight
      self.expected_weight_shapes.extend(
          [(self.hidden_units,)] * self.local_num_layers
      )  # input layernorm bias
      self.expected_weight_shapes.extend(
          [(self.hidden_units, self.local_hidden_units * 3)]
          * self.local_num_layers
      )  # attention qkv weight
      self.expected_weight_shapes.extend(
          [(self.local_hidden_units * 3,)] * self.local_num_layers
      )  # attention qkv bias
      self.expected_weight_shapes.extend(
          [(self.local_hidden_units, self.hidden_units)]
          * self.local_num_layers
      )  # attention dense weight
      self.expected_weight_shapes.extend(
          [(self.hidden_units,)] * self.local_num_layers
      )  # attention dense bias
      self.expected_weight_shapes.extend(
          [(self.hidden_units,)] * self.local_num_layers
      )  # post attention layernorm weight
      self.expected_weight_shapes.extend(
          [(self.hidden_units,)] * self.local_num_layers
      )  # post attention layernorm bias
      self.expected_weight_shapes.extend(
          [(self.hidden_units, self.local_inter_size)]
          * self.local_num_layers
      )  # ffn_kernel1
      self.expected_weight_shapes.extend(
          [(self.local_inter_size,)] * self.local_num_layers
      )  # ffn_bias1
      self.expected_weight_shapes.extend(
          [(self.local_inter_size, self.hidden_units)]
          * self.local_num_layers
      )  # ffn_kernel2
      self.expected_weight_shapes.extend(
          [(self.hidden_units,)] * self.local_num_layers
      )  # ffn_bias2
  
      # Adapters
      if self.has_adapters:
          self.expected_weight_shapes.extend(
              [(self.hidden_units, self.local_adapter_inter_size)]
              * self.local_num_layers
          )  # adaptor1_kernel1
          self.expected_weight_shapes.extend(
              [(self.local_adapter_inter_size,)] * self.local_num_layers
          )  # adaptor1_bias1
          self.expected_weight_shapes.extend(
              [(self.local_adapter_inter_size, self.hidden_units)]
              * self.local_num_layers
          )  # adaptor1_kernel2
          self.expected_weight_shapes.extend(
              [(self.hidden_units,)] * self.local_num_layers
          )  # adaptor1_bias2
          self.expected_weight_shapes.extend(
              [(self.hidden_units, self.local_adapter_inter_size)]
              * self.local_num_layers
          )  # adaptor2_kernel1
          self.expected_weight_shapes.extend(
              [(self.local_adapter_inter_size,)] * self.local_num_layers
          )  # adaptor2_bias1
          self.expected_weight_shapes.extend(
              [(self.local_adapter_inter_size, self.hidden_units)]
              * self.local_num_layers
          )  # adaptor2_kernel2
          self.expected_weight_shapes.extend(
              [(self.hidden_units,)] * self.local_num_layers
          )
  
imports: import math
  from abc import abstractmethod
  from pathlib import Path
  from typing import List, Literal, Optional, Union
  import os
  import numpy
  from .None import comm
  from .None import profiler
  from .gpt import GptInitModelParameters
  from nebullvm.optional_modules.torch import torch
label_desc: Use image node_id 1 to create a new GptLayerWeights object  with example: obj = GptLayerWeights(num_heads, size_per_head, inter_size, num_layers, tensor_para_size, pipeline_para_size, has_adapters, adapter_inter_size, int8_mode)