class_docstr: Linear layer with row parallelism.
  This layer is used to fit `Conv1D` layer (Fused QKV) in gpt2 of huggingface.
  
  Args:
      in_features (int): size of each input sample.
      out_features (int): size of each output sample.
      bias (bool, optional): If set to ``False``, the layer will not learn an additive bias, defaults to ``True``.
      dtype (`torch.dtype`): The dtype of parameters, defaults to None.
      parallel_input (bool): If set to ``True``, it's assumed that the input is split, defaults to False.
      skip_bias_add (bool): If set to ``True``, it will skip bias add for linear layer,
      seq_parallel (`bool`): If set to ``True``, it will use sequence parallel, defaults to False.
          which is preserved for kernel fusion, defaults to False
      weight_initializer (:class:`typing.Callable`, optional):
          The initializer of weight, defaults to kaiming uniform initializer.
      bias_initializer (:class:`typing.Callable`, optional):
          The initializer of bias, defaults to xavier uniform initializer.
  
  More details about ``initializer`` please refer to
  `init <https://github.com/hpcaitech/ColossalAI/blob/main/colossalai/nn/init.py>`_.
class_docstr_tok: ["Linear","layer","with","row","parallelism",".","This","layer","is","used","to","fit","`","Conv1D","`","layer","(","Fused","QKV",")","in","gpt2","of","huggingface",".","Args",":","in_features","(","int",")",":","size","of","each","input","sample",".","out_features","(","int",")",":","size","of","each","output","sample",".","bias","(","bool",",","optional",")",":","If","set","to","``","False","``",",","the","layer","will","not","learn","an","additive","bias",",","defaults","to","``","True","``",".","dtype","(","`","torch.dtype","`",")",":","The","dtype","of","parameters",",","defaults","to","None",".","parallel_input","(","bool",")",":","If","set","to","``","True","``",",","it","'s","assumed","that","the","input","is","split",",","defaults","to","False",".","skip_bias_add","(","bool",")",":","If","set","to","``","True","``",",","it","will","skip","bias","add","for","linear","layer",",","seq_parallel","(","`","bool","`",")",":","If","set","to","``","True","``",",","it","will","use","sequence","parallel",",","defaults","to","False",".","which","is","preserved","for","kernel","fusion",",","defaults","to","False","weight_initializer","(",":","class",":","`","typing.Callable","`",",","optional",")",":","The","initializer","of","weight",",","defaults","to","kaiming","uniform","initializer",".","bias_initializer","(",":","class",":","`","typing.Callable","`",",","optional",")",":","The","initializer","of","bias",",","defaults","to","xavier","uniform","initializer",".","More","details","about","``","initializer","``","please","refer","to","`","init","<","https",":","\/\/github.com\/hpcaitech\/ColossalAI\/blob\/main\/colossalai\/nn\/init.py",">","`","_","."]
docstr: Convert a native PyTorch linear layer to a parallelized linear layer.
docstr_tok: ["Convert","a","native","PyTorch","linear","layer","to","a","parallelized","linear","layer","."]
code: def from_native_module(
      module: nn.Linear,
      process_group: Union[ProcessGroup, List[ProcessGroup]],
      *args,
      **kwargs,
  ) -> ParallelModule:
      r"""
      Convert a native PyTorch linear layer to a parallelized linear layer.
      """
      LazyInitContext.materialize(module)
      # get the attributes
      in_features = module.weight.shape[0]
      out_features = module.weight.shape[1]
      bias = module.bias is not None
      device = module.weight.device
  
      # ensure only one process group is passed
      if isinstance(process_group, (list, tuple)):
          assert (
              len(process_group) == 1
          ), f"Expected only one process group, got {len(process_group)}."
          process_group = process_group[0]
  
      tp_size = dist.get_world_size(process_group)
      if in_features < tp_size:
          return module
  
      if in_features % tp_size != 0:
          raise ValueError(
              f"The size of in_features:{in_features} is not integer multiples of tensor parallel size: {tp_size}!"
          )
  
      linear_1d = GPT2FusedLinearConv1D_Row(
          in_features=in_features,
          out_features=out_features,
          bias=bias,
          device=device,
          process_group=process_group,
          weight=module.weight,
          bias_=module.bias,
          *args,
          **kwargs,
      )
  
      return linear_1d
  
imports: import math
  from typing import Callable, List, Optional, Tuple, Union
  import torch
  import torch.distributed
  import torch.nn
  from torch import Tensor
  from torch.distributed import ProcessGroup
  from torch.nn.parameter import Parameter
  from colossalai.lazy import LazyInitContext
  from colossalai.nn import init
  from colossalai.nn.layer.utils import divide
  from colossalai.tensor.d_tensor.api import customized_distributed_tensor_to_existing_param, distribute_tensor_with_customization, is_customized_distributed_tensor, is_distributed_tensor, shard_rowwise, sharded_tensor_to_existing_param
  from ._operation import gather_forward_split_backward, linear_reducescatter_forward_gather_backward, linear_with_async_comm, matmul_gather_forward_reducescatter_backward, matmul_with_async_comm, reduce_backward, reduce_forward, split_forward_gather_backward
  from .parallel_module import ParallelModule
  from .utils import create_randomizer_with_offset
label_desc: Use image node_id 2 for calling the GPT2FusedLinearConv1D_Row obj's underlying member method code with example usage: obj.from_native_module(module, process_group) and returns: linear_1d, module