class_docstr: A PyTorch RoBERTa implementation
class_docstr_tok: ["A","PyTorch","RoBERTa","implementation"]
code: def __init__(  # noqa C901
      self,
      config: Config,
      output_encoded_layers: bool,
      token_embedding: nn.Embedding = None,
      **kwarg,
  ) -> None:
      super().__init__(
          config, output_encoded_layers=output_encoded_layers
      )
  
      # map to the real model_path
      config.model_path = (
          resources.roberta.RESOURCE_MAP[config.model_path]
          if config.model_path in resources.roberta.RESOURCE_MAP
          else config.model_path
      )
      # assert config.pretrained_encoder.load_path, "Load path cannot be empty."
      # sharing compression across each layers
  
      # create compress layer if use linear multihead attention
      if config.use_linformer_encoder:
          compress_layer = nn.Linear(
              config.max_seq_len - 2,
              (config.max_seq_len - 2)
              // config.linformer_compressed_ratio,
          )
  
      self.use_selfie_encoder = config.use_selfie_encoder
      self.skip_token_embed = config.skip_token_embed
  
      if config.use_linformer_encoder:
          if config.linformer_quantize:
              layers = [
                  TransformerLayer(
                      embedding_dim=config.embedding_dim,
                      attention=QuantizedMultiheadLinearAttention(
                          embed_dim=config.embedding_dim,
                          num_heads=config.num_attention_heads,
                          compress_layer=compress_layer,
                      ),
                  )
                  for _ in range(config.num_encoder_layers)
              ]
          else:
              layers = [
                  TransformerLayer(
                      embedding_dim=config.embedding_dim,
                      attention=MultiheadLinearAttention(
                          embed_dim=config.embedding_dim,
                          num_heads=config.num_attention_heads,
                          compress_layer=compress_layer,
                      ),
                  )
                  for _ in range(config.num_encoder_layers)
              ]
      else:
          layers = [
              TransformerLayer(
                  embedding_dim=config.embedding_dim,
                  attention=MultiheadSelfAttention(
                      embed_dim=config.embedding_dim,
                      num_heads=config.num_attention_heads,
                      scaling=config.scaling,
                  ),
                  normalize_before=config.normalize_before,
              )
              for _ in range(config.num_encoder_layers)
          ]
      if not config.skip_token_embed:
          self.encoder = (
              SentenceEncoder(
                  transformer=Transformer(
                      vocab_size=config.vocab_size,
                      embedding_dim=config.embedding_dim,
                      layers=layers,
                      max_seq_len=config.max_seq_len,
                      normalize_before=config.normalize_before,
                      token_embedding=token_embedding,
                      use_mixup=config.use_mixup,
                  )
              )
              if not self.use_selfie_encoder
              else PostEncoder(
                  transformer=SELFIETransformer(
                      vocab_size=config.vocab_size,
                      embedding_dim=config.embedding_dim,
                      layers=layers,
                      max_seq_len=config.max_seq_len,
                  )
              )
          )
      else:
          self.encoder = PassthroughEncoder(
              transformer=PassthroughTransformer(
                  vocab_size=config.vocab_size,
                  embedding_dim=config.embedding_dim,
                  layers=layers,
                  max_seq_len=config.max_seq_len,
                  normalize_before=config.normalize_before,
              )
          )
      self.apply(init_params)
  
      if config.prune_before_load:
          self._prune_transformer_layers_and_heads(config)
  
      if config.model_path:
          with PathManager.open(config.model_path, "rb") as f:
              roberta_state = torch.load(
                  f,
                  map_location=lambda s, l: default_restore_location(
                      s, "cpu"
                  ),
              )
          # In case the model has previously been loaded in PyText and finetuned,
          # then we dont need to do the special state dict translation. Load
          # it directly
          if not config.is_finetuned:
              self.encoder.load_roberta_state_dict(
                  roberta_state["model"]
              )
          elif config.load_partial_model is not None:
              roberta_state = {
                  k.replace(config.load_partial_model + ".", ""): v
                  for k, v in roberta_state["model_state"].items()
                  if k.startswith(config.load_partial_model)
              }
              self.load_state_dict(roberta_state)
          else:
              self.load_state_dict(roberta_state)
  
      if config.use_bias_finetuning:
          for n, p in self.encoder.named_parameters():
              # "encoder.transformer.layers.0.attention.input_projection.weight" -> false
              # "encoder.transformer.layers.0.attention.input_projection.bias" -> true
              if n.split(".")[-1] != "bias":
                  p.requires_grad_(False)
  
      if not config.prune_before_load:
          self._prune_transformer_layers_and_heads(config)
  
      self.export_encoder = config.export_encoder
      self.variable_size_embedding = config.variable_size_embedding
      self.use_linformer_encoder = config.use_linformer_encoder
      log_class_usage(__class__)
  
imports: import logging
  from typing import Dict, List, Optional, Tuple
  import torch
  from pytext import resources
  from pytext.common.constants import Stage, TORCH_VERSION
  from pytext.config import ConfigBase
  from pytext.data.roberta_tensorizer import RoBERTaTensorizer, RoBERTaTokenLevelTensorizer
  from pytext.data.tensorizers import FloatListTensorizer, LabelTensorizer, NumericLabelTensorizer, Tensorizer
  from pytext.models.bert_classification_models import NewBertModel
  from pytext.models.bert_regression_model import NewBertRegressionModel
  from pytext.models.decoders.mlp_decoder import MLPDecoder
  from pytext.models.model import BaseModel
  from pytext.models.module import create_module, Module
  from pytext.models.output_layers import WordTaggingOutputLayer
  from pytext.models.representations.transformer import MultiheadLinearAttention, PassthroughEncoder, PassthroughTransformer, PostEncoder, QuantizedMultiheadLinearAttention, SELFIETransformer, SentenceEncoder
  from pytext.models.representations.transformer_sentence_encoder_base import PoolingMethod, TransformerSentenceEncoderBase
  from pytext.models.utils import normalize_embeddings
  from pytext.torchscript.module import ScriptPyTextEmbeddingModuleIndex, ScriptPyTextModule, ScriptPyTextModuleWithDense
  from pytext.utils.file_io import PathManager
  from pytext.utils.usage import log_class_usage
  from pytorch.text.fb.nn.modules.multihead_attention import MultiheadSelfAttention
  from pytorch.text.fb.nn.modules.transformer import DEFAULT_MAX_SEQUENCE_LENGTH, Transformer, TransformerLayer
  from torch import nn
  from torch.serialization import default_restore_location
  from .r3f_models import R3FConfigOptions, R3FPyTextMixin
label_desc: Use image node_id 1 to create a new RoBERTaEncoder object from inherited base classes: RoBERTaEncoderBase with example: obj = RoBERTaEncoder(config, output_encoded_layers, token_embedding)