class_docstr: Tokenizer used for SacreBLEU calculation.
  
  Source: https://github.com/mjpost/sacrebleu/tree/master/sacrebleu/tokenizers
class_docstr_tok: ["Tokenizer","used","for","SacreBLEU","calculation",".","Source",":","https",":","\/\/github.com\/mjpost\/sacrebleu\/tree\/master\/sacrebleu\/tokenizers"]
docstr: Tokenizes an input line with the tokenizer.
  
  Args:
      line: a segment to tokenize
  
  Return:
      the tokenized line
docstr_tok: ["Tokenizes","an","input","line","with","the","tokenizer",".","Args",":","line",":","a","segment","to","tokenize","Return",":","the","tokenized","line"]
code: def _tokenize_base(
      cls: Type["_SacreBLEUTokenizer"], line: str
  ) -> str:
      """Tokenizes an input line with the tokenizer.
  
      Args:
          line: a segment to tokenize
  
      Return:
          the tokenized line
  
      """
      return line
  
imports: import os
  import re
  import tempfile
  from functools import partial
  from typing import Any, ClassVar, Dict, Optional, Sequence, Type
  import torch
  from torch import Tensor, tensor
  from typing_extensions import Literal
  from torchmetrics.functional.text.bleu import _bleu_score_compute, _bleu_score_update
  from torchmetrics.utilities.imports import _IPADIC_AVAILABLE, _MECAB_AVAILABLE, _MECAB_KO_AVAILABLE, _MECAB_KO_DIC_AVAILABLE, _REGEX_AVAILABLE, _SENTENCEPIECE_AVAILABLE
label_desc: Use image node_id 6 for calling the _SacreBLEUTokenizer obj's underlying member method code with example usage: obj._tokenize_base(cls, line) and returns: line