class_docstr: Tokenizer used for SacreBLEU calculation.
  
  Source: https://github.com/mjpost/sacrebleu/tree/master/sacrebleu/tokenizers
class_docstr_tok: ["Tokenizer","used","for","SacreBLEU","calculation",".","Source",":","https",":","\/\/github.com\/mjpost\/sacrebleu\/tree\/master\/sacrebleu\/tokenizers"]
code: def tokenize(
      cls: Type["_SacreBLEUTokenizer"],
      line: str,
      tokenize: _TokenizersLiteral,
      lowercase: bool = False,
  ) -> Sequence[str]:
      cls._check_tokenizers_validity(tokenize)
  
      tokenize_fn = getattr(cls, cls._TOKENIZE_FN[tokenize])
      tokenized_line = tokenize_fn(line)
      return cls._lower(tokenized_line, lowercase).split()
  
imports: import os
  import re
  import tempfile
  from functools import partial
  from typing import Any, ClassVar, Dict, Optional, Sequence, Type
  import torch
  from torch import Tensor, tensor
  from typing_extensions import Literal
  from torchmetrics.functional.text.bleu import _bleu_score_compute, _bleu_score_update
  from torchmetrics.utilities.imports import _IPADIC_AVAILABLE, _MECAB_AVAILABLE, _MECAB_KO_AVAILABLE, _MECAB_KO_DIC_AVAILABLE, _REGEX_AVAILABLE, _SENTENCEPIECE_AVAILABLE
label_desc: Use image node_id 3 for calling the _SacreBLEUTokenizer obj's underlying member method code with example usage: obj.tokenize(cls, line, tokenize, lowercase) and returns: cls