class_docstr: Tokenizer used for SacreBLEU calculation.
  
  Source: https://github.com/mjpost/sacrebleu/tree/master/sacrebleu/tokenizers
class_docstr_tok: ["Tokenizer","used","for","SacreBLEU","calculation",".","Source",":","https",":","\/\/github.com\/mjpost\/sacrebleu\/tree\/master\/sacrebleu\/tokenizers"]
docstr: Tokenizes a string following the official BLEU implementation.
  
  See github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L954-L983
  
  In our case, the input string is expected to be just one line.
  We just tokenize on punctuation and symbols,
  except when a punctuation is preceded and followed by a digit
  (e.g. a comma/dot as a thousand/decimal separator).
  We do not recover escaped forms of punctuations such as &apos; or &gt;
  as these should never appear in MT system outputs (see issue #138)
  
  Note that a number (e.g., a year) followed by a dot at the end of
  sentence is NOT tokenized, i.e. the dot stays with the number because
  `s/(\\p{P})(\\P{N})/ $1 $2/g` does not match this case (unless we add a
  space after each sentence). However, this error is already in the
  original mteval-v14.pl and we want to be consistent with it.
  The error is not present in the non-international version,
  which uses `$norm_text = " $norm_text "`.
  
  Args:
      line: the input string to tokenize.
  
  Return:
      The tokenized string.
docstr_tok: ["Tokenizes","a","string","following","the","official","BLEU","implementation",".","See","github.com\/moses-smt\/mosesdecoder\/blob\/master\/scripts\/generic\/mteval-v14.pl","#","L954-L983","In","our","case",",","the","input","string","is","expected","to","be","just","one","line",".","We","just","tokenize","on","punctuation","and","symbols",",","except","when","a","punctuation","is","preceded","and","followed","by","a","digit","(","e.g",".","a","comma\/dot","as","a","thousand\/decimal","separator",")",".","We","do","not","recover","escaped","forms","of","punctuations","such","as","&","apos",";","or","&","gt",";","as","these","should","never","appear","in","MT","system","outputs","(","see","issue","#","138",")","Note","that","a","number","(","e.g.",",","a","year",")","followed","by","a","dot","at","the","end","of","sentence","is","NOT","tokenized",",","i.e",".","the","dot","stays","with","the","number","because","`","s\/","(","\\\\p","{","P","}",")","(","\\\\P","{","N","}",")","\/","$","1","$","2\/g","`","does","not","match","this","case","(","unless","we","add","a","space","after","each","sentence",")",".","However",",","this","error","is","already","in","the","original","mteval-v14.pl","and","we","want","to","be","consistent","with","it",".","The","error","is","not","present","in","the","non-international","version",",","which","uses","`","$","norm_text","=","``","$","norm_text","``","`",".","Args",":","line",":","the","input","string","to","tokenize",".","Return",":","The","tokenized","string","."]
code: def _tokenize_international(
      cls: Type["_SacreBLEUTokenizer"], line: str
  ) -> str:
      r"""Tokenizes a string following the official BLEU implementation.
  
      See github.com/moses-smt/mosesdecoder/blob/master/scripts/generic/mteval-v14.pl#L954-L983
  
      In our case, the input string is expected to be just one line.
      We just tokenize on punctuation and symbols,
      except when a punctuation is preceded and followed by a digit
      (e.g. a comma/dot as a thousand/decimal separator).
      We do not recover escaped forms of punctuations such as &apos; or &gt;
      as these should never appear in MT system outputs (see issue #138)
  
      Note that a number (e.g., a year) followed by a dot at the end of
      sentence is NOT tokenized, i.e. the dot stays with the number because
      `s/(\\p{P})(\\P{N})/ $1 $2/g` does not match this case (unless we add a
      space after each sentence). However, this error is already in the
      original mteval-v14.pl and we want to be consistent with it.
      The error is not present in the non-international version,
      which uses `$norm_text = " $norm_text "`.
  
      Args:
          line: the input string to tokenize.
  
      Return:
          The tokenized string.
  
      """
      for _re, repl in cls._INT_REGEX:
          line = _re.sub(repl, line)
  
      return " ".join(line.split())
  
imports: import os
  import re
  import tempfile
  from functools import partial
  from typing import Any, ClassVar, Dict, Optional, Sequence, Type
  import torch
  from torch import Tensor, tensor
  from typing_extensions import Literal
  from torchmetrics.functional.text.bleu import _bleu_score_compute, _bleu_score_update
  from torchmetrics.utilities.imports import _IPADIC_AVAILABLE, _MECAB_AVAILABLE, _MECAB_KO_AVAILABLE, _MECAB_KO_DIC_AVAILABLE, _REGEX_AVAILABLE, _SENTENCEPIECE_AVAILABLE
label_desc: Use image node_id 9 for calling the _SacreBLEUTokenizer obj's underlying member method code with example usage: obj._tokenize_international(cls, line) and returns: str